Uname:Linux ip-10-0-5-176 6.8.0-1057-aws #60~22.04.1-Ubuntu SMP Wed May 27 08:16:59 UTC 2026 x86_64

403WebShell
403Webshell
Server IP : 3.111.61.48  /  Your IP : 216.73.216.67
Web Server : Apache
System : Linux ip-10-0-5-176 6.8.0-1057-aws #60~22.04.1-Ubuntu SMP Wed May 27 08:16:59 UTC 2026 x86_64
User : ubuntu ( 1000)
PHP Version : 8.2.31
Disable Function : exec,passthru,shell_exec,system
MySQL : OFF  |  cURL : ON  |  WGET : ON  |  Perl : ON  |  Python : OFF  |  Sudo : ON  |  Pkexec : ON
Directory :  /usr/lib/python3.10/test/__pycache__/

Upload File :
current_dir [ Writeable ] document_root [ Writeable ]

 

Command :

[ Back ]     

Current File : /usr/lib/python3.10/test/__pycache__/test_tokenize.cpython-310.pyc
o

�9j�@s�ddlmZddlmZddlmZmZmZmZmZm	Z	m
Z
mZmZm
Z
mZmZmZmZmZddlmZmZddlZddlmZddlmZmZddlmZmZddlmZdd	lm Z m!Z!ddl"Z"ddl#Z#d
d�Z$Gdd
�d
e�Z%Gdd�de%�Z&dd�Z'Gdd�de�Z(Gdd�de�Z)Gdd�de�Z*Gdd�de�Z+Gdd�de�Z,Gdd�de�Z-Gdd�de�Z.Gd d!�d!ej�Z/e0d"kr�e�1�dSdS)#�)�support)�	os_helper)�tokenize�	_tokenize�
untokenize�NUMBER�NAME�OP�STRING�	ENDMARKER�ENCODING�tok_name�detect_encoding�open�Untokenizer�generate_tokens�NEWLINE)�BytesIO�StringION)�dedent)�TestCase�mock)�VALID_UNDERSCORE_LITERALS�INVALID_UNDERSCORE_LITERALS)�run_test_script�make_scriptc
Cs�g}t|���}|ddv}|D]2\}}}}}	|tkr|S|r+|tkr+|d|kr+qt|}|�d|d�d|d�d|�d|���q|S)N���z
r�    �10� �13)�len�
splitlinesrrr
�append)
�token_generator�
source_string�result�	num_lines�missing_trailing_nl�type�token�start�end�line�r.�)/usr/lib/python3.10/test/test_tokenize.py�stringify_tokens_from_sources�(r0c@�eZdZdd�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dd�Zdd�Zdd�Z
dd�Zdd�Zdd�Zdd�Zdd �Zd!d"�Zd#d$�Zd%d&�Zd'd(�Zd)S)*�TokenizeTestcCs<t|�d��}tt|j�|�}|�|dg|�����dS)N�utf-8z*    ENCODING   'utf-8'       (0, 0) (0, 0))r�encoder0r�readline�assertEqual�rstripr"��self�s�expected�fr&r.r.r/�check_tokenize.s
��zTokenizeTest.check_tokenizecCsDtd�d��}tt|j��}|�|djt�|�|djt�dS)N�xr3���r)	rr4�listrr5r6r)rr)r9r<�tokensr.r.r/�test_implicit_newline7sz"TokenizeTest.test_implicit_newlinecCsf|�dd�|�dd�d}t|�j}|�td��t|�D]}qWd�dS1s,wYdS)Nz1 + 1z�    NUMBER     '1'           (1, 0) (1, 1)
    OP         '+'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    z3if False:
    # NL
    
    True = False # NEWLINE
af    NAME       'if'          (1, 0) (1, 2)
    NAME       'False'       (1, 3) (1, 8)
    OP         ':'           (1, 8) (1, 9)
    NEWLINE    '\n'          (1, 9) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NL         '\n'          (3, 4) (3, 5)
    INDENT     '    '        (4, 0) (4, 4)
    NAME       'True'        (4, 4) (4, 8)
    OP         '='           (4, 9) (4, 10)
    NAME       'False'       (4, 11) (4, 16)
    COMMENT    '# NEWLINE'   (4, 17) (4, 26)
    NEWLINE    '\n'          (4, 26) (4, 27)
    DEDENT     ''            (5, 0) (5, 0)
    sdef k(x):
    x += 2
  x += 5
z3unindent does not match any outer indentation level)r=rr5�assertRaisesRegex�IndentationErrorr)r9�indent_error_filer5�tokr.r.r/�
test_basic?s�
��"�zTokenizeTest.test_basiccCsp|�dd�|�dd�|�dd�|�dd�|�d	d
�|�dd�|�d
d�|�dd�|�dd�dS)Nz0xff <= 255z�    NUMBER     '0xff'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    z0b10 <= 255z�    NUMBER     '0b10'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    z0o123 <= 0O123z�    NUMBER     '0o123'       (1, 0) (1, 5)
    OP         '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    z1234567 > ~0x15z�    NUMBER     '1234567'     (1, 0) (1, 7)
    OP         '>'           (1, 8) (1, 9)
    OP         '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    z2134568 != 1231515z�    NUMBER     '2134568'     (1, 0) (1, 7)
    OP         '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    z(-124561-1) & 200000000ac    OP         '('           (1, 0) (1, 1)
    OP         '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    OP         '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         ')'           (1, 10) (1, 11)
    OP         '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    z0xdeadbeef != -1z�    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    OP         '!='          (1, 11) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    z0xdeadc0de & 12345z�    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    OP         '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    z0xFF & 0x15 | 1234z�    NUMBER     '0xFF'        (1, 0) (1, 4)
    OP         '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    OP         '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    �r=�r9r.r.r/�test_intds
zTokenizeTest.test_intcCs4|�dd�|�dd�|�dd�|�dd�dS)	Nzx = 0z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0'           (1, 4) (1, 5)
    zx = 0xfffffffffffz�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0xfffffffffff' (1, 4) (1, 17)
    zx = 123141242151251616110z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '123141242151251616110' (1, 4) (1, 25)
    zx = -15921590215012591z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '-'           (1, 4) (1, 5)
    NUMBER     '15921590215012591' (1, 5) (1, 22)
    rHrIr.r.r/�	test_long�szTokenizeTest.test_longcCsX|�dd�|�dd�|�dd�|�dd�|�d	d
�|�dd�|�d
d�dS)Nzx = 3.14159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    zx = 314159.z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    zx = .314159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    zx = 3e14159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    z	x = 3E123z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    z
x+y = 3e-1230z�    NAME       'x'           (1, 0) (1, 1)
    OP         '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    OP         '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    zx = 3.14e159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    rHrIr.r.r/�
test_float�szTokenizeTest.test_floatcCsJdd�}tD]}d|vr
q|�||�|�qtD]
}|�||�|�qdS)NcSs<t|�d��}t|j�D]\}}}}}|tkr|SqdS)Nr3z
invalid token)rr4rr5r)r:r<�toktyper*r+r,r-r.r.r/�number_token�s�z;TokenizeTest.test_underscore_literals.<locals>.number_token�()rr6r�assertNotEqual)r9rN�litr.r.r/�test_underscore_literals�s�z%TokenizeTest.test_underscore_literalscCs<|�dd�|�dd�|�dd�|�dd�|�d	d
�|�dd�|�d
d�|�dd�|�dd�|�dd�|�dd�|�dd�|�dd�|�dd�|�dd�|�dd �|�d!d"�|�d#d$�|�d%d&�|�d'd(�|�d)d*�|�d+d,�|�d-d.�|�d/d0�|�d1d2�|�d3d4�dS)5Nzx = ''; y = ""a5    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    OP         ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    zx = '"'; y = "'"a6    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    OP         ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    OP         '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    zx = "doesn't "shrink", does it"z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    zx = 'abc' + 'ABC'z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    zy = "ABC" + "ABC"z�    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    z%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    z%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a�    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    zu'abc' + U'abc'z�    STRING     "u'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    zu"abc" + U"abc"z�    STRING     'u"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    zb'abc' + B'abc'z�    STRING     "b'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    zb"abc" + B"abc"z�    STRING     'b"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    z%br'abc' + bR'abc' + Br'abc' + BR'abc'a;    STRING     "br'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    z%br"abc" + bR"abc" + Br"abc" + BR"abc"a;    STRING     'br"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    z%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a;    STRING     "rb'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    z%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a;    STRING     'rb"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    z"a\
de\
fg"z3    STRING     '"a\\\nde\\\nfg"' (1, 0) (3, 3)
    zu"a\
de"z/    STRING     'u"a\\\nde"'  (1, 0) (2, 3)
    zrb"a\
d"z/    STRING     'rb"a\\\nd"'  (1, 0) (2, 2)
    z
"""a\
b"""z0    STRING     '"""a\\\nb"""' (1, 0) (2, 4)
    zu"""a\
b"""z1    STRING     'u"""a\\\nb"""' (1, 0) (2, 4)
    zrb"""a\
b\
c"""z7    STRING     'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4)
    zf"abc"z/    STRING     'f"abc"'      (1, 0) (1, 6)
    z	fR"a{b}c"z/    STRING     'fR"a{b}c"'   (1, 0) (1, 9)
    z
f"""abc"""z0    STRING     'f"""abc"""'  (1, 0) (1, 10)
    zf"abc\
def"z1    STRING     'f"abc\\\ndef"' (1, 0) (2, 4)
    zRf"abc\
def"z2    STRING     'Rf"abc\\\ndef"' (1, 0) (2, 4)
    rHrIr.r.r/�test_string�sT					
��������zTokenizeTest.test_stringcCs(|�dd�|�dd�|�dd�dS)Nz!def d22(a, b, c=2, d=2, *k): passa    NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    OP         ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    OP         '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    OP         '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    OP         ','           (1, 22) (1, 23)
    OP         '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    OP         ')'           (1, 26) (1, 27)
    OP         ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    zdef d01v_(a=1, *k, **w): passa�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    OP         '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         ','           (1, 13) (1, 14)
    OP         '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    OP         '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    OP         ')'           (1, 22) (1, 23)
    OP         ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    z&def d23(a: str, b: int=3) -> int: passa�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    OP         ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    OP         '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    OP         ')'           (1, 24) (1, 25)
    OP         '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    OP         ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    rHrIr.r.r/�
test_function�szTokenizeTest.test_functioncC�|�dd�dS)NzXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa�    NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    OP         '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    OP         '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    OP         '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    OP         '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    OP         ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    rHrIr.r.r/�test_comparison�s�zTokenizeTest.test_comparisoncCrU)Nzx = 1 << 1 >> 5a6    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '<<'          (1, 6) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '>>'          (1, 11) (1, 13)
    NUMBER     '5'           (1, 14) (1, 15)
    rHrIr.r.r/�
test_shift��zTokenizeTest.test_shiftcCrU)Nz%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    OP         '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    OP         '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    OP         '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    OP         '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    OP         '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    OP         ']'           (1, 36) (1, 37)
    rHrIr.r.r/�
test_additiverXzTokenizeTest.test_additivecCrU)Nzx = 1//1*1/5*12%0x12@42a�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    OP         '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    OP         '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    OP         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    rHrIr.r.r/�test_multiplicativerXz TokenizeTest.test_multiplicativecCs|�dd�|�dd�dS)Nz~1 ^ 1 & 1 |1 ^ -1a�    OP         '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '^'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    z-1*1/1+1*1//1 - ---1**1aH    OP         '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    OP         '-'           (1, 17) (1, 18)
    OP         '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    OP         '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    rHrIr.r.r/�
test_unary+s
zTokenizeTest.test_unarycCrU)Nz/import sys, time
x = sys.modules['time'].time()a�    NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    OP         ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'x'           (2, 0) (2, 1)
    OP         '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    OP         '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    OP         '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    OP         ']'           (2, 22) (2, 23)
    OP         '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    OP         '('           (2, 28) (2, 29)
    OP         ')'           (2, 29) (2, 30)
    rHrIr.r.r/�
test_selectorPrXzTokenizeTest.test_selectorcCrU)Nz @staticmethod
def foo(x,y): passa    OP         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    '\n'          (1, 13) (1, 14)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    OP         '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    OP         ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    OP         ')'           (2, 11) (2, 12)
    OP         ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    rHrIr.r.r/�test_methodfrXzTokenizeTest.test_methodcCrU)Nzdef f():
	if x
        	passa_    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    INDENT     '\t'          (2, 0) (2, 1)
    NAME       'if'          (2, 1) (2, 3)
    NAME       'x'           (2, 4) (2, 5)
    NEWLINE    '\n'          (2, 5) (2, 6)
    INDENT     '        \t'  (3, 0) (3, 9)
    NAME       'pass'        (3, 9) (3, 13)
    DEDENT     ''            (4, 0) (4, 0)
    DEDENT     ''            (4, 0) (4, 0)
    rHrIr.r.r/�	test_tabsws�zTokenizeTest.test_tabscCrU)Nu!Örter = 'places'
grün = 'green'u7    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "'places'"    (1, 8) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "'green'"     (2, 7) (2, 14)
    rHrIr.r.r/�test_non_ascii_identifiers�rXz'TokenizeTest.test_non_ascii_identifierscCrU)Nu#Örter = u'places'
grün = U'green'u7    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    '\n'          (1, 17) (1, 18)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    rHrIr.r.r/�test_unicode�rXzTokenizeTest.test_unicodecCs|�dd�|�dd�|�dd�|�dd�|�d	d
�|�dd�|�d
d�|�dd�|�dd�|�dd�|�dd�|�dd�|�dd�|�dd�|�dd�|�dd �|�d!d"�|�d#d$�|�d%d&�|�d'd(�|�d)d*�dS)+Nz	async = 1z�    NAME       'async'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    za = (async = 1)a8    NAME       'a'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '('           (1, 4) (1, 5)
    NAME       'async'       (1, 5) (1, 10)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    zasync()z�    NAME       'async'       (1, 0) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    zclass async(Bar):passa<    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    OP         ')'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    zclass async:passz�    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    z	await = 1z�    NAME       'await'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    z	foo.asyncz�    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    zasync for a in b: passa;    NAME       'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    zasync with a as b: passa<    NAME       'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    z	async.fooz�    NAME       'async'       (1, 0) (1, 5)
    OP         '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    �asyncz/    NAME       'async'       (1, 0) (1, 5)
    zasync
#comment
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    COMMENT    '#comment'    (2, 0) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NAME       'await'       (3, 0) (3, 5)
    zasync
...
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    OP         '...'         (2, 0) (2, 3)
    NEWLINE    '\n'          (2, 3) (2, 4)
    NAME       'await'       (3, 0) (3, 5)
    zasync
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    NAME       'await'       (2, 0) (2, 5)
    z
foo.async + 1z�    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    zasync def foo(): passa;    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    zNasync def foo():
  def foo(await):
    await = 1
  if 1:
    await
async += 1
a�    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    OP         '('           (2, 9) (2, 10)
    NAME       'await'       (2, 10) (2, 15)
    OP         ')'           (2, 15) (2, 16)
    OP         ':'           (2, 16) (2, 17)
    NEWLINE    '\n'          (2, 17) (2, 18)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'await'       (3, 4) (3, 9)
    OP         '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    '\n'          (3, 13) (3, 14)
    DEDENT     ''            (4, 2) (4, 2)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    OP         ':'           (4, 6) (4, 7)
    NEWLINE    '\n'          (4, 7) (4, 8)
    INDENT     '    '        (5, 0) (5, 4)
    NAME       'await'       (5, 4) (5, 9)
    NEWLINE    '\n'          (5, 9) (5, 10)
    DEDENT     ''            (6, 0) (6, 0)
    DEDENT     ''            (6, 0) (6, 0)
    NAME       'async'       (6, 0) (6, 5)
    OP         '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    '\n'          (6, 10) (6, 11)
    z)async def foo():
  async for i in 1: passa�    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    OP         ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (3, 0) (3, 0)
    zasync def foo(async): awaitah    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    NAME       'async'       (1, 14) (1, 19)
    OP         ')'           (1, 19) (1, 20)
    OP         ':'           (1, 20) (1, 21)
    NAME       'await'       (1, 22) (1, 27)
    z@def f():

  def baz(): pass
  async def bar(): pass

  await = 2a�    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    zFasync def f():

  def baz(): pass
  async def bar(): pass

  await = 2a    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    OP         '('           (1, 11) (1, 12)
    OP         ')'           (1, 12) (1, 13)
    OP         ':'           (1, 13) (1, 14)
    NEWLINE    '\n'          (1, 14) (1, 15)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    rHrIr.r.r/�
test_async�s:




�,��%�zTokenizeTest.test_asyncN)�__name__�
__module__�__qualname__r=rBrGrJrKrLrRrSrTrVrWrYrZr[r\r]r^r_r`rbr.r.r.r/r2's,	%9('<&%r2c@�eZdZdd�ZdS)�GenerateTokensTestcCs0t|�}tt|j�|�}|�||�����dS�N)rr0rr5r6r7r"r8r.r.r/r=�sz!GenerateTokensTest.check_tokenizeN)rcrdrer=r.r.r.r/rg��rgcCs�g}tt|�d��j�}|D]*\}}}}}|tkr1d|vr1|�tdftdftt	|�ftdfg�q|�
||f�qt|��d�S)Nr3�.�DecimalrO�))
rrr4r5r�extendrr	r
�reprr#r�decode)r:r&�g�toknum�tokval�_r.r.r/�decistmt�s
�rtc@rf)�TestMisccCsPddlm}d}|�t|�d�|�tt|��d�|�tt|��|d��dS)Nr)rkz+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)�decimalrkr6rt�assertRegexrn�eval)r9rkr:r.r.r/�
test_decistmt�s
��zTestMisc.test_decistmtN)rcrdreryr.r.r.r/ru�sruc@s@eZdZdZdd�Zdd�Zdd�Zdd	�Zd
d�Zdd
�Z	dS)�TestTokenizerAdheresToPep0263zU
    Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263.
    cCs,tj�tj�t�|�}t�|t|d��dS)N�rb)�os�path�join�dirname�__file__�
TestRoundtrip�check_roundtripr)r9�filenamer}r.r.r/�	_testFile�sz'TestTokenizerAdheresToPep0263._testFilecC�d}|�|�dS)Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt�r��r9r<r.r.r/�'test_utf8_coding_cookie_and_no_utf8_bom��zETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bomcCsd}|�t|j|�dS)a�
        As per PEP 0263, if a file starts with a utf-8 BOM signature, the only
        allowed encoding for the comment is 'utf-8'.  The text file used in
        this test starts with a BOM signature, but specifies latin1 as the
        coding, so verify that a SyntaxError is raised, which matches the
        behaviour of the interpreter when it encounters a similar condition.
        z8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtN��assertRaises�SyntaxErrorr�r�r.r.r/�&test_latin1_coding_cookie_and_utf8_bom�szDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bomcCr�)Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtr�r�r.r.r/�"test_no_coding_cookie_and_utf8_bom�r�z@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bomcCr�)Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtr�r�r.r.r/�$test_utf8_coding_cookie_and_utf8_bomr�zBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bomcCs$|�t|jd�|�t|jd�dS)Nz
bad_coding.pyzbad_coding2.pyr�rIr.r.r/�test_bad_coding_cookiesz4TestTokenizerAdheresToPep0263.test_bad_coding_cookieN)
rcrdre�__doc__r�r�r�r�r�r�r.r.r.r/rz�srzc@seZdZdd�Zdd�ZdS)�
Test_TokenizecsPd}|�d��d���fdd�}tt|dd��dd�}d	g}|�||d
�dS)N�"ЉЊЈЁЂ"r3Fc��sd��SdS�NT�r.r.��firstr-r.r/r5�zNTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readline��encoding�r?��r�)r�r)r��r�zbytes not decoded with encoding)r4r@rr6)r9�literalr5rA�expected_tokensr.r�r/�.test__tokenize_decodes_with_specified_encodings
	�z<Test_Tokenize.test__tokenize_decodes_with_specified_encodingcsFd�d���fdd�}tt|dd��dd�}dg}|�||d�dS)	Nr�Fcr�r�r.r.�r�r�r.r/r5!r�zQTest_Tokenize.test__tokenize_does_not_decode_with_encoding_none.<locals>.readliner�r?r�z*string not tokenized when encoding is None)r@rr6)r9r5rAr�r.r�r/�1test__tokenize_does_not_decode_with_encoding_nones	�z?Test_Tokenize.test__tokenize_does_not_decode_with_encoding_noneN)rcrdrer�r�r.r.r.r/r�
sr�c@r1)*�TestDetectEncodingcsd���fdd�}|S)Nrcs$�t��krt���}�d7�|S�Nr�)r!�
StopIteration)r-��index�linesr.r/r54s
z1TestDetectEncoding.get_readline.<locals>.readliner.�r9r�r5r.r�r/�get_readline2szTestDetectEncoding.get_readlinecCs>d}t|�|��\}}|�|d�|�|t|dd���dS)N)�# something
�print(something)
�do_something(else)
r3�)rr�r6r@�r9r�r��consumed_linesr.r.r/�test_no_bom_no_encoding_cookie=sz1TestDetectEncoding.test_no_bom_no_encoding_cookiecC�6d}t|�|��\}}|�|d�|�|ddg�dS)N)s# something
r�r��	utf-8-sigr�r��rr�r6r�r.r.r/�test_bom_no_cookieGs�z%TestDetectEncoding.test_bom_no_cookiecC�4d}t|�|��\}}|�|d�|�|dg�dS)N)�# -*- coding: latin-1 -*-
r�r��
iso-8859-1r�r�r�r.r.r/�test_cookie_first_line_no_bomR�z0TestDetectEncoding.test_cookie_first_line_no_bomcCr�)N)s# coding=utf-8
r�r�r�s# coding=utf-8
r�r�r.r.r/�&test_matched_bom_and_cookie_first_line\r�z9TestDetectEncoding.test_matched_bom_and_cookie_first_linecC� d}|�|�}|�tt|�dS)N)s## vim: set fileencoding=ascii :
r�r��r�r�r�rr�r.r.r/�<test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorfs
zOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorcC�:d}t|�|��\}}|�|d�ddg}|�||�dS)N)�
#! something
� # vim: set fileencoding=ascii :
r�r��asciir�r�r��r9r�r�r�r;r.r.r/�test_cookie_second_line_no_bomos
z1TestDetectEncoding.test_cookie_second_line_no_bomcCr�)N)�#! something
�f# coding=utf-8
r�r�r�r�r�r�r�r.r.r/�'test_matched_bom_and_cookie_second_line{s�z:TestDetectEncoding.test_matched_bom_and_cookie_second_linecCr�)N)r�r�r�r�r�r�r.r.r/�=test_mismatched_bom_and_cookie_second_line_raises_syntaxerror�s
zPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorcCs8d}t|�|��\}}|�|d�dg}|�||�dS)N)�print('£')
�%# vim: set fileencoding=iso8859-15 :
�
print('€')
r3r�r�r�r.r.r/�/test_cookie_second_line_noncommented_first_line�s
zBTestDetectEncoding.test_cookie_second_line_noncommented_first_linecCr�)N)�
#print('£')
r�r��
iso8859-15r�r�r�r�r.r.r/�,test_cookie_second_line_commented_first_line��
z?TestDetectEncoding.test_cookie_second_line_commented_first_linecCr�)N)�
r�r�r�r�r�r�r�r.r.r/�(test_cookie_second_line_empty_first_line�r�z;TestDetectEncoding.test_cookie_second_line_empty_first_linec	Csdd}|D]+}dD]&}|�d|�}dd|�d�ddd	f}|�|�}t|�\}}|�|d
�qqdS)N)zlatin-1r�ziso-latin-1zlatin-1-unixziso-8859-1-unixziso-latin-1-mac��-rsr��#!/usr/bin/python
�
# coding: r�r�sprint(things)
sdo_something += 4
r���replacer4r�rr6�	r9�	encodingsr��rep�encr��rl�foundr�r.r.r/�test_latin1_normalization�s�
��z,TestDetectEncoding.test_latin1_normalizationcCr�)N�s
print("�")r�r�r.r.r/�test_syntaxerror_latin1�s
z*TestDetectEncoding.test_syntaxerror_latin1c	Csbd}|D]*}dD]%}|�d|�}dd|�d�ddf}|�|�}t|�\}}|�|d	�qqdS)
N)r3z	utf-8-macz
utf-8-unixr�r�r�r�r�r�s1 + 3
r3r�r�r.r.r/�test_utf8_normalization�s�
��z*TestDetectEncoding.test_utf8_normalizationcCs�|�d�}t|�\}}|�|d�|�|dg�t|�d��\}}|�|d�|�|g�|�d�}t|�\}}|�|d�|�|dg�|�d�}t|�\}}|�|d�|�|g�|�d�}|�tt|�dS)	N)r�r3r�r.)sprint(something)
r�)s)s# coding: bad
)r�rr6r�r��r9r5r�r�r.r.r/�test_short_files�s"



z#TestDetectEncoding.test_short_filescCs4|�d�}t|�\}}|�|d�|�|dg�dS)N)�print("#coding=fake")r3r�)r�rr6r�r.r.r/�test_false_encoding�s
z&TestDetectEncoding.test_false_encodingc	Cs4tjd}|�tj|�dD]I}t|d|d��}td||d�td|d�Wd�n1s0wYt|��}|�|j|�|�|j	d�Wd�n1sRwYqt|dd	d��}td|d�Wd�n1spwYt|��}|�|jd	�|�|j	d�Wd�dS1s�wYdS)
Nz.py)ziso-8859-15r3�wr�z# coding: %s��fileuprint('euro:€')�rr�)
r�TESTFN�
addCleanup�unlinkr�print�
tokenize_openr6r��mode)r9r�r��fpr.r.r/�	test_open�s&
�
���
"�zTestDetectEncoding.test_opencs�d}d�G�fdd�d�}|�t��|�|�}|`t|j�Wd�n1s)wY|�td�|���|�|�}t|j�Wd�dS1sMwYdS)N�some_file_pathr�cs eZdZdd�Z�fdd�ZdS)z;TestDetectEncoding.test_filename_in_exception.<locals>.BunkcSs||_||_d|_dS)Nr)�name�_lines�_index)r9r�r}r.r.r/�__init__s
zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__cs.|jt��kr	t��|j}|jd7_|Sr�)r�r!r�)r9r-�r�r.r/r5s

zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readlineN)rcrdrer�r5r.r�r.r/�Bunksr�z.*{})r�r�r�rr5rC�format)r9r}r��insr.r�r/�test_filename_in_exception
s
�
"�z-TestDetectEncoding.test_filename_in_exceptioncCsTtd�}tjd|d��|�ttd�Wd�n1swY|�|j�dS)Ns#coding:xxxztokenize._builtin_open)�return_value�foobar)rr�patchr�r�r��
assertTrue�closed)r9�mr.r.r/�test_open_error&s
�z"TestDetectEncoding.test_open_errorN)rcrdrer�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�rr.r.r.r/r�0s*


	


r�c@s<eZdZdd�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
S)�TestTokenizecs�ddl}t��d��fdd�}�fdd�}d��fdd�}|j}|j}||_||_zt|�}|�t|�gd��W||_||_n||_||_w|����dS)	Nrcs�ddgfS)N�first�secondr.)r5r�r.r/�mock_detect_encoding4sz8TestTokenize.test_tokenize.<locals>.mock_detect_encodingcs$|�g}	|�}|r|�|�q|Srh)r#)r5r��out�	next_line)�
encoding_usedr.r/�mock__tokenize7s
z2TestTokenize.test_tokenize.<locals>.mock__tokenizecs �d7��dkr
dSt����S)Nr��r�)�strr4r.)�counterr.r/�
mock_readlineCsz1TestTokenize.test_tokenize.<locals>.mock_readline)rr�1�2�3�4)r�objectrrr6r@)r9�tokenize_moduler	r
r�orig_detect_encoding�orig__tokenize�resultsr.)rr�rr/�
test_tokenize0s*
��zTestTokenize.test_tokenizecCsfg}td�D]}|�dj|d��q|�d�d�|�}ttt|�d��j��}|�	|dj
d�dS)Ni�zdef i{i}(): return {i})�i�OK�
r3���)�ranger#r�r~r@rrr4r5r6�string)r9�bufr�toksr.r.r/�test_oneline_defsXs

zTestTokenize.test_oneline_defscGs�ttt|�d��j��}t|�}|�t|�d|�|�t|djtt	�t
|�D]}|�t||djt||�q+|�t|d|jttj�|�t|d|jttj
�dS)Nr3r�rr�r�)r@rrr4r5r!r6r
�
exact_typerr r*rr)r9�opstr�optypesrA�num_optypesrr.r.r/�assertExactTypeEqualds �
���z!TestTokenize.assertExactTypeEqualcCs�|�dtjtj�|�dtjtj�|�dtj�|�dtj�|�dtj�|�dtj	�|�dtj
�|�dtj�|�d	tj�|�d
tj
�|�dtj�|�dtj�|�d
tj�|�dtj�|�dtj�|�dtj�|�dtjtj�|�dtj�|�dtj�|�dtj�|�dtj�|�dtj�|�dtj�|�dtj�|�dtj�|�dtj�|�dtj�|�dtj �|�dtj!�|�dtj"�|�dtj#�|�d tj$�|�d!tj%�|�d"tj&�|�d"tj&�|�d#tj'�|�d$tj(�|�d%tj)�|�d&tj*�|�d'tj+�|�d(tj,�|�d)tj-�|�d*tj.�|�d+tj/�|�d,tj0�|�d-t1tjt2tj	t1tjt2tjt1tjt2�|�d.tjtj2tjtj2tjtj2tj�|�d/tjtjtj1tjtj2tj�dS)0Nz()z[]�:�,�;�+r��*�/�|�&�<�>�=rj�%z{}z==z!=z<=z>=�~�^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->�@z@=za**2+b**2==c**2z	{1, 2, 3}z
^(x & 0x1))3r)r*�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENT�LBRACE�RBRACE�EQEQUAL�NOTEQUAL�	LESSEQUAL�GREATEREQUAL�TILDE�
CIRCUMFLEX�	LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR�	PLUSEQUAL�MINEQUAL�	STAREQUAL�
SLASHEQUAL�PERCENTEQUAL�
AMPEREQUAL�	VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL�DOUBLESLASH�DOUBLESLASHEQUAL�
COLONEQUAL�ELLIPSIS�RARROW�AT�ATEQUALrrrIr.r.r/�test_exact_typers����zTestTokenize.test_exact_typecCs|�dtj�dS)Nz@          )r)r*rfrIr.r.r/�%test_pathological_trailing_whitespace�sz2TestTokenize.test_pathological_trailing_whitespacecCs�d}tjtjtjtjtjtjg}ttt	|�
d��j��}|�t
|djt
t�td�D]}|�t
||djt
||�q-|�t
|djt
tj�dS)Nzb = 1

#testr3r�r�r)r*rrHrr�NL�COMMENTr@rrr4r5r6r
r%rr r)r9�sourcer�rArr.r.r/�5test_comment_at_the_end_of_the_source_without_newline�s$ zBTestTokenize.test_comment_at_the_end_of_the_source_without_newlineN)	rcrdrerr$r)rhrirnr.r.r.r/r.s(Arc@s$eZdZdd�Zdd�Zdd�ZdS)�UntokenizeTestcCsnt�}d|_d|_|�t��
}|�d�Wd�n1swY|�|jjdd�|�t|jd�dS)Nr�)r�r�rz'start (1,3) precedes previous end (2,2))r�r�)	r�prev_row�prev_colr��
ValueError�add_whitespacer6�	exception�args)r9�u�cmr.r.r/�test_bad_input_order�s��z#UntokenizeTest.test_bad_input_ordercCsdt�}d|_d|_g|_|�d�|�|jdg�d|_|�d�|�|jgd��t�|d�dS)Nr�)r�r�\
r�)�rz)ryz\
\
rza
  b
    c
  \
  c
)rrprqrArsr6r�r�)r9rvr.r.r/�test_backslash_continuation�s

z*UntokenizeTest.test_backslash_continuationcCs�t�}tdf}tdf|g}|�|tg��|�|jdg�t�}|�|�t|g��d�t�}|�|�t|��d�|�|jd�|�tt|��d�dS)N�Hellor3zHello sHello )	rrr�compat�iterr6rArr�)r9rvr*rAr.r.r/�test_iter_compat�szUntokenizeTest.test_iter_compatN)rcrdrerxr{rr.r.r.r/ro�s
roc@sLeZdZdd�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dS)r�cCs�t|t�r|�d�}n|��}|��t|jdd��j}tt	|��}dd�|D�}t
|�}t|jdd��j}dd�t	|�D�}|�||�t
|�}	t|	jdd��j}
dd�t	|
�D�}|�||�dS)	a)
        Test roundtrip for `untokenize`. `f` is an open file or a string.
        The source code in f is tokenized to both 5- and 2-tuples.
        Both sequences are converted back to source code via
        tokenize.untokenize(), and the latter tokenized again to 2-tuples.
        The test fails if the 3 pair tokenizations do not match.

        When untokenize bugs are fixed, untokenize with 5-tuples should
        reproduce code that does not contain a backslash continuation
        following spaces.  A proper test should test this.
        r3T)�keependscS�g|]}|dd��qS�Nr�r.��.0rFr.r.r/�
<listcomp>�z1TestRoundtrip.check_roundtrip.<locals>.<listcomp>cSr�r�r.r�r.r.r/r�r�cSr�r�r.r�r.r.r/r�
r�N)�
isinstancerr4�read�closer~r"�__next__r@rrr6)r9r<�coder5�tokens5�tokens2�bytes_from2�	readline2�
tokens2_from2�bytes_from5�	readline5�
tokens2_from5r.r.r/r��s

zTestRoundtrip.check_roundtripcCs�|�d�|�d�|�d�t�d�}t|d��
}|�|�Wd�n1s)wY|�d�|�d�|�d�|�d	�dS)
Nzif x == 1:
    print(x)
z # This is a comment
# This also
zif x == 1 : 
  print(x)
�tokenize_tests.txtr{zoif x == 1:
    # A comment by itself.
    print(x) # Comment here, too.
    # Another comment.
after_if = True
zPif (x # The comments need to go in the right place
    == 1):
    print('x==1')
z�class Test: # A comment here
  # A comment with weird indent
  after_com = 5
  def x(m): return m*5 # a one liner
  def y(m): # A whitespace after the colon
     return y*4 # 3-space indent
zttry: import somemodule
except ImportError: # comment
    print('Can not import' # comment2
)else:   print('Loaded')
)r�r�findfiler)r9�fnr<r.r.r/�test_roundtrip
s



�


zTestRoundtrip.test_roundtripcC�|�d�dS)Nzna = (3,4, 
5,6)
y = [3, 4,
5]
z = {'a': 5,
'b':15, 'c':True}
x = len(y) + 5 - a[
3] - a[2]
+ len(z) - z[
'b']
�r�rIr.r.r/�test_continuation4�zTestRoundtrip.test_continuationcCs|�d�|�d�dS)Nz)x=1+\
1
# This is a comment\
# This also
z# Comment \
x = 0r�rIr.r.r/r{As
z)TestRoundtrip.test_backslash_continuationcCr�)Nz'' ''r�rIr.r.r/�test_string_concatenationJr�z'TestRoundtrip.test_string_concatenationc
Csddl}ddl}t�d�}tj�|�ptj}|�tj�|�	|�d��}|�
tj�|d��dD]}|�
tj�|d�|�q/t�d�sI|�|d�}|D]<}tj
d	krWtd
|�t|d��#}|j|d��
|�|�Wd�n1sswYWd�n1s�wYqKdS)
Nrr�ztest*.pyztest_unicode_identifiers.py)�buffer�builtin�fileio�inspectr|�platform�sysz
test_%s.py�cpu�
r�rr{r�)�glob�randomrr�r|r}r�curdirr~�escape�remove�is_resource_enabled�sample�verboser�r�subTestr�)r9r�r�r��tempdir�	testfilesr<�testfiler.r.r/�test_random_filesNs(



�����zTestRoundtrip.test_random_filescCs,t|t�r
|�d�}ttt|�j���d�S)Nr3)r�rr4rrrr5ro)r9r�r.r.r/�	roundtripks

zTestRoundtrip.roundtripcCs6d}|�|��d�}|�|d|d�|�|�dS)z�
        Ensure that although whitespace might be mutated in a roundtrip,
        the semantic meaning of the indentation remains consistent.
        zif False:
	x=3
	x=3
rr�r�N)r��splitr6r�)r9r��	codelinesr.r.r/�#test_indentation_semantics_retainedpsz1TestRoundtrip.test_indentation_semantics_retainedN)rcrdrer�r�r�r{r�r�r�r�r.r.r.r/r��s '
	r�c@rf)�CTokenizerBufferTestscCsZddd�ddd��}t���}t|d|�}t|�Wd�dS1s&wYdS)Nz"        #coding: latin-1
        #�ai'z

        #i'�foo)r�temp_dirrr)r9�test_scriptr��	file_namer.r.r/�!test_newline_at_the_end_of_buffer|s��

"�z7CTokenizerBufferTests.test_newline_at_the_end_of_bufferN)rcrdrer�r.r.r.r/r�{rir��__main__)2�testr�test.supportrrrrrrr	r
rrr
rrr�rrr�iorr�unittest�textwraprrr�test.test_grammarrr�test.support.script_helperrrr|r*r0r2rgrtrurzr�r�rror�r�rc�mainr.r.r.r/�<module>sJD	%&)�

Youez - 2016 - github.com/yon3zu
LinuXploit