IndexError: string index out of range

[pmvd]

IndexError: string index out of range

[hhatto]

@pmvd
Could you provide a minimum code example where the problem occurs?

[hhatto]

📝

Example code to be reproduced

# target.py
doc, doc2 = ("""
World:
1 + "programming"
+ " pythonあ""",
"""abc"""
)

with Python3.12 (3.12.4)

$ python --version
Python 3.12.4
$ pycodestyle --version
2.12.0
$ pycodestyle target.py
Traceback (most recent call last):
  File "/Users/hideohattori/.pyenv/versions/common3124/bin/pycodestyle", line 8, in <module>
    sys.exit(_main())
             ^^^^^^^
  File "/Users/hideohattori/.pyenv/versions/3.12.4/envs/common3124/lib/python3.12/site-packages/pycodestyle.py", line 2671, in _main
    report = style_guide.check_files()
             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/hideohattori/.pyenv/versions/3.12.4/envs/common3124/lib/python3.12/site-packages/pycodestyle.py", line 2371, in check_files
    runner(path)
  File "/Users/hideohattori/.pyenv/versions/3.12.4/envs/common3124/lib/python3.12/site-packages/pycodestyle.py", line 2383, in input_file
    return fchecker.check_all(expected=expected, line_offset=line_offset)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/hideohattori/.pyenv/versions/3.12.4/envs/common3124/lib/python3.12/site-packages/pycodestyle.py", line 2137, in check_all
    self.check_logical()
  File "/Users/hideohattori/.pyenv/versions/3.12.4/envs/common3124/lib/python3.12/site-packages/pycodestyle.py", line 1993, in check_logical
    mapping = self.build_tokens_line()
              ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/hideohattori/.pyenv/versions/3.12.4/envs/common3124/lib/python3.12/site-packages/pycodestyle.py", line 1973, in build_tokens_line
    prev_text = self.lines[prev_row - 1][prev_col - 1]
                ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^
IndexError: string index out of range

with Python3.11 (3.11.7)

$ python --version
Python 3.11.7
$ pycodestyle --version
2.12.0
$ pycodestyle target.py
t3_2.py:6:1: E128 continuation line under-indented for visual indent
t3_2.py:7:1: E124 closing bracket does not match visual indentation

Why?

The behaviour of tokenize.generate_tokens in Python 3.12 may have changed.

# sample1.py
import tokenize
from io import StringIO

code = """doc, doc2 = (\"\"\"
World:
1 + "programming"
+ " pythonあ\"\"\",
'''abc'''
)
"""

def tokenize_code(code):
    tokens = []
    for tok in tokenize.generate_tokens(StringIO(code).readline):
        tokens.append((tok.type, tok.string, tok.start, tok.end, tok.line))
    return tokens

tokens_311 = tokenize_code(code)
for token in tokens_311:
    print(token)

print(tokenize.untokenize(tokens_311))

with Python3.11.7

~/.pyenv/versions/3.11.7/bin/python sample1.py
(1, 'doc', (1, 0), (1, 3), 'doc, doc2 = ("""\n')
(54, ',', (1, 3), (1, 4), 'doc, doc2 = ("""\n')
(1, 'doc2', (1, 5), (1, 9), 'doc, doc2 = ("""\n')
(54, '=', (1, 10), (1, 11), 'doc, doc2 = ("""\n')
(54, '(', (1, 12), (1, 13), 'doc, doc2 = ("""\n')
(3, '"""\nWorld:\n1 + "programming"\n+ " pythonあ"""', (1, 13), (4, 14), 'doc, doc2 = ("""\nWorld:\n1 + "programming"\n+ " pythonあ""",\n')
(54, ',', (4, 14), (4, 15), '+ " pythonあ""",\n')
(62, '\n', (4, 15), (4, 16), '+ " pythonあ""",\n')
(3, "'''abc'''", (5, 0), (5, 9), "'''abc'''\n")
(62, '\n', (5, 9), (5, 10), "'''abc'''\n")
(54, ')', (6, 0), (6, 1), ')\n')
(4, '\n', (6, 1), (6, 2), ')\n')
(0, '', (7, 0), (7, 0), '')
doc, doc2 = ("""
World:
1 + "programming"
+ " pythonあ""",
'''abc'''
)

with Python3.12.4

~/.pyenv/versions/3.12.4/bin/python sample1.py
(1, 'doc', (1, 0), (1, 3), 'doc, doc2 = ("""\n')
(55, ',', (1, 3), (1, 4), 'doc, doc2 = ("""\n')
(1, 'doc2', (1, 5), (1, 9), 'doc, doc2 = ("""\n')
(55, '=', (1, 10), (1, 11), 'doc, doc2 = ("""\n')
(55, '(', (1, 12), (1, 13), 'doc, doc2 = ("""\n')
(3, '"""\nWorld:\n1 + "programming"\n+ " pythonあ"""', (1, 13), (4, 14), 'doc, doc2 = ("""\nWorld:\n1 + "programming"\n+ " pythonあ""",\n')
(55, ',', (4, 16), (4, 17), '+ " pythonあ""",\n')
(65, '\n', (4, 17), (4, 18), '+ " pythonあ""",\n')
(3, "'''abc'''", (5, 0), (5, 9), "'''abc'''\n")
(65, '\n', (5, 9), (5, 10), "'''abc'''\n")
(55, ')', (6, 0), (6, 1), ')\n')
(4, '\n', (6, 1), (6, 2), ')\n')
(0, '', (7, 0), (7, 0), '')
doc, doc2 = ("""
World:
1 + "programming"
+ " pythonあ"""  ,
'''abc'''
)