Python 3.6 introduced a DeprecationWarning for invalid escape sequences.
This is upgraded to a SyntaxWarning in Python 3.12 (3.12.0rc1), and is
intended to eventually be a syntax error.
This series aims to fix these now to get ahead of it before it's an error.
Most of the patches are generated using the below Python script. It
uses the builtin ast module to parse a Python file, locate all strings,
find the corresponding source code, and check for bad escape sequences.
If it finds things to fix then it applies the fixes and reparses the
file into a fixed AST. It finally compares the original and fixed ASTs
to ensure no semantic difference has been introduced (dumping is done to
remove node location information, which is expected to be different).
There are some limitations of the ast module, in particular it throws
away a lot of information about the string source. f-strings especially
interact poorly here (the slices between formats are presented as
separate strings, but the node range of each is the entire f-string),
so are skipped. f-strings are handled manually in the final patch.
A lot of the fixes are for regex patterns, which could be changed to use
r-strings instead. But that is less easy to automate, so I avoided doing
so in this series. AST verification should still be possible though,
because being a plain or r-string is stripped away in the AST.
---
#!/usr/bin/env python3
"""
Fix all bad escape characters in strings
"""
import ast
from pathlib import Path
def get_offset(source: str, row: int, col: int) -> int:
"""
Turn a row + column pair into a byte offset
"""
offset = 0
cur_row = 1 # 1-indexed rows
cur_col = 0 # 0-indexed columns
for c in source:
if cur_row == row and cur_col == col:
return offset
offset += 1
if c == "\n":
cur_row += 1
cur_col = 0
else:
cur_col += 1
raise Exception("Failed to get offset")
parse_failures: list[Path] = []
fix_failures = 0
bad_escapes = 0
fstrings: set[Path] = set()
for pyfile in Path(".").glob("**/*.py"):
content = pyfile.read_text("utf-8")
try:
syntax = ast.parse(content, filename=pyfile)
except:
print(f"{pyfile}: ERROR Failed to parse, is it Python3?")
parse_failures.append(pyfile)
continue
fixes: list[tuple[int, int, str]] = []
for node in ast.walk(syntax):
if not isinstance(node, ast.Constant):
continue
if not isinstance(node.value, str):
continue
if node.value.count("\\") == 0:
continue
assert(isinstance(node.end_lineno, int))
assert(isinstance(node.end_col_offset, int))
start = get_offset(content, node.lineno, node.col_offset)
end = get_offset(content, node.end_lineno, node.end_col_offset)
raw = content[start:end]
# backslashes in r-strings are already literal
if raw.startswith("r"):
continue
# f-strings are difficult to handle correctly
if raw.startswith("f"):
fstrings.add(pyfile)
continue
fixed = "" # The fixed representation of the string
escaped = False # If the current character is escaped by a previous backslash
allowed = '\n\\\'"abfnrtv01234567xNuU' # characters allowed after a backslash
for c in raw:
if escaped:
if c not in allowed:
fixed += '\\'
fixed += c
escaped = False
continue
fixed += c
if c == '\\':
escaped = True
if fixed != raw:
print(f"{pyfile}:{node.lineno}:{node.col_offset}: FOUND {raw}")
fixes.append((start, end, fixed))
if len(fixes) == 0:
continue
bad_escapes += len(fixes)
# Apply fixes in reverse order to keep offsets valid
for start, end, fix in reversed(sorted(fixes, key=lambda k: k[0])):
print(f"{pyfile}:[{start}-{end}]: APPLY {fix}")
content = content[:start] + fix + content[end:]
fixed_syntax = ast.parse(content, filename=f"{pyfile}+fixed")
if ast.dump(syntax) != ast.dump(fixed_syntax):
print(f"{pyfile}: ERROR Fixed syntax tree yields different AST")
fix_failures += 1
continue
pyfile.write_text(content)
print(f"---------------------------------")
print(f"Parse failures: {len(parse_failures)}")
for f in sorted(parse_failures):
print(f" - {f}")
print(f"Bad escapes fixed: {bad_escapes}")
print(f"Fixes that broke the AST: {fix_failures}")
print(f"Files with skipped f-strings: {len(fstrings)}")
for f in sorted(fstrings):
print(f" - {f}")
---
Benjamin Gray (8):
ia64: fix Python string escapes
Documentation/sphinx: fix Python string escapes
drivers/comedi: fix Python string escapes
scripts: fix Python string escapes
tools/perf: fix Python string escapes
tools/power: fix Python string escapes
selftests/bpf: fix Python string escapes
selftests/bpf: fix Python string escapes in f-strings
Documentation/sphinx/cdomain.py | 2 +-
Documentation/sphinx/kernel_abi.py | 2 +-
Documentation/sphinx/kernel_feat.py | 2 +-
Documentation/sphinx/kerneldoc.py | 2 +-
Documentation/sphinx/maintainers_include.py | 8 +--
arch/ia64/scripts/unwcheck.py | 2 +-
.../ni_routing/tools/convert_csv_to_c.py | 2 +-
scripts/bpf_doc.py | 56 +++++++++----------
scripts/clang-tools/gen_compile_commands.py | 2 +-
scripts/gdb/linux/symbols.py | 2 +-
tools/perf/pmu-events/jevents.py | 2 +-
.../scripts/python/arm-cs-trace-disasm.py | 4 +-
tools/perf/scripts/python/compaction-times.py | 2 +-
.../scripts/python/exported-sql-viewer.py | 4 +-
tools/power/pm-graph/bootgraph.py | 12 ++--
.../selftests/bpf/test_bpftool_synctypes.py | 26 ++++-----
tools/testing/selftests/bpf/test_offload.py | 2 +-
17 files changed, 66 insertions(+), 66 deletions(-)
--
2.41.0
