With python 3.12, we now get following warning when running the script:
SyntaxWarning: invalid escape sequence '\#'
This and other minor issues have been fixed upstream since we first
imported the script, so let's sync and import the content of following
Linux v6.11-rc6 commits:
074075aea2ff scripts/clang-tools: switch explicitly to Python 3
585d32f9b053 gen_compile_commands: prune some directories
265264b814c2 gen_compile_commands: extract compiler command from a series of commands
ec783c7cb249 gen_compile_commands: fix missing 'sys' package
a4ab14e1d8fe gen_compile_commands: handle multiple lines per .mod file
f65a486821cf kbuild: change module.order to list *.o instead of *.ko
92215e7a801d kbuild: rename cmd_$@ to savedcmd_$@ in *.cmd files
1c67921444bf gen_compile_commands: add assembly files to compilation database
52c15e7e7928 gen_compile_commands: Allow the line prefix to still be cmd_
9e56d3be4bfd gen_compile_commands: Sort output compile commands by file name
880946158b01 gen_compile_commands.py: fix path resolve with symlinks in it
dae4a0171e25 gen_compile_commands: fix invalid escape sequence warning
There is one commit that we don't take over though as that doesn't apply
to barebox yet:
321648455061 kbuild: use obj-y instead extra-y for objects placed at the head
Signed-off-by: Ahmad Fatoum <a.fatoum@xxxxxxxxxxxxxx>
---
scripts/clang-tools/gen_compile_commands.py | 50 ++++++++++++---------
1 file changed, 29 insertions(+), 21 deletions(-)
diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py
index 7ed3919f453a..b8ec35e65ae6 100755
--- a/scripts/clang-tools/gen_compile_commands.py
+++ b/scripts/clang-tools/gen_compile_commands.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0
#
# Copyright (C) Google LLC, 2018
@@ -11,17 +11,19 @@ import argparse
import json
import logging
import os
-import sys
import re
import subprocess
+import sys
_DEFAULT_OUTPUT = 'compile_commands.json'
_DEFAULT_LOG_LEVEL = 'WARNING'
_FILENAME_PATTERN = r'^\..*\.cmd$'
-_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$'
+_LINE_PATTERN = r'^(saved)?cmd_[^ ]*\.o := (?P<command_prefix>.* )(?P<file_path>[^ ]*\.[cS]) *(;|$)'
_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
-
+# The tools/ directory adopts a different build system, and produces .cmd
+# files in a different format. Do not support it.
+_EXCLUDE_DIRS = ['.git', 'Documentation', 'include', 'tools']
def parse_arguments():
"""Sets up and parses command-line arguments.
@@ -62,7 +64,7 @@ def parse_arguments():
args = parser.parse_args()
return (args.log_level,
- os.path.abspath(args.directory),
+ os.path.realpath(args.directory),
args.output,
args.ar,
args.paths if len(args.paths) > 0 else [args.directory])
@@ -81,8 +83,14 @@ def cmdfiles_in_dir(directory):
"""
filename_matcher = re.compile(_FILENAME_PATTERN)
+ exclude_dirs = [ os.path.join(directory, d) for d in _EXCLUDE_DIRS ]
+
+ for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
+ # Prune unwanted directories.
+ if dirpath in exclude_dirs:
+ dirnames[:] = []
+ continue
- for dirpath, _, filenames in os.walk(directory):
for filename in filenames:
if filename_matcher.match(filename):
yield os.path.join(dirpath, filename)
@@ -144,15 +152,15 @@ def cmdfiles_for_modorder(modorder):
"""
with open(modorder) as f:
for line in f:
- ko = line.rstrip()
- base, ext = os.path.splitext(ko)
- if ext != '.ko':
- sys.exit('{}: module path must end with .ko'.format(ko))
+ obj = line.rstrip()
+ base, ext = os.path.splitext(obj)
+ if ext != '.o':
+ sys.exit('{}: module path must end with .o'.format(obj))
mod = base + '.mod'
- # The first line of *.mod lists the objects that compose the module.
+ # Read from *.mod, to get a list of objects that compose the module.
with open(mod) as m:
- for obj in m.readline().split():
- yield to_cmdfile(obj)
+ for mod_line in m:
+ yield to_cmdfile(mod_line.rstrip())
def process_line(root_directory, command_prefix, file_path):
@@ -176,10 +184,10 @@ def process_line(root_directory, command_prefix, file_path):
# escape the pound sign '#', either as '\#' or '$(pound)' (depending on the
# kernel version). The compile_commands.json file is not interepreted
# by Make, so this code replaces the escaped version with '#'.
- prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#')
+ prefix = command_prefix.replace(r'\#', '#').replace('$(pound)', '#')
- # Use os.path.abspath() to normalize the path resolving '.' and '..' .
- abs_path = os.path.abspath(os.path.join(root_directory, file_path))
+ # Return the canonical path, eliminating any symbolic links encountered in the path.
+ abs_path = os.path.realpath(os.path.join(root_directory, file_path))
if not os.path.exists(abs_path):
raise ValueError('File %s not found' % abs_path)
return {
@@ -208,10 +216,10 @@ def main():
# Modules are listed in modules.order.
if os.path.isdir(path):
cmdfiles = cmdfiles_in_dir(path)
- elif path.endswith('.o'):
- cmdfiles = cmdfiles_for_o(path)
elif path.endswith('.a'):
cmdfiles = cmdfiles_for_a(path, ar)
+ elif path.endswith('.o'):
+ cmdfiles = cmdfiles_for_o(path)
elif path.endswith('modules.order'):
cmdfiles = cmdfiles_for_modorder(path)
else:
@@ -222,15 +230,15 @@ def main():
result = line_matcher.match(f.readline())
if result:
try:
- entry = process_line(directory, result.group(1),
- result.group(2))
+ entry = process_line(directory, result.group('command_prefix'),
+ result.group('file_path'))
compile_commands.append(entry)
except ValueError as err:
logging.info('Could not add line from %s: %s',
cmdfile, err)
with open(output, 'wt') as f:
- json.dump(compile_commands, f, indent=2, sort_keys=True)
+ json.dump(sorted(compile_commands, key=lambda x: x["file"]), f, indent=2, sort_keys=True)
if __name__ == '__main__':
--
2.39.2
