diff --git a/tools/targets/cdk.py b/tools/targets/cdk.py index 73fb06559ca..63e79e8eb99 100644 --- a/tools/targets/cdk.py +++ b/tools/targets/cdk.py @@ -31,6 +31,9 @@ from utils import _make_path_relative from utils import xml_indent +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import target_utils + def SDKAddGroup(ProjectFiles, parent, name, files, project_path): # don't add an empty group if len(files) == 0: @@ -41,11 +44,11 @@ def SDKAddGroup(ProjectFiles, parent, name, files, project_path): for f in files: fn = f.rfile() name = fn.name - path = os.path.dirname(fn.abspath) + dir_path = os.path.dirname(fn.abspath) - basename = os.path.basename(path) - path = _make_path_relative(project_path, path) - elm_attr_name = os.path.join(path, name) + # stable, single-separator relative path (was _make_path_relative + + # os.path.join, which mixed '/' and '\\'); ElementTree escapes on write + elm_attr_name = target_utils.normalize_group_file_path(project_path, dir_path, name) file = SubElement(group, 'File', attrib={'Name': elm_attr_name}) @@ -112,11 +115,14 @@ def _CDKProject(tree, target, script): IncludePath = tree.find('BuildConfigs/BuildConfig/Asm/IncludePath') IncludePath.text = text + # fold tuple macros to FOO=1, order-preserving de-dup. + # '; '.join(set(...)) raised TypeError on a ('FOO','1') tuple and reordered. + defines = target_utils.normalize_defines(CPPDEFINES) Define = tree.find('BuildConfigs/BuildConfig/Compiler/Define') - Define.text = '; '.join(set(CPPDEFINES)) - + Define.text = '; '.join(defines) + Define = tree.find('BuildConfigs/BuildConfig/Asm/Define') - Define.text = '; '.join(set(CPPDEFINES)) + Define.text = '; '.join(defines) CC_Misc = tree.find('BuildConfigs/BuildConfig/Compiler/OtherFlags') CC_Misc.text = CCFLAGS diff --git a/tools/targets/cmake.py b/tools/targets/cmake.py index e1840e68bd2..deb1e8ef9c5 100644 --- a/tools/targets/cmake.py +++ b/tools/targets/cmake.py @@ -29,6 +29,9 @@ from utils import _make_path_relative from collections import defaultdict, Counter +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import target_utils + def GenerateCFiles(env, project, project_name): """ @@ -63,15 +66,13 @@ def GenerateCFiles(env, project, project_name): OBJCOPY = tool_path_conv["CMAKE_OBJCOPY"]["path"] FROMELF = tool_path_conv["CMAKE_FROMELF"]["path"] - CFLAGS = "".join(env['CFLAGS']) - CFLAGS = CFLAGS.replace('\\', "/").replace('\"', "\\\"") + CFLAGS = target_utils.escape_quoted_flags("".join(env['CFLAGS'])) if 'CXXFLAGS' in dir(rtconfig): - cflag_str=''.join(env['CXXFLAGS']) - CXXFLAGS = cflag_str.replace('\\', "/").replace('\"', "\\\"") + CXXFLAGS = target_utils.escape_quoted_flags(''.join(env['CXXFLAGS'])) else: CXXFLAGS = CFLAGS - AFLAGS = env['ASFLAGS'].replace('\\', "/").replace('\"', "\\\"") - LFLAGS = env['LINKFLAGS'].replace('\\', "/").replace('\"', "\\\"") + AFLAGS = target_utils.escape_quoted_flags(env['ASFLAGS']) + LFLAGS = target_utils.escape_quoted_flags(env['LINKFLAGS']) POST_ACTION = rtconfig.POST_ACTION # replace the tool name with the cmake variable @@ -179,34 +180,19 @@ def GenerateCFiles(env, project, project_name): cm_file.write('\n') + # include paths: relative, de-duplicated, and quoted so spaces / ';' + # cannot break the CMake list + inc_paths = target_utils.normalize_paths( + [_make_path_relative(os.getcwd(), i) for i in env['CPPPATH']]) cm_file.write("INCLUDE_DIRECTORIES(\n") - for i in env['CPPPATH']: - # use relative path - path = _make_path_relative(os.getcwd(), i) - cm_file.write( "\t" + path.replace("\\", "/") + "\n") - cm_file.write(")\n\n") + cm_file.write(target_utils.cmake_list(inc_paths)) + cm_file.write("\n)\n\n") + # defines: fold SCons str/tuple/list into NAME / NAME=value, stable order + defines = ["-D" + d for d in target_utils.normalize_defines(env['CPPDEFINES'])] cm_file.write("ADD_DEFINITIONS(\n") - for i in env['CPPDEFINES']: - # Handle CPPDEFINES from SCons (str / tuple) - if isinstance(i, tuple): - # e.g. ('STM32F407xx',) - if len(i) == 1: - cm_file.write("\t-D" + str(i[0]) + "\n") - # e.g. ('FOO', None) - elif len(i) == 2: - if i[1] is None: - cm_file.write("\t-D" + str(i[0]) + "\n") - # e.g. ('FOO', 1) - else: - cm_file.write("\t-D{}={}\n".format(i[0], i[1])) - else: - # unexpected form, fallback to name only - cm_file.write("\t-D" + str(i[0]) + "\n") - else: - # generic macro (commonly a string), ensure robust string conversion - cm_file.write("\t-D" + str(i) + "\n") - cm_file.write(")\n\n") + cm_file.write(target_utils.cmake_list(defines)) + cm_file.write("\n)\n\n") libgroups = [] interfacelibgroups = [] @@ -236,12 +222,13 @@ def GenerateCFiles(env, project, project_name): cm_file.write("# Library source files\n") for group in project: + src_paths = target_utils.normalize_paths( + [os.path.normpath(f.rfile().abspath) for f in group['src']], + ) + src_paths = [_make_path_relative(os.getcwd(), p) for p in src_paths] cm_file.write("SET(RT_{:s}_SOURCES\n".format(group['name'].upper())) - for f in group['src']: - # use relative path - path = _make_path_relative(os.getcwd(), os.path.normpath(f.rfile().abspath)) - cm_file.write( "\t" + path.replace("\\", "/") + "\n" ) - cm_file.write(")\n\n") + cm_file.write(target_utils.cmake_list(src_paths)) + cm_file.write("\n)\n\n") cm_file.write("# Library search paths\n") for group in libgroups + interfacelibgroups: @@ -251,10 +238,10 @@ def GenerateCFiles(env, project, project_name): if len(group['LIBPATH']) == 0: continue + lib_dirs = target_utils.normalize_paths(group['LIBPATH']) cm_file.write("SET(RT_{:s}_LINK_DIRS\n".format(group['name'].upper())) - for f in group['LIBPATH']: - cm_file.write("\t"+ f.replace("\\", "/") + "\n" ) - cm_file.write(")\n\n") + cm_file.write(target_utils.cmake_list(lib_dirs)) + cm_file.write("\n)\n\n") cm_file.write("# Library local macro definitions\n") for group in libgroups: @@ -264,10 +251,10 @@ def GenerateCFiles(env, project, project_name): if len(group['LOCAL_CPPDEFINES']) == 0: continue + local_defines = target_utils.normalize_defines(group['LOCAL_CPPDEFINES']) cm_file.write("SET(RT_{:s}_DEFINES\n".format(group['name'].upper())) - for f in group['LOCAL_CPPDEFINES']: - cm_file.write("\t"+ f.replace("\\", "/") + "\n" ) - cm_file.write(")\n\n") + cm_file.write(target_utils.cmake_list(local_defines)) + cm_file.write("\n)\n\n") cm_file.write("# Library dependencies\n") for group in libgroups + interfacelibgroups: @@ -277,10 +264,10 @@ def GenerateCFiles(env, project, project_name): if len(group['LIBS']) == 0: continue + libs = [target_utils.normalize_path(f) for f in group['LIBS']] cm_file.write("SET(RT_{:s}_LIBS\n".format(group['name'].upper())) - for f in group['LIBS']: - cm_file.write("\t"+ "{}\n".format(f.replace("\\", "/"))) - cm_file.write(")\n\n") + cm_file.write(target_utils.cmake_list(libs)) + cm_file.write("\n)\n\n") cm_file.write("# Libraries\n") for group in libgroups: diff --git a/tools/targets/codeblocks.py b/tools/targets/codeblocks.py index 8e29f438005..d8787d077b5 100644 --- a/tools/targets/codeblocks.py +++ b/tools/targets/codeblocks.py @@ -35,6 +35,9 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import building +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import target_utils + import xml.etree.ElementTree as etree fs_encoding = sys.getfilesystemencoding() @@ -48,21 +51,24 @@ def CB_AddHeadFiles(program, elem, project_path): # print utils.source_list for f in utils.source_list: - path = _make_path_relative(project_path, f) + # stable, single-separator path; ElementTree escapes & < > " on write. + # (was path.decode(fs_encoding) -- str has no .decode on Python 3) + path = target_utils.normalize_group_file_path(project_path, f) Unit = SubElement(elem, 'Unit') - Unit.set('filename', path.decode(fs_encoding)) + Unit.set('filename', path) def CB_AddCFiles(ProjectFiles, parent, gname, files, project_path): for f in files: fn = f.rfile() name = fn.name - path = os.path.dirname(fn.abspath) + dir_path = os.path.dirname(fn.abspath) - path = _make_path_relative(project_path, path) - path = os.path.join(path, name) + # normalized relative path (was _make_path_relative + os.path.join, + # which mixed '/' and '\\'; and path.decode crashed on Python 3) + path = target_utils.normalize_group_file_path(project_path, dir_path, name) Unit = SubElement(parent, 'Unit') - Unit.set('filename', path.decode(fs_encoding)) + Unit.set('filename', path) Option = SubElement(Unit, 'Option') Option.set('compilerVar', "CC") @@ -95,12 +101,9 @@ def CBProject(target, script, program): # write head include path if 'CPPPATH' in building.Env: cpp_path = building.Env['CPPPATH'] - paths = set() - for path in cpp_path: - inc = _make_path_relative(project_path, os.path.normpath(path)) - paths.add(inc) #.replace('\\', '/') - - paths = [i for i in paths] + # order-preserving de-dup, then sort (set() reordered non-deterministically) + paths = target_utils.normalize_paths( + [_make_path_relative(project_path, os.path.normpath(path)) for path in cpp_path]) paths.sort() # write include path, definitions for elem in tree.iter(tag='Compiler'): @@ -109,10 +112,13 @@ def CBProject(target, script, program): Add = SubElement(elem, 'Add') Add.set('directory', path) - for macro in building.Env.get('CPPDEFINES', []): + # one per define. The old `for d in macro` + # iterated the *characters* of a string macro (leaving only -D) + # and dropped the name of a ('STM32','1') tuple; normalize_defines folds + # tuples to STM32=1 and yields one complete macro per entry. + for macro in target_utils.normalize_defines(building.Env.get('CPPDEFINES', [])): Add = SubElement(elem, 'Add') - for d in macro: - Add.set('option', "-D"+d) + Add.set('option', "-D" + macro) # write link flags ''' @@ -139,5 +145,7 @@ def CBProject(target, script, program): elem.set('AdditionalLibraryDirectories', lib_paths) ''' xml_indent(root) - out.write(etree.tostring(root, encoding='utf-8')) + # encoding='unicode' yields str for the text-mode file; encoding='utf-8' + # returns bytes and raised TypeError on write under Python 3 + out.write(etree.tostring(root, encoding='unicode')) out.close() diff --git a/tools/targets/eclipse.py b/tools/targets/eclipse.py index 050d8a184f3..74477a5ecca 100644 --- a/tools/targets/eclipse.py +++ b/tools/targets/eclipse.py @@ -24,6 +24,9 @@ from utils import _make_path_relative from utils import xml_indent +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import target_utils + MODULE_VER_NUM = 6 source_pattern = ['*.c', '*.cpp', '*.cxx', '*.cc', '*.s', '*.S', '*.asm','*.cmd'] @@ -179,7 +182,9 @@ def HandleToolOption(tools, env, project, reset): is_cpp_prj = IsCppProject() BSP_ROOT = os.path.abspath(env['BSP_ROOT']) - CPPDEFINES = project['CPPDEFINES'] + # fold tuple/list macros into NAME / NAME=value strings up front so the + # downstream str ops (.replace('=', ' '), set diff) can never hit a tuple + CPPDEFINES = target_utils.normalize_defines(project['CPPDEFINES']) paths = [ConverToRttEclipsePathFormat(RelativeProjectPath(env, os.path.normpath(i)).replace('\\', '/')) for i in project['CPPPATH']] compile_include_paths_options = [] @@ -282,7 +287,8 @@ def HandleToolOption(tools, env, project, reset): else: project_defs += [item.get('value')] if len(project_defs) > 0: - cproject_defs = set(CPPDEFINES) - set(project_defs) + existing = set(project_defs) + cproject_defs = [d for d in CPPDEFINES if d not in existing] else: cproject_defs = CPPDEFINES @@ -295,10 +301,10 @@ def HandleToolOption(tools, env, project, reset): if linker_scriptfile_option is not None : option = linker_scriptfile_option linker_script = 'link.lds' - items = env['LINKFLAGS'].split(' ') - if '-T' in items: - linker_script = items[items.index('-T') + 1] - linker_script = ConverToRttEclipsePathFormat(linker_script) + # robust parse: handles -Tfoo, -T foo, -T "dir with space/x", -Wl,-T,x + scripts = target_utils.normalize_link_script_flags(env['LINKFLAGS']) + if scripts: + linker_script = ConverToRttEclipsePathFormat(scripts[0]) listOptionValue = option.find('listOptionValue') if listOptionValue != None: @@ -309,14 +315,16 @@ def HandleToolOption(tools, env, project, reset): # scriptfile in stm32cubeIDE if linker_script_option is not None : option = linker_script_option - items = env['LINKFLAGS'].split(' ') - if '-T' in items: - linker_script = ConverToRttEclipsePathFormat(items[items.index('-T') + 1]).strip('"') + scripts = target_utils.normalize_link_script_flags(env['LINKFLAGS']) + if scripts: + linker_script = ConverToRttEclipsePathFormat(scripts[0]).strip('"') option.set('value', linker_script) # update nostartfiles config if linker_nostart_option is not None : option = linker_nostart_option - if env['LINKFLAGS'].find('-nostartfiles') != -1: + # match the flag as a whole token, not a substring of another flag + link_tokens = target_utils.normalize_flags(env['LINKFLAGS']) + if '-nostartfiles' in link_tokens: option.set('value', 'true') else: option.set('value', 'false') diff --git a/tools/targets/esp_idf.py b/tools/targets/esp_idf.py index b6bdf8856af..d9790cd33db 100644 --- a/tools/targets/esp_idf.py +++ b/tools/targets/esp_idf.py @@ -1,8 +1,12 @@ import os import re +import sys import utils from utils import _make_path_relative +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import target_utils + def GenerateCFiles(env,project): """ Generate CMakeLists.txt files @@ -16,22 +20,27 @@ def GenerateCFiles(env,project): cm_file.write("idf_component_register(\n") cm_file.write("\tSRCS\n") + src_paths = [] for group in project: for f in group['src']: path = _make_path_relative(main_component_dir, os.path.normpath(f.rfile().abspath)) - cm_file.write( "\t" + path.replace("\\", "/") + "\n" ) + src_paths.append(path.replace("\\", "/")) src = open(f.rfile().abspath, 'r') for line in src.readlines(): if re.match(r'INIT_(BOARD|PREV|DEVICE|COMPONENT|ENV|APP)_EXPORT\(.+\)', line): init_export.append(re.search(r'\(.+\)', line).group(0)[1:-1]) src.close() + # double-quote each item so a path with a space is one CMake argument, + # and cmake_quote escapes an embedded ';' / '"' / '\\'; also de-duplicated + for p in target_utils.ordered_unique(src_paths): + cm_file.write('\t"' + target_utils.cmake_quote(p) + '"\n') cm_file.write("\n") cm_file.write("\tINCLUDE_DIRS\n") - for i in info['CPPPATH']: - path = _make_path_relative(main_component_dir, i) - cm_file.write( "\t" + path.replace("\\", "/") + "\n") + inc_paths = [_make_path_relative(main_component_dir, i).replace("\\", "/") for i in info['CPPPATH']] + for p in target_utils.ordered_unique(inc_paths): + cm_file.write('\t"' + target_utils.cmake_quote(p) + '"\n') cm_file.write(")\n\n") n = len(init_export) diff --git a/tools/targets/iar.py b/tools/targets/iar.py index ee53ad2e001..edfc76cb0af 100644 --- a/tools/targets/iar.py +++ b/tools/targets/iar.py @@ -34,6 +34,9 @@ from utils import _make_path_relative from utils import xml_indent +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import target_utils + fs_encoding = sys.getfilesystemencoding() iar_workspace = r''' @@ -56,24 +59,26 @@ def IARAddGroup(parent, name, files, project_path): for f in files: fn = f.rfile() name = fn.name - path = os.path.dirname(fn.abspath) - basename = os.path.basename(path) - path = _make_path_relative(project_path, path) - path = os.path.join(path, name) + dir_path = os.path.dirname(fn.abspath) + # stable relative path using IAR's '\\' convention (was mixed '/' + '\\' + # from _make_path_relative + os.path.join); ElementTree escapes the text + path = target_utils.normalize_group_file_path(project_path, dir_path, name, sep='\\') file = SubElement(group, 'file') file_name = SubElement(file, 'name') if os.path.isabs(path): - file_name.text = path # path.decode(fs_encoding) + file_name.text = path else: - file_name.text = '$PROJ_DIR$\\' + path # ('$PROJ_DIR$\\' + path).decode(fs_encoding) + file_name.text = '$PROJ_DIR$\\' + path def IARWorkspace(target): # make an workspace workspace = target.replace('.ewp', '.eww') out = open(workspace, 'w') - xml = iar_workspace % target + # the target path is interpolated into raw XML text under the $WS_DIR$\ + # convention: normalize to '\\' separators and escape &, <, >, quotes + xml = iar_workspace % target_utils.xml_path_attr(target, sep='\\') out.write(xml) out.close() @@ -152,22 +157,26 @@ def searchLib(group): state.text = '$PROJ_DIR$\\' + path if name.text == 'CCDefines': - for define in CPPDEFINES: + # fold tuple/list macros to FOO=1; a raw tuple in state.text would + # crash ElementTree serialization + for define in target_utils.normalize_defines(CPPDEFINES): state = SubElement(option, 'state') state.text = define - for define in LOCAL_CPPDEFINES: + for define in target_utils.normalize_defines(LOCAL_CPPDEFINES): state = SubElement(option, 'state') state.text = define if name.text == 'IlinkAdditionalLibs': for path in Libs: state = SubElement(option, 'state') + # normalize separators; the old .decode(fs_encoding) crashed on + # Python 3 (str has no decode). ElementTree escapes the text. + path = target_utils.normalize_path(path, force_posix=False) if os.path.isabs(path) or path.startswith('$'): - path = path.decode(fs_encoding) + state.text = path else: - path = ('$PROJ_DIR$\\' + path).decode(fs_encoding) - state.text = path + state.text = '$PROJ_DIR$\\' + path xml_indent(root) out.write(etree.tostring(root, encoding='utf-8').decode()) diff --git a/tools/targets/keil.py b/tools/targets/keil.py index 452f273dffd..c6a867a1345 100644 --- a/tools/targets/keil.py +++ b/tools/targets/keil.py @@ -32,6 +32,9 @@ from utils import _make_path_relative from utils import xml_indent +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import target_utils + fs_encoding = sys.getfilesystemencoding() def _get_filetype(fn): @@ -64,11 +67,12 @@ def MDK4AddGroupForFN(ProjectFiles, parent, name, filename, project_path): group_name.text = name name = os.path.basename(filename) - path = os.path.dirname (filename) + dir_path = os.path.dirname(filename) - basename = os.path.basename(path) - path = _make_path_relative(project_path, path) - path = os.path.join(path, name) + basename = os.path.basename(dir_path) + # stable, single-separator relative path (was _make_path_relative + os.path.join, + # which mixed '/' and '\\' in the serialized FilePath depending on the OS) + path = target_utils.normalize_group_file_path(project_path, dir_path, name) files = SubElement(group, 'Files') file = SubElement(files, 'File') file_name = SubElement(file, 'FileName') @@ -88,28 +92,22 @@ def MDK4AddGroupForFN(ProjectFiles, parent, name, filename, project_path): if ProjectFiles.count(obj_name): name = basename + '_' + name ProjectFiles.append(obj_name) - try: # python 2 - file_name.text = name.decode(fs_encoding) - except: # python 3 - file_name.text = name + # ElementTree escapes text on write; a name/path with & < > " is now safe + file_name.text = name file_type = SubElement(file, 'FileType') file_type.text = '%d' % _get_filetype(name) file_path = SubElement(file, 'FilePath') - try: # python 2 - file_path.text = path.decode(fs_encoding) - except: # python 3 - file_path.text = path - + file_path.text = path return group def MDK4AddLibToGroup(ProjectFiles, group, name, filename, project_path): name = os.path.basename(filename) - path = os.path.dirname (filename) + dir_path = os.path.dirname(filename) - basename = os.path.basename(path) - path = _make_path_relative(project_path, path) - path = os.path.join(path, name) + basename = os.path.basename(dir_path) + # normalized relative path (fixes mixed '/' + '\\' separators in FilePath) + path = target_utils.normalize_group_file_path(project_path, dir_path, name) files = SubElement(group, 'Files') file = SubElement(files, 'File') file_name = SubElement(file, 'FileName') @@ -129,18 +127,11 @@ def MDK4AddLibToGroup(ProjectFiles, group, name, filename, project_path): if ProjectFiles.count(obj_name): name = basename + '_' + name ProjectFiles.append(obj_name) - try: - file_name.text = name.decode(fs_encoding) - except: - file_name.text = name + file_name.text = name file_type = SubElement(file, 'FileType') file_type.text = '%d' % _get_filetype(name) file_path = SubElement(file, 'FilePath') - - try: - file_path.text = path.decode(fs_encoding) - except: - file_path.text = path + file_path.text = path return group @@ -156,11 +147,11 @@ def MDK4AddGroup(ProjectFiles, parent, name, files, project_path, group_scons): for f in files: fn = f.rfile() name = fn.name - path = os.path.dirname(fn.abspath) + dir_path = os.path.dirname(fn.abspath) - basename = os.path.basename(path) - path = _make_path_relative(project_path, path) - path = os.path.join(path, name) + basename = os.path.basename(dir_path) + # normalized relative path (fixes mixed '/' + '\\' separators in FilePath) + path = target_utils.normalize_group_file_path(project_path, dir_path, name) files = SubElement(group, 'Files') file = SubElement(files, 'File') @@ -179,11 +170,11 @@ def MDK4AddGroup(ProjectFiles, parent, name, files, project_path, group_scons): if ProjectFiles.count(obj_name): name = basename + '_' + name ProjectFiles.append(obj_name) - file_name.text = name # name.decode(fs_encoding) + file_name.text = name # ElementTree escapes & < > " on write file_type = SubElement(file, 'FileType') file_type.text = '%d' % _get_filetype(name) file_path = SubElement(file, 'FilePath') - file_path.text = path # path.decode(fs_encoding) + file_path.text = path # for local LOCAL_CFLAGS/LOCAL_CXXFLAGS/LOCAL_CCFLAGS/LOCAL_CPPPATH/LOCAL_CPPDEFINES MiscControls_text = ' ' @@ -202,14 +193,17 @@ def MDK4AddGroup(ProjectFiles, parent, name, files, project_path, group_scons): MiscControls.text = MiscControls_text Define = SubElement(VariousControls, 'Define') if 'LOCAL_CPPDEFINES' in group_scons: - Define.text = ', '.join(set(group_scons['LOCAL_CPPDEFINES'])) + # tuple macros fold to FOO=1, stable order (was set(): reordered) + Define.text = ', '.join(target_utils.normalize_defines(group_scons['LOCAL_CPPDEFINES'])) else: Define.text = ' ' Undefine = SubElement(VariousControls, 'Undefine') Undefine.text = ' ' IncludePath = SubElement(VariousControls, 'IncludePath') if 'LOCAL_CPPPATH' in group_scons: - IncludePath.text = ';'.join([_make_path_relative(project_path, os.path.normpath(i)) for i in group_scons['LOCAL_CPPPATH']]) + local_incs = target_utils.ordered_unique( + [_make_path_relative(project_path, os.path.normpath(i)) for i in group_scons['LOCAL_CPPPATH']]) + IncludePath.text = target_utils.xml_list_value(local_incs) else: IncludePath.text = ' ' @@ -285,11 +279,15 @@ def MDK45Project(env, tree, target, script): group_tree = MDK4AddGroupForFN(ProjectFiles, groups, group['name'], full_path, project_path) # write include path, definitions and link flags + # (ordered de-dup keeps stable output; set() reordered it every run) IncludePath = tree.find('Targets/Target/TargetOption/TargetArmAds/Cads/VariousControls/IncludePath') - IncludePath.text = ';'.join([_make_path_relative(project_path, os.path.normpath(i)) for i in set(CPPPATH)]) + inc_paths = target_utils.ordered_unique( + [_make_path_relative(project_path, os.path.normpath(i)) for i in CPPPATH]) + IncludePath.text = target_utils.xml_list_value(inc_paths) Define = tree.find('Targets/Target/TargetOption/TargetArmAds/Cads/VariousControls/Define') - Define.text = ', '.join(set(CPPDEFINES)) + # tuple macros fold to FOO=1 -- ', '.join(set(...)) crashed on tuples + Define.text = ', '.join(target_utils.normalize_defines(CPPDEFINES)) if 'c99' in CXXFLAGS or 'c99' in CCFLAGS or 'c99' in CFLAGS: uC99 = tree.find('Targets/Target/TargetOption/TargetArmAds/Cads/uC99') @@ -427,10 +425,10 @@ def MDK2Project(env, target, script): for node in group['src']: fn = node.rfile() name = fn.name - path = os.path.dirname(fn.abspath) - basename = os.path.basename(path) - path = _make_path_relative(project_path, path) - path = os.path.join(path, name) + dir_path = os.path.dirname(fn.abspath) + basename = os.path.basename(dir_path) + # normalized relative path (was _make_path_relative + os.path.join) + path = target_utils.normalize_group_file_path(project_path, dir_path, name) if ProjectFiles.count(name): name = basename + '_' + name ProjectFiles.append(name) @@ -443,17 +441,13 @@ def MDK2Project(env, target, script): lines.insert(line_index, '\r\n') line_index += 1 - # remove repeat path - paths = set() - for path in CPPPATH: - inc = _make_path_relative(project_path, os.path.normpath(path)) - paths.add(inc) #.replace('\\', '/') - - paths = [i for i in paths] - CPPPATH = string.join(paths, ';') + # remove repeat path (ordered de-dup; string.join was Python-2 only) + paths = target_utils.ordered_unique( + [_make_path_relative(project_path, os.path.normpath(path)) for path in CPPPATH]) + CPPPATH = target_utils.xml_list_value(paths) - definitions = [i for i in set(CPPDEFINES)] - CPPDEFINES = string.join(definitions, ', ') + # fold tuple macros to FOO=1; ', '.join replaces the py2 string.join(set(...)) + CPPDEFINES = ', '.join(target_utils.normalize_defines(CPPDEFINES)) while line_index < len(lines): if lines[line_index].startswith(' ADSCINCD '): diff --git a/tools/targets/makefile.py b/tools/targets/makefile.py index 8be19fa96a2..0ec3f864780 100644 --- a/tools/targets/makefile.py +++ b/tools/targets/makefile.py @@ -30,6 +30,9 @@ from utils import _make_path_relative import rtconfig +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)))) +import target_utils + makefile = '''phony := all all: @@ -79,10 +82,11 @@ def TargetMakefile(env): if ('LIBS' in env): make.write('EXTERN_LIB := ') for tlib in env['LIBS']: - make.write('-l%s ' % (tlib)) + # quote so a lib name / path with a space cannot break the recipe + make.write('-l%s ' % target_utils.make_quote(tlib)) if ('LIBPATH' in env): for tlibpath in env['LIBPATH']: - make.write('-L%s ' % (tlibpath)) + make.write('-L%s ' % target_utils.make_quote(tlibpath)) make.write('\n') make.write('\n') @@ -111,7 +115,8 @@ def TargetMakefile(env): path = '' paths = CPPPATH for item in paths: - path += '\t-I%s \\\n' % item + # make_quote escapes spaces / '#' but keeps the $(BSP_ROOT) prefix intact + path += '\t-I%s \\\n' % target_utils.make_quote(item) make.write('CPPPATHS :=') if path[0] == '\t': path = path[1:] @@ -121,11 +126,11 @@ def TargetMakefile(env): make.write('\n') make.write('\n') - defines = '' - for item in project['CPPDEFINES']: - defines += ' -D%s' % item + # -D list, make-quoted so a value with a space cannot split the flag + define_flags = ['-D' + d for d in target_utils.normalize_defines(project['CPPDEFINES'])] make.write('DEFINES :=') - make.write(defines) + if define_flags: + make.write(' ' + target_utils.make_join(define_flags)) make.write('\n') files = Files @@ -150,7 +155,9 @@ def TargetMakefile(env): files = Files src.write('SRC_FILES :=\n') for item in files: - src.write('SRC_FILES +=%s\n' % item.replace('\\', '/')) + # forward slashes, then make-quote so a space / '#' in the path is safe + # while the leading $(BSP_ROOT)/$(RTT_ROOT) variable is preserved + src.write('SRC_FILES +=%s\n' % target_utils.make_quote(item.replace('\\', '/'))) make = open('Makefile', 'w') make.write(makefile) diff --git a/tools/targets/ses.py b/tools/targets/ses.py index 120d889e867..66e8468db92 100644 --- a/tools/targets/ses.py +++ b/tools/targets/ses.py @@ -9,6 +9,9 @@ from utils import xml_indent from utils import ProjectInfo +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import target_utils + def SDKAddGroup(parent, name, files, project_path): # don't add an empty group if len(files) == 0: @@ -43,7 +46,7 @@ def SESProject(env) : script = env['project'] root = tree.getroot() - out = file(target, 'w') + out = open(target, 'w', encoding='utf-8') out.write('\n') CPPPATH = [] @@ -73,9 +76,11 @@ def SESProject(env) : LINKFLAGS += group['LINKFLAGS'] # write include path, definitions and link flags - path = ';'.join([_make_path_relative(project_path, os.path.normpath(i)) for i in project['CPPPATH']]) - path = path.replace('\\', '/') - defines = ';'.join(set(project['CPPDEFINES'])) + inc_paths = target_utils.normalize_paths( + [_make_path_relative(project_path, os.path.normpath(i)) for i in project['CPPPATH']]) + path = target_utils.semicolon_list(inc_paths) + # normalize (NAME / NAME=value) and keep a stable order instead of set() + defines = target_utils.semicolon_list(target_utils.normalize_defines(project['CPPDEFINES'])) node = tree.findall('project/configuration') for item in node: @@ -86,7 +91,8 @@ def SESProject(env) : item.set('c_user_include_directories', path) xml_indent(root) - out.write(etree.tostring(root, encoding='utf-8')) + # encoding='unicode' yields str for the text-mode file (utf-8 bytes would fail) + out.write(etree.tostring(root, encoding='unicode')) out.close() return diff --git a/tools/targets/target_utils.py b/tools/targets/target_utils.py new file mode 100644 index 00000000000..29beeaae7de --- /dev/null +++ b/tools/targets/target_utils.py @@ -0,0 +1,432 @@ +""" +Common serialization helpers for the tools/targets project generators. + +The project generators (CMake / Makefile / xmake / Zig / SES / Eclipse / VS / +Keil / IAR) each take the same SCons build data (CPPDEFINES, CPPPATH, source +files, LIBS, LIBPATH, C/ASM/link flags) and serialize it into a different +project-file syntax. Historically every generator re-implemented that +serialization inline, which produced a family of related bugs: + + * tuple/list CPPDEFINES such as ('FOO', '1') rendered as two macros + ("FOO;1") or crashed on ``str.join`` / ``set()``; + * ``set(...)`` was used for ordering, giving non-deterministic output; + * include/source paths containing spaces, ';' or quotes broke the target + file's list syntax; + * linker-script flags (``-T link.lds``) were parsed with ``str.split`` and + fell apart on quoted paths or the ``-Wl,-T,x`` form. + +These helpers centralize that logic so every generator normalizes and escapes +its output the same, correct way. +""" + +import os +import shlex +import xml.sax.saxutils as saxutils + + +# --------------------------------------------------------------------------- +# ordering / de-duplication +# --------------------------------------------------------------------------- + +def ordered_unique(items): + """Return items with duplicates removed, preserving first-seen order. + + Generators must never use ``set()`` for this: it reorders the output and + makes generated project files churn between runs. + """ + result = [] + seen = set() + for item in items or []: + try: + key = item + hash(key) + except TypeError: + key = repr(item) + if key in seen: + continue + seen.add(key) + result.append(item) + return result + + +# --------------------------------------------------------------------------- +# preprocessor defines +# --------------------------------------------------------------------------- + +def normalize_define(item): + """Normalize a single SCons CPPDEFINES entry to a ``NAME`` / ``NAME=value`` string. + + "FOO" -> "FOO" + ("FOO", "1") -> "FOO=1" + ["FOO", "1"] -> "FOO=1" + ("FOO", None) -> "FOO" + "FOO=bar" -> "FOO=bar" + 'FOO="a b"' -> 'FOO="a b"' + + Returns None for entries that carry no macro name (so callers can filter). + """ + if item is None: + return None + + if isinstance(item, (tuple, list)): + parts = list(item) + if len(parts) == 0: + return None + name = parts[0] + if name is None: + return None + if len(parts) == 1 or parts[1] is None: + return str(name) + return "{}={}".format(name, parts[1]) + + return str(item) + + +def normalize_defines(defines): + """Normalize a list of CPPDEFINES entries, filtering empties and de-duplicating. + + Order is preserved (see :func:`ordered_unique`); the tuple form is folded + into ``NAME=value`` so no generator ever emits ``FOO`` and ``1`` as two + separate macros. + """ + normalized = [] + for item in defines or []: + value = normalize_define(item) + if value is None or value == "": + continue + normalized.append(value) + return ordered_unique(normalized) + + +# --------------------------------------------------------------------------- +# paths +# --------------------------------------------------------------------------- + +def normalize_path(path, force_posix=True): + """Normalize a filesystem path for inclusion in a project file. + + Applies ``os.path.normpath`` and, by default, converts backslashes to + forward slashes so generated files are stable across platforms. + """ + text = os.path.normpath(str(path)) + if force_posix: + text = text.replace('\\', '/') + return text + + +def normalize_paths(paths, force_posix=True): + """Normalize and de-duplicate a list of paths, preserving order.""" + return ordered_unique([normalize_path(p, force_posix) for p in paths or []]) + + +# --------------------------------------------------------------------------- +# CMake +# --------------------------------------------------------------------------- + +def cmake_quote(value): + """Escape a value for a CMake double-quoted string / list element. + + Handles backslash, double quote and the semicolon that CMake would + otherwise treat as a list separator. + """ + text = str(value) + text = text.replace('\\', '\\\\') + text = text.replace('"', '\\"') + text = text.replace(';', '\\;') + return text + + +def cmake_list(values, indent='\t'): + """Format values as one indented, quote-safe CMake list element per line.""" + return "\n".join("{}{}".format(indent, cmake_quote(v)) for v in values) + + +# --------------------------------------------------------------------------- +# Makefile +# --------------------------------------------------------------------------- + +def make_quote(value): + """Escape a value for use in a Makefile assignment. + + Escapes the characters that carry meaning to make/the shell (space, ``#`` + comment, backslash) while deliberately preserving ``$(VAR)`` / ``${VAR}`` + make-variable references, which the generators intentionally embed in + paths. A bare ``$`` that is not part of a make variable is escaped to + ``$$`` so it survives expansion. + """ + text = str(value) + text = text.replace('#', '\\#') + + out = [] + i = 0 + length = len(text) + while i < length: + ch = text[i] + if ch == '$': + nxt = text[i + 1] if i + 1 < length else '' + if nxt in ('(', '{'): + # keep a make variable reference such as $(BSP_ROOT) intact + out.append(ch) + else: + out.append('$$') + elif ch == ' ': + out.append('\\ ') + else: + out.append(ch) + i += 1 + return ''.join(out) + + +def make_join(values, sep=' '): + """Quote each value for make and join them with sep.""" + return sep.join(make_quote(v) for v in values) + + +# --------------------------------------------------------------------------- +# Lua (xmake) +# --------------------------------------------------------------------------- + +def lua_quote(value): + """Return a Lua double-quoted string literal for value.""" + text = str(value) + text = text.replace('\\', '\\\\') + text = text.replace('"', '\\"') + text = text.replace('\n', '\\n') + return '"{}"'.format(text) + + +def lua_list(values, indent='\t'): + """Format values as a comma-separated list of Lua string literals. + + One quoted item per line, no trailing comma (matches the xmake.lua + template's ``add_xxx( ... )`` block layout). + """ + return ",\n".join("{}{}".format(indent, lua_quote(v)) for v in values) + + +# --------------------------------------------------------------------------- +# Zig +# --------------------------------------------------------------------------- + +def zig_quote(value): + """Return a Zig double-quoted string literal for value.""" + text = str(value) + text = text.replace('\\', '\\\\') + text = text.replace('"', '\\"') + text = text.replace('\n', '\\n') + return '"{}"'.format(text) + + +def zig_list(values, indent='\t'): + """Format values as Zig array elements, one quoted item per line. + + A trailing comma is emitted on every line, which Zig array literals allow. + """ + return "".join("{}{},\n".format(indent, zig_quote(v)) for v in values) + + +# --------------------------------------------------------------------------- +# XML (VS / Keil / IAR / Eclipse / SES) +# --------------------------------------------------------------------------- + +def xml_attr(value): + """Escape a value for safe inclusion as an XML attribute value. + + Escapes ``&``, ``<``, ``>`` plus the ``"`` and ``'`` quote characters, so + the result is safe both as attribute content and as element text. Use this + only when building XML text by hand (e.g. string templates); when assigning + through ElementTree's ``set()``/``.text`` the library escapes for you. + """ + return saxutils.escape(str(value), {'"': '"', "'": '''}) + + +def xml_path_attr(path, sep='/'): + """Normalize a path's separators, then escape it for an XML attribute. + + Combines :func:`normalize_path` (stable, single separator style) with + :func:`xml_attr` so a path holding ``&``/``<``/quotes cannot break the + generated XML when written into a hand-built attribute string. + """ + return xml_attr(normalize_path(path, force_posix=(sep == '/'))) + + +def normalize_group_file_path(project_path, file_path, filename=None, sep='/'): + """Build a stable, relative file path for a file-group emitter. + + Keil / IAR / VS / VS2012 all turn a source file into a project-relative + path that becomes an XML attribute. Historically they called + ``_make_path_relative`` (which returns '/') and then ``os.path.join`` the + file name (which re-introduces '\\' on Windows), yielding mixed, OS-dependent + separators in the serialized file. This centralizes that: make the path + relative to ``project_path``, optionally append ``filename``, and normalize + to a single separator so the output is deterministic. ``sep`` selects '/' + (VS / Keil) or '\\' (IAR's ``$PROJ_DIR$\\`` convention). + + Note: the returned value is the raw path -- callers that hand it to + ElementTree let the library escape it; callers building XML by hand should + wrap it with :func:`xml_attr`. + """ + from utils import _make_path_relative + + rel = _make_path_relative(project_path, file_path) + if filename is not None: + rel = os.path.join(rel, filename) + rel = os.path.normpath(rel) + if sep == '/': + rel = rel.replace('\\', '/') + else: + rel = rel.replace('/', '\\') + return rel + + +def xml_list_value(values, sep=';'): + """Join values into a single delimited string for an XML node. + + The result is meant to be handed to ElementTree (``.set()`` / ``.text``), + which performs XML escaping itself, so this only concatenates -- it does + not pre-escape (that would double-escape ``&`` into ``&amp;``). Order + is preserved and duplicate values removed. + """ + return sep.join(str(v) for v in ordered_unique(list(values))) + + +def semicolon_list(values): + """Join values with ';', order-preserving de-dup -- for IDE list fields.""" + return xml_list_value(values, ';') + + +# --------------------------------------------------------------------------- +# compiler command / flag parsing +# --------------------------------------------------------------------------- + +def shlex_split_flags(text): + """Split a flag/command string into tokens, robust on Windows paths. + + Tuned so backslash path separators are kept verbatim and quotes around + ``"dir with space"`` are honored and stripped. + """ + if text is None: + return [] + lex = shlex.shlex(str(text), posix=True) + lex.whitespace_split = True + lex.escape = '' + lex.commenters = '' + return list(lex) + + +def split_command(command): + """Split a compile_commands.json ``command`` into argv tokens. + + ``arguments`` lists are already tokenized and returned as-is; string + commands are tokenized with :func:`shlex_split_flags`. + """ + if command is None: + return [] + if isinstance(command, (list, tuple)): + return list(command) + return shlex_split_flags(command) + + +def extract_include_args(parts): + """Extract include directories from tokenized compiler arguments. + + Supports ``-Ifoo``, ``-I foo``, ``/Ifoo`` and ``/I foo`` without running + past the end of the token list. + """ + includes = [] + i = 0 + length = len(parts) + while i < length: + part = parts[i] + if part in ('-I', '/I'): + if i + 1 < length: + includes.append(parts[i + 1]) + i += 2 + continue + elif part.startswith('-I') or part.startswith('/I'): + includes.append(part[2:]) + i += 1 + return includes + + +def escape_quoted_flags(flags): + """Normalize a raw flags string for embedding inside a ``"..."`` literal. + + CMake, Lua (xmake) and Zig all write compiler flags as a single + double-quoted string (e.g. ``SET(CMAKE_C_FLAGS "")``). Two things + can break that literal: a backslash path separator and an embedded double + quote. This converts backslashes to ``/`` (accepted by every compiler we + target) and escapes embedded quotes, so the flags cannot terminate the + string early. Centralizes a pattern that was duplicated across the + generators. + """ + text = str(flags) + text = text.replace('\\', '/') + text = text.replace('"', '\\"') + return text + + +def normalize_flags(flags): + """Return a stable, de-duplicated token list from a flags string or list. + + Accepts either a raw flags string or an already-split sequence; splitting + honors quoted paths. Order is preserved and duplicate tokens removed. + """ + if flags is None: + return [] + if isinstance(flags, (list, tuple)): + tokens = [] + for item in flags: + tokens.extend(shlex_split_flags(item)) + else: + tokens = shlex_split_flags(flags) + return ordered_unique(tokens) + + +def normalize_link_script_flags(flags): + """Extract linker-script paths from link flags. + + Handles the real forms seen in ``LINKFLAGS``: + ``-Tlink.lds`` (attached) + ``-T link.lds`` (separate token) + ``-T "dir with space/x"`` (quoted path, kept whole by the tokenizer) + ``-Wl,-T,link.lds`` (passed through the compiler to the linker) + ``-Wl,-T link.lds`` (compiler ``-Wl,-T`` then the path token) + + Returns the script paths in first-seen order. + """ + tokens = flags if isinstance(flags, (list, tuple)) else shlex_split_flags(flags) + tokens = list(tokens) + + scripts = [] + i = 0 + length = len(tokens) + while i < length: + tok = tokens[i] + if tok == '-T': + if i + 1 < length: + scripts.append(tokens[i + 1]) + i += 2 + continue + elif tok.startswith('-T') and tok != '-T': + scripts.append(tok[2:]) + elif tok.startswith('-Wl,'): + # e.g. -Wl,-T,link.lds or -Wl,-T + wl_parts = tok.split(',')[1:] + j = 0 + while j < len(wl_parts): + p = wl_parts[j] + if p == '-T': + if j + 1 < len(wl_parts): + scripts.append(wl_parts[j + 1]) + j += 2 + continue + # path is the following top-level token + elif i + 1 < length: + scripts.append(tokens[i + 1]) + i += 1 + elif p.startswith('-T') and p != '-T': + scripts.append(p[2:]) + j += 1 + i += 1 + return ordered_unique([s for s in scripts if s]) diff --git a/tools/targets/vs.py b/tools/targets/vs.py index 4cde4f0fc70..64a909a1031 100644 --- a/tools/targets/vs.py +++ b/tools/targets/vs.py @@ -35,6 +35,9 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import building +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import target_utils + import xml.etree.ElementTree as etree fs_encoding = sys.getfilesystemencoding() @@ -45,29 +48,20 @@ def VS_AddGroup(ProjectFiles, parent, name, files, libs, project_path): for f in files: fn = f.rfile() name = fn.name - path = os.path.dirname(fn.abspath) - - path = _make_path_relative(project_path, path) - path = os.path.join(path, name) - try: - path = path.decode(fs_encoding) - except: - path = path + dir_path = os.path.dirname(fn.abspath) + + # stable, single-separator RelativePath (was _make_path_relative + + # os.path.join, which mixed '/' and '\\'); ElementTree escapes on write + path = target_utils.normalize_group_file_path(project_path, dir_path, name) File = SubElement(Filter, 'File') File.set('RelativePath', path) for lib in libs: name = os.path.basename(lib) - path = os.path.dirname(lib) - - path = _make_path_relative(project_path, path) - path = os.path.join(path, name) + dir_path = os.path.dirname(lib) + path = target_utils.normalize_group_file_path(project_path, dir_path, name) File = SubElement(Filter, 'File') - try: - path = path.decode(fs_encoding) - except: - path = path File.set('RelativePath', path) def VS_AddHeadFilesGroup(program, elem, project_path): @@ -79,12 +73,8 @@ def VS_AddHeadFilesGroup(program, elem, project_path): # print utils.source_list for f in utils.source_list: - path = _make_path_relative(project_path, f) + path = target_utils.normalize_group_file_path(project_path, f) File = SubElement(elem, 'File') - try: - path = path.decode(fs_encoding) - except: - path = path File.set('RelativePath', path) def VSProject(target, script, program): @@ -128,14 +118,11 @@ def VSProject(target, script, program): # write head include path if 'CPPPATH' in building.Env: cpp_path = building.Env['CPPPATH'] - paths = set() - for path in cpp_path: - inc = _make_path_relative(project_path, os.path.normpath(path)) - paths.add(inc) #.replace('\\', '/') - - paths = [i for i in paths] - paths.sort() - cpp_path = ';'.join(paths) + paths = [_make_path_relative(project_path, os.path.normpath(path)) for path in cpp_path] + # de-duplicate (keep order) then sort for stable output; ElementTree + # escapes the attribute, so quotes/'&' in a path cannot break the XML + paths = sorted(target_utils.ordered_unique(paths)) + cpp_path = target_utils.xml_list_value(paths) # write include path, definitions for elem in tree.iter(tag='Tool'): @@ -146,14 +133,10 @@ def VSProject(target, script, program): # write cppdefinitons flags if 'CPPDEFINES' in building.Env: - CPPDEFINES = building.Env['CPPDEFINES'] - definitions = [] - if type(CPPDEFINES[0]) == type(()): - for item in CPPDEFINES: - definitions += [i for i in item] - definitions = ';'.join(definitions) - else: - definitions = ';'.join(building.Env['CPPDEFINES']) + # fold ('FOO','1') into 'FOO=1' -- the old code flattened tuples into + # separate 'FOO' and '1' entries, emitting a bogus "FOO;1" macro + definitions = target_utils.xml_list_value( + target_utils.normalize_defines(building.Env['CPPDEFINES'])) elem.set('PreprocessorDefinitions', definitions) # write link flags @@ -169,14 +152,9 @@ def VSProject(target, script, program): # write lib include path if 'LIBPATH' in building.Env: lib_path = building.Env['LIBPATH'] - paths = set() - for path in lib_path: - inc = _make_path_relative(project_path, os.path.normpath(path)) - paths.add(inc) #.replace('\\', '/') - - paths = [i for i in paths] - paths.sort() - lib_paths = ';'.join(paths) + paths = [_make_path_relative(project_path, os.path.normpath(path)) for path in lib_path] + paths = sorted(target_utils.ordered_unique(paths)) + lib_paths = target_utils.xml_list_value(paths) elem.set('AdditionalLibraryDirectories', lib_paths) xml_indent(root) diff --git a/tools/targets/vs2012.py b/tools/targets/vs2012.py index 07fdcef0fca..8db8eef1ef2 100644 --- a/tools/targets/vs2012.py +++ b/tools/targets/vs2012.py @@ -35,6 +35,9 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import building +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import target_utils + import xml.etree.ElementTree as etree fs_encoding = sys.getfilesystemencoding() @@ -59,18 +62,13 @@ def VS2012_AddGroup(parent, group_name, files, project_path): for f in files: fn = f.rfile() name = fn.name - path = os.path.dirname(fn.abspath) + dir_path = os.path.dirname(fn.abspath) - path = _make_path_relative(project_path, path) - path = os.path.join(path, name) + # stable, single-separator Include; ElementTree escapes on write + path = target_utils.normalize_group_file_path(project_path, dir_path, name) ClCompile = SubElement(parent, 'ClCompile') - - if sys.version > '3': - ClCompile.set('Include', path) - else: - # python3 is no decode function - ClCompile.set('Include', path.decode(fs_encoding)) + ClCompile.set('Include', path) Filter = SubElement(ClCompile, 'Filter') Filter.text='Source Files\\'+group_name @@ -128,16 +126,12 @@ def VS_add_ItemGroup(parent, file_type, files, project_path): objpath = ''.join('kernel'+objpath[len(RTT_ROOT):]) else : objpath = ''.join('bsp'+objpath[len(project_path):]) - path = _make_path_relative(project_path, path) - path = os.path.join(path, name) + # stable, single-separator Include (objpath above is untouched); + # ElementTree escapes & < > " on write + rel_path = target_utils.normalize_group_file_path(project_path, path, name) File = SubElement(ItemGroup, item_tag) - - if sys.version > '3': - File.set('Include', path) - else: - # python3 is no decode function - File.set('Include', path.decode(fs_encoding)) + File.set('Include', rel_path) if file_type == 'C' : ObjName = SubElement(File, 'ObjectFileName') @@ -154,23 +148,14 @@ def VS_add_HeadFiles(program, elem, project_path): filter_h_ItemGroup = SubElement(filter_project, 'ItemGroup') for f in utils.source_list: - path = _make_path_relative(project_path, f) + # stable, single-separator Include; ElementTree escapes on write + path = target_utils.normalize_group_file_path(project_path, f) File = SubElement(ItemGroup, 'ClInclude') - - if sys.version > '3': - File.set('Include', path) - else: - # python3 is no decode function - File.set('Include', path.decode(fs_encoding)) + File.set('Include', path) # add project.vcxproj.filter ClInclude = SubElement(filter_h_ItemGroup, 'ClInclude') - - if sys.version > '3': - ClInclude.set('Include', path) - else: - # python3 is no decode function - ClInclude.set('Include', path.decode(fs_encoding)) + ClInclude.set('Include', path) Filter = SubElement(ClInclude, 'Filter') Filter.text='Header Files' @@ -200,14 +185,9 @@ def VS2012Project(target, script, program): # write head include path if 'CPPPATH' in building.Env: cpp_path = building.Env['CPPPATH'] - paths = set() - for path in cpp_path: - inc = _make_path_relative(project_path, os.path.normpath(path)) - paths.add(inc) #.replace('\\', '/') - - paths = [i for i in paths] - paths.sort() - cpp_path = ';'.join(paths) + ';%(AdditionalIncludeDirectories)' + paths = [_make_path_relative(project_path, os.path.normpath(path)) for path in cpp_path] + paths = sorted(target_utils.ordered_unique(paths)) + cpp_path = target_utils.xml_list_value(paths) + ';%(AdditionalIncludeDirectories)' # write include path for elem in tree.iter(tag='AdditionalIncludeDirectories'): @@ -217,15 +197,9 @@ def VS2012Project(target, script, program): # write cppdefinitons flags if 'CPPDEFINES' in building.Env: for elem in tree.iter(tag='PreprocessorDefinitions'): - CPPDEFINES = building.Env['CPPDEFINES'] - definitions = [] - if type(CPPDEFINES[0]) == type(()): - for item in CPPDEFINES: - definitions += [i for i in item] - definitions = ';'.join(definitions) - else: - definitions = ';'.join(building.Env['CPPDEFINES']) - + # fold ('FOO','1') into 'FOO=1' instead of two macros 'FOO';'1' + definitions = target_utils.xml_list_value( + target_utils.normalize_defines(building.Env['CPPDEFINES'])) definitions = definitions + ';%(PreprocessorDefinitions)' elem.text = definitions break @@ -242,14 +216,9 @@ def VS2012Project(target, script, program): # write lib include path if 'LIBPATH' in building.Env: lib_path = building.Env['LIBPATH'] - paths = set() - for path in lib_path: - inc = _make_path_relative(project_path, os.path.normpath(path)) - paths.add(inc) - - paths = [i for i in paths] - paths.sort() - lib_paths = ';'.join(paths) + ';%(AdditionalLibraryDirectories)' + paths = [_make_path_relative(project_path, os.path.normpath(path)) for path in lib_path] + paths = sorted(target_utils.ordered_unique(paths)) + lib_paths = target_utils.xml_list_value(paths) + ';%(AdditionalLibraryDirectories)' for elem in tree.iter(tag='AdditionalLibraryDirectories'): elem.text = lib_paths break diff --git a/tools/targets/vsc.py b/tools/targets/vsc.py index f6ea8e37dfe..695c34989d4 100644 --- a/tools/targets/vsc.py +++ b/tools/targets/vsc.py @@ -29,12 +29,16 @@ """ import os +import sys import json import utils import rtconfig from SCons.Script import GetLaunchDir from utils import _make_path_relative + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import target_utils def find_first_node_with_two_children(tree): for key, subtree in tree.items(): if len(subtree) >= 2: @@ -95,17 +99,11 @@ def extract_source_dirs(compile_commands): # command or arguments command = entry.get('command') or entry.get('arguments') - if isinstance(command, str): - parts = command.split() - else: - parts = command - # 读取-I或者/I - for i, part in enumerate(parts): - if part.startswith('-I'): - include_dir = part[2:] if len(part) > 2 else parts[i + 1] - source_dirs.add(os.path.abspath(include_dir)) - elif part.startswith('/I'): - include_dir = part[2:] if len(part) > 2 else parts[i + 1] + # split the command safely (keeps quoted "-I dir with space") + parts = target_utils.split_command(command) + # read -I / /I include directories + for include_dir in target_utils.extract_include_args(parts): + if include_dir: source_dirs.add(os.path.abspath(include_dir)) return sorted(source_dirs) @@ -202,6 +200,13 @@ def command_json_to_workspace(root_path,command_json_path): print("Filtered Directory Tree:") #print_tree(filtered_tree) + # An empty compile_commands.json (or a tree without a common node) leaves + # nothing to exclude; emit an empty exclude config instead of crashing. + if not filtered_tree: + print("No source directories found, generating empty exclude config.") + generate_code_workspace_file(set(), command_json_abs_path, root_path) + return + # 打印filtered_tree的root节点的相对路径 root_key = list(filtered_tree.keys())[0] print(f"Root node relative path: {root_key}") @@ -227,9 +232,16 @@ def command_json_to_workspace(root_path,command_json_path): generate_code_workspace_file(exclude_fold,command_json_abs_path,root_path) def delete_repeatelist(data): - temp_dict = set([str(item) for item in data]) - data = [eval(i) for i in temp_dict] - return data + # order-preserving de-duplication (do not use eval / set on repr strings) + result = [] + seen = set() + for item in data: + key = json.dumps(item, sort_keys=True) + if key in seen: + continue + seen.add(key) + result.append(item) + return result def GenerateCFiles(env): """ diff --git a/tools/targets/xmake.py b/tools/targets/xmake.py index ac9208a0939..fecd3790301 100644 --- a/tools/targets/xmake.py +++ b/tools/targets/xmake.py @@ -4,12 +4,16 @@ """ import os +import sys import utils from string import Template import rtconfig from utils import _make_path_relative +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import target_utils + class XmakeProject: def __init__(self, env, project): @@ -34,34 +38,29 @@ def set_toolchain_path(self): def set_target_config(self): info = utils.ProjectInfo(self.env) - # 1. config src path + # 1. config src path -- relative, de-duplicated, Lua-quoted list + src_files = [] for group in self.project: for f in group['src']: - # use relative path path = _make_path_relative(os.getcwd(), os.path.normpath(f.rfile().abspath)) - self.src_path += "\t\"{0}\",\n".format(path.replace("\\", "/")) - self.src_path = self.src_path[:-2] + src_files.append(path.replace("\\", "/")) + self.src_path = target_utils.lua_list(target_utils.ordered_unique(src_files)) # 2. config dir path - for i in info['CPPPATH']: - # use relative path - path = _make_path_relative(os.getcwd(), i) - self.inc_path += "\t\"{0}\",\n".format(path.replace("\\", "/")) - self.inc_path = self.inc_path[:-2] + inc_dirs = [_make_path_relative(os.getcwd(), i).replace("\\", "/") for i in info['CPPPATH']] + self.inc_path = target_utils.lua_list(target_utils.ordered_unique(inc_dirs)) # 3. config cflags - self.cflags = rtconfig.CFLAGS.replace('\\', "/").replace('\"', "\\\"") + self.cflags = target_utils.escape_quoted_flags(rtconfig.CFLAGS) # 4. config cxxflags if 'CXXFLAGS' in dir(rtconfig): - self.cxxflags = rtconfig.CXXFLAGS.replace('\\', "/").replace('\"', "\\\"") + self.cxxflags = target_utils.escape_quoted_flags(rtconfig.CXXFLAGS) else: self.cxxflags = self.cflags # 5. config asflags - self.asflags = rtconfig.AFLAGS.replace('\\', "/").replace('\"', "\\\"") + self.asflags = target_utils.escape_quoted_flags(rtconfig.AFLAGS) # 6. config lflags - self.ldflags = rtconfig.LFLAGS.replace('\\', "/").replace('\"', "\\\"") - # 7. config define - for i in info['CPPDEFINES']: - self.define += "\t\"{0}\",\n".format(i) - self.define = self.define[:-2] + self.ldflags = target_utils.escape_quoted_flags(rtconfig.LFLAGS) + # 7. config define -- tuple macros fold to FOO=1 (never FOO, 1) + self.define = target_utils.lua_list(target_utils.normalize_defines(info['CPPDEFINES'])) def generate_xmake_file(self): if os.getenv('RTT_ROOT'): diff --git a/tools/targets/zigbuild.py b/tools/targets/zigbuild.py index 26063de36e3..426efa5b7d3 100644 --- a/tools/targets/zigbuild.py +++ b/tools/targets/zigbuild.py @@ -11,6 +11,9 @@ import rtconfig from utils import _make_path_relative +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import target_utils + def get_zig_version(): try: @@ -33,8 +36,8 @@ def GenerateCFiles(env,project): ARCH = ".thumb" if rtconfig.CPU in ['cortex-m0', 'cortex-m3', 'cortex-m4', 'cortex-m7','cortex-m23','cortex-m33','cortex-m85'] else ".arm" - CFLAGS = rtconfig.CFLAGS.replace('\\', "/").replace('\"', "\\\"") - LFLAGS = rtconfig.LFLAGS.replace('\\', "/").replace('\"', "\\\"") + CFLAGS = target_utils.escape_quoted_flags(rtconfig.CFLAGS) + LFLAGS = target_utils.escape_quoted_flags(rtconfig.LFLAGS) zig_file = open('build.zig', 'w') if zig_file: @@ -58,33 +61,31 @@ def GenerateCFiles(env,project): zig_file.write(" .abi = .eabi,\n") zig_file.write("};\n\n") + # include dirs: relative, de-duplicated, Zig-quoted list + inc_dirs = [_make_path_relative(os.getcwd(), i).replace("\\", "/") for i in info['CPPPATH']] zig_file.write("const c_includes = [_][]const u8{\n") - for i in info['CPPPATH']: - # use relative path - path = _make_path_relative(os.getcwd(), i) - zig_file.write("\t\"{}\",\n".format(path.replace("\\", "/"))) + zig_file.write(target_utils.zig_list(target_utils.ordered_unique(inc_dirs))) zig_file.write("};\n\n") - zig_file.write("const c_sources = [_][]const u8{\n") + src_files = [] for group in project: for f in group['src']: - # use relative path path = _make_path_relative(os.getcwd(), os.path.normpath(f.rfile().abspath)) - zig_file.write("\t\"{}\",\n".format(path.replace("\\", "/"))) + src_files.append(path.replace("\\", "/")) + zig_file.write("const c_sources = [_][]const u8{\n") + zig_file.write(target_utils.zig_list(target_utils.ordered_unique(src_files))) zig_file.write("};\n\n") + # -D flags: tuple macros fold to FOO=1, global then per-group locals + flag_defs = ["-D" + d for d in target_utils.normalize_defines(info['CPPDEFINES'])] + for group in project: + if 'LOCAL_CPPDEFINES' in group and group['LOCAL_CPPDEFINES']: + flag_defs += ["-D" + d for d in target_utils.normalize_defines(group['LOCAL_CPPDEFINES'])] zig_file.write("const c_flags = [_][]const u8{\n") zig_file.write("\t\"-std=c99\",\n") zig_file.write("\t\"-ffunction-sections\",\n") zig_file.write("\t\"-fdata-sections\",\n") - # conver CDefines to CFlags - for i in info['CPPDEFINES']: - zig_file.write("\t\"-D{}\",\n".format(i)) - # conver LocalCDefines to CFlags - for group in project: - if 'LOCAL_CPPDEFINES' in group and group['LOCAL_CPPDEFINES']: - for i in group['LOCAL_CPPDEFINES']: - zig_file.write("\t\"-D{}\",\n".format(i)) + zig_file.write(target_utils.zig_list(target_utils.ordered_unique(flag_defs))) zig_file.write("};\n\n") zig_file.write("pub fn build(b: *std.Build) void {\n") @@ -117,9 +118,12 @@ def GenerateCFiles(env,project): zig_file.write(" elf.entry = .{ .symbol_name = \"Reset_Handler\" };\n\n") - # find link script in rtconfig.LFLAGS - LINK_SCRIPT = re.search(r'-T\s*(\S+)', LFLAGS) - zig_file.write(" elf.setLinkerScript(b.path(\"{}\"));\n".format(LINK_SCRIPT.group(1))) + # find the linker script in LFLAGS -- handles -Tfoo, -T foo, + # -T "dir with space/link.lds" and -Wl,-T,foo without truncating + link_scripts = target_utils.normalize_link_script_flags(LFLAGS) + link_script = link_scripts[0].replace("\\", "/") if link_scripts else "" + zig_file.write(" elf.setLinkerScript(b.path({}));\n".format( + target_utils.zig_quote(link_script))) zig_file.write(" const copy_elf = b.addInstallArtifact(elf, .{});\n") zig_file.write(" b.default_step.dependOn(©_elf.step);\n\n") diff --git a/tools/testcases/test_target_serialization.py b/tools/testcases/test_target_serialization.py new file mode 100644 index 00000000000..f3b08c66291 --- /dev/null +++ b/tools/testcases/test_target_serialization.py @@ -0,0 +1,464 @@ +import importlib.util +import json +import os +import sys +import tempfile +import types +import unittest + + +RTT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) + + +class _FakeRFile: + def __init__(self, abspath, name): + self.abspath = abspath + self.name = name + + +class _FakeNode: + """Minimal stand-in for a SCons File node used by the group emitters.""" + + def __init__(self, abspath): + self._r = _FakeRFile(abspath, os.path.basename(abspath)) + + def rfile(self): + return self._r + + +def load_module_from_path(name, relative_path, stub_rtconfig=False): + sys.path.insert(0, os.path.join(RTT_ROOT, "tools")) + + old_rtconfig = sys.modules.get("rtconfig") + if stub_rtconfig: + rtconfig = types.ModuleType("rtconfig") + rtconfig.EXEC_PATH = "" + rtconfig.CC = "gcc" + rtconfig.CXX = "g++" + rtconfig.AS = "gcc" + rtconfig.AR = "ar" + rtconfig.LINK = "gcc" + rtconfig.TARGET_EXT = "elf" + sys.modules["rtconfig"] = rtconfig + + try: + path = os.path.join(RTT_ROOT, relative_path) + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + finally: + if stub_rtconfig: + if old_rtconfig is None: + sys.modules.pop("rtconfig", None) + else: + sys.modules["rtconfig"] = old_rtconfig + + +class TargetSerializationTest(unittest.TestCase): + + def test_target_utils_normalizes_scons_defines(self): + target_utils = load_module_from_path( + "target_utils_under_test", + os.path.join("tools", "targets", "target_utils.py"), + ) + + defines = target_utils.normalize_defines([ + "PLAIN", + ("WITH_VALUE", "1"), + "FROM_STRING=2", + 'QUOTED="a b"', + ]) + + self.assertEqual( + defines, + [ + "PLAIN", + "WITH_VALUE=1", + "FROM_STRING=2", + 'QUOTED="a b"', + ], + ) + + def test_vsc_empty_compile_commands_does_not_crash(self): + vsc = load_module_from_path( + "vsc_under_test", + os.path.join("tools", "targets", "vsc.py"), + stub_rtconfig=True, + ) + + with tempfile.TemporaryDirectory() as temp_dir: + root_path = os.path.join(temp_dir, "project") + build_dir = os.path.join(root_path, "build") + os.makedirs(build_dir) + + command_json_path = os.path.join(build_dir, "compile_commands.json") + with open(command_json_path, "w", encoding="utf-8") as f: + json.dump([], f) + + old_cwd = os.getcwd() + os.chdir(temp_dir) + try: + vsc.command_json_to_workspace(root_path, command_json_path) + finally: + os.chdir(old_cwd) + + def test_vsc_extract_source_dirs_handles_quoted_include_paths(self): + vsc = load_module_from_path( + "vsc_under_test_quoted", + os.path.join("tools", "targets", "vsc.py"), + stub_rtconfig=True, + ) + + with tempfile.TemporaryDirectory() as temp_dir: + source_dir = os.path.join(temp_dir, "src") + include_dir = os.path.join(temp_dir, "dir with space", "inc") + os.makedirs(source_dir) + os.makedirs(include_dir) + + source_file = os.path.join(source_dir, "main.c") + with open(source_file, "w", encoding="utf-8") as f: + f.write("int main(void) { return 0; }\n") + + compile_commands = [ + { + "directory": temp_dir, + "file": source_file, + "command": f'gcc -I"{include_dir}" -c "{source_file}"', + } + ] + + dirs = vsc.extract_source_dirs(compile_commands) + self.assertIn(os.path.abspath(include_dir), dirs) + + def _target_utils(self): + return load_module_from_path( + "target_utils_helpers", + os.path.join("tools", "targets", "target_utils.py"), + ) + + def test_normalize_defines_supports_tuple_list_value_quoted(self): + t = self._target_utils() + self.assertEqual(t.normalize_define("FOO"), "FOO") + self.assertEqual(t.normalize_define(("FOO", "1")), "FOO=1") + self.assertEqual(t.normalize_define(["FOO", "1"]), "FOO=1") + self.assertEqual(t.normalize_define(("FOO", None)), "FOO") + self.assertEqual(t.normalize_define("FOO=bar"), "FOO=bar") + self.assertEqual(t.normalize_define('FOO="a b"'), 'FOO="a b"') + self.assertIsNone(t.normalize_define(None)) + self.assertEqual( + t.normalize_defines(["A", ("B", "1"), ["C", "2"], "A", None, ("D", None)]), + ["A", "B=1", "C=2", "D"], + ) + + def test_ordered_unique_preserves_order(self): + t = self._target_utils() + self.assertEqual( + t.ordered_unique(["b", "a", "b", "c", "a", "d"]), + ["b", "a", "c", "d"], + ) + + def test_cmake_quote_handles_space_semicolon_quote(self): + t = self._target_utils() + self.assertEqual(t.cmake_quote('a b'), 'a b') + self.assertEqual(t.cmake_quote('a;b'), 'a\\;b') + self.assertEqual(t.cmake_quote('a"b'), 'a\\"b') + self.assertEqual(t.cmake_quote('a\\b'), 'a\\\\b') + + def test_make_quote_handles_space_hash_dollar_and_keeps_make_var(self): + t = self._target_utils() + self.assertEqual(t.make_quote('a b'), 'a\\ b') + self.assertEqual(t.make_quote('a#b'), 'a\\#b') + # a bare '$' is escaped, but a $(VAR)/${VAR} reference is preserved + self.assertEqual(t.make_quote('a$b'), 'a$$b') + self.assertEqual(t.make_quote('$(BSP_ROOT)/x'), '$(BSP_ROOT)/x') + self.assertEqual(t.make_quote('${RTT_ROOT}/x'), '${RTT_ROOT}/x') + + def test_lua_quote_handles_quote_backslash(self): + t = self._target_utils() + self.assertEqual(t.lua_quote('a"b'), '"a\\"b"') + self.assertEqual(t.lua_quote('a\\b'), '"a\\\\b"') + + def test_zig_quote_handles_quote_backslash(self): + t = self._target_utils() + self.assertEqual(t.zig_quote('a"b'), '"a\\"b"') + self.assertEqual(t.zig_quote('a\\b'), '"a\\\\b"') + + def test_xml_attr_and_list_value(self): + t = self._target_utils() + # xml_attr is attribute-safe: escapes & < > and both quote chars + self.assertEqual(t.xml_attr('a&b"d"'), 'a&b<c>"d"') + # xml_list_value only concatenates (ElementTree escapes on write) + self.assertEqual(t.xml_list_value(["A", "B=1", "C"]), "A;B=1;C") + self.assertEqual(t.semicolon_list(["x", "y", "z"]), "x;y;z") + + def test_normalize_link_script_flags_forms(self): + t = self._target_utils() + self.assertEqual( + t.normalize_link_script_flags('-Tlink.lds -nostartfiles'), ['link.lds']) + self.assertEqual( + t.normalize_link_script_flags('-T link.lds'), ['link.lds']) + self.assertEqual( + t.normalize_link_script_flags('-T "dir with space/link.lds"'), + ['dir with space/link.lds']) + self.assertEqual( + t.normalize_link_script_flags('-Wl,-T,build/link.lds'), + ['build/link.lds']) + self.assertEqual(t.normalize_link_script_flags(''), []) + + def test_xml_attr_and_path_attr_escape_special_chars(self): + t = self._target_utils() + # xml_attr now escapes the quote characters too (attribute-safe) + self.assertEqual(t.xml_attr('a&b"d"\'e'), + 'a&b<c>"d"'e') + # xml_path_attr normalizes separators then escapes + self.assertEqual(t.xml_path_attr('a\\b/c&d'), 'a/b/c&d') + self.assertEqual(t.xml_path_attr('dir with space/x"y'), + 'dir with space/x"y') + # backslash convention keeps '\\' as separator + self.assertEqual(t.xml_path_attr('a/b', sep='\\'), 'a\\b') + + def test_normalize_group_file_path_is_stable_and_relative(self): + t = self._target_utils() + with tempfile.TemporaryDirectory() as temp_dir: + project = os.path.join(temp_dir, "proj") + src_dir = os.path.join(project, "sub dir", "drivers") + os.makedirs(src_dir) + + rel = t.normalize_group_file_path(project, src_dir, "main.c") + # relative, single '/' separator, no backslashes, no drive/abs prefix + self.assertEqual(rel, "sub dir/drivers/main.c") + self.assertNotIn("\\", rel) + self.assertFalse(os.path.isabs(rel)) + + # backslash convention for IAR's $PROJ_DIR$ paths + rel_bs = t.normalize_group_file_path(project, src_dir, "main.c", sep="\\") + self.assertEqual(rel_bs, "sub dir\\drivers\\main.c") + self.assertNotIn("/", rel_bs) + + def test_elementtree_escapes_group_attr_round_trip(self): + import xml.etree.ElementTree as etree + t = self._target_utils() + # the file-group emitters rely on ElementTree escaping the raw path we + # assign; verify a path with XML-special chars round-trips intact + raw = t.normalize_group_file_path("/proj", "/proj/a&b/", 'x".c') + root = etree.Element("File") + root.set("RelativePath", raw) + serialized = etree.tostring(root, encoding="unicode") + self.assertIn("&", serialized) + parsed = etree.fromstring(serialized) + self.assertEqual(parsed.get("RelativePath"), raw) + + def test_group_define_serialization_keeps_foo_1(self): + t = self._target_utils() + # Keil/IAR join defines; VS/VS2012 use semicolon list -- all must keep + # the tuple ('FOO','1') as FOO=1, never split into FOO and 1 + normalized = t.normalize_defines(["A", ("FOO", "1"), ("BAR", None)]) + self.assertEqual(normalized, ["A", "FOO=1", "BAR"]) + self.assertEqual(t.semicolon_list(normalized), "A;FOO=1;BAR") + self.assertEqual(", ".join(normalized), "A, FOO=1, BAR") + self.assertNotIn(";1", t.semicolon_list(normalized)) + + def test_escape_quoted_flags_normalizes_backslash_and_quote(self): + t = self._target_utils() + # backslash -> '/', embedded quote escaped so the "..." literal is safe + self.assertEqual( + t.escape_quoted_flags('-Ic:\\a\\b -DNAME=\"x\"'), + '-Ic:/a/b -DNAME=\\"x\\"', + ) + self.assertEqual(t.escape_quoted_flags('-O2 -g'), '-O2 -g') + + def test_normalize_flags_tokenizes_and_dedupes(self): + t = self._target_utils() + self.assertEqual( + t.normalize_flags('-O2 -g -O2 "-DA=1"'), + ['-O2', '-g', '-DA=1'], + ) + self.assertIn('-nostartfiles', t.normalize_flags('-T link.lds -nostartfiles')) + + def test_split_command_and_extract_include_args(self): + t = self._target_utils() + # string command with a quoted include path containing a space + parts = t.split_command('gcc -I"/dir with space/inc" -c "/x/main.c"') + self.assertEqual(t.extract_include_args(parts), ['/dir with space/inc']) + # arguments list with the split "-I", "path" form + args = ["gcc", "-I", "/a/inc", "-Ib/inc", "/Ic/inc", "-c", "main.c"] + self.assertEqual( + t.extract_include_args(t.split_command(args)), + ['/a/inc', 'b/inc', 'c/inc'], + ) + + def test_tuple_define_serializes_as_foo_1_across_generators(self): + t = self._target_utils() + defines = ["PLAIN", ("STM32", "1"), ("EMPTY", None)] + normalized = t.normalize_defines(defines) + self.assertEqual(normalized, ["PLAIN", "STM32=1", "EMPTY"]) + # cmake: -D list, no bare "1" + cmake_text = t.cmake_list(["-D" + d for d in normalized]) + self.assertIn("-DSTM32=1", cmake_text) + self.assertNotIn("-D1", cmake_text) + # xmake (lua) and zig list quote each define, tuple folded to FOO=1 + self.assertIn('"STM32=1"', t.lua_list(normalized)) + self.assertIn('"STM32=1"', t.zig_list(normalized)) + # vs / keil semicolon or comma joined XML value + self.assertEqual(t.xml_list_value(normalized), "PLAIN;STM32=1;EMPTY") + self.assertNotIn(";1;", t.xml_list_value(normalized)) + + def test_codeblocks_define_and_python3_xml(self): + import xml.etree.ElementTree as etree + + codeblocks = load_module_from_path( + "codeblocks_under_test", + os.path.join("tools", "targets", "codeblocks.py"), + stub_rtconfig=True, + ) + import building + import utils + + with tempfile.TemporaryDirectory() as temp_dir: + src_dir = os.path.join(temp_dir, "app dir") + os.makedirs(src_dir) + src_file = os.path.join(src_dir, "main.c") + with open(src_file, "w", encoding="utf-8") as f: + f.write("int main(void){return 0;}\n") + + template = os.path.join(temp_dir, "template.cbp") + with open(template, "w", encoding="utf-8") as f: + f.write("" + "") + + target = os.path.join(temp_dir, "project.cbp") + script = [{"name": "Applications", "src": [_FakeNode(src_file)]}] + + building.Env = { + "CPPPATH": [os.path.join(temp_dir, "inc dir")], + "CPPDEFINES": ["RT_USING_FINSH", ("STM32", "1")], + } + utils.source_list = [] + + old_cwd = os.getcwd() + os.chdir(temp_dir) + try: + # must not raise AttributeError (str.decode) or TypeError (bytes write) + codeblocks.CBProject(target, script, []) + finally: + os.chdir(old_cwd) + + # output must be valid XML + tree = etree.parse(target) + options = [a.get("option") for a in tree.iter("Add") if a.get("option")] + self.assertIn("-DRT_USING_FINSH", options) + self.assertIn("-DSTM32=1", options) # tuple folded, name kept + self.assertNotIn("-DH", options) # not char-by-char truncated + self.assertNotIn("-D1", options) # tuple name not dropped + + # file path uses forward slashes, no mixed separators + filenames = [u.get("filename") for u in tree.iter("Unit")] + self.assertTrue(any("app dir/main.c" in fn for fn in filenames)) + self.assertFalse(any("\\" in fn for fn in filenames)) + + def test_cdk_tuple_define_serialization(self): + import xml.etree.ElementTree as etree + + cdk = load_module_from_path( + "cdk_under_test", + os.path.join("tools", "targets", "cdk.py"), + ) + + template_xml = ( + "" + "" + "" + "" + "" + ) + tree = etree.ElementTree(etree.fromstring(template_xml)) + + script = [{ + "name": "kernel", + "src": [], + "CPPPATH": [], + "CPPDEFINES": ["A", ("FOO", "1"), "A"], + "CCFLAGS": "", + "LINKFLAGS": "", + }] + + with tempfile.TemporaryDirectory() as temp_dir: + target = os.path.join(temp_dir, "project.cdkproj") + # must not raise TypeError on the ('FOO','1') tuple + cdk._CDKProject(tree, target, script) + + parsed = etree.parse(target) + define = parsed.find("BuildConfigs/BuildConfig/Compiler/Define").text + self.assertEqual(define, "A; FOO=1") # folded + order-preserving de-dup + self.assertNotIn(";1;", define) + + def test_esp_idf_cmake_paths_are_quoted(self): + esp_idf = load_module_from_path( + "esp_idf_under_test", + os.path.join("tools", "targets", "esp_idf.py"), + stub_rtconfig=True, + ) + + with tempfile.TemporaryDirectory() as temp_dir: + src_dir = os.path.join(temp_dir, "src space") + os.makedirs(src_dir) + src_file = os.path.join(src_dir, "main.c") + with open(src_file, "w", encoding="utf-8") as f: + f.write("/* no init export */\n") + + inc_dir = os.path.join(temp_dir, "inc space") + esp_idf.utils.ProjectInfo = lambda env: {"CPPPATH": [inc_dir]} + + project = [{"name": "app", "src": [_FakeNode(src_file)]}] + + os.makedirs(os.path.join(temp_dir, "main")) + old_cwd = os.getcwd() + os.chdir(temp_dir) + try: + esp_idf.GenerateCFiles({}, project) + with open(os.path.join(temp_dir, "main", "CMakeLists.txt"), + encoding="utf-8") as f: + content = f.read() + finally: + os.chdir(old_cwd) + + # the space-containing paths must be a single quoted CMake list item + self.assertIn('"', content) + self.assertRegex(content, r'"[^"\n]*src space/main\.c"') + self.assertRegex(content, r'"[^"\n]*inc space"') + + def test_ses_project_generation_works_on_python3(self): + ses = load_module_from_path( + "ses_under_test", + os.path.join("tools", "targets", "ses.py"), + ) + + ses.ProjectInfo = lambda env: { + "CPPPATH": [], + "CPPDEFINES": [], + } + + class FakeEnv(dict): + pass + + with tempfile.TemporaryDirectory() as temp_dir: + old_cwd = os.getcwd() + os.chdir(temp_dir) + try: + with open("template.emProject", "w", encoding="utf-8") as f: + f.write("") + + env = FakeEnv() + env["BSP_ROOT"] = temp_dir + env["project"] = [] + + ses.SESProject(env) + + self.assertTrue(os.path.exists("project.emProject")) + finally: + os.chdir(old_cwd) + + +if __name__ == "__main__": + unittest.main()