tools: harden target generator serialization

tools/targets/cdk.py

@@ -31,6 +31,9 @@
 from utils import _make_path_relative
 from utils import xml_indent
 
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import target_utils
+
 def SDKAddGroup(ProjectFiles, parent, name, files, project_path):
     # don't add an empty group
     if len(files) == 0:
@@ -41,11 +44,11 @@ def SDKAddGroup(ProjectFiles, parent, name, files, project_path):
     for f in files:
         fn = f.rfile()
         name = fn.name
-        path = os.path.dirname(fn.abspath)
+        dir_path = os.path.dirname(fn.abspath)
 
-        basename = os.path.basename(path)
-        path = _make_path_relative(project_path, path)
-        elm_attr_name = os.path.join(path, name)
+        # stable, single-separator relative path (was _make_path_relative +
+        # os.path.join, which mixed '/' and '\\'); ElementTree escapes on write
+        elm_attr_name = target_utils.normalize_group_file_path(project_path, dir_path, name)
 
         file = SubElement(group, 'File', attrib={'Name': elm_attr_name})
 
@@ -112,11 +115,14 @@ def _CDKProject(tree, target, script):
     IncludePath = tree.find('BuildConfigs/BuildConfig/Asm/IncludePath')
     IncludePath.text = text
 
+    # fold tuple macros to FOO=1, order-preserving de-dup.
+    # '; '.join(set(...)) raised TypeError on a ('FOO','1') tuple and reordered.
+    defines = target_utils.normalize_defines(CPPDEFINES)
     Define = tree.find('BuildConfigs/BuildConfig/Compiler/Define')
-    Define.text = '; '.join(set(CPPDEFINES))
-    
+    Define.text = '; '.join(defines)
+
     Define = tree.find('BuildConfigs/BuildConfig/Asm/Define')
-    Define.text = '; '.join(set(CPPDEFINES))
+    Define.text = '; '.join(defines)
 
     CC_Misc = tree.find('BuildConfigs/BuildConfig/Compiler/OtherFlags')
     CC_Misc.text = CCFLAGS

tools/targets/cmake.py

@@ -29,6 +29,9 @@
 from utils import _make_path_relative
 from collections import defaultdict, Counter
 
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import target_utils
+
 
 def GenerateCFiles(env, project, project_name):
     """
@@ -63,15 +66,13 @@ def GenerateCFiles(env, project, project_name):
     OBJCOPY = tool_path_conv["CMAKE_OBJCOPY"]["path"]
     FROMELF = tool_path_conv["CMAKE_FROMELF"]["path"]
 
-    CFLAGS = "".join(env['CFLAGS'])
-    CFLAGS = CFLAGS.replace('\\', "/").replace('\"', "\\\"")   
+    CFLAGS = target_utils.escape_quoted_flags("".join(env['CFLAGS']))
     if 'CXXFLAGS' in dir(rtconfig):
-        cflag_str=''.join(env['CXXFLAGS'])
-        CXXFLAGS = cflag_str.replace('\\', "/").replace('\"', "\\\"")
+        CXXFLAGS = target_utils.escape_quoted_flags(''.join(env['CXXFLAGS']))
     else:
         CXXFLAGS = CFLAGS
-    AFLAGS = env['ASFLAGS'].replace('\\', "/").replace('\"', "\\\"")
-    LFLAGS = env['LINKFLAGS'].replace('\\', "/").replace('\"', "\\\"")
+    AFLAGS = target_utils.escape_quoted_flags(env['ASFLAGS'])
+    LFLAGS = target_utils.escape_quoted_flags(env['LINKFLAGS'])
 
     POST_ACTION = rtconfig.POST_ACTION
     # replace the tool name with the cmake variable
@@ -179,34 +180,19 @@ def GenerateCFiles(env, project, project_name):
 
         cm_file.write('\n')
 
+        # include paths: relative, de-duplicated, and quoted so spaces / ';'
+        # cannot break the CMake list
+        inc_paths = target_utils.normalize_paths(
+            [_make_path_relative(os.getcwd(), i) for i in env['CPPPATH']])
         cm_file.write("INCLUDE_DIRECTORIES(\n")
-        for i in env['CPPPATH']:
-            # use relative path
-            path = _make_path_relative(os.getcwd(), i)
-            cm_file.write( "\t" + path.replace("\\", "/") + "\n")
-        cm_file.write(")\n\n")
+        cm_file.write(target_utils.cmake_list(inc_paths))
+        cm_file.write("\n)\n\n")
 
+        # defines: fold SCons str/tuple/list into NAME / NAME=value, stable order
+        defines = ["-D" + d for d in target_utils.normalize_defines(env['CPPDEFINES'])]
         cm_file.write("ADD_DEFINITIONS(\n")
-        for i in env['CPPDEFINES']:
-            # Handle CPPDEFINES from SCons (str / tuple)
-            if isinstance(i, tuple):
-                # e.g. ('STM32F407xx',)
-                if len(i) == 1:
-                    cm_file.write("\t-D" + str(i[0]) + "\n")
-                # e.g. ('FOO', None)
-                elif len(i) == 2:
-                    if i[1] is None:
-                        cm_file.write("\t-D" + str(i[0]) + "\n")
-                    # e.g. ('FOO', 1)
-                    else:
-                        cm_file.write("\t-D{}={}\n".format(i[0], i[1]))
-                else:
-                    # unexpected form, fallback to name only
-                    cm_file.write("\t-D" + str(i[0]) + "\n")
-            else:
-                # generic macro (commonly a string), ensure robust string conversion
-                cm_file.write("\t-D" + str(i) + "\n")
-        cm_file.write(")\n\n")
+        cm_file.write(target_utils.cmake_list(defines))
+        cm_file.write("\n)\n\n")
 
         libgroups = []
         interfacelibgroups = []
@@ -236,12 +222,13 @@ def GenerateCFiles(env, project, project_name):
 
         cm_file.write("# Library source files\n")
         for group in project:
+            src_paths = target_utils.normalize_paths(
+                [os.path.normpath(f.rfile().abspath) for f in group['src']],
+            )
+            src_paths = [_make_path_relative(os.getcwd(), p) for p in src_paths]
             cm_file.write("SET(RT_{:s}_SOURCES\n".format(group['name'].upper()))
-            for f in group['src']:
-                # use relative path
-                path = _make_path_relative(os.getcwd(), os.path.normpath(f.rfile().abspath))
-                cm_file.write( "\t" + path.replace("\\", "/") + "\n" )
-            cm_file.write(")\n\n")
+            cm_file.write(target_utils.cmake_list(src_paths))
+            cm_file.write("\n)\n\n")
 
         cm_file.write("# Library search paths\n")
         for group in libgroups + interfacelibgroups:
@@ -251,10 +238,10 @@ def GenerateCFiles(env, project, project_name):
             if len(group['LIBPATH']) == 0:
                 continue
 
+            lib_dirs = target_utils.normalize_paths(group['LIBPATH'])
             cm_file.write("SET(RT_{:s}_LINK_DIRS\n".format(group['name'].upper()))
-            for f in group['LIBPATH']:
-                cm_file.write("\t"+ f.replace("\\", "/") + "\n" )
-            cm_file.write(")\n\n")
+            cm_file.write(target_utils.cmake_list(lib_dirs))
+            cm_file.write("\n)\n\n")
 
         cm_file.write("# Library local macro definitions\n")
         for group in libgroups:
@@ -264,10 +251,10 @@ def GenerateCFiles(env, project, project_name):
             if len(group['LOCAL_CPPDEFINES']) == 0:
                 continue
 
+            local_defines = target_utils.normalize_defines(group['LOCAL_CPPDEFINES'])
             cm_file.write("SET(RT_{:s}_DEFINES\n".format(group['name'].upper()))
-            for f in group['LOCAL_CPPDEFINES']:
-                cm_file.write("\t"+ f.replace("\\", "/") + "\n" )
-            cm_file.write(")\n\n")
+            cm_file.write(target_utils.cmake_list(local_defines))
+            cm_file.write("\n)\n\n")
 
         cm_file.write("# Library dependencies\n")
         for group in libgroups + interfacelibgroups:
@@ -277,10 +264,10 @@ def GenerateCFiles(env, project, project_name):
             if len(group['LIBS']) == 0:
                 continue
 
+            libs = [target_utils.normalize_path(f) for f in group['LIBS']]
             cm_file.write("SET(RT_{:s}_LIBS\n".format(group['name'].upper()))
-            for f in group['LIBS']:
-                cm_file.write("\t"+ "{}\n".format(f.replace("\\", "/")))
-            cm_file.write(")\n\n")
+            cm_file.write(target_utils.cmake_list(libs))
+            cm_file.write("\n)\n\n")
 
         cm_file.write("# Libraries\n")
         for group in libgroups:

tools/targets/codeblocks.py

@@ -35,6 +35,9 @@
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import building
 
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import target_utils
+
 import xml.etree.ElementTree as etree
 
 fs_encoding = sys.getfilesystemencoding()
@@ -48,21 +51,24 @@ def CB_AddHeadFiles(program, elem, project_path):
     # print utils.source_list
 
     for f in utils.source_list:
-        path = _make_path_relative(project_path, f)
+        # stable, single-separator path; ElementTree escapes & < > " on write.
+        # (was path.decode(fs_encoding) -- str has no .decode on Python 3)
+        path = target_utils.normalize_group_file_path(project_path, f)
         Unit = SubElement(elem, 'Unit')
-        Unit.set('filename', path.decode(fs_encoding))
+        Unit.set('filename', path)
 
 def CB_AddCFiles(ProjectFiles, parent, gname, files, project_path):
     for f in files:
         fn = f.rfile()
         name = fn.name
-        path = os.path.dirname(fn.abspath)
+        dir_path = os.path.dirname(fn.abspath)
 
-        path = _make_path_relative(project_path, path)
-        path = os.path.join(path, name)
+        # normalized relative path (was _make_path_relative + os.path.join,
+        # which mixed '/' and '\\'; and path.decode crashed on Python 3)
+        path = target_utils.normalize_group_file_path(project_path, dir_path, name)
 
         Unit = SubElement(parent, 'Unit')
-        Unit.set('filename', path.decode(fs_encoding))
+        Unit.set('filename', path)
         Option = SubElement(Unit, 'Option')
         Option.set('compilerVar', "CC")
 
@@ -95,12 +101,9 @@ def CBProject(target, script, program):
     # write head include path
     if 'CPPPATH' in building.Env:
         cpp_path = building.Env['CPPPATH']
-        paths  = set()
-        for path in cpp_path:
-            inc = _make_path_relative(project_path, os.path.normpath(path))
-            paths.add(inc) #.replace('\\', '/')
-
-        paths = [i for i in paths]
+        # order-preserving de-dup, then sort (set() reordered non-deterministically)
+        paths = target_utils.normalize_paths(
+            [_make_path_relative(project_path, os.path.normpath(path)) for path in cpp_path])
         paths.sort()
         # write include path, definitions
         for elem in tree.iter(tag='Compiler'):
@@ -109,10 +112,13 @@ def CBProject(target, script, program):
             Add = SubElement(elem, 'Add')
             Add.set('directory', path)
 
-        for macro in building.Env.get('CPPDEFINES', []):
+        # one <Add option="-D<macro>"/> per define. The old `for d in macro`
+        # iterated the *characters* of a string macro (leaving only -D<last char>)
+        # and dropped the name of a ('STM32','1') tuple; normalize_defines folds
+        # tuples to STM32=1 and yields one complete macro per entry.
+        for macro in target_utils.normalize_defines(building.Env.get('CPPDEFINES', [])):
             Add = SubElement(elem, 'Add')
-            for d in macro:
-                Add.set('option', "-D"+d)
+            Add.set('option', "-D" + macro)
 
         # write link flags
     '''
@@ -139,5 +145,7 @@ def CBProject(target, script, program):
             elem.set('AdditionalLibraryDirectories', lib_paths)
     '''
     xml_indent(root)
-    out.write(etree.tostring(root, encoding='utf-8'))
+    # encoding='unicode' yields str for the text-mode file; encoding='utf-8'
+    # returns bytes and raised TypeError on write under Python 3
+    out.write(etree.tostring(root, encoding='unicode'))
     out.close()

tools/targets/eclipse.py

@@ -24,6 +24,9 @@
 from utils import _make_path_relative
 from utils import xml_indent
 
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import target_utils
+
 MODULE_VER_NUM = 6
 
 source_pattern = ['*.c', '*.cpp', '*.cxx', '*.cc', '*.s', '*.S', '*.asm','*.cmd']
@@ -179,7 +182,9 @@ def HandleToolOption(tools, env, project, reset):
     is_cpp_prj = IsCppProject()
     BSP_ROOT = os.path.abspath(env['BSP_ROOT'])
 
-    CPPDEFINES = project['CPPDEFINES']
+    # fold tuple/list macros into NAME / NAME=value strings up front so the
+    # downstream str ops (.replace('=', ' '), set diff) can never hit a tuple
+    CPPDEFINES = target_utils.normalize_defines(project['CPPDEFINES'])
     paths = [ConverToRttEclipsePathFormat(RelativeProjectPath(env, os.path.normpath(i)).replace('\\', '/')) for i in project['CPPPATH']]
 
     compile_include_paths_options = []
@@ -282,7 +287,8 @@ def HandleToolOption(tools, env, project, reset):
                 else:
                     project_defs += [item.get('value')]
             if len(project_defs) > 0:
-                cproject_defs = set(CPPDEFINES) - set(project_defs)
+                existing = set(project_defs)
+                cproject_defs = [d for d in CPPDEFINES if d not in existing]
             else:
                 cproject_defs = CPPDEFINES
 
@@ -295,10 +301,10 @@ def HandleToolOption(tools, env, project, reset):
     if linker_scriptfile_option is not None :
         option = linker_scriptfile_option
         linker_script = 'link.lds'
-        items = env['LINKFLAGS'].split(' ')
-        if '-T' in items:
-            linker_script = items[items.index('-T') + 1]
-            linker_script = ConverToRttEclipsePathFormat(linker_script)
+        # robust parse: handles -Tfoo, -T foo, -T "dir with space/x", -Wl,-T,x
+        scripts = target_utils.normalize_link_script_flags(env['LINKFLAGS'])
+        if scripts:
+            linker_script = ConverToRttEclipsePathFormat(scripts[0])
 
         listOptionValue = option.find('listOptionValue')
         if listOptionValue != None:
@@ -309,14 +315,16 @@ def HandleToolOption(tools, env, project, reset):
     # scriptfile in stm32cubeIDE
     if linker_script_option is not None :
         option = linker_script_option
-        items = env['LINKFLAGS'].split(' ')
-        if '-T' in items:
-            linker_script = ConverToRttEclipsePathFormat(items[items.index('-T') + 1]).strip('"')
+        scripts = target_utils.normalize_link_script_flags(env['LINKFLAGS'])
+        if scripts:
+            linker_script = ConverToRttEclipsePathFormat(scripts[0]).strip('"')
             option.set('value', linker_script)
     # update nostartfiles config
     if linker_nostart_option is not None :
         option = linker_nostart_option
-        if env['LINKFLAGS'].find('-nostartfiles') != -1:
+        # match the flag as a whole token, not a substring of another flag
+        link_tokens = target_utils.normalize_flags(env['LINKFLAGS'])
+        if '-nostartfiles' in link_tokens:
             option.set('value', 'true')
         else:
             option.set('value', 'false')

tools/targets/esp_idf.py

@@ -1,8 +1,12 @@
 import os
 import re
+import sys
 import utils
 from utils import _make_path_relative
 
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import target_utils
+
 def GenerateCFiles(env,project):
     """
     Generate CMakeLists.txt files
@@ -16,22 +20,27 @@ def GenerateCFiles(env,project):
         cm_file.write("idf_component_register(\n")
 
         cm_file.write("\tSRCS\n")
+        src_paths = []
         for group in project:
             for f in group['src']:
                 path = _make_path_relative(main_component_dir, os.path.normpath(f.rfile().abspath))
-                cm_file.write( "\t" + path.replace("\\", "/") + "\n" ) 
+                src_paths.append(path.replace("\\", "/"))
                 src = open(f.rfile().abspath, 'r')
                 for line in src.readlines():
                     if re.match(r'INIT_(BOARD|PREV|DEVICE|COMPONENT|ENV|APP)_EXPORT\(.+\)', line):
                         init_export.append(re.search(r'\(.+\)', line).group(0)[1:-1])
                 src.close()
+        # double-quote each item so a path with a space is one CMake argument,
+        # and cmake_quote escapes an embedded ';' / '"' / '\\'; also de-duplicated
+        for p in target_utils.ordered_unique(src_paths):
+            cm_file.write('\t"' + target_utils.cmake_quote(p) + '"\n')
 
         cm_file.write("\n")
 
         cm_file.write("\tINCLUDE_DIRS\n")
-        for i in info['CPPPATH']:
-            path = _make_path_relative(main_component_dir, i)
-            cm_file.write( "\t" + path.replace("\\", "/") + "\n")
+        inc_paths = [_make_path_relative(main_component_dir, i).replace("\\", "/") for i in info['CPPPATH']]
+        for p in target_utils.ordered_unique(inc_paths):
+            cm_file.write('\t"' + target_utils.cmake_quote(p) + '"\n')
         cm_file.write(")\n\n")
 
         n = len(init_export)