Add asrc, csrc, cxxsrc, cppasrc taken from the cpu definition in the makefile
[bertos.git] / wizard / bertos_utils.py
1 #!/usr/bin/env python
2 # encoding: utf-8
3 #
4 # Copyright 2008 Develer S.r.l. (http://www.develer.com/)
5 # All rights reserved.
6 #
7 # $Id:$
8 #
9 # Author: Lorenzo Berni <duplo@develer.com>
10 #
11
12 import os
13 import fnmatch
14 import glob
15 import re
16 import shutil
17 import pickle
18
19 import const
20 from plugins import codelite_project
21 import DefineException
22
23 def isBertosDir(directory):
24    return os.path.exists(directory + "/VERSION")
25
26 def bertosVersion(directory):
27    return open(directory + "/VERSION").readline().strip()
28
29 def createBertosProject(project_info):
30     directory = project_info.info("PROJECT_PATH")
31     sources_dir = project_info.info("SOURCES_PATH")
32     if not os.path.isdir(directory):
33         os.mkdir(directory)
34     f = open(directory + "/project.bertos", "w")
35     f.write(pickle.dumps(project_info))
36     f.close()
37     # Destination source dir
38     srcdir = directory + "/bertos"
39     shutil.rmtree(srcdir, True)
40     shutil.copytree(sources_dir + "/bertos", srcdir)
41     # Destination makefile
42     makefile = directory + "/Makefile"
43     if os.path.exists(makefile):
44         os.remove(makefile)
45     makefile = open("mktemplates/Makefile").read()
46     makefile = makefileGenerator(project_info, makefile)
47     open(directory + "/Makefile", "w").write(makefile)
48     # Destination project dir
49     prjdir = directory + "/" + os.path.basename(directory)
50     shutil.rmtree(prjdir, True)
51     os.mkdir(prjdir)
52     # Destination configurations files
53     cfgdir = prjdir + "/cfg"
54     shutil.rmtree(cfgdir, True)
55     os.mkdir(cfgdir)
56     for configuration, information in project_info.info("CONFIGURATIONS").items():
57         string = open(sources_dir + "/" + configuration, "r").read()
58         for start, parameter in information["paramlist"]:
59             infos = information[parameter]
60             value = infos["value"]
61             if "type" in infos["informations"] and infos["informations"]["type"] == "autoenabled":
62                 value = "1"
63             if "unsigned" in infos["informations"].keys() and infos["informations"]["unsigned"]:
64                 value += "U"
65             if "long" in infos["informations"].keys() and infos["informations"]["long"]:
66                 value += "L"
67             string = sub(string, parameter, value)
68         f = open(cfgdir + "/" + os.path.basename(configuration), "w")
69         f.write(string)
70         f.close()
71     # Destinatio mk file
72     makefile = open("mktemplates/template.mk", "r").read()
73     makefile = mkGenerator(project_info, makefile)
74     open(prjdir + "/" + os.path.basename(prjdir) + ".mk", "w").write(makefile)
75     # Destination main.c file
76     main = open("srctemplates/main.c", "r").read()
77     open(prjdir + "/main.c", "w").write(main)
78     # Codelite project files
79     if "codelite" in project_info.info("OUTPUT"):
80         codelite_project.createProject(project_info)
81
82 def mkGenerator(project_info, makefile):
83     """
84     Generates the mk file for the current project.
85     """
86     mk_data = {}
87     mk_data["$pname"] = os.path.basename(project_info.info("PROJECT_PATH"))
88     mk_data["$cpuflag"] = project_info.info("CPU_INFOS")["CPU_FLAG_NAME"]
89     mk_data["$cpuname"] = project_info.info("CPU_INFOS")["CORE_CPU"]
90     mk_data["$cflags"] = " ".join(project_info.info("CPU_INFOS")["C_FLAGS"])
91     mk_data["$ldflags"] = " ".join(project_info.info("CPU_INFOS")["LD_FLAGS"])
92     mk_data["$cppflags"] = " ".join(project_info.info("CPU_INFOS")["CPP_FLAGS"])
93     mk_data["$cppaflags"] = " ".join(project_info.info("CPU_INFOS")["CPPA_FLAGS"])
94     mk_data["$cxxflags"] = " ".join(project_info.info("CPU_INFOS")["CXX_FLAGS"])
95     mk_data["$asflags"] = " ".join(project_info.info("CPU_INFOS")["AS_FLAGS"])
96     mk_data["$arflags"] = " ".join(project_info.info("CPU_INFOS")["AR_FLAGS"])
97     mk_data["$csrc"], mk_data["$pcsrc"], mk_data["$cppasrc"], mk_data["$cxxsrc"], mk_data["$asrc"], mk_data["$constants"] = csrcGenerator(project_info)
98     mk_data["$prefix"] = project_info.info("TOOLCHAIN")["path"].split("gcc")[0]
99     mk_data["$suffix"] = project_info.info("TOOLCHAIN")["path"].split("gcc")[1]
100     mk_data["$main"] = os.path.basename(project_info.info("PROJECT_PATH")) + "/main.c"
101     for key in mk_data:
102         while makefile.find(key) != -1:
103             makefile = makefile.replace(key, mk_data[key])
104     return makefile
105
106 def makefileGenerator(project_info, makefile):
107     """
108     Generate the Makefile for the current project.
109     """
110     # TODO write a general function that works for both the mk file and the Makefile
111     while makefile.find("project_name") != -1:
112         makefile = makefile.replace("project_name", os.path.basename(project_info.info("PROJECT_PATH")))
113     return makefile
114
115 def csrcGenerator(project_info):
116     modules = project_info.info("MODULES")
117     files = project_info.info("FILES")
118     if "harvard" in project_info.info("CPU_INFOS")["CPU_TAGS"]:
119         harvard = True
120     else:
121         harvard = False
122     # file to be included in CSRC variable
123     csrc = []
124     # file to be included in PCSRC variable
125     pcsrc = []
126     # files to be included in CPPASRC variable
127     cppasrc = []
128     # files to be included in CXXSRC variable
129     cxxsrc = []
130     # files to be included in ASRC variable
131     asrc = []
132     # constants to be included at the beginning of the makefile
133     constants = {}
134     for module, information in modules.items():
135         module_files = set([])
136         dependency_files = set([])
137         # assembly sources
138         asm_files = set([])
139         if information["enabled"]:
140             if "constants" in information:
141                 constants.update(information["constants"])
142             cfiles, sfiles = findModuleFiles(module, project_info)
143             module_files |= set(cfiles)
144             asm_files |= set(sfiles)
145             for file_dependency in information["depends"]:
146                 if file_dependency in files:
147                     dependencyCFiles, dependencySFiles = findModuleFiles(file_dependency, project_info)
148                     dependency_files |= set(dependencyCFiles)
149                     asm_files |= set(dependencySFiles)
150             for file in module_files:
151                 if not harvard or "harvard" not in information or information["harvard"] == "both":
152                     csrc.append(file)
153                 if harvard and "harvard" in information:
154                     pcsrc.append(file)
155             for file in dependency_files:
156                 csrc.append(file)
157             for file in project_info.info("CPU_INFOS")["C_SRC"]:
158                 csrc.append(file)
159             for file in project_info.info("CPU_INFOS")["PC_SRC"]:
160                 pcsrc.append(file)
161             for file in asm_files:
162                 cppasrc.append(file)
163     for file in project_info.info("CPU_INFOS")["CPPA_SRC"]:
164         cppasrc.append(file)
165     for file in project_info.info("CPU_INFOS")["CXX_SRC"]:
166         cxxsrc.append(file)
167     for file in project_info.info("CPU_INFOS")["ASRC"]:
168         asrc.append(file)
169     csrc = " \\\n\t".join(csrc) + " \\"
170     pcsrc = " \\\n\t".join(pcsrc) + " \\"
171     cppasrc = " \\\n\t".join(cppasrc) + " \\"
172     cxxsrc = " \\\n\t".join(cxxsrc) + " \\"
173     asrc = " \\\n\t".join(asrc) + " \\"
174     constants = "\n".join([os.path.basename(project_info.info("PROJECT_PATH")) + "_" + key + " = " + str(value) for key, value in constants.items()])
175     return csrc, pcsrc, cppasrc, cxxsrc, asrc, constants
176
177 def findModuleFiles(module, project_info):
178     # Find the files related to the selected module
179     cfiles = []
180     sfiles = []
181     # .c files related to the module and the cpu architecture
182     for filename, path in findDefinitions(module + ".c", project_info) + \
183             findDefinitions(module + "_" + project_info.info("CPU_INFOS")["TOOLCHAIN"] + ".c", project_info):
184         path = path.replace(project_info.info("SOURCES_PATH") + os.sep, "")
185         if os.sep != "/":
186             path = replaceSeparators(path)
187         cfiles.append(path + "/" + filename)
188     # .s files related to the module and the cpu architecture
189     for filename, path in findDefinitions(module + ".s", project_info) + \
190             findDefinitions(module + "_" + project_info.info("CPU_INFOS")["TOOLCHAIN"] + ".s", project_info) + \
191             findDefinitions(module + ".S", project_info) + \
192             findDefinitions(module + "_" + project_info.info("CPU_INFOS")["TOOLCHAIN"] + ".S", project_info):
193         path = path.replace(project_info.info("SOURCES_PATH") + os.sep, "")
194         if os.sep != "/":
195             path = replaceSeparators(path)
196         sfiles.append(path + "/" + filename)
197     # .c and .s files related to the module and the cpu tags
198     for tag in project_info.info("CPU_INFOS")["CPU_TAGS"]:
199         for filename, path in findDefinitions(module + "_" + tag + ".c", project_info):
200             path = path.replace(project_info.info("SOURCES_PATH") + os.sep, "")
201             if os.sep != "/":
202                 path = replaceSeparators(path)
203             cfiles.append(path + "/" + filename)
204         for filename, path in findDefinitions(module + "_" + tag + ".s", project_info) + \
205                 findDefinitions(module + "_" + tag + ".S", project_info):
206             path = path.replace(project_info.info("SOURCES_PATH") + os.sep, "")
207             if os.sep != "/":
208                 path = replaceSeparators(path)
209             sfiles.append(path + "/" + filename)
210     return cfiles, sfiles
211
212 def replaceSeparators(path):
213     """
214     Replace the separators in the given path with unix standard separator.
215     """
216     while path.find(os.sep) != -1:
217         path = path.replace(os.sep, "/")
218     return path
219
220 def getSystemPath():
221     path = os.environ["PATH"]
222     if os.name == "nt":
223         path = path.split(";")
224     else:
225         path = path.split(":")
226     return path
227
228 def findToolchains(path_list):
229     toolchains = []
230     for element in path_list:
231         for toolchain in glob.glob(element+ "/" + const.GCC_NAME):
232             toolchains.append(toolchain)
233     return list(set(toolchains))
234
235 def getToolchainInfo(output):
236     info = {}
237     expr = re.compile("Target: .*")
238     target = expr.findall(output)
239     if len(target) == 1:
240         info["target"] = target[0].split("Target: ")[1]
241     expr = re.compile("gcc version [0-9,.]*")
242     version = expr.findall(output)
243     if len(version) == 1:
244         info["version"] = version[0].split("gcc version ")[1]
245     expr = re.compile("gcc version [0-9,.]* \(.*\)")
246     build = expr.findall(output)
247     if len(build) == 1:
248         build = build[0].split("gcc version ")[1]
249         build = build[build.find("(") + 1 : build.find(")")]
250         info["build"] = build
251     expr = re.compile("Configured with: .*")
252     configured = expr.findall(output)
253     if len(configured) == 1:
254         info["configured"] = configured[0].split("Configured with: ")[1]
255     expr = re.compile("Thread model: .*")
256     thread = expr.findall(output)
257     if len(thread) == 1:
258         info["thread"] = thread[0].split("Thread model: ")[1]
259     return info
260
261 def loadSourceTree(project):
262     fileList = [f for f in os.walk(project.info("SOURCES_PATH"))]
263     project.setInfo("FILE_LIST", fileList)
264
265 def findDefinitions(ftype, project):
266     L = project.info("FILE_LIST")
267     definitions = []
268     for element in L:
269         for filename in element[2]:
270             if fnmatch.fnmatch(filename, ftype):
271                 definitions.append((filename, element[0]))
272     return definitions
273
274 def loadCpuInfos(project):
275     cpuInfos = []
276     for definition in findDefinitions(const.CPU_DEFINITION, project):
277         cpuInfos.append(getInfos(definition))
278     return cpuInfos
279
280 def getInfos(definition):
281     D = {}
282     D.update(const.CPU_DEF)
283     def include(filename, dict = D, directory=definition[1]):
284         execfile(directory + "/" + filename, {}, D)
285     D["include"] = include
286     include(definition[0], D)
287     D["CPU_NAME"] = definition[0].split(".")[0]
288     D["DEFINITION_PATH"] = definition[1] + "/" + definition[0]
289     del D["include"]
290     return D
291
292 def getCommentList(string):
293     comment_list = re.findall(r"/\*{2}\s*([^*]*\*(?:[^/*][^*]*\*+)*)/", string)
294     comment_list = [re.findall(r"^\s*\* *(.*?)$", comment, re.MULTILINE) for comment in comment_list]
295     return comment_list
296
297 def loadModuleDefinition(first_comment):
298     to_be_parsed = False
299     module_definition = {}
300     for num, line in enumerate(first_comment):
301         index = line.find("$WIZ$")
302         if index != -1:
303             to_be_parsed = True
304             try:
305                 exec line[index + len("$WIZ$ "):] in {}, module_definition
306             except:
307                 raise ParseError(num, line[index:])
308         elif line.find("\\brief") != -1:
309             module_definition["module_description"] = line[line.find("\\brief") + len("\\brief "):]
310     module_dict = {}
311     if "module_name" in module_definition.keys():
312         module_name = module_definition[const.MODULE_DEFINITION["module_name"]]
313         del module_definition[const.MODULE_DEFINITION["module_name"]]
314         module_dict[module_name] = {}
315         if const.MODULE_DEFINITION["module_depends"] in module_definition.keys():
316             if type(module_definition[const.MODULE_DEFINITION["module_depends"]]) == str:
317                 module_definition[const.MODULE_DEFINITION["module_depends"]] = (module_definition[const.MODULE_DEFINITION["module_depends"]],)
318             module_dict[module_name]["depends"] = module_definition[const.MODULE_DEFINITION["module_depends"]]
319             del module_definition[const.MODULE_DEFINITION["module_depends"]]
320         else:
321             module_dict[module_name]["depends"] = ()
322         if const.MODULE_DEFINITION["module_configuration"] in module_definition.keys():
323             module_dict[module_name]["configuration"] = module_definition[const.MODULE_DEFINITION["module_configuration"]]
324             del module_definition[const.MODULE_DEFINITION["module_configuration"]]
325         else:
326             module_dict[module_name]["configuration"] = ""
327         if "module_description" in module_definition.keys():
328             module_dict[module_name]["description"] = module_definition["module_description"]
329             del module_definition["module_description"]
330         if const.MODULE_DEFINITION["module_harvard"] in module_definition.keys():
331             harvard = module_definition[const.MODULE_DEFINITION["module_harvard"]]
332             if harvard == "both" or harvard == "pgm_memory":
333                 module_dict[module_name]["harvard"] = harvard
334             del module_definition[const.MODULE_DEFINITION["module_harvard"]]
335         module_dict[module_name]["constants"] = module_definition
336         module_dict[module_name]["enabled"] = False
337     return to_be_parsed, module_dict
338
339 def loadDefineLists(comment_list):
340     define_list = {}
341     for comment in comment_list:
342         for num, line in enumerate(comment):
343             index = line.find("$WIZ$")
344             if index != -1:
345                 try:
346                     exec line[index + len("$WIZ$ "):] in {}, define_list
347                 except:
348                     raise ParseError(num, line[index:])
349     for key, value in define_list.items():
350         if type(value) == str:
351             define_list[key] = (value,)
352     return define_list
353
354 def getDescriptionInformations(comment):
355     """
356     Take the doxygen comment and strip the wizard informations, returning the tuple
357     (comment, wizard_information)
358     """
359     brief = ""
360     description = ""
361     information = {}
362     for num, line in enumerate(comment):
363         index = line.find("$WIZ$")
364         if index != -1:
365             if len(brief) == 0:
366                 brief += line[:index].strip()
367             else:
368                 description += " " + line[:index]
369             try:
370                 exec line[index + len("$WIZ$ "):] in {}, information
371             except:
372                 raise ParseError(num, line[index:])
373         else:
374             if len(brief) == 0:
375                 brief += line.strip()
376             else:
377                 description += " " + line
378                 description = description.strip()
379     return brief.strip(), description.strip(), information
380
381 def getDefinitionBlocks(text):
382     """
383     Take a text and return a list of tuple (description, name-value).
384     """
385     block = []
386     block_tmp = re.finditer(r"/\*{2}\s*([^*]*\*(?:[^/*][^*]*\*+)*)/\s*#define\s+((?:[^/]*?/?)+)\s*?(?:/{2,3}[^<].*?)?$", text, re.MULTILINE)
387     for match in block_tmp:
388         # Only the first element is needed
389         comment = match.group(1)
390         define = match.group(2)
391         start = match.start()
392         block.append(([re.findall(r"^\s*\* *(.*?)$", line, re.MULTILINE)[0] for line in comment.splitlines()], define, start))
393     for match in re.finditer(r"/{3}\s*([^<].*?)\s*#define\s+((?:[^/]*?/?)+)\s*?(?:/{2,3}[^<].*?)?$", text, re.MULTILINE):
394         comment = match.group(1)
395         define = match.group(2)
396         start = match.start()
397         block.append(([comment], define, start))
398     for match in re.finditer(r"#define\s*(.*?)\s*/{3}<\s*(.+?)\s*?(?:/{2,3}[^<].*?)?$", text, re.MULTILINE):
399         comment = match.group(2)
400         define = match.group(1)
401         start = match.start()
402         block.append(([comment], define, start))
403     return block
404
405 def loadModuleData(project):
406     module_info_dict = {}
407     list_info_dict = {}
408     configuration_info_dict = {}
409     file_dict = {}
410     for filename, path in findDefinitions("*.h", project) + findDefinitions("*.c", project) + findDefinitions("*.s", project) + findDefinitions("*.S", project):
411         comment_list = getCommentList(open(path + "/" + filename, "r").read())
412         if len(comment_list) > 0:
413             module_info = {}
414             configuration_info = {}
415             try:
416                 to_be_parsed, module_dict = loadModuleDefinition(comment_list[0])
417             except ParseError, err:
418                 raise DefineException.ModuleDefineException(path, err.line_number, err.line)
419             for module, information in module_dict.items():
420                 if "depends" not in information:
421                     information["depends"] = ()
422                 information["depends"] += (filename.split(".")[0],)
423                 information["category"] = os.path.basename(path)
424                 if "configuration" in information.keys() and len(information["configuration"]):
425                     configuration = module_dict[module]["configuration"]
426                     try:
427                         configuration_info[configuration] = loadConfigurationInfos(project.info("SOURCES_PATH") + "/" + configuration)
428                     except ParseError, err:
429                         raise DefineException.ConfigurationDefineException(project.info("SOURCES_PATH") + "/" + configuration, err.line_number, err.line)
430             module_info_dict.update(module_dict)
431             configuration_info_dict.update(configuration_info)
432             if to_be_parsed:
433                 try:
434                     list_dict = loadDefineLists(comment_list[1:])
435                     list_info_dict.update(list_dict)
436                 except ParseError, err:
437                     raise DefineException.EnumDefineException(path, err.line_number, err.line)
438     for filename, path in findDefinitions("*_" + project.info("CPU_INFOS")["TOOLCHAIN"] + ".h", project):
439         comment_list = getCommentList(open(path + "/" + filename, "r").read())
440         list_info_dict.update(loadDefineLists(comment_list))
441     for tag in project.info("CPU_INFOS")["CPU_TAGS"]:
442         for filename, path in findDefinitions("*_" + tag + ".h", project):
443             comment_list = getCommentList(open(path + "/" + filename, "r").read())
444             list_info_dict.update(loadDefineLists(comment_list))
445     project.setInfo("MODULES", module_info_dict)
446     project.setInfo("LISTS", list_info_dict)
447     project.setInfo("CONFIGURATIONS", configuration_info_dict)
448     project.setInfo("FILES", file_dict)
449
450 def formatParamNameValue(text):
451     """
452     Take the given string and return a tuple with the name of the parameter in the first position
453     and the value in the second.
454     """
455     block = re.findall("\s*([^\s]+)\s*(.+?)\s*$", text, re.MULTILINE)
456     return block[0]
457
458 def loadConfigurationInfos(path):
459     """
460     Return the module configurations found in the given file as a dict with the
461     parameter name as key and a dict containig the fields above as value:
462         "value": the value of the parameter
463         "description": the description of the parameter
464         "informations": a dict containig optional informations:
465             "type": "int" | "boolean" | "enum"
466             "min": the minimum value for integer parameters
467             "max": the maximum value for integer parameters
468             "long": boolean indicating if the num is a long
469             "unsigned": boolean indicating if the num is an unsigned
470             "value_list": the name of the enum for enum parameters
471     """
472     configuration_infos = {}
473     configuration_infos["paramlist"] = []
474     for comment, define, start in getDefinitionBlocks(open(path, "r").read()):
475         name, value = formatParamNameValue(define)
476         brief, description, informations = getDescriptionInformations(comment)
477         configuration_infos["paramlist"].append((start, name))
478         configuration_infos[name] = {}
479         configuration_infos[name]["value"] = value
480         configuration_infos[name]["informations"] = informations
481         if not "type" in configuration_infos[name]["informations"]:
482             configuration_infos[name]["informations"]["type"] = findParameterType(configuration_infos[name])
483         if ("type" in configuration_infos[name]["informations"].keys() and
484                 configuration_infos[name]["informations"]["type"] == "int" and
485                 configuration_infos[name]["value"].find("L") != -1):
486             configuration_infos[name]["informations"]["long"] = True
487             configuration_infos[name]["value"] = configuration_infos[name]["value"].replace("L", "")
488         if ("type" in configuration_infos[name]["informations"].keys() and
489                 configuration_infos[name]["informations"]["type"] == "int" and
490                 configuration_infos[name]["value"].find("U") != -1):
491             configuration_infos[name]["informations"]["unsigned"] = True
492             configuration_infos[name]["value"] = configuration_infos[name]["value"].replace("U", "")
493         configuration_infos[name]["description"] = description
494         configuration_infos[name]["brief"] = brief
495     return configuration_infos
496
497 def findParameterType(parameter):
498     if "value_list" in parameter["informations"]:
499         return "enum"
500     if "min" in parameter["informations"] or "max" in parameter["informations"] or re.match(r"^\d+U?L?$", parameter["value"]) != None:
501         return "int"
502
503 def sub(string, parameter, value):
504     """
505     Substitute the given value at the given parameter define in the given string
506     """
507     return re.sub(r"(?P<define>#define\s+" + parameter + r"\s+)([^\s]+)", r"\g<define>" + value, string)
508
509 def isInt(informations):
510     """
511     Return True if the value is a simple int.
512     """
513     if ("long" not in informatios.keys() or not informations["long"]) and ("unsigned" not in informations.keys() or informations["unsigned"]):
514         return True
515     else:
516         return False
517
518 def isLong(informations):
519     """
520     Return True if the value is a long.
521     """
522     if "long" in informations.keys() and informations["long"] and "unsigned" not in informations.keys():
523         return True
524     else:
525         return False
526
527 def isUnsigned(informations):
528     """
529     Return True if the value is an unsigned.
530     """
531     if "unsigned" in informations.keys() and informations["unsigned"] and "long" not in informations.keys():
532         return True
533     else:
534         return False
535
536 def isUnsignedLong(informations):
537     """
538     Return True if the value is an unsigned long.
539     """
540     if "unsigned" in informations.keys() and "long" in informations.keys() and informations["unsigned"] and informations["long"]:
541         return True
542     else:
543         return False
544
545 class ParseError(Exception):
546     def __init__(self, line_number, line):
547         Exception.__init__(self)
548         self.line_number = line_number
549         self.line = line