wizard/bertos_utils.py

   1 #!/usr/bin/env python
   2 # encoding: utf-8
   3 #
   4 # Copyright 2008 Develer S.r.l. (http://www.develer.com/)
   5 # All rights reserved.
   6 #
   7 # $Id:$
   8 #
   9 # Author: Lorenzo Berni <duplo@develer.com>
  10 #
  11
  12 import os
  13 import fnmatch
  14 import glob
  15 import re
  16 import shutil
  17
  18 import const
  19 import codelite_project
  20 import DefineException
  21
  22 def isBertosDir(directory):
  23    return os.path.exists(directory + "/VERSION")
  24
  25 def bertosVersion(directory):
  26    return open(directory + "/VERSION").readline().strip()
  27
  28 def createBertosProject(projectInfo):
  29     directory = projectInfo.info("PROJECT_PATH")
  30     sourcesDir = projectInfo.info("SOURCES_PATH")
  31     if not os.path.isdir(directory):
  32         os.mkdir(directory)
  33     f = open(directory + "/project.bertos", "w")
  34     f.write(repr(projectInfo))
  35     f.close()
  36     ## Destination source dir
  37     srcdir = directory + "/bertos"
  38     shutil.rmtree(srcdir, True)
  39     shutil.copytree(sourcesDir + "/bertos", srcdir)
  40     ## Destination makefile
  41     makefile = directory + "/Makefile"
  42     if os.path.exists(makefile):
  43         os.remove(makefile)
  44     makefile = open("mktemplates/Makefile").read()
  45     makefile = makefileGenerator(projectInfo, makefile)
  46     open(directory + "/Makefile", "w").write(makefile)
  47     ## Destination project dir
  48     prjdir = directory + "/" + os.path.basename(directory)
  49     shutil.rmtree(prjdir, True)
  50     os.mkdir(prjdir)
  51     ## Destination configurations files
  52     cfgdir = prjdir + "/cfg"
  53     shutil.rmtree(cfgdir, True)
  54     os.mkdir(cfgdir)
  55     for key, value in projectInfo.info("CONFIGURATIONS").items():
  56         string = open(sourcesDir + "/" + key, "r").read()
  57         for parameter, infos in value.items():
  58             value = infos["value"]
  59             if "type" in infos["informations"] and infos["informations"]["type"] == "autoenabled":
  60                 value = "1"
  61             if "unsigned" in infos["informations"].keys() and infos["informations"]["unsigned"]:
  62                 value += "U"
  63             if "long" in infos["informations"].keys() and infos["informations"]["long"]:
  64                 value += "L"
  65             string = sub(string, parameter, value)
  66         f = open(cfgdir + "/" + os.path.basename(key), "w")
  67         f.write(string)
  68         f.close()
  69     ## Destinatio mk file
  70     makefile = open("mktemplates/template.mk", "r").read()
  71     makefile = mkGenerator(projectInfo, makefile)
  72     open(prjdir + "/" + os.path.basename(prjdir) + ".mk", "w").write(makefile)
  73     ## Destination main.c file
  74     main = open("srctemplates/main.c", "r").read()
  75     open(prjdir + "/main.c", "w").write(main)
  76     if "codelite" in projectInfo.info("OUTPUT"):
  77         workspace = codeliteWorkspaceGenerator(projectInfo)
  78         open(directory + "/" + os.path.basename(prjdir) + ".workspace", "w").write(workspace)
  79         project = codeliteProjectGenerator(projectInfo)
  80         open(directory + "/" + os.path.basename(prjdir) + ".project", "w").write(project)
  81
  82 def mkGenerator(projectInfo, makefile):
  83     """
  84     Generates the mk file for the current project.
  85     """
  86     mkData = {}
  87     mkData["$pname"] = os.path.basename(projectInfo.info("PROJECT_PATH"))
  88     mkData["$cpuname"] = projectInfo.info("CPU_INFOS")["CORE_CPU"]
  89     mkData["$cflags"] = " ".join(projectInfo.info("CPU_INFOS")["C_FLAGS"])
  90     mkData["$ldflags"] = " ".join(projectInfo.info("CPU_INFOS")["LD_FLAGS"])
  91     mkData["$csrc"], mkData["$pcsrc"], mkData["$constants"] = csrcGenerator(projectInfo)
  92     mkData["$prefix"] = projectInfo.info("TOOLCHAIN")["path"].split("gcc")[0]
  93     mkData["$suffix"] = projectInfo.info("TOOLCHAIN")["path"].split("gcc")[1]
  94     mkData["$cross"] = projectInfo.info("TOOLCHAIN")["path"].split("gcc")[0]
  95     mkData["$main"] = os.path.basename(projectInfo.info("PROJECT_PATH")) + "/main.c"
  96     for key in mkData:
  97         while makefile.find(key) != -1:
  98             makefile = makefile.replace(key, mkData[key])
  99     return makefile
 100
 101 def makefileGenerator(projectInfo, makefile):
 102     """
 103     Generate the Makefile for the current project.
 104     """
 105     # TODO: write a general function that works for both the mk file and the Makefile
 106     while makefile.find("project_name") != -1:
 107         makefile = makefile.replace("project_name", os.path.basename(projectInfo.info("PROJECT_PATH")))
 108     return makefile
 109
 110 def csrcGenerator(projectInfo):
 111     modules = projectInfo.info("MODULES")
 112     files = projectInfo.info("FILES")
 113     if "harvard" in projectInfo.info("CPU_INFOS")["CPU_TAGS"]:
 114         harvard = True
 115     else:
 116         harvard = False
 117     ## file to be included in CSRC variable
 118     csrc = []
 119     ## file to be included in PCSRC variable
 120     pcsrc = []
 121     ## constants to be included at the beginning of the makefile
 122     constants = {}
 123     moduleFiles = set([])
 124     dependencyFiles = set([])
 125     ## assembly sources
 126     asmFiles = set([])
 127     for module, information in modules.items():
 128         if information["enabled"]:
 129             if "constants" in information:
 130                 constants.update(information["constants"])
 131             cfiles, sfiles = findModuleFiles(module, projectInfo)
 132             moduleFiles |= set(cfiles)
 133             asmFiles |= set(sfiles)
 134             for fileDependency in information["depends"]:
 135                 if fileDependency in files:
 136                     dependencyCFiles, dependencySFiles = findModuleFiles(fileDependency, projectInfo)
 137                     dependencyFiles |= set(dependencyCFiles)
 138                     asmFiles |= set(dependencySFiles)
 139             for file in moduleFiles:
 140                 if not harvard or "harvard" not in information or information["harvard"] == "both":
 141                     csrc.append(file)
 142                 if harvard and "harvard" in information:
 143                     pcsrc.append(file)
 144             for file in dependencyFiles:
 145                 csrc.append(file)
 146     csrc = " \\\n\t".join(csrc) + " \\"
 147     pcsrc = " \\\n\t".join(pcsrc) + " \\"
 148     constants = "\n".join([os.path.basename(projectInfo.info("PROJECT_PATH")) + "_" + key + " = " + str(value) for key, value in constants.items()])
 149     return csrc, pcsrc, constants
 150
 151 def findModuleFiles(module, projectInfo):
 152     ## Find the files related to the selected module
 153     cfiles = []
 154     sfiles = []
 155     ## .c files related to the module and the cpu architecture
 156     for filename, path in findDefinitions(module + ".c", projectInfo) + \
 157             findDefinitions(module + "_" + projectInfo.info("CPU_INFOS")["TOOLCHAIN"] + ".c", projectInfo):
 158         path = path.replace(projectInfo.info("SOURCES_PATH") + "/", "")
 159         cfiles.append(path + "/" + filename)
 160     ## .s files related to the module and the cpu architecture
 161     for filename, path in findDefinitions(module + ".s", projectInfo) + \
 162             findDefinitions(module + "_" + projectInfo.info("CPU_INFOS")["TOOLCHAIN"] + ".s", projectInfo) + \
 163             findDefinitions(module + ".S", projectInfo) + \
 164             findDefinitions(module + "_" + projectInfo.info("CPU_INFOS")["TOOLCHAIN"] + ".S", projectInfo):
 165         path = path.replace(projectInfo.info("SOURCES_PATH") + "/", "")
 166         sfiles.append(path + "/" + filename)
 167     ## .c and .s files related to the module and the cpu tags
 168     for tag in projectInfo.info("CPU_INFOS")["CPU_TAGS"]:
 169         for filename, path in findDefinitions(module + "_" + tag + ".c", projectInfo):
 170             path = path.replace(projectInfo.info("SOURCES_PATH") + "/", "")
 171             cfiles.append(path + "/" + filename)
 172         for filename, path in findDefinitions(module + "_" + tag + ".s", projectInfo) + \
 173                 findDefinitions(module + "_" + tag + ".S", projectInfo):
 174             path = path.replace(projectInfo.info("SOURCES_PATH") + "/", "")
 175             sfiles.append(path + "/" + filename)
 176     return cfiles, sfiles
 177
 178 def codeliteProjectGenerator(projectInfo):
 179     template = open("cltemplates/bertos.project").read()
 180     filelist = "\n".join(codelite_project.clFiles(codelite_project.findSources(projectInfo.info("PROJECT_PATH")), projectInfo.info("PROJECT_PATH")))
 181     while template.find("$filelist") != -1:
 182         template = template.replace("$filelist", filelist)
 183     projectName = os.path.basename(projectInfo.info("PROJECT_PATH"))
 184     while template.find("$project") != -1:
 185         template = template.replace("$project", projectName)
 186     return template
 187
 188 def codeliteWorkspaceGenerator(projectInfo):
 189     template = open("cltemplates/bertos.workspace").read()
 190     projectName = os.path.basename(projectInfo.info("PROJECT_PATH"))
 191     while template.find("$project") != -1:
 192         template = template.replace("$project", projectName)
 193     return template
 194
 195 def getSystemPath():
 196     path = os.environ["PATH"]
 197     if os.name == "nt":
 198         path = path.split(";")
 199     else:
 200         path = path.split(":")
 201     return path
 202
 203 def findToolchains(pathList):
 204     toolchains = []
 205     for element in pathList:
 206         for toolchain in glob.glob(element+ "/" + const.GCC_NAME):
 207             toolchains.append(toolchain)
 208     return list(set(toolchains))
 209
 210 def getToolchainInfo(output):
 211     info = {}
 212     expr = re.compile("Target: .*")
 213     target = expr.findall(output)
 214     if len(target) == 1:
 215         info["target"] = target[0].split("Target: ")[1]
 216     expr = re.compile("gcc version [0-9,.]*")
 217     version = expr.findall(output)
 218     if len(version) == 1:
 219         info["version"] = version[0].split("gcc version ")[1]
 220     expr = re.compile("gcc version [0-9,.]* \(.*\)")
 221     build = expr.findall(output)
 222     if len(build) == 1:
 223         build = build[0].split("gcc version ")[1]
 224         build = build[build.find("(") + 1 : build.find(")")]
 225         info["build"] = build
 226     expr = re.compile("Configured with: .*")
 227     configured = expr.findall(output)
 228     if len(configured) == 1:
 229         info["configured"] = configured[0].split("Configured with: ")[1]
 230     expr = re.compile("Thread model: .*")
 231     thread = expr.findall(output)
 232     if len(thread) == 1:
 233         info["thread"] = thread[0].split("Thread model: ")[1]
 234     return info
 235
 236 def loadSourceTree(project):
 237     fileList = [f for f in os.walk(project.info("SOURCES_PATH"))]
 238     project.setInfo("FILE_LIST", fileList)
 239
 240 def findDefinitions(ftype, project):
 241     L = project.info("FILE_LIST")
 242     definitions = []
 243     for element in L:
 244         for filename in element[2]:
 245             if fnmatch.fnmatch(filename, ftype):
 246                 definitions.append((filename, element[0]))
 247     return definitions
 248
 249 def loadCpuInfos(project):
 250     cpuInfos = []
 251     for definition in findDefinitions(const.CPU_DEFINITION, project):
 252         cpuInfos.append(getInfos(definition))
 253     return cpuInfos
 254
 255 def getInfos(definition):
 256     D = {}
 257     D.update(const.CPU_DEF)
 258     def include(filename, dict = D, directory=definition[1]):
 259         execfile(directory + "/" + filename, {}, D)
 260     D["include"] = include
 261     include(definition[0], D)
 262     D["CPU_NAME"] = definition[0].split(".")[0]
 263     D["DEFINITION_PATH"] = definition[1] + "/" + definition[0]
 264     del D["include"]
 265     return D
 266
 267 def getCommentList(string):
 268     commentList = re.findall(r"/\*{2}\s*([^*]*\*(?:[^/*][^*]*\*+)*)/", string)
 269     commentList = [re.findall(r"^\s*\* *(.*?)$", comment, re.MULTILINE) for comment in commentList]
 270     return commentList
 271
 272 def loadModuleDefinition(first_comment):
 273     toBeParsed = False
 274     moduleDefinition = {}
 275     for num, line in enumerate(first_comment):
 276         index = line.find("$WIZ$")
 277         if index != -1:
 278             toBeParsed = True
 279             try:
 280                 exec line[index + len("$WIZ$ "):] in {}, moduleDefinition
 281             except:
 282                 raise ParseError(num, line[index:])
 283         elif line.find("\\brief") != -1:
 284             moduleDefinition["module_description"] = line[line.find("\\brief") + len("\\brief "):]
 285     moduleDict = {}
 286     if "module_name" in moduleDefinition.keys():
 287         moduleName = moduleDefinition[const.MODULE_DEFINITION["module_name"]]
 288         del moduleDefinition[const.MODULE_DEFINITION["module_name"]]
 289         moduleDict[moduleName] = {}
 290         if const.MODULE_DEFINITION["module_depends"] in moduleDefinition.keys():
 291             if type(moduleDefinition[const.MODULE_DEFINITION["module_depends"]]) == str:
 292                 moduleDefinition[const.MODULE_DEFINITION["module_depends"]] = (moduleDefinition[const.MODULE_DEFINITION["module_depends"]],)
 293             moduleDict[moduleName]["depends"] = moduleDefinition[const.MODULE_DEFINITION["module_depends"]]
 294             del moduleDefinition[const.MODULE_DEFINITION["module_depends"]]
 295         else:
 296             moduleDict[moduleName]["depends"] = ()
 297         if const.MODULE_DEFINITION["module_configuration"] in moduleDefinition.keys():
 298             moduleDict[moduleName]["configuration"] = moduleDefinition[const.MODULE_DEFINITION["module_configuration"]]
 299             del moduleDefinition[const.MODULE_DEFINITION["module_configuration"]]
 300         else:
 301             moduleDict[moduleName]["configuration"] = ""
 302         if "module_description" in moduleDefinition.keys():
 303             moduleDict[moduleName]["description"] = moduleDefinition["module_description"]
 304             del moduleDefinition["module_description"]
 305         if const.MODULE_DEFINITION["module_harvard"] in moduleDefinition.keys():
 306             harvard = moduleDefinition[const.MODULE_DEFINITION["module_harvard"]]
 307             if harvard == "both" or harvard == "pgm_memory":
 308                 moduleDict[moduleName]["harvard"] = harvard
 309             del moduleDefinition[const.MODULE_DEFINITION["module_harvard"]]
 310         moduleDict[moduleName]["constants"] = moduleDefinition
 311         moduleDict[moduleName]["enabled"] = False
 312     return toBeParsed, moduleDict
 313
 314 def loadDefineLists(commentList):
 315     defineList = {}
 316     for comment in commentList:
 317         for num, line in enumerate(comment):
 318             index = line.find("$WIZ$")
 319             if index != -1:
 320                 try:
 321                     exec line[index + len("$WIZ$ "):] in {}, defineList
 322                 except:
 323                     raise ParseError(num, line[index:])
 324     for key, value in defineList.items():
 325         if type(value) == str:
 326             defineList[key] = (value,)
 327     return defineList
 328
 329 def getDescriptionInformations(comment):
 330     """
 331     Take the doxygen comment and strip the wizard informations, returning the tuple
 332     (comment, wizard_information)
 333     """
 334     brief = ""
 335     description = ""
 336     information = {}
 337     for num, line in enumerate(comment):
 338         index = line.find("$WIZ$")
 339         if index != -1:
 340             if len(brief) == 0:
 341                 brief += line[:index].strip()
 342             else:
 343                 description += " " + line[:index]
 344             try:
 345                 exec line[index + len("$WIZ$ "):] in {}, information
 346             except:
 347                 raise ParseError(num, line[index:])
 348         else:
 349             if len(brief) == 0:
 350                 brief += line.strip()
 351             else:
 352                 description += " " + line
 353                 description = description.strip()
 354     return brief.strip(), description.strip(), information
 355
 356 def getDefinitionBlocks(text):
 357     """
 358     Take a text and return a list of tuple (description, name-value).
 359     """
 360     block = []
 361     block_tmp = re.findall(r"/\*{2}\s*([^*]*\*(?:[^/*][^*]*\*+)*)/\s*#define\s+((?:[^/]*?/?)+)\s*?(?:/{2,3}[^<].*?)?$", text, re.MULTILINE)
 362     for comment, define in block_tmp:
 363         # Only the first element is needed
 364         block.append(([re.findall(r"^\s*\* *(.*?)$", line, re.MULTILINE)[0] for line in comment.splitlines()], define))
 365     for comment, define in re.findall(r"/{3}\s*([^<].*?)\s*#define\s+((?:[^/]*?/?)+)\s*?(?:/{2,3}[^<].*?)?$", text, re.MULTILINE):
 366         block.append(([comment], define))
 367     for define, comment in re.findall(r"#define\s*(.*?)\s*/{3}<\s*(.+?)\s*?(?:/{2,3}[^<].*?)?$", text, re.MULTILINE):
 368         block.append(([comment], define))
 369     return block
 370
 371 def loadModuleData(project):
 372     moduleInfoDict = {}
 373     listInfoDict = {}
 374     configurationInfoDict = {}
 375     fileDict = {}
 376     for filename, path in findDefinitions("*.h", project) + findDefinitions("*.c", project) + findDefinitions("*.s", project) + findDefinitions("*.S", project):
 377         commentList = getCommentList(open(path + "/" + filename, "r").read())
 378         if len(commentList) > 0:
 379             moduleInfo = {}
 380             configurationInfo = {}
 381             try:
 382                 toBeParsed, moduleDict = loadModuleDefinition(commentList[0])
 383             except ParseError, err:
 384                 raise DefineException.ModuleDefineException(path, err.line_number, err.line)
 385             for module, information in moduleDict.items():
 386                 information["category"] = os.path.basename(path)
 387                 if "configuration" in information.keys() and len(information["configuration"]):
 388                     configuration = moduleDict[module]["configuration"]
 389                     try:
 390                         configurationInfo[configuration] = loadConfigurationInfos(project.info("SOURCES_PATH") + "/" + configuration)
 391                     except ParseError, err:
 392                         raise DefineException.ConfigurationDefineException(project.info("SOURCES_PATH") + "/" + configuration, err.line_number, err.line)
 393             moduleInfoDict.update(moduleDict)
 394             configurationInfoDict.update(configurationInfo)
 395             if toBeParsed:
 396                 try:
 397                     listDict = loadDefineLists(commentList[1:])
 398                     listInfoDict.update(listDict)
 399                 except ParseError, err:
 400                     raise DefineException.EnumDefineException(path, err.line_number, err.line)
 401     for filename, path in findDefinitions("*_" + project.info("CPU_INFOS")["TOOLCHAIN"] + ".h", project):
 402         commentList = getCommentList(open(path + "/" + filename, "r").read())
 403         listInfoDict.update(loadDefineLists(commentList))
 404     for tag in project.info("CPU_INFOS")["CPU_TAGS"]:
 405         for filename, path in findDefinitions("*_" + tag + ".h", project):
 406             commentList = getCommentList(open(path + "/" + filename, "r").read())
 407             listInfoDict.update(loadDefineLists(commentList))
 408     project.setInfo("MODULES", moduleInfoDict)
 409     project.setInfo("LISTS", listInfoDict)
 410     project.setInfo("CONFIGURATIONS", configurationInfoDict)
 411     project.setInfo("FILES", fileDict)
 412
 413 def formatParamNameValue(text):
 414     """
 415     Take the given string and return a tuple with the name of the parameter in the first position
 416     and the value in the second.
 417     """
 418     block = re.findall("\s*([^\s]+)\s*(.+?)\s*$", text, re.MULTILINE)
 419     return block[0]
 420
 421 def loadConfigurationInfos(path):
 422     """
 423     Return the module configurations found in the given file as a dict with the
 424     parameter name as key and a dict containig the fields above as value:
 425         "value": the value of the parameter
 426         "description": the description of the parameter
 427         "informations": a dict containig optional informations:
 428             "type": "int" | "boolean" | "enum"
 429             "min": the minimum value for integer parameters
 430             "max": the maximum value for integer parameters
 431             "long": boolean indicating if the num is a long
 432             "unsigned": boolean indicating if the num is an unsigned
 433             "value_list": the name of the enum for enum parameters
 434     """
 435     configurationInfos = {}
 436     for comment, define in getDefinitionBlocks(open(path, "r").read()):
 437         name, value = formatParamNameValue(define)
 438         brief, description, informations = getDescriptionInformations(comment)
 439         configurationInfos[name] = {}
 440         configurationInfos[name]["value"] = value
 441         configurationInfos[name]["informations"] = informations
 442         if not "type" in configurationInfos[name]["informations"]:
 443             configurationInfos[name]["informations"]["type"] = findParameterType(configurationInfos[name])
 444         if ("type" in configurationInfos[name]["informations"].keys() and
 445                 configurationInfos[name]["informations"]["type"] == "int" and
 446                 configurationInfos[name]["value"].find("L") != -1):
 447             configurationInfos[name]["informations"]["long"] = True
 448             configurationInfos[name]["value"] = configurationInfos[name]["value"].replace("L", "")
 449         if ("type" in configurationInfos[name]["informations"].keys() and
 450                 configurationInfos[name]["informations"]["type"] == "int" and
 451                 configurationInfos[name]["value"].find("U") != -1):
 452             configurationInfos[name]["informations"]["unsigned"] = True
 453             configurationInfos[name]["value"] = configurationInfos[name]["value"].replace("U", "")
 454         configurationInfos[name]["description"] = description
 455         configurationInfos[name]["brief"] = brief
 456     return configurationInfos
 457
 458 def findParameterType(parameter):
 459     if "value_list" in parameter["informations"]:
 460         return "enum"
 461     if "min" in parameter["informations"] or "max" in parameter["informations"] or re.match(r"^\d+U?L?$", parameter["value"]) != None:
 462         return "int"
 463
 464 def sub(string, parameter, value):
 465     """
 466     Substitute the given value at the given parameter define in the given string
 467     """
 468     return re.sub(r"(?P<define>#define\s+" + parameter + r"\s+)([^\s]+)", r"\g<define>" + value, string)
 469
 470 def isInt(informations):
 471     """
 472     Return True if the value is a simple int.
 473     """
 474     if ("long" not in informatios.keys() or not informations["long"]) and ("unsigned" not in informations.keys() or informations["unsigned"]):
 475         return True
 476     else:
 477         return False
 478
 479 def isLong(informations):
 480     """
 481     Return True if the value is a long.
 482     """
 483     if "long" in informations.keys() and informations["long"] and "unsigned" not in informations.keys():
 484         return True
 485     else:
 486         return False
 487
 488 def isUnsigned(informations):
 489     """
 490     Return True if the value is an unsigned.
 491     """
 492     if "unsigned" in informations.keys() and informations["unsigned"] and "long" not in informations.keys():
 493         return True
 494     else:
 495         return False
 496
 497 def isUnsignedLong(informations):
 498     """
 499     Return True if the value is an unsigned long.
 500     """
 501     if "unsigned" in informations.keys() and "long" in informations.keys() and informations["unsigned"] and informations["long"]:
 502         return True
 503     else:
 504         return False
 505
 506 class ParseError(Exception):
 507     def __init__(self, line_number, line):
 508         Exception.__init__(self)
 509         self.line_number = line_number
 510         self.line = line