Add new _wizard_version for differentiate the newest ones that use "BERTOS_PATH"...
[bertos.git] / wizard / BProject.py
1 #!/usr/bin/env python
2 # encoding: utf-8
3 #
4 # This file is part of BeRTOS.
5 #
6 # Bertos is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
19 #
20 # As a special exception, you may use this file as part of a free software
21 # library without restriction.  Specifically, if other files instantiate
22 # templates or use macros or inline functions from this file, or you compile
23 # this file and link it with other files to produce an executable, this
24 # file does not by itself cause the resulting executable to be covered by
25 # the GNU General Public License.  This exception does not however
26 # invalidate any other reasons why the executable file might be covered by
27 # the GNU General Public License.
28 #
29 # Copyright 2008 Develer S.r.l. (http://www.develer.com/)
30 #
31 # $Id$
32 #
33 # Author: Lorenzo Berni <duplo@develer.com>
34 #
35
36 import os
37 import fnmatch
38 import copy
39 import pickle
40 import shutil
41 import copytree
42
43 import DefineException
44
45 from LoadException import VersionException, ToolchainException
46
47 import const
48
49 from bertos_utils import (
50                             # Utility functions
51                             isBertosDir, getTagSet, getInfos, updateConfigurationValues,
52                             loadConfigurationInfos, loadDefineLists, loadModuleDefinition,
53                             getCommentList, sub,
54
55                             # Project creation functions
56                             projectFileGenerator, versionFileGenerator, makefileGenerator,
57                             userMkGenerator, mkGenerator, loadPlugin, mergeSources,
58
59                             # Custom exceptions
60                             ParseError, SupportedException
61                         )
62
63 class BProject(object):
64     """
65     Simple class for store and retrieve project informations.
66     """
67
68     def __init__(self, project_file="", info_dict={}):
69         self.infos = {}
70         self._cached_queries = {}
71         self.edit = False
72         if project_file:
73             self.edit = True
74             self.loadBertosProject(project_file, info_dict)
75
76     #--- Load methods (methods that loads data into project) ------------------#
77
78     def loadBertosProject(self, project_file, info_dict):
79         project_dir = os.path.dirname(project_file)
80         project_data = pickle.loads(open(project_file, "r").read())
81         # If PROJECT_NAME is not defined it use the directory name as PROJECT_NAME
82         # NOTE: this can throw an Exception if the user has changed the directory containing the project
83         self.infos["PROJECT_NAME"] = project_data.get("PROJECT_NAME", os.path.basename(project_dir))
84         self.infos["PROJECT_PATH"] = os.path.dirname(project_file)
85         project_src_path = project_data.get("PROJECT_SRC_PATH", None)
86         project_src_path = os.path.join(project_dir, project_data.get("PROJECT_SRC_PATH", None))
87         if project_src_path:
88             self.infos["PROJECT_SRC_PATH"] = project_src_path
89             
90         else:
91             # In projects created with older versions of the Wizard this metadata doesn't exist
92             self.infos["PROJECT_SRC_PATH"] = os.path.join(self.infos["PROJECT_PATH"], self.infos["PROJECT_NAME"])
93
94         wizard_version = project_data.get("WIZARD_VERSION", 0)
95         if wizard_version == 0:
96             # Ignore the BERTOS_PATH inside the project file for older project
97             project_data["BERTOS_PATH"] = project_dir
98         elif wizard_version == 1:
99             # Use SOURCES_PATH instead of BERTOS_PATH for backward compatibility
100             project_data["BERTOS_PATH"] = project_data["SOURCES_PATH"]
101         else:
102             linked_sources_path = project_data["BERTOS_PATH"]
103             sources_abspath = os.path.abspath(os.path.join(project_dir, linked_sources_path))
104             project_data["BERTOS_PATH"] = sources_abspath
105         
106         self._loadBertosSourceStuff(project_data["BERTOS_PATH"], info_dict.get("BERTOS_PATH", None))
107         
108         self.infos["PRESET"] = project_data.get("PRESET", False)
109
110         # For those projects that don't have a VERSION file create a dummy one.
111         if not isBertosDir(project_dir):
112             version_file = open(os.path.join(const.DATA_DIR, "vtemplates/VERSION"), "r").read()
113             open(os.path.join(project_dir, "VERSION"), "w").write(version_file.replace("$version", "").strip())
114
115         self.loadSourceTree()
116         self._loadCpuStuff(project_data["CPU_NAME"], project_data["SELECTED_FREQ"])
117         self._loadToolchainStuff(project_data["TOOLCHAIN"], info_dict.get("TOOLCHAIN", None))
118         self.infos["OUTPUT"] = project_data["OUTPUT"]
119         self.loadModuleData(True)
120         self.setEnabledModules(project_data["ENABLED_MODULES"])
121
122     def _loadBertosSourceStuff(self, sources_path, forced_version=None):
123         if forced_version:
124             sources_path = forced_version
125         if os.path.exists(sources_path):
126             self.infos["BERTOS_PATH"] = sources_path
127         else:
128             raise VersionException(self)
129
130     def _loadCpuStuff(self, cpu_name, cpu_frequency):
131         self.infos["CPU_NAME"] = cpu_name
132         cpu_info = self.getCpuInfos()
133         for cpu in cpu_info:
134             if cpu["CPU_NAME"] == cpu_name:
135                 self.infos["CPU_INFOS"] = cpu
136                 break
137         tag_list = getTagSet(cpu_info)
138         # Create, fill and store the dict with the tags
139         tag_dict = {}
140         for element in tag_list:
141             tag_dict[element] = False
142         infos = self.info("CPU_INFOS")
143         for tag in tag_dict:
144             if tag in infos["CPU_TAGS"] + [infos["CPU_NAME"], infos["TOOLCHAIN"]]:
145                 tag_dict[tag] = True
146             else:
147                 tag_dict[tag] = False
148         self.infos["ALL_CPU_TAGS"] = tag_dict
149         self.infos["SELECTED_FREQ"] = cpu_frequency
150
151     def _loadToolchainStuff(self, toolchain, forced_toolchain=None):
152         toolchain = toolchain
153         if forced_toolchain:
154             toolchain = forced_toolchain
155         if os.path.exists(toolchain["path"]):
156             self.infos["TOOLCHAIN"] = toolchain
157         else:
158             raise ToolchainException(self)
159
160     def loadProjectFromPreset(self, preset):
161         """
162         Load a project from a preset.
163         NOTE: this is a stub.
164         """
165         project_file = os.path.join(preset, "project.bertos")
166         project_data = pickle.loads(open(project_file, "r").read())
167         self.loadSourceTree()
168         self._loadCpuStuff(project_data["CPU_NAME"], project_data["SELECTED_FREQ"])
169         self._loadToolchainStuff(project_data["TOOLCHAIN"])
170         # NOTE: this is a HACK!!!
171         # TODO: find a better way to reuse loadModuleData
172         preset_project_name = project_data.get("PROJECT_NAME", os.path.basename(preset))
173         preset_prj_src_path = os.path.join(preset, project_data.get("PROJECT_SRC_PATH", os.path.join(preset, preset_project_name)))
174
175         old_project_name = self.infos["PROJECT_NAME"]
176         old_project_path = self.infos["PROJECT_PATH"]
177         old_project_src_path = self.infos["PROJECT_SRC_PATH"]
178
179         self.infos["PROJECT_NAME"] = preset_project_name
180         self.infos["PROJECT_PATH"] = preset
181         self.infos["PROJECT_SRC_PATH"] = preset_prj_src_path
182
183         self.loadModuleData(True)
184         self.setEnabledModules(project_data["ENABLED_MODULES"])
185
186         self.infos["PROJECT_NAME"] = old_project_name
187         self.infos["PROJECT_PATH"] = old_project_path
188         self.infos["PROJECT_SRC_PATH"] = old_project_src_path
189
190         self.infos["PRESET_NAME"] = preset_project_name
191         self.infos["PRESET_PATH"] = preset
192         self.infos["PRESET_SRC_PATH"] = preset_prj_src_path
193
194     def loadProjectPresets(self):
195         """
196         Load the default presets (into the const.PREDEFINED_BOARDS_DIR).
197         """
198         # NOTE: this method does nothing (for now).
199         preset_path = os.path.join(self.infos["BERTOS_PATH"], const.PREDEFINED_BOARDS_DIR)
200         preset_tree = {}
201         if os.path.exists(preset_path):
202             preset_tree = self._loadProjectPresetTree(preset_path)
203         self.infos["PRESET_TREE"] = preset_tree
204
205     def _loadProjectPresetTree(self, path):
206         _tree = {}
207         _tree["info"] = self._loadPresetInfo(os.path.join(path, const.PREDEFINED_BOARD_SPEC_FILE))
208         _tree["info"]["filename"] = os.path.basename(path)
209         _tree["info"]["path"] = path
210         _tree["children"] = []
211         entries = set(os.listdir(path))
212         for entry in entries:
213             _path = os.path.join(path, entry)
214             if os.path.isdir(_path):
215                 sub_entries = set(os.listdir(_path))
216                 if const.PREDEFINED_BOARD_SPEC_FILE in sub_entries:
217                     _tree["children"].append(self._loadProjectPresetTree(_path))
218         # Add into the info dict the dir type (dir/project)
219         if _tree["children"]:
220             _tree["info"]["type"] = "dir"
221         else:
222             _tree["info"]["type"] = "project"
223         return _tree
224
225     def _loadPresetInfo(self, preset_spec_file):
226         D = {}
227         execfile(preset_spec_file, {}, D)
228         return D
229
230     def loadModuleData(self, edit=False):
231         module_info_dict = {}
232         list_info_dict = {}
233         configuration_info_dict = {}
234         file_dict = {}
235         for filename, path in self.findDefinitions("*.h") + self.findDefinitions("*.c") + self.findDefinitions("*.s") + self.findDefinitions("*.S"):
236             comment_list = getCommentList(open(path + "/" + filename, "r").read())
237             if len(comment_list) > 0:
238                 module_info = {}
239                 configuration_info = {}
240                 try:
241                     to_be_parsed, module_dict = loadModuleDefinition(comment_list[0])
242                 except ParseError, err:
243                     raise DefineException.ModuleDefineException(path, err.line_number, err.line)
244                 for module, information in module_dict.items():
245                     if "depends" not in information:
246                         information["depends"] = ()
247                     information["depends"] += (filename.split(".")[0],)
248                     information["category"] = os.path.basename(path)
249                     if "configuration" in information and len(information["configuration"]):
250                         configuration = module_dict[module]["configuration"]
251                         try:
252                             configuration_info[configuration] = loadConfigurationInfos(self.infos["BERTOS_PATH"] + "/" + configuration)
253                         except ParseError, err:
254                             raise DefineException.ConfigurationDefineException(self.infos["BERTOS_PATH"] + "/" + configuration, err.line_number, err.line)
255                         if edit:
256                             try:
257                                 path = self.infos["PROJECT_SRC_PATH"]
258                                 user_configuration = loadConfigurationInfos(configuration.replace("bertos", path))
259                                 configuration_info[configuration] = updateConfigurationValues(configuration_info[configuration], user_configuration)
260                             except ParseError, err:
261                                 raise DefineException.ConfigurationDefineException(configuration.replace("bertos", path))
262                 module_info_dict.update(module_dict)
263                 configuration_info_dict.update(configuration_info)
264                 if to_be_parsed:
265                     try:
266                         list_dict = loadDefineLists(comment_list[1:])
267                         list_info_dict.update(list_dict)
268                     except ParseError, err:
269                         raise DefineException.EnumDefineException(path, err.line_number, err.line)
270         for tag in self.infos["CPU_INFOS"]["CPU_TAGS"]:
271             for filename, path in self.findDefinitions("*_" + tag + ".h"):
272                 comment_list = getCommentList(open(path + "/" + filename, "r").read())
273                 list_info_dict.update(loadDefineLists(comment_list))
274         self.infos["MODULES"] = module_info_dict
275         self.infos["LISTS"] = list_info_dict
276         self.infos["CONFIGURATIONS"] = configuration_info_dict
277         self.infos["FILES"] = file_dict
278
279     def loadSourceTree(self):
280         """
281         Index BeRTOS source and load it in memory.
282         """
283         # Index only the BERTOS_PATH/bertos content
284         bertos_sources_dir = os.path.join(self.info("BERTOS_PATH"), "bertos")
285         file_dict = {}
286         if os.path.exists(bertos_sources_dir):
287             for element in os.walk(bertos_sources_dir):
288                 for f in element[2]:
289                     file_dict[f] = file_dict.get(f, []) + [element[0]]
290         self.infos["FILE_DICT"] = file_dict
291
292     def reloadCpuInfo(self):
293         for cpu_info in self.getCpuInfos():
294             if cpu_info["CPU_NAME"] == self.infos["CPU_NAME"]:
295                 self.infos["CPU_INFOS"] = cpu_info
296
297     #-------------------------------------------------------------------------#
298
299     def createBertosProject(self):
300         # NOTE: Temporary hack.
301         if self.edit:
302             self._editBertosProject()
303         else:
304             if not self.from_preset:
305                 self._newCustomBertosProject()
306             else:
307                 self._newBertosProjectFromPreset()
308
309     def _newBertosProject(self):
310         for directory in (self.maindir, self.srcdir, self.prjdir, self.cfgdir):
311             self._createDirectory(directory)
312         # Write the project file
313         self._writeProjectFile(os.path.join(self.maindir, "project.bertos"))
314         # VERSION file
315         self._writeVersionFile(os.path.join(self.maindir, "VERSION"))
316         # Destination makefile
317         self._writeMakefile(os.path.join(self.maindir, "Makefile"))
318         # Copy the sources
319         self._copySources(self.sources_dir, self.srcdir)
320         # Set properly the autoenabled parameters
321         self._setupAutoenabledParameters()
322         # Copy all the configuration files
323         self._writeCfgFiles(self.sources_dir, self.cfgdir)
324         # Destination wizard mk file
325         self._writeWizardMkFile(os.path.join(self.prjdir, os.path.basename(self.prjdir) + "_wiz.mk"))
326
327     def _newCustomBertosProject(self):
328         # Create/write/copy the common things
329         self._newBertosProject()
330         # Copy the clean hw files
331         self._createDirectory(self.hwdir)
332         # Copy all the hw files
333         self._writeHwFiles(self.sources_dir, self.hwdir)
334         # Destination user mk file
335         self._writeUserMkFile(os.path.join(self.prjdir, os.path.basename(self.prjdir) + ".mk"))
336         # Destination main.c file
337         self._writeMainFile(self.prjdir + "/main.c")
338         # Create project files for selected plugins
339         self._createProjectFiles()
340
341     def _newBertosProjectFromPreset(self):
342         # Create/write/copy the common things
343         self._newBertosProject()
344
345         # Copy all the files and dirs except cfg/hw/*_wiz.mk
346         self._writeCustomSrcFiles()
347
348         if self.infos["EMPTY_MAIN"]:
349             # Create and empty main.c file only if the user check the box
350             self._writeMainFile(self.prjdir + "/main.c")
351
352         # Create project files for selected plugins
353         self._createProjectFiles()
354
355     def _editBertosProject(self):
356         # Write the project file
357         self._writeProjectFile(os.path.join(self.maindir, "project.bertos"))
358         # VERSION file
359         self._writeVersionFile(os.path.join(self.maindir, "VERSION"))
360         # Destination makefile
361         self._writeMakefile(os.path.join(self.maindir, "Makefile"))
362         # Merge sources
363         self._mergeSources(self.sources_dir, self.srcdir, self.old_srcdir)
364         # Copy all the hw files
365         self._writeHwFiles(self.sources_dir, self.hwdir)
366         # Set properly the autoenabled parameters
367         self._setupAutoenabledParameters()
368         # Copy all the configuration files
369         self._writeCfgFiles(self.sources_dir, self.cfgdir)
370         # Destination wizard mk file
371         self._writeWizardMkFile(os.path.join(self.prjdir, os.path.basename(self.prjdir) + "_wiz.mk"))
372         # Create project files for selected plugins
373         self._createProjectFiles()
374
375     def _createProjectFiles(self):
376         # Files for selected plugins
377         relevants_files = {}
378         for plugin in self.infos["OUTPUT"]:
379             module = loadPlugin(plugin)
380             relevants_files[plugin] = module.createProject(self)
381         self.infos["RELEVANT_FILES"] = relevants_files
382
383     def _writeVersionFile(self, filename):
384         version_file = open(os.path.join(const.DATA_DIR, "vtemplates/VERSION"), "r").read()
385         open(filename, "w").write(versionFileGenerator(self, version_file))
386
387     def _writeProjectFile(self, filename):
388         f = open(filename, "w")
389         f.write(projectFileGenerator(self))
390         f.close()
391
392     def _writeMakefile(self, filename):
393         makefile = open(os.path.join(const.DATA_DIR, "mktemplates/Makefile"), "r").read()
394         makefile = makefileGenerator(self, makefile)
395         open(filename, "w").write(makefile)
396
397     def _writeUserMkFile(self, filename):
398         makefile = open(os.path.join(const.DATA_DIR, "mktemplates/template.mk"), "r").read()
399         # Deadly performances loss was here :(
400         makefile = userMkGenerator(self, makefile)
401         open(filename, "w").write(makefile)
402
403     def _writeWizardMkFile(self, filename):
404         makefile = open(os.path.join(const.DATA_DIR, "mktemplates/template_wiz.mk"), "r").read()
405         makefile = mkGenerator(self, makefile)
406         open(filename, "w").write(makefile)
407
408     def _writeMainFile(self, filename):
409         main = open(os.path.join(const.DATA_DIR, "srctemplates/main.c"), "r").read()
410         open(filename, "w").write(main)
411
412     def _writeHwFiles(self, source_dir, destination_dir):
413         for module, information in self.infos["MODULES"].items():
414             for hwfile in information["hw"]:
415                 string = open(source_dir + "/" + hwfile, "r").read()
416                 hwfile_path = destination_dir + "/" + os.path.basename(hwfile)
417                 if not self.edit or not os.path.exists(hwfile_path):
418                     # If not in editing mode it copies all the hw files. If in
419                     # editing mode it copies only the files that don't exist yet
420                     open(os.path.join(destination_dir,os.path.basename(hwfile)), "w").write(string)
421
422     def _writeCfgFiles(self, source_dir, destination_dir):
423         for configuration, information in self.infos["CONFIGURATIONS"].items():
424             string = open(source_dir + "/" + configuration, "r").read()
425             for start, parameter in information["paramlist"]:
426                 infos = information[parameter]
427                 value = infos["value"]
428                 if "unsigned" in infos["informations"] and infos["informations"]["unsigned"]:
429                     value += "U"
430                 if "long" in infos["informations"] and infos["informations"]["long"]:
431                     value += "L"
432                 string = sub(string, parameter, value)
433             f = open(os.path.join(destination_dir, os.path.basename(configuration)), "w")
434             f.write(string)
435             f.close()
436
437     def _writeCustomSrcFiles(self):
438         origin = self.infos["PRESET_SRC_PATH"]
439         # Files to be ignored (all project files, cfg dir, wizard mk file, all global ignored dirs)
440         project_related_stuff = (
441             "cfg",
442             self.infos["PROJECT_NAME"] + "_wiz.mk",
443             "project.bertos",
444             self.infos["PROJECT_NAME"] + ".project",
445             self.infos["PROJECT_NAME"] + ".workspace",
446         ) + const.IGNORE_LIST
447         for element in os.listdir(origin):
448             if element not in project_related_stuff:
449                 full_path = os.path.join(origin, element)
450                 if os.path.isdir(full_path):
451                     copytree.copytree(full_path, os.path.join(self.prjdir, element), ignore_list=const.IGNORE_LIST)
452                 else:
453                     shutil.copy(full_path, self.prjdir)
454
455     def _setupAutoenabledParameters(self):
456         for module, information in self.infos["MODULES"].items():
457             if "configuration" in information and information["configuration"] != "":
458                 configurations = self.infos["CONFIGURATIONS"]
459                 configuration = configurations[information["configuration"]]
460                 for start, parameter in configuration["paramlist"]:
461                     if "type" in configuration[parameter]["informations"] and configuration[parameter]["informations"]["type"] == "autoenabled":
462                         configuration[parameter]["value"] = "1" if information["enabled"] else "0"
463                 self.infos["CONFIGURATIONS"] = configurations
464
465     @property
466     def maindir(self):
467         return self.infos.get("PROJECT_PATH", None)
468
469     @property
470     def srcdir(self):
471         if self.maindir:
472             return os.path.join(self.maindir, "bertos")
473         else:
474             return None
475
476     @property
477     def prjdir(self):
478         return self.infos.get("PROJECT_SRC_PATH", None)
479
480     @property
481     def hwdir(self):
482         if self.prjdir:
483             return os.path.join(self.prjdir, "hw")
484         else:
485             return None
486
487     @property
488     def cfgdir(self):
489         if self.prjdir:
490             return os.path.join(self.prjdir, "cfg")
491         else:
492             return None
493
494     @property
495     def old_srcdir(self):
496         return self.infos.get("OLD_BERTOS_PATH", None)
497
498     @property
499     def sources_dir(self):
500         return self.infos.get("BERTOS_PATH", None)
501
502     @property
503     def from_preset(self):
504         return self.infos.get("PROJECT_FROM_PRESET", False)
505
506     def _createDirectory(self, directory):
507         if not directory:
508             return
509         if os.path.isdir(directory):
510             shutil.rmtree(directory, True)
511         os.makedirs(directory)
512
513     def _copySources(self, origin, destination):
514         # If not in editing mode it copies all the bertos sources in the /bertos subdirectory of the project
515         shutil.rmtree(destination, True)
516         copytree.copytree(origin + "/bertos", destination, ignore_list=const.IGNORE_LIST)
517
518     def _mergeSources(self, origin, destination, old_sources_dir):
519         if old_sources_dir:
520             mergeSources(destination, origin, old_sources_dir)
521
522     def setInfo(self, key, value):
523         """
524         Store the given value with the name key.
525         """
526         self.infos[key] = value
527
528     def info(self, key, default=None):
529         """
530         Retrieve the value associated with the name key.
531         """
532         if key in self.infos:
533             return copy.deepcopy(self.infos[key])
534         return default
535
536     def getCpuInfos(self):
537         cpuInfos = []
538         for definition in self.findDefinitions(const.CPU_DEFINITION):
539             cpuInfos.append(getInfos(definition))
540         return cpuInfos
541
542     def searchFiles(self, filename):
543         file_dict = self.infos["FILE_DICT"]
544         return [(filename, dirname) for dirname in file_dict.get(filename, [])]
545
546     def findDefinitions(self, ftype):
547         # Maintain a cache for every scanned BERTOS_PATH
548         definitions_dict = self._cached_queries.get(self.infos["BERTOS_PATH"], {})
549         definitions = definitions_dict.get(ftype, None)
550         if definitions is not None:
551             return definitions
552         file_dict = self.infos["FILE_DICT"]
553         definitions = []
554         for filename in file_dict:
555             if fnmatch.fnmatch(filename, ftype):
556                 definitions += [(filename, dirname) for dirname in file_dict.get(filename, [])]
557
558         # If no cache for the current BERTOS_PATH create an empty one
559         if not definitions_dict:
560             self._cached_queries[self.infos["BERTOS_PATH"]] = {}
561         # Fill the empty cache with the result
562         self._cached_queries[self.infos["BERTOS_PATH"]][ftype] = definitions
563         return definitions
564
565     def setEnabledModules(self, enabled_modules):
566         modules = self.infos["MODULES"]
567         files = {}
568         for module, information in modules.items():
569             information["enabled"] = module in enabled_modules
570             if information["enabled"]:
571                 for dependency in information["depends"]:
572                     if not dependency in modules:
573                         files[dependency] = files.get(dependency, 0) + 1
574         self.infos["MODULES"] = modules
575         self.infos["FILES"] = files
576
577     def __repr__(self):
578         return "<BProject:instance %d>%s" %(id(self), repr(self.infos))