Use PRESET_NAME instead of PROJECT_NAME when creating a project from a preset (to...
[bertos.git] / wizard / BProject.py
1 #!/usr/bin/env python
2 # encoding: utf-8
3 #
4 # This file is part of BeRTOS.
5 #
6 # Bertos is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
19 #
20 # As a special exception, you may use this file as part of a free software
21 # library without restriction.  Specifically, if other files instantiate
22 # templates or use macros or inline functions from this file, or you compile
23 # this file and link it with other files to produce an executable, this
24 # file does not by itself cause the resulting executable to be covered by
25 # the GNU General Public License.  This exception does not however
26 # invalidate any other reasons why the executable file might be covered by
27 # the GNU General Public License.
28 #
29 # Copyright 2008 Develer S.r.l. (http://www.develer.com/)
30 #
31 # $Id$
32 #
33 # Author: Lorenzo Berni <duplo@develer.com>
34 #
35
36 import os
37 import fnmatch
38 import copy
39 import pickle
40 import shutil
41 import copytree
42
43 import DefineException
44
45 from LoadException import VersionException, ToolchainException
46
47 import const
48
49 from bertos_utils import (
50                             # Utility functions
51                             isBertosDir, getTagSet, getInfos, updateConfigurationValues,
52                             loadConfigurationInfos, loadDefineLists, loadModuleDefinition,
53                             getCommentList, sub,
54
55                             # Project creation functions
56                             projectFileGenerator, versionFileGenerator, makefileGenerator,
57                             userMkGenerator, mkGenerator, loadPlugin, mergeSources,
58
59                             # Custom exceptions
60                             ParseError, SupportedException
61                         )
62
63 class BProject(object):
64     """
65     Simple class for store and retrieve project informations.
66     """
67
68     def __init__(self, project_file="", info_dict={}):
69         self.infos = {}
70         self._cached_queries = {}
71         self.edit = False
72         if project_file:
73             self.edit = True
74             self.loadBertosProject(project_file, info_dict)
75
76     #--- Load methods (methods that loads data into project) ------------------#
77
78     def loadBertosProject(self, project_file, info_dict):
79         project_dir = os.path.dirname(project_file)
80         project_data = pickle.loads(open(project_file, "r").read())
81         # If PROJECT_NAME is not defined it use the directory name as PROJECT_NAME
82         # NOTE: this can throw an Exception if the user has changed the directory containing the project
83         self.infos["PROJECT_NAME"] = project_data.get("PROJECT_NAME", os.path.basename(project_dir))
84         self.infos["PROJECT_PATH"] = os.path.dirname(project_file)
85         project_src_path = project_data.get("PROJECT_SRC_PATH", None)
86         project_src_path = os.path.join(project_dir, project_data.get("PROJECT_SRC_PATH", None))
87         if project_src_path:
88             self.infos["PROJECT_SRC_PATH"] = project_src_path
89             
90         else:
91             # In projects created with older versions of the Wizard this metadata doesn't exist
92             self.infos["PROJECT_SRC_PATH"] = os.path.join(self.infos["PROJECT_PATH"], self.infos["PROJECT_NAME"])
93
94         wizard_version = project_data.get("WIZARD_VERSION", 0)
95         if wizard_version == 0:
96             # Ignore the BERTOS_PATH inside the project file for older project
97             project_data["BERTOS_PATH"] = project_dir
98         elif wizard_version == 1:
99             # Use SOURCES_PATH instead of BERTOS_PATH for backward compatibility
100             project_data["BERTOS_PATH"] = project_data["SOURCES_PATH"]
101         linked_sources_path = project_data["BERTOS_PATH"]
102         sources_abspath = os.path.abspath(os.path.join(project_dir, linked_sources_path))
103         project_data["BERTOS_PATH"] = sources_abspath
104         
105         self._loadBertosSourceStuff(project_data["BERTOS_PATH"], info_dict.get("BERTOS_PATH", None))
106         
107         self.infos["PRESET"] = project_data.get("PRESET", False)
108
109         # For those projects that don't have a VERSION file create a dummy one.
110         if not isBertosDir(project_dir) and not self.is_preset:
111             version_file = open(os.path.join(const.DATA_DIR, "vtemplates/VERSION"), "r").read()
112             open(os.path.join(project_dir, "VERSION"), "w").write(version_file.replace("$version", "").strip())
113
114         self.loadSourceTree()
115         self._loadCpuStuff(project_data["CPU_NAME"], project_data["SELECTED_FREQ"])
116         self._loadToolchainStuff(project_data["TOOLCHAIN"], info_dict.get("TOOLCHAIN", None))
117         self.infos["OUTPUT"] = project_data["OUTPUT"]
118         self.loadModuleData(True)
119         self.setEnabledModules(project_data["ENABLED_MODULES"])
120
121     def _loadBertosSourceStuff(self, sources_path, forced_version=None):
122         if forced_version:
123             sources_path = forced_version
124         if os.path.exists(sources_path):
125             self.infos["BERTOS_PATH"] = sources_path
126         else:
127             raise VersionException(self)
128
129     def _loadCpuStuff(self, cpu_name, cpu_frequency):
130         self.infos["CPU_NAME"] = cpu_name
131         cpu_info = self.getCpuInfos()
132         for cpu in cpu_info:
133             if cpu["CPU_NAME"] == cpu_name:
134                 self.infos["CPU_INFOS"] = cpu
135                 break
136         tag_list = getTagSet(cpu_info)
137         # Create, fill and store the dict with the tags
138         tag_dict = {}
139         for element in tag_list:
140             tag_dict[element] = False
141         infos = self.info("CPU_INFOS")
142         for tag in tag_dict:
143             if tag in infos["CPU_TAGS"] + [infos["CPU_NAME"], infos["TOOLCHAIN"]]:
144                 tag_dict[tag] = True
145             else:
146                 tag_dict[tag] = False
147         self.infos["ALL_CPU_TAGS"] = tag_dict
148         self.infos["SELECTED_FREQ"] = cpu_frequency
149
150     def _loadToolchainStuff(self, toolchain, forced_toolchain=None):
151         toolchain = toolchain
152         if forced_toolchain:
153             toolchain = forced_toolchain
154         if os.path.exists(toolchain["path"]):
155             self.infos["TOOLCHAIN"] = toolchain
156         else:
157             raise ToolchainException(self)
158
159     def loadProjectFromPreset(self, preset):
160         """
161         Load a project from a preset.
162         NOTE: this is a stub.
163         """
164         project_file = os.path.join(preset, "project.bertos")
165         project_data = pickle.loads(open(project_file, "r").read())
166         self.loadSourceTree()
167         self._loadCpuStuff(project_data["CPU_NAME"], project_data["SELECTED_FREQ"])
168         self._loadToolchainStuff(project_data["TOOLCHAIN"])
169
170         # NOTE: this is a HACK!!!
171         # TODO: find a better way to reuse loadModuleData
172         preset_project_name = project_data.get("PROJECT_NAME", os.path.basename(preset))
173         preset_prj_src_path = os.path.join(preset, project_data.get("PROJECT_SRC_PATH", os.path.join(preset, preset_project_name)))
174
175         old_project_name = self.infos["PROJECT_NAME"]
176         old_project_path = self.infos["PROJECT_PATH"]
177         old_project_src_path = self.infos["PROJECT_SRC_PATH"]
178
179         self.infos["PROJECT_NAME"] = preset_project_name
180         self.infos["PROJECT_PATH"] = preset
181         self.infos["PROJECT_SRC_PATH"] = preset_prj_src_path
182
183         self.loadModuleData(True)
184         self.setEnabledModules(project_data["ENABLED_MODULES"])
185
186         self.infos["PROJECT_NAME"] = old_project_name
187         self.infos["PROJECT_PATH"] = old_project_path
188         self.infos["PROJECT_SRC_PATH"] = old_project_src_path
189         # End of the ugly HACK!
190
191         self.infos["PRESET_NAME"] = preset_project_name
192         self.infos["PRESET_PATH"] = preset
193         self.infos["PRESET_SRC_PATH"] = preset_prj_src_path
194
195     def loadProjectPresets(self):
196         """
197         Load the default presets (into the const.PREDEFINED_BOARDS_DIR).
198         """
199         # NOTE: this method does nothing (for now).
200         preset_path = os.path.join(self.infos["BERTOS_PATH"], const.PREDEFINED_BOARDS_DIR)
201         preset_tree = {}
202         if os.path.exists(preset_path):
203             preset_tree = self._loadProjectPresetTree(preset_path)
204         self.infos["PRESET_TREE"] = preset_tree
205
206     def _loadProjectPresetTree(self, path):
207         _tree = {}
208         _tree["info"] = self._loadPresetInfo(os.path.join(path, const.PREDEFINED_BOARD_SPEC_FILE))
209         _tree["info"]["filename"] = os.path.basename(path)
210         _tree["info"]["path"] = path
211         _tree["children"] = []
212         entries = set(os.listdir(path))
213         for entry in entries:
214             _path = os.path.join(path, entry)
215             if os.path.isdir(_path):
216                 sub_entries = set(os.listdir(_path))
217                 if const.PREDEFINED_BOARD_SPEC_FILE in sub_entries:
218                     _tree["children"].append(self._loadProjectPresetTree(_path))
219         # Add into the info dict the dir type (dir/project)
220         if _tree["children"]:
221             _tree["info"]["type"] = "dir"
222         else:
223             _tree["info"]["type"] = "project"
224         return _tree
225
226     def _loadPresetInfo(self, preset_spec_file):
227         D = {}
228         execfile(preset_spec_file, {}, D)
229         return D
230
231     def loadModuleData(self, edit=False):
232         module_info_dict = {}
233         list_info_dict = {}
234         configuration_info_dict = {}
235         file_dict = {}
236         for filename, path in self.findDefinitions("*.h") + self.findDefinitions("*.c") + self.findDefinitions("*.s") + self.findDefinitions("*.S"):
237             comment_list = getCommentList(open(path + "/" + filename, "r").read())
238             if len(comment_list) > 0:
239                 module_info = {}
240                 configuration_info = {}
241                 try:
242                     to_be_parsed, module_dict = loadModuleDefinition(comment_list[0])
243                 except ParseError, err:
244                     raise DefineException.ModuleDefineException(path, err.line_number, err.line)
245                 for module, information in module_dict.items():
246                     if "depends" not in information:
247                         information["depends"] = ()
248                     information["depends"] += (filename.split(".")[0],)
249                     information["category"] = os.path.basename(path)
250                     if "configuration" in information and len(information["configuration"]):
251                         configuration = module_dict[module]["configuration"]
252                         try:
253                             configuration_info[configuration] = loadConfigurationInfos(self.infos["BERTOS_PATH"] + "/" + configuration)
254                         except ParseError, err:
255                             raise DefineException.ConfigurationDefineException(self.infos["BERTOS_PATH"] + "/" + configuration, err.line_number, err.line)
256                         if edit:
257                             try:
258                                 path = self.infos["PROJECT_SRC_PATH"]
259                                 user_configuration = loadConfigurationInfos(configuration.replace("bertos", path))
260                                 configuration_info[configuration] = updateConfigurationValues(configuration_info[configuration], user_configuration)
261                             except ParseError, err:
262                                 raise DefineException.ConfigurationDefineException(configuration.replace("bertos", path))
263                 module_info_dict.update(module_dict)
264                 configuration_info_dict.update(configuration_info)
265                 if to_be_parsed:
266                     try:
267                         list_dict = loadDefineLists(comment_list[1:])
268                         list_info_dict.update(list_dict)
269                     except ParseError, err:
270                         raise DefineException.EnumDefineException(path, err.line_number, err.line)
271         for tag in self.infos["CPU_INFOS"]["CPU_TAGS"]:
272             for filename, path in self.findDefinitions("*_" + tag + ".h"):
273                 comment_list = getCommentList(open(path + "/" + filename, "r").read())
274                 list_info_dict.update(loadDefineLists(comment_list))
275         self.infos["MODULES"] = module_info_dict
276         self.infos["LISTS"] = list_info_dict
277         self.infos["CONFIGURATIONS"] = configuration_info_dict
278         self.infos["FILES"] = file_dict
279
280     def loadSourceTree(self):
281         """
282         Index BeRTOS source and load it in memory.
283         """
284         # Index only the BERTOS_PATH/bertos content
285         bertos_sources_dir = os.path.join(self.info("BERTOS_PATH"), "bertos")
286         file_dict = {}
287         if os.path.exists(bertos_sources_dir):
288             for element in os.walk(bertos_sources_dir):
289                 for f in element[2]:
290                     file_dict[f] = file_dict.get(f, []) + [element[0]]
291         self.infos["FILE_DICT"] = file_dict
292
293     def reloadCpuInfo(self):
294         for cpu_info in self.getCpuInfos():
295             if cpu_info["CPU_NAME"] == self.infos["CPU_NAME"]:
296                 self.infos["CPU_INFOS"] = cpu_info
297
298     #-------------------------------------------------------------------------#
299
300     def createBertosProject(self):
301         # NOTE: Temporary hack.
302         if self.edit:
303             self._editBertosProject()
304         else:
305             if not self.from_preset:
306                 self._newCustomBertosProject()
307             else:
308                 self._newBertosProjectFromPreset()
309
310     def _newBertosProject(self):
311         for directory in (self.maindir, self.srcdir, self.prjdir, self.cfgdir):
312             self._createDirectory(directory)
313         # Write the project file
314         self._writeProjectFile(os.path.join(self.maindir, "project.bertos"))
315         # VERSION file
316         self._writeVersionFile(os.path.join(self.maindir, "VERSION"))
317         # Destination makefile
318         self._writeMakefile(os.path.join(self.maindir, "Makefile"))
319         # Copy the sources
320         self._copySources(self.sources_dir, self.srcdir)
321         # Set properly the autoenabled parameters
322         self._setupAutoenabledParameters()
323         # Copy all the configuration files
324         self._writeCfgFiles(self.sources_dir, self.cfgdir)
325         # Destination wizard mk file
326         self._writeWizardMkFile(os.path.join(self.prjdir, os.path.basename(self.prjdir) + "_wiz.mk"))
327
328     def _newCustomBertosProject(self):
329         # Create/write/copy the common things
330         self._newBertosProject()
331         # Copy the clean hw files
332         self._createDirectory(self.hwdir)
333         # Copy all the hw files
334         self._writeHwFiles(self.sources_dir, self.hwdir)
335         # Destination user mk file
336         self._writeUserMkFile(os.path.join(self.prjdir, os.path.basename(self.prjdir) + ".mk"))
337         # Destination main.c file
338         self._writeMainFile(self.prjdir + "/main.c")
339         # Create project files for selected plugins
340         self._createProjectFiles()
341
342     def _newBertosProjectFromPreset(self):
343         # Create/write/copy the common things
344         self._newBertosProject()
345
346         # Copy all the files and dirs except cfg/hw/*_wiz.mk
347         self._writeCustomSrcFiles()
348
349         if self.infos["EMPTY_MAIN"]:
350             # Create and empty main.c file only if the user check the box
351             self._writeMainFile(self.prjdir + "/main.c")
352
353         # Create project files for selected plugins
354         self._createProjectFiles()
355
356     def _editBertosProject(self):
357         # Write the project file
358         self._writeProjectFile(os.path.join(self.maindir, "project.bertos"))
359         if not self.is_preset:
360             # Generate this files only if the project isn't a preset
361             # VERSION file
362             self._writeVersionFile(os.path.join(self.maindir, "VERSION"))
363             # Destination makefile
364             self._writeMakefile(os.path.join(self.maindir, "Makefile"))
365             # Merge sources
366             self._mergeSources(self.sources_dir, self.srcdir, self.old_srcdir)
367             # Copy all the hw files
368             self._writeHwFiles(self.sources_dir, self.hwdir)
369             # Destination wizard mk file
370             self._writeWizardMkFile(os.path.join(self.prjdir, os.path.basename(self.prjdir) + "_wiz.mk"))
371         # Set properly the autoenabled parameters
372         self._setupAutoenabledParameters()
373         # Copy all the configuration files
374         self._writeCfgFiles(self.sources_dir, self.cfgdir)
375         if not self.is_preset:
376             # Create project files for selected plugins only if the project isn't a preset
377             self._createProjectFiles()
378
379     def _createProjectFiles(self):
380         # Files for selected plugins
381         relevants_files = {}
382         for plugin in self.infos["OUTPUT"]:
383             module = loadPlugin(plugin)
384             relevants_files[plugin] = module.createProject(self)
385         self.infos["RELEVANT_FILES"] = relevants_files
386
387     def _writeVersionFile(self, filename):
388         version_file = open(os.path.join(const.DATA_DIR, "vtemplates/VERSION"), "r").read()
389         open(filename, "w").write(versionFileGenerator(self, version_file))
390
391     def _writeProjectFile(self, filename):
392         f = open(filename, "w")
393         f.write(projectFileGenerator(self))
394         f.close()
395
396     def _writeMakefile(self, filename):
397         makefile = open(os.path.join(const.DATA_DIR, "mktemplates/Makefile"), "r").read()
398         makefile = makefileGenerator(self, makefile)
399         open(filename, "w").write(makefile)
400
401     def _writeUserMkFile(self, filename):
402         makefile = open(os.path.join(const.DATA_DIR, "mktemplates/template.mk"), "r").read()
403         # Deadly performances loss was here :(
404         makefile = userMkGenerator(self, makefile)
405         open(filename, "w").write(makefile)
406
407     def _writeWizardMkFile(self, filename):
408         makefile = open(os.path.join(const.DATA_DIR, "mktemplates/template_wiz.mk"), "r").read()
409         makefile = mkGenerator(self, makefile)
410         open(filename, "w").write(makefile)
411
412     def _writeMainFile(self, filename):
413         main = open(os.path.join(const.DATA_DIR, "srctemplates/main.c"), "r").read()
414         open(filename, "w").write(main)
415
416     def _writeHwFiles(self, source_dir, destination_dir):
417         for module, information in self.infos["MODULES"].items():
418             for hwfile in information["hw"]:
419                 string = open(source_dir + "/" + hwfile, "r").read()
420                 hwfile_path = destination_dir + "/" + os.path.basename(hwfile)
421                 if not self.edit or not os.path.exists(hwfile_path):
422                     # If not in editing mode it copies all the hw files. If in
423                     # editing mode it copies only the files that don't exist yet
424                     open(os.path.join(destination_dir,os.path.basename(hwfile)), "w").write(string)
425
426     def _writeCfgFiles(self, source_dir, destination_dir):
427         for configuration, information in self.infos["CONFIGURATIONS"].items():
428             string = open(source_dir + "/" + configuration, "r").read()
429             for start, parameter in information["paramlist"]:
430                 infos = information[parameter]
431                 value = infos["value"]
432                 if "unsigned" in infos["informations"] and infos["informations"]["unsigned"]:
433                     value += "U"
434                 if "long" in infos["informations"] and infos["informations"]["long"]:
435                     value += "L"
436                 string = sub(string, parameter, value)
437             f = open(os.path.join(destination_dir, os.path.basename(configuration)), "w")
438             f.write(string)
439             f.close()
440
441     def _writeCustomSrcFiles(self):
442         origin = self.infos["PRESET_SRC_PATH"]
443         # Files to be ignored (all project files, cfg dir, wizard mk file, all global ignored dirs)
444         project_related_stuff = (
445             "cfg",
446             self.infos["PRESET_NAME"] + "_wiz.mk",
447             "project.bertos",
448             self.infos["PRESET_NAME"] + ".project",
449             self.infos["PRESET_NAME"] + ".workspace",
450         ) + const.IGNORE_LIST
451         for element in os.listdir(origin):
452             if element not in project_related_stuff:
453                 full_path = os.path.join(origin, element)
454                 if os.path.isdir(full_path):
455                     copytree.copytree(full_path, os.path.join(self.prjdir, element), ignore_list=const.IGNORE_LIST)
456                 else:
457                     shutil.copy(full_path, self.prjdir)
458
459     def _setupAutoenabledParameters(self):
460         for module, information in self.infos["MODULES"].items():
461             if "configuration" in information and information["configuration"] != "":
462                 configurations = self.infos["CONFIGURATIONS"]
463                 configuration = configurations[information["configuration"]]
464                 for start, parameter in configuration["paramlist"]:
465                     if "type" in configuration[parameter]["informations"] and configuration[parameter]["informations"]["type"] == "autoenabled":
466                         configuration[parameter]["value"] = "1" if information["enabled"] else "0"
467                 self.infos["CONFIGURATIONS"] = configurations
468
469     @property
470     def maindir(self):
471         return self.infos.get("PROJECT_PATH", None)
472
473     @property
474     def srcdir(self):
475         if self.maindir:
476             return os.path.join(self.maindir, "bertos")
477         else:
478             return None
479
480     @property
481     def prjdir(self):
482         return self.infos.get("PROJECT_SRC_PATH", None)
483
484     @property
485     def hwdir(self):
486         if self.prjdir:
487             return os.path.join(self.prjdir, "hw")
488         else:
489             return None
490
491     @property
492     def cfgdir(self):
493         if self.prjdir:
494             return os.path.join(self.prjdir, "cfg")
495         else:
496             return None
497
498     @property
499     def old_srcdir(self):
500         return self.infos.get("OLD_BERTOS_PATH", None)
501
502     @property
503     def sources_dir(self):
504         return self.infos.get("BERTOS_PATH", None)
505
506     @property
507     def from_preset(self):
508         return self.infos.get("PROJECT_FROM_PRESET", False)
509
510     @property
511     def is_preset(self):
512         return self.infos.get("PRESET", False)
513
514     def _createDirectory(self, directory):
515         if not directory:
516             return
517         if os.path.isdir(directory):
518             shutil.rmtree(directory, True)
519         os.makedirs(directory)
520
521     def _copySources(self, origin, destination):
522         # If not in editing mode it copies all the bertos sources in the /bertos subdirectory of the project
523         shutil.rmtree(destination, True)
524         copytree.copytree(origin + "/bertos", destination, ignore_list=const.IGNORE_LIST)
525
526     def _mergeSources(self, origin, destination, old_sources_dir):
527         if old_sources_dir:
528             mergeSources(destination, origin, old_sources_dir)
529
530     def setInfo(self, key, value):
531         """
532         Store the given value with the name key.
533         """
534         self.infos[key] = value
535
536     def info(self, key, default=None):
537         """
538         Retrieve the value associated with the name key.
539         """
540         if key in self.infos:
541             return copy.deepcopy(self.infos[key])
542         return default
543
544     def getCpuInfos(self):
545         cpuInfos = []
546         for definition in self.findDefinitions(const.CPU_DEFINITION):
547             cpuInfos.append(getInfos(definition))
548         return cpuInfos
549
550     def searchFiles(self, filename):
551         file_dict = self.infos["FILE_DICT"]
552         return [(filename, dirname) for dirname in file_dict.get(filename, [])]
553
554     def findDefinitions(self, ftype):
555         # Maintain a cache for every scanned BERTOS_PATH
556         definitions_dict = self._cached_queries.get(self.infos["BERTOS_PATH"], {})
557         definitions = definitions_dict.get(ftype, None)
558         if definitions is not None:
559             return definitions
560         file_dict = self.infos["FILE_DICT"]
561         definitions = []
562         for filename in file_dict:
563             if fnmatch.fnmatch(filename, ftype):
564                 definitions += [(filename, dirname) for dirname in file_dict.get(filename, [])]
565
566         # If no cache for the current BERTOS_PATH create an empty one
567         if not definitions_dict:
568             self._cached_queries[self.infos["BERTOS_PATH"]] = {}
569         # Fill the empty cache with the result
570         self._cached_queries[self.infos["BERTOS_PATH"]][ftype] = definitions
571         return definitions
572
573     def setEnabledModules(self, enabled_modules):
574         modules = self.infos["MODULES"]
575         files = {}
576         for module, information in modules.items():
577             information["enabled"] = module in enabled_modules
578             if information["enabled"]:
579                 for dependency in information["depends"]:
580                     if not dependency in modules:
581                         files[dependency] = files.get(dependency, 0) + 1
582         self.infos["MODULES"] = modules
583         self.infos["FILES"] = files
584
585     def __repr__(self):
586         return "<BProject:instance %d>%s" %(id(self), repr(self.infos))