Add support for parametric hw dir.
[bertos.git] / wizard / BProject.py
1 #!/usr/bin/env python
2 # encoding: utf-8
3 #
4 # This file is part of BeRTOS.
5 #
6 # Bertos is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
19 #
20 # As a special exception, you may use this file as part of a free software
21 # library without restriction.  Specifically, if other files instantiate
22 # templates or use macros or inline functions from this file, or you compile
23 # this file and link it with other files to produce an executable, this
24 # file does not by itself cause the resulting executable to be covered by
25 # the GNU General Public License.  This exception does not however
26 # invalidate any other reasons why the executable file might be covered by
27 # the GNU General Public License.
28 #
29 # Copyright 2008 Develer S.r.l. (http://www.develer.com/)
30 #
31 # $Id$
32 #
33 # Author: Lorenzo Berni <duplo@develer.com>
34 #
35
36 import os
37 import fnmatch
38 import copy
39 import pickle
40 import shutil
41 import copytree
42
43 import DefineException
44
45 from LoadException import VersionException, ToolchainException
46
47 import const
48
49 from bertos_utils import (
50                             # Utility functions
51                             isBertosDir, getTagSet, getInfos, updateConfigurationValues,
52                             loadConfigurationInfos, loadDefineLists, loadModuleDefinition,
53                             getCommentList, sub,
54
55                             # Project creation functions
56                             projectFileGenerator, versionFileGenerator, loadPlugin, 
57                             mergeSources,
58
59                             # Custom exceptions
60                             ParseError, SupportedException
61                         )
62 import bertos_utils
63
64 from compatibility import updateProject
65
66 class BProject(object):
67     """
68     Simple class for store and retrieve project informations.
69     """
70
71     def __init__(self, project_file="", info_dict={}):
72         self.infos = {}
73         self._cached_queries = {}
74         self.edit = False
75         if project_file:
76             self.edit = True
77             self.loadBertosProject(project_file, info_dict)
78
79     #--- Load methods (methods that loads data into project) ------------------#
80
81     def loadBertosProject(self, project_file, info_dict):
82         project_dir = os.path.dirname(project_file)
83         project_data = pickle.loads(open(project_file, "r").read())
84         updateProject(project_data)
85         # If PROJECT_NAME is not defined it use the directory name as PROJECT_NAME
86         # NOTE: this can throw an Exception if the user has changed the directory containing the project
87         self.infos["PROJECT_NAME"] = project_data.get("PROJECT_NAME", os.path.basename(project_dir))
88         self.infos["PROJECT_PATH"] = os.path.dirname(project_file)
89         project_src_path = os.path.join(project_dir, project_data.get("PROJECT_SRC_PATH", project_data["PROJECT_NAME"]))
90         if project_src_path:
91             self.infos["PROJECT_SRC_PATH"] = project_src_path
92             
93         else:
94             # In projects created with older versions of the Wizard this metadata doesn't exist
95             self.infos["PROJECT_SRC_PATH"] = os.path.join(self.infos["PROJECT_PATH"], self.infos["PROJECT_NAME"])
96         self.infos["HW_PATH"] = os.path.join(self.infos["PROJECT_PATH"], project_data.get("HW_PATH", self.infos["PROJECT_PATH"]))
97
98         linked_sources_path = project_data["BERTOS_PATH"]
99         sources_abspath = os.path.abspath(os.path.join(project_dir, linked_sources_path))
100         project_data["BERTOS_PATH"] = sources_abspath
101         
102         self._loadBertosSourceStuff(project_data["BERTOS_PATH"], info_dict.get("BERTOS_PATH", None))
103         
104         self.infos["PRESET"] = project_data.get("PRESET", False)
105
106         # For those projects that don't have a VERSION file create a dummy one.
107         if not isBertosDir(project_dir) and not self.is_preset:
108             version_file = open(os.path.join(const.DATA_DIR, "vtemplates/VERSION"), "r").read()
109             open(os.path.join(project_dir, "VERSION"), "w").write(version_file.replace("$version", "").strip())
110
111         self.loadSourceTree()
112         self._loadCpuStuff(project_data["CPU_NAME"], project_data["SELECTED_FREQ"])
113         self._loadToolchainStuff(project_data["TOOLCHAIN"], info_dict.get("TOOLCHAIN", None))
114         self.infos["OUTPUT"] = project_data["OUTPUT"]
115         self.loadModuleData(True)
116         self.setEnabledModules(project_data["ENABLED_MODULES"])
117
118     def _loadBertosSourceStuff(self, sources_path, forced_version=None):
119         if forced_version:
120             sources_path = forced_version
121         if os.path.exists(sources_path):
122             self.infos["BERTOS_PATH"] = sources_path
123         else:
124             raise VersionException(self)
125
126     def _loadCpuStuff(self, cpu_name, cpu_frequency):
127         self.infos["CPU_NAME"] = cpu_name
128         cpu_info = self.getCpuInfos()
129         for cpu in cpu_info:
130             if cpu["CPU_NAME"] == cpu_name:
131                 self.infos["CPU_INFOS"] = cpu
132                 break
133         tag_list = getTagSet(cpu_info)
134         # Create, fill and store the dict with the tags
135         tag_dict = {}
136         for element in tag_list:
137             tag_dict[element] = False
138         infos = self.info("CPU_INFOS")
139         for tag in tag_dict:
140             if tag in infos["CPU_TAGS"] + [infos["CPU_NAME"], infos["TOOLCHAIN"]]:
141                 tag_dict[tag] = True
142             else:
143                 tag_dict[tag] = False
144         self.infos["ALL_CPU_TAGS"] = tag_dict
145         self.infos["SELECTED_FREQ"] = cpu_frequency
146
147     def _loadToolchainStuff(self, toolchain, forced_toolchain=None):
148         toolchain = toolchain
149         if forced_toolchain:
150             toolchain = forced_toolchain
151         if os.path.exists(toolchain["path"]):
152             self.infos["TOOLCHAIN"] = toolchain
153         else:
154             raise ToolchainException(self)
155
156     def loadProjectFromPreset(self, preset):
157         """
158         Load a project from a preset.
159         NOTE: this is a stub.
160         """
161         project_file = os.path.join(preset, "project.bertos")
162         project_data = pickle.loads(open(project_file, "r").read())
163         self.loadSourceTree()
164         self._loadCpuStuff(project_data["CPU_NAME"], project_data["SELECTED_FREQ"])
165         self._loadToolchainStuff(project_data["TOOLCHAIN"])
166
167         # NOTE: this is a HACK!!!
168         # TODO: find a better way to reuse loadModuleData
169         preset_project_name = project_data.get("PROJECT_NAME", os.path.basename(preset))
170         preset_prj_src_path = os.path.join(preset, project_data.get("PROJECT_SRC_PATH", os.path.join(preset, preset_project_name)))
171         preset_hw_path = os.path.join(preset, project_data.get("PROJET_HW_DIR", preset))
172
173         old_project_name = self.infos["PROJECT_NAME"]
174         old_project_path = self.infos["PROJECT_PATH"]
175         old_project_src_path = self.infos["PROJECT_SRC_PATH"]
176
177         self.infos["PROJECT_NAME"] = preset_project_name
178         self.infos["PROJECT_PATH"] = preset
179         self.infos["PROJECT_SRC_PATH"] = preset_prj_src_path
180
181         self.loadModuleData(True)
182         self.setEnabledModules(project_data["ENABLED_MODULES"])
183
184         self.infos["PROJECT_NAME"] = old_project_name
185         self.infos["PROJECT_PATH"] = old_project_path
186         self.infos["PROJECT_SRC_PATH"] = old_project_src_path
187         # End of the ugly HACK!
188
189         self.infos["PRESET_NAME"] = preset_project_name
190         self.infos["PRESET_PATH"] = preset
191         self.infos["PRESET_SRC_PATH"] = preset_prj_src_path
192         self.infos["PRESET_HW_DIR"] = preset_hw_path
193
194     def loadProjectPresets(self):
195         """
196         Load the default presets (into the const.PREDEFINED_BOARDS_DIR).
197         """
198         # NOTE: this method does nothing (for now).
199         preset_path = os.path.join(self.infos["BERTOS_PATH"], const.PREDEFINED_BOARDS_DIR)
200         preset_tree = {}
201         if os.path.exists(preset_path):
202             preset_tree = self._loadProjectPresetTree(preset_path)
203         self.infos["PRESET_TREE"] = preset_tree
204
205     def _loadProjectPresetTree(self, path):
206         _tree = {}
207         _tree["info"] = self._loadPresetInfo(os.path.join(path, const.PREDEFINED_BOARD_SPEC_FILE))
208         _tree["info"]["filename"] = os.path.basename(path)
209         _tree["info"]["path"] = path
210         _tree["children"] = []
211         entries = set(os.listdir(path))
212         for entry in entries:
213             _path = os.path.join(path, entry)
214             if os.path.isdir(_path):
215                 sub_entries = set(os.listdir(_path))
216                 if const.PREDEFINED_BOARD_SPEC_FILE in sub_entries:
217                     _tree["children"].append(self._loadProjectPresetTree(_path))
218         # Add into the info dict the dir type (dir/project)
219         if _tree["children"]:
220             _tree["info"]["type"] = "dir"
221         else:
222             _tree["info"]["type"] = "project"
223         return _tree
224
225     def _loadPresetInfo(self, preset_spec_file):
226         D = {}
227         execfile(preset_spec_file, {}, D)
228         return D
229
230     def loadModuleData(self, edit=False):
231         module_info_dict = {}
232         list_info_dict = {}
233         configuration_info_dict = {}
234         file_dict = {}
235         for filename, path in self.findDefinitions("*.h") + self.findDefinitions("*.c") + self.findDefinitions("*.s") + self.findDefinitions("*.S"):
236             comment_list = getCommentList(open(path + "/" + filename, "r").read())
237             if len(comment_list) > 0:
238                 module_info = {}
239                 configuration_info = {}
240                 try:
241                     to_be_parsed, module_dict = loadModuleDefinition(comment_list[0])
242                 except ParseError, err:
243                     raise DefineException.ModuleDefineException(path, err.line_number, err.line)
244                 for module, information in module_dict.items():
245                     if "depends" not in information:
246                         information["depends"] = ()
247                     information["depends"] += (filename.split(".")[0],)
248                     information["category"] = os.path.basename(path)
249                     if "configuration" in information and len(information["configuration"]):
250                         configuration = module_dict[module]["configuration"]
251                         try:
252                             configuration_info[configuration] = loadConfigurationInfos(self.infos["BERTOS_PATH"] + "/" + configuration)
253                         except ParseError, err:
254                             raise DefineException.ConfigurationDefineException(self.infos["BERTOS_PATH"] + "/" + configuration, err.line_number, err.line)
255                         if edit:
256                             try:
257                                 path = self.infos["PROJECT_SRC_PATH"]
258                                 user_configuration = loadConfigurationInfos(configuration.replace("bertos", path))
259                                 configuration_info[configuration] = updateConfigurationValues(configuration_info[configuration], user_configuration)
260                             except ParseError, err:
261                                 raise DefineException.ConfigurationDefineException(configuration.replace("bertos", path))
262                 module_info_dict.update(module_dict)
263                 configuration_info_dict.update(configuration_info)
264                 if to_be_parsed:
265                     try:
266                         list_dict = loadDefineLists(comment_list[1:])
267                         list_info_dict.update(list_dict)
268                     except ParseError, err:
269                         raise DefineException.EnumDefineException(path, err.line_number, err.line)
270         for tag in self.infos["CPU_INFOS"]["CPU_TAGS"]:
271             for filename, path in self.findDefinitions("*_" + tag + ".h"):
272                 comment_list = getCommentList(open(path + "/" + filename, "r").read())
273                 list_info_dict.update(loadDefineLists(comment_list))
274         self.infos["MODULES"] = module_info_dict
275         self.infos["LISTS"] = list_info_dict
276         self.infos["CONFIGURATIONS"] = configuration_info_dict
277         self.infos["FILES"] = file_dict
278
279     def loadSourceTree(self):
280         """
281         Index BeRTOS source and load it in memory.
282         """
283         # Index only the BERTOS_PATH/bertos content
284         bertos_sources_dir = os.path.join(self.info("BERTOS_PATH"), "bertos")
285         file_dict = {}
286         if os.path.exists(bertos_sources_dir):
287             for element in os.walk(bertos_sources_dir):
288                 for f in element[2]:
289                     file_dict[f] = file_dict.get(f, []) + [element[0]]
290         self.infos["FILE_DICT"] = file_dict
291
292     def reloadCpuInfo(self):
293         for cpu_info in self.getCpuInfos():
294             if cpu_info["CPU_NAME"] == self.infos["CPU_NAME"]:
295                 self.infos["CPU_INFOS"] = cpu_info
296
297     #-------------------------------------------------------------------------#
298
299     def createBertosProject(self):
300         # NOTE: Temporary hack.
301         if self.edit:
302             self._editBertosProject()
303         else:
304             if not self.from_preset:
305                 self._newCustomBertosProject()
306             else:
307                 self._newBertosProjectFromPreset()
308
309     def _newBertosProject(self):
310         for directory in (self.maindir, self.srcdir, self.prjdir, self.cfgdir, self.hwdir):
311             self._createDirectory(directory)
312         # Write the project file
313         self._writeProjectFile(os.path.join(self.maindir, "project.bertos"))
314         # VERSION file
315         self._writeVersionFile(os.path.join(self.maindir, "VERSION"))
316         # Destination makefile
317         self._writeMakefile()
318         # Copy the sources
319         self._copySources(self.sources_dir, self.srcdir)
320         # Set properly the autoenabled parameters
321         self._setupAutoenabledParameters()
322         # Copy all the configuration files
323         self._writeCfgFiles(self.sources_dir, self.cfgdir)
324         # Destination wizard mk file
325         self._writeWizardMkFile()
326
327     def _newCustomBertosProject(self):
328         # Create/write/copy the common things
329         self._newBertosProject()
330         # Copy the clean hw files
331         self._createDirectory(self.hwdir)
332         # Copy all the hw files
333         self._writeHwFiles(self.sources_dir, self.hwdir)
334         # Destination user mk file
335         self._writeUserMkFile()
336         # Destination main.c file
337         self._writeMainFile(self.prjdir + "/main.c")
338         # Create project files for selected plugins
339         self._createProjectFiles()
340
341     def _newBertosProjectFromPreset(self):
342         # Create/write/copy the common things
343         self._newBertosProject()
344
345         # Copy all the files and dirs except cfg/hw/*.mk
346         self._writeCustomSrcFiles()
347         
348         # Copy the hw files
349         self._writeHwFiles(self.src_hwdir, self.hwdir)
350
351         # Copyt the new *_user.mk file
352         self._writeUserMkFileFromPreset()
353
354         if self.infos["EMPTY_MAIN"]:
355             # Create and empty main.c file only if the user check the box
356             self._writeMainFile(self.prjdir + "/main.c")
357
358         # Create project files for selected plugins
359         self._createProjectFiles()
360
361     def _editBertosProject(self):
362         # Write the project file
363         self._writeProjectFile(os.path.join(self.maindir, "project.bertos"))
364         if not self.is_preset:
365             # Generate this files only if the project isn't a preset
366             # VERSION file
367             self._writeVersionFile(os.path.join(self.maindir, "VERSION"))
368             # Destination makefile
369             self._writeMakefile()
370             # Merge sources
371             self._mergeSources(self.sources_dir, self.srcdir, self.old_srcdir)
372             # Copy all the hw files
373             self._writeHwFiles(self.sources_dir, self.hwdir)
374             # Destination wizard mk file
375             self._writeWizardMkFile()
376         # Set properly the autoenabled parameters
377         self._setupAutoenabledParameters()
378         # Copy all the configuration files
379         self._writeCfgFiles(self.sources_dir, self.cfgdir)
380         if not self.is_preset:
381             # Create project files for selected plugins only if the project isn't a preset
382             self._createProjectFiles()
383
384     def _createProjectFiles(self):
385         # Files for selected plugins
386         relevants_files = {}
387         for plugin in self.infos["OUTPUT"]:
388             module = loadPlugin(plugin)
389             relevants_files[plugin] = module.createProject(self)
390         self.infos["RELEVANT_FILES"] = relevants_files
391
392     def _writeVersionFile(self, filename):
393         version_file = open(os.path.join(const.DATA_DIR, "vtemplates/VERSION"), "r").read()
394         open(filename, "w").write(versionFileGenerator(self, version_file))
395
396     def _writeProjectFile(self, filename):
397         f = open(filename, "w")
398         f.write(projectFileGenerator(self))
399         f.close()
400
401     def _writeMakefile(self):
402         bertos_utils.makefileGenerator(self)
403
404     def _writeUserMkFile(self):
405         bertos_utils.userMkGenerator(self)
406
407     def _writeUserMkFileFromPreset(self):
408         bertos_utils.userMkGeneratorFromPreset(self)
409
410     def _writeWizardMkFile(self):
411         bertos_utils.mkGenerator(self)
412
413     def _writeMainFile(self, filename):
414         main = open(os.path.join(const.DATA_DIR, "srctemplates/main.c"), "r").read()
415         open(filename, "w").write(main)
416
417     def _writeHwFiles(self, source_dir, destination_dir):
418         for module, information in self.infos["MODULES"].items():
419             for hwfile in information["hw"]:
420                 string = open(source_dir + "/" + hwfile, "r").read()
421                 hwfile_path = destination_dir + "/" + os.path.basename(hwfile)
422                 if not self.edit or not os.path.exists(hwfile_path):
423                     # If not in editing mode it copies all the hw files. If in
424                     # editing mode it copies only the files that don't exist yet
425                     open(os.path.join(destination_dir,os.path.basename(hwfile)), "w").write(string)
426
427     def _writeCfgFiles(self, source_dir, destination_dir):
428         for configuration, information in self.infos["CONFIGURATIONS"].items():
429             string = open(source_dir + "/" + configuration, "r").read()
430             for start, parameter in information["paramlist"]:
431                 infos = information[parameter]
432                 value = infos["value"]
433                 if "unsigned" in infos["informations"] and infos["informations"]["unsigned"]:
434                     value += "U"
435                 if "long" in infos["informations"] and infos["informations"]["long"]:
436                     value += "L"
437                 string = sub(string, parameter, value)
438             f = open(os.path.join(destination_dir, os.path.basename(configuration)), "w")
439             f.write(string)
440             f.close()
441
442     def _writeCustomSrcFiles(self):
443         origin = self.infos["PRESET_SRC_PATH"]
444         # Files to be ignored (all project files, cfg dir, wizard mk file, all global ignored dirs)
445         project_related_stuff = (
446             "cfg",
447             "hw",
448             self.infos["PRESET_NAME"] + ".mk",
449             self.infos["PRESET_NAME"] + "_user.mk",
450             "project.bertos",
451             self.infos["PRESET_NAME"] + ".project",
452             self.infos["PRESET_NAME"] + ".workspace",
453         ) + const.IGNORE_LIST
454         for element in os.listdir(origin):
455             if element not in project_related_stuff:
456                 full_path = os.path.join(origin, element)
457                 if os.path.isdir(full_path):
458                     copytree.copytree(full_path, os.path.join(self.prjdir, element), ignore_list=const.IGNORE_LIST)
459                 else:
460                     shutil.copy(full_path, self.prjdir)
461
462     def _setupAutoenabledParameters(self):
463         for module, information in self.infos["MODULES"].items():
464             if "configuration" in information and information["configuration"] != "":
465                 configurations = self.infos["CONFIGURATIONS"]
466                 configuration = configurations[information["configuration"]]
467                 for start, parameter in configuration["paramlist"]:
468                     if "type" in configuration[parameter]["informations"] and configuration[parameter]["informations"]["type"] == "autoenabled":
469                         configuration[parameter]["value"] = "1" if information["enabled"] else "0"
470                 self.infos["CONFIGURATIONS"] = configurations
471
472     @property
473     def maindir(self):
474         return self.infos.get("PROJECT_PATH", None)
475
476     @property
477     def srcdir(self):
478         if self.maindir:
479             return os.path.join(self.maindir, "bertos")
480         else:
481             return None
482
483     @property
484     def prjdir(self):
485         return self.infos.get("PROJECT_SRC_PATH", None)
486
487     @property
488     def hwdir(self):
489         if self.prjdir:
490             return os.path.join(self.prjdir, "hw")
491         else:
492             return None
493
494     @property
495     def cfgdir(self):
496         if self.prjdir:
497             return os.path.join(self.prjdir, "cfg")
498         else:
499             return None
500
501     @property
502     def old_srcdir(self):
503         return self.infos.get("OLD_BERTOS_PATH", None)
504
505     @property
506     def sources_dir(self):
507         return self.infos.get("BERTOS_PATH", None)
508
509     @property
510     def src_hwdir(self):
511         if self.from_preset:
512             return os.path.join(self.infos["PRESET_PATH"], self.infos["PRESET_HW_DIR"])
513         else:
514             return self.sources_dir
515
516     @property
517     def from_preset(self):
518         return self.infos.get("PROJECT_FROM_PRESET", False)
519
520     @property
521     def is_preset(self):
522         return self.infos.get("PRESET", False)
523
524     def _createDirectory(self, directory):
525         if not directory:
526             return
527         if os.path.isdir(directory):
528             shutil.rmtree(directory, True)
529         os.makedirs(directory)
530
531     def _copySources(self, origin, destination):
532         # If not in editing mode it copies all the bertos sources in the /bertos subdirectory of the project
533         shutil.rmtree(destination, True)
534         copytree.copytree(origin + "/bertos", destination, ignore_list=const.IGNORE_LIST)
535
536     def _mergeSources(self, origin, destination, old_sources_dir):
537         if old_sources_dir:
538             mergeSources(destination, origin, old_sources_dir)
539
540     def setInfo(self, key, value):
541         """
542         Store the given value with the name key.
543         """
544         self.infos[key] = value
545
546     def info(self, key, default=None):
547         """
548         Retrieve the value associated with the name key.
549         """
550         if key in self.infos:
551             return copy.deepcopy(self.infos[key])
552         return default
553
554     def getCpuInfos(self):
555         cpuInfos = []
556         for definition in self.findDefinitions(const.CPU_DEFINITION):
557             cpuInfos.append(getInfos(definition))
558         return cpuInfos
559
560     def searchFiles(self, filename):
561         file_dict = self.infos["FILE_DICT"]
562         return [(filename, dirname) for dirname in file_dict.get(filename, [])]
563
564     def findDefinitions(self, ftype):
565         # Maintain a cache for every scanned BERTOS_PATH
566         definitions_dict = self._cached_queries.get(self.infos["BERTOS_PATH"], {})
567         definitions = definitions_dict.get(ftype, None)
568         if definitions is not None:
569             return definitions
570         file_dict = self.infos["FILE_DICT"]
571         definitions = []
572         for filename in file_dict:
573             if fnmatch.fnmatch(filename, ftype):
574                 definitions += [(filename, dirname) for dirname in file_dict.get(filename, [])]
575
576         # If no cache for the current BERTOS_PATH create an empty one
577         if not definitions_dict:
578             self._cached_queries[self.infos["BERTOS_PATH"]] = {}
579         # Fill the empty cache with the result
580         self._cached_queries[self.infos["BERTOS_PATH"]][ftype] = definitions
581         return definitions
582
583     def setEnabledModules(self, enabled_modules):
584         modules = self.infos["MODULES"]
585         files = {}
586         for module, information in modules.items():
587             information["enabled"] = module in enabled_modules
588             if information["enabled"]:
589                 for dependency in information["depends"]:
590                     if not dependency in modules:
591                         files[dependency] = files.get(dependency, 0) + 1
592         self.infos["MODULES"] = modules
593         self.infos["FILES"] = files
594
595     def __repr__(self):
596         return "<BProject:instance %d>%s" %(id(self), repr(self.infos))