More logic into makefiles generation functions
[bertos.git] / wizard / BProject.py
1 #!/usr/bin/env python
2 # encoding: utf-8
3 #
4 # This file is part of BeRTOS.
5 #
6 # Bertos is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
19 #
20 # As a special exception, you may use this file as part of a free software
21 # library without restriction.  Specifically, if other files instantiate
22 # templates or use macros or inline functions from this file, or you compile
23 # this file and link it with other files to produce an executable, this
24 # file does not by itself cause the resulting executable to be covered by
25 # the GNU General Public License.  This exception does not however
26 # invalidate any other reasons why the executable file might be covered by
27 # the GNU General Public License.
28 #
29 # Copyright 2008 Develer S.r.l. (http://www.develer.com/)
30 #
31 # $Id$
32 #
33 # Author: Lorenzo Berni <duplo@develer.com>
34 #
35
36 import os
37 import fnmatch
38 import copy
39 import pickle
40 import shutil
41 import copytree
42
43 import DefineException
44
45 from LoadException import VersionException, ToolchainException
46
47 import const
48
49 from bertos_utils import (
50                             # Utility functions
51                             isBertosDir, getTagSet, getInfos, updateConfigurationValues,
52                             loadConfigurationInfos, loadDefineLists, loadModuleDefinition,
53                             getCommentList, sub,
54
55                             # Project creation functions
56                             projectFileGenerator, versionFileGenerator, makefileGenerator,
57                             userMkGenerator, mkGenerator, loadPlugin, mergeSources,
58
59                             # Custom exceptions
60                             ParseError, SupportedException
61                         )
62
63 class BProject(object):
64     """
65     Simple class for store and retrieve project informations.
66     """
67
68     def __init__(self, project_file="", info_dict={}):
69         self.infos = {}
70         self._cached_queries = {}
71         self.edit = False
72         if project_file:
73             self.edit = True
74             self.loadBertosProject(project_file, info_dict)
75
76     #--- Load methods (methods that loads data into project) ------------------#
77
78     def loadBertosProject(self, project_file, info_dict):
79         project_dir = os.path.dirname(project_file)
80         project_data = pickle.loads(open(project_file, "r").read())
81         updateProject(project_data)
82         # If PROJECT_NAME is not defined it use the directory name as PROJECT_NAME
83         # NOTE: this can throw an Exception if the user has changed the directory containing the project
84         self.infos["PROJECT_NAME"] = project_data.get("PROJECT_NAME", os.path.basename(project_dir))
85         self.infos["PROJECT_PATH"] = os.path.dirname(project_file)
86         project_src_path = project_data.get("PROJECT_SRC_PATH", None)
87         project_src_path = os.path.join(project_dir, project_data.get("PROJECT_SRC_PATH", None))
88         if project_src_path:
89             self.infos["PROJECT_SRC_PATH"] = project_src_path
90             
91         else:
92             # In projects created with older versions of the Wizard this metadata doesn't exist
93             self.infos["PROJECT_SRC_PATH"] = os.path.join(self.infos["PROJECT_PATH"], self.infos["PROJECT_NAME"])
94
95         linked_sources_path = project_data["BERTOS_PATH"]
96         sources_abspath = os.path.abspath(os.path.join(project_dir, linked_sources_path))
97         project_data["BERTOS_PATH"] = sources_abspath
98         
99         self._loadBertosSourceStuff(project_data["BERTOS_PATH"], info_dict.get("BERTOS_PATH", None))
100         
101         self.infos["PRESET"] = project_data.get("PRESET", False)
102
103         # For those projects that don't have a VERSION file create a dummy one.
104         if not isBertosDir(project_dir) and not self.is_preset:
105             version_file = open(os.path.join(const.DATA_DIR, "vtemplates/VERSION"), "r").read()
106             open(os.path.join(project_dir, "VERSION"), "w").write(version_file.replace("$version", "").strip())
107
108         self.loadSourceTree()
109         self._loadCpuStuff(project_data["CPU_NAME"], project_data["SELECTED_FREQ"])
110         self._loadToolchainStuff(project_data["TOOLCHAIN"], info_dict.get("TOOLCHAIN", None))
111         self.infos["OUTPUT"] = project_data["OUTPUT"]
112         self.loadModuleData(True)
113         self.setEnabledModules(project_data["ENABLED_MODULES"])
114
115     def _loadBertosSourceStuff(self, sources_path, forced_version=None):
116         if forced_version:
117             sources_path = forced_version
118         if os.path.exists(sources_path):
119             self.infos["BERTOS_PATH"] = sources_path
120         else:
121             raise VersionException(self)
122
123     def _loadCpuStuff(self, cpu_name, cpu_frequency):
124         self.infos["CPU_NAME"] = cpu_name
125         cpu_info = self.getCpuInfos()
126         for cpu in cpu_info:
127             if cpu["CPU_NAME"] == cpu_name:
128                 self.infos["CPU_INFOS"] = cpu
129                 break
130         tag_list = getTagSet(cpu_info)
131         # Create, fill and store the dict with the tags
132         tag_dict = {}
133         for element in tag_list:
134             tag_dict[element] = False
135         infos = self.info("CPU_INFOS")
136         for tag in tag_dict:
137             if tag in infos["CPU_TAGS"] + [infos["CPU_NAME"], infos["TOOLCHAIN"]]:
138                 tag_dict[tag] = True
139             else:
140                 tag_dict[tag] = False
141         self.infos["ALL_CPU_TAGS"] = tag_dict
142         self.infos["SELECTED_FREQ"] = cpu_frequency
143
144     def _loadToolchainStuff(self, toolchain, forced_toolchain=None):
145         toolchain = toolchain
146         if forced_toolchain:
147             toolchain = forced_toolchain
148         if os.path.exists(toolchain["path"]):
149             self.infos["TOOLCHAIN"] = toolchain
150         else:
151             raise ToolchainException(self)
152
153     def loadProjectFromPreset(self, preset):
154         """
155         Load a project from a preset.
156         NOTE: this is a stub.
157         """
158         project_file = os.path.join(preset, "project.bertos")
159         project_data = pickle.loads(open(project_file, "r").read())
160         self.loadSourceTree()
161         self._loadCpuStuff(project_data["CPU_NAME"], project_data["SELECTED_FREQ"])
162         self._loadToolchainStuff(project_data["TOOLCHAIN"])
163
164         # NOTE: this is a HACK!!!
165         # TODO: find a better way to reuse loadModuleData
166         preset_project_name = project_data.get("PROJECT_NAME", os.path.basename(preset))
167         preset_prj_src_path = os.path.join(preset, project_data.get("PROJECT_SRC_PATH", os.path.join(preset, preset_project_name)))
168
169         old_project_name = self.infos["PROJECT_NAME"]
170         old_project_path = self.infos["PROJECT_PATH"]
171         old_project_src_path = self.infos["PROJECT_SRC_PATH"]
172
173         self.infos["PROJECT_NAME"] = preset_project_name
174         self.infos["PROJECT_PATH"] = preset
175         self.infos["PROJECT_SRC_PATH"] = preset_prj_src_path
176
177         self.loadModuleData(True)
178         self.setEnabledModules(project_data["ENABLED_MODULES"])
179
180         self.infos["PROJECT_NAME"] = old_project_name
181         self.infos["PROJECT_PATH"] = old_project_path
182         self.infos["PROJECT_SRC_PATH"] = old_project_src_path
183         # End of the ugly HACK!
184
185         self.infos["PRESET_NAME"] = preset_project_name
186         self.infos["PRESET_PATH"] = preset
187         self.infos["PRESET_SRC_PATH"] = preset_prj_src_path
188
189     def loadProjectPresets(self):
190         """
191         Load the default presets (into the const.PREDEFINED_BOARDS_DIR).
192         """
193         # NOTE: this method does nothing (for now).
194         preset_path = os.path.join(self.infos["BERTOS_PATH"], const.PREDEFINED_BOARDS_DIR)
195         preset_tree = {}
196         if os.path.exists(preset_path):
197             preset_tree = self._loadProjectPresetTree(preset_path)
198         self.infos["PRESET_TREE"] = preset_tree
199
200     def _loadProjectPresetTree(self, path):
201         _tree = {}
202         _tree["info"] = self._loadPresetInfo(os.path.join(path, const.PREDEFINED_BOARD_SPEC_FILE))
203         _tree["info"]["filename"] = os.path.basename(path)
204         _tree["info"]["path"] = path
205         _tree["children"] = []
206         entries = set(os.listdir(path))
207         for entry in entries:
208             _path = os.path.join(path, entry)
209             if os.path.isdir(_path):
210                 sub_entries = set(os.listdir(_path))
211                 if const.PREDEFINED_BOARD_SPEC_FILE in sub_entries:
212                     _tree["children"].append(self._loadProjectPresetTree(_path))
213         # Add into the info dict the dir type (dir/project)
214         if _tree["children"]:
215             _tree["info"]["type"] = "dir"
216         else:
217             _tree["info"]["type"] = "project"
218         return _tree
219
220     def _loadPresetInfo(self, preset_spec_file):
221         D = {}
222         execfile(preset_spec_file, {}, D)
223         return D
224
225     def loadModuleData(self, edit=False):
226         module_info_dict = {}
227         list_info_dict = {}
228         configuration_info_dict = {}
229         file_dict = {}
230         for filename, path in self.findDefinitions("*.h") + self.findDefinitions("*.c") + self.findDefinitions("*.s") + self.findDefinitions("*.S"):
231             comment_list = getCommentList(open(path + "/" + filename, "r").read())
232             if len(comment_list) > 0:
233                 module_info = {}
234                 configuration_info = {}
235                 try:
236                     to_be_parsed, module_dict = loadModuleDefinition(comment_list[0])
237                 except ParseError, err:
238                     raise DefineException.ModuleDefineException(path, err.line_number, err.line)
239                 for module, information in module_dict.items():
240                     if "depends" not in information:
241                         information["depends"] = ()
242                     information["depends"] += (filename.split(".")[0],)
243                     information["category"] = os.path.basename(path)
244                     if "configuration" in information and len(information["configuration"]):
245                         configuration = module_dict[module]["configuration"]
246                         try:
247                             configuration_info[configuration] = loadConfigurationInfos(self.infos["BERTOS_PATH"] + "/" + configuration)
248                         except ParseError, err:
249                             raise DefineException.ConfigurationDefineException(self.infos["BERTOS_PATH"] + "/" + configuration, err.line_number, err.line)
250                         if edit:
251                             try:
252                                 path = self.infos["PROJECT_SRC_PATH"]
253                                 user_configuration = loadConfigurationInfos(configuration.replace("bertos", path))
254                                 configuration_info[configuration] = updateConfigurationValues(configuration_info[configuration], user_configuration)
255                             except ParseError, err:
256                                 raise DefineException.ConfigurationDefineException(configuration.replace("bertos", path))
257                 module_info_dict.update(module_dict)
258                 configuration_info_dict.update(configuration_info)
259                 if to_be_parsed:
260                     try:
261                         list_dict = loadDefineLists(comment_list[1:])
262                         list_info_dict.update(list_dict)
263                     except ParseError, err:
264                         raise DefineException.EnumDefineException(path, err.line_number, err.line)
265         for tag in self.infos["CPU_INFOS"]["CPU_TAGS"]:
266             for filename, path in self.findDefinitions("*_" + tag + ".h"):
267                 comment_list = getCommentList(open(path + "/" + filename, "r").read())
268                 list_info_dict.update(loadDefineLists(comment_list))
269         self.infos["MODULES"] = module_info_dict
270         self.infos["LISTS"] = list_info_dict
271         self.infos["CONFIGURATIONS"] = configuration_info_dict
272         self.infos["FILES"] = file_dict
273
274     def loadSourceTree(self):
275         """
276         Index BeRTOS source and load it in memory.
277         """
278         # Index only the BERTOS_PATH/bertos content
279         bertos_sources_dir = os.path.join(self.info("BERTOS_PATH"), "bertos")
280         file_dict = {}
281         if os.path.exists(bertos_sources_dir):
282             for element in os.walk(bertos_sources_dir):
283                 for f in element[2]:
284                     file_dict[f] = file_dict.get(f, []) + [element[0]]
285         self.infos["FILE_DICT"] = file_dict
286
287     def reloadCpuInfo(self):
288         for cpu_info in self.getCpuInfos():
289             if cpu_info["CPU_NAME"] == self.infos["CPU_NAME"]:
290                 self.infos["CPU_INFOS"] = cpu_info
291
292     #-------------------------------------------------------------------------#
293
294     def createBertosProject(self):
295         # NOTE: Temporary hack.
296         if self.edit:
297             self._editBertosProject()
298         else:
299             if not self.from_preset:
300                 self._newCustomBertosProject()
301             else:
302                 self._newBertosProjectFromPreset()
303
304     def _newBertosProject(self):
305         for directory in (self.maindir, self.srcdir, self.prjdir, self.cfgdir):
306             self._createDirectory(directory)
307         # Write the project file
308         self._writeProjectFile(os.path.join(self.maindir, "project.bertos"))
309         # VERSION file
310         self._writeVersionFile(os.path.join(self.maindir, "VERSION"))
311         # Destination makefile
312         self._writeMakefile(os.path.join(self.maindir, "Makefile"))
313         # Copy the sources
314         self._copySources(self.sources_dir, self.srcdir)
315         # Set properly the autoenabled parameters
316         self._setupAutoenabledParameters()
317         # Copy all the configuration files
318         self._writeCfgFiles(self.sources_dir, self.cfgdir)
319         # Destination wizard mk file
320         self._writeWizardMkFile(os.path.join(self.prjdir, os.path.basename(self.prjdir) + "_wiz.mk"))
321
322     def _newCustomBertosProject(self):
323         # Create/write/copy the common things
324         self._newBertosProject()
325         # Copy the clean hw files
326         self._createDirectory(self.hwdir)
327         # Copy all the hw files
328         self._writeHwFiles(self.sources_dir, self.hwdir)
329         # Destination user mk file
330         self._writeUserMkFile(os.path.join(self.prjdir, os.path.basename(self.prjdir) + ".mk"))
331         # Destination main.c file
332         self._writeMainFile(self.prjdir + "/main.c")
333         # Create project files for selected plugins
334         self._createProjectFiles()
335
336     def _newBertosProjectFromPreset(self):
337         # Create/write/copy the common things
338         self._newBertosProject()
339
340         # Copy all the files and dirs except cfg/hw/*_wiz.mk
341         self._writeCustomSrcFiles()
342
343         if self.infos["EMPTY_MAIN"]:
344             # Create and empty main.c file only if the user check the box
345             self._writeMainFile(self.prjdir + "/main.c")
346
347         # Create project files for selected plugins
348         self._createProjectFiles()
349
350     def _editBertosProject(self):
351         # Write the project file
352         self._writeProjectFile(os.path.join(self.maindir, "project.bertos"))
353         if not self.is_preset:
354             # Generate this files only if the project isn't a preset
355             # VERSION file
356             self._writeVersionFile(os.path.join(self.maindir, "VERSION"))
357             # Destination makefile
358             self._writeMakefile(os.path.join(self.maindir, "Makefile"))
359             # Merge sources
360             self._mergeSources(self.sources_dir, self.srcdir, self.old_srcdir)
361             # Copy all the hw files
362             self._writeHwFiles(self.sources_dir, self.hwdir)
363             # Destination wizard mk file
364             self._writeWizardMkFile(os.path.join(self.prjdir, os.path.basename(self.prjdir) + "_wiz.mk"))
365         # Set properly the autoenabled parameters
366         self._setupAutoenabledParameters()
367         # Copy all the configuration files
368         self._writeCfgFiles(self.sources_dir, self.cfgdir)
369         if not self.is_preset:
370             # Create project files for selected plugins only if the project isn't a preset
371             self._createProjectFiles()
372
373     def _createProjectFiles(self):
374         # Files for selected plugins
375         relevants_files = {}
376         for plugin in self.infos["OUTPUT"]:
377             module = loadPlugin(plugin)
378             relevants_files[plugin] = module.createProject(self)
379         self.infos["RELEVANT_FILES"] = relevants_files
380
381     def _writeVersionFile(self, filename):
382         version_file = open(os.path.join(const.DATA_DIR, "vtemplates/VERSION"), "r").read()
383         open(filename, "w").write(versionFileGenerator(self, version_file))
384
385     def _writeProjectFile(self, filename):
386         f = open(filename, "w")
387         f.write(projectFileGenerator(self))
388         f.close()
389
390     def _writeMakefile(self, filename):
391         makefileGenerator(self, filename)
392
393     def _writeUserMkFile(self, filename):
394         userMkGenerator(self, filename)
395
396     def _writeWizardMkFile(self, filename):
397         mkGenerator(self, filename)
398
399     def _writeMainFile(self, filename):
400         main = open(os.path.join(const.DATA_DIR, "srctemplates/main.c"), "r").read()
401         open(filename, "w").write(main)
402
403     def _writeHwFiles(self, source_dir, destination_dir):
404         for module, information in self.infos["MODULES"].items():
405             for hwfile in information["hw"]:
406                 string = open(source_dir + "/" + hwfile, "r").read()
407                 hwfile_path = destination_dir + "/" + os.path.basename(hwfile)
408                 if not self.edit or not os.path.exists(hwfile_path):
409                     # If not in editing mode it copies all the hw files. If in
410                     # editing mode it copies only the files that don't exist yet
411                     open(os.path.join(destination_dir,os.path.basename(hwfile)), "w").write(string)
412
413     def _writeCfgFiles(self, source_dir, destination_dir):
414         for configuration, information in self.infos["CONFIGURATIONS"].items():
415             string = open(source_dir + "/" + configuration, "r").read()
416             for start, parameter in information["paramlist"]:
417                 infos = information[parameter]
418                 value = infos["value"]
419                 if "unsigned" in infos["informations"] and infos["informations"]["unsigned"]:
420                     value += "U"
421                 if "long" in infos["informations"] and infos["informations"]["long"]:
422                     value += "L"
423                 string = sub(string, parameter, value)
424             f = open(os.path.join(destination_dir, os.path.basename(configuration)), "w")
425             f.write(string)
426             f.close()
427
428     def _writeCustomSrcFiles(self):
429         origin = self.infos["PRESET_SRC_PATH"]
430         # Files to be ignored (all project files, cfg dir, wizard mk file, all global ignored dirs)
431         project_related_stuff = (
432             "cfg",
433             self.infos["PRESET_NAME"] + "_wiz.mk",
434             "project.bertos",
435             self.infos["PRESET_NAME"] + ".project",
436             self.infos["PRESET_NAME"] + ".workspace",
437         ) + const.IGNORE_LIST
438         for element in os.listdir(origin):
439             if element not in project_related_stuff:
440                 full_path = os.path.join(origin, element)
441                 if os.path.isdir(full_path):
442                     copytree.copytree(full_path, os.path.join(self.prjdir, element), ignore_list=const.IGNORE_LIST)
443                 else:
444                     shutil.copy(full_path, self.prjdir)
445
446     def _setupAutoenabledParameters(self):
447         for module, information in self.infos["MODULES"].items():
448             if "configuration" in information and information["configuration"] != "":
449                 configurations = self.infos["CONFIGURATIONS"]
450                 configuration = configurations[information["configuration"]]
451                 for start, parameter in configuration["paramlist"]:
452                     if "type" in configuration[parameter]["informations"] and configuration[parameter]["informations"]["type"] == "autoenabled":
453                         configuration[parameter]["value"] = "1" if information["enabled"] else "0"
454                 self.infos["CONFIGURATIONS"] = configurations
455
456     @property
457     def maindir(self):
458         return self.infos.get("PROJECT_PATH", None)
459
460     @property
461     def srcdir(self):
462         if self.maindir:
463             return os.path.join(self.maindir, "bertos")
464         else:
465             return None
466
467     @property
468     def prjdir(self):
469         return self.infos.get("PROJECT_SRC_PATH", None)
470
471     @property
472     def hwdir(self):
473         if self.prjdir:
474             return os.path.join(self.prjdir, "hw")
475         else:
476             return None
477
478     @property
479     def cfgdir(self):
480         if self.prjdir:
481             return os.path.join(self.prjdir, "cfg")
482         else:
483             return None
484
485     @property
486     def old_srcdir(self):
487         return self.infos.get("OLD_BERTOS_PATH", None)
488
489     @property
490     def sources_dir(self):
491         return self.infos.get("BERTOS_PATH", None)
492
493     @property
494     def from_preset(self):
495         return self.infos.get("PROJECT_FROM_PRESET", False)
496
497     @property
498     def is_preset(self):
499         return self.infos.get("PRESET", False)
500
501     def _createDirectory(self, directory):
502         if not directory:
503             return
504         if os.path.isdir(directory):
505             shutil.rmtree(directory, True)
506         os.makedirs(directory)
507
508     def _copySources(self, origin, destination):
509         # If not in editing mode it copies all the bertos sources in the /bertos subdirectory of the project
510         shutil.rmtree(destination, True)
511         copytree.copytree(origin + "/bertos", destination, ignore_list=const.IGNORE_LIST)
512
513     def _mergeSources(self, origin, destination, old_sources_dir):
514         if old_sources_dir:
515             mergeSources(destination, origin, old_sources_dir)
516
517     def setInfo(self, key, value):
518         """
519         Store the given value with the name key.
520         """
521         self.infos[key] = value
522
523     def info(self, key, default=None):
524         """
525         Retrieve the value associated with the name key.
526         """
527         if key in self.infos:
528             return copy.deepcopy(self.infos[key])
529         return default
530
531     def getCpuInfos(self):
532         cpuInfos = []
533         for definition in self.findDefinitions(const.CPU_DEFINITION):
534             cpuInfos.append(getInfos(definition))
535         return cpuInfos
536
537     def searchFiles(self, filename):
538         file_dict = self.infos["FILE_DICT"]
539         return [(filename, dirname) for dirname in file_dict.get(filename, [])]
540
541     def findDefinitions(self, ftype):
542         # Maintain a cache for every scanned BERTOS_PATH
543         definitions_dict = self._cached_queries.get(self.infos["BERTOS_PATH"], {})
544         definitions = definitions_dict.get(ftype, None)
545         if definitions is not None:
546             return definitions
547         file_dict = self.infos["FILE_DICT"]
548         definitions = []
549         for filename in file_dict:
550             if fnmatch.fnmatch(filename, ftype):
551                 definitions += [(filename, dirname) for dirname in file_dict.get(filename, [])]
552
553         # If no cache for the current BERTOS_PATH create an empty one
554         if not definitions_dict:
555             self._cached_queries[self.infos["BERTOS_PATH"]] = {}
556         # Fill the empty cache with the result
557         self._cached_queries[self.infos["BERTOS_PATH"]][ftype] = definitions
558         return definitions
559
560     def setEnabledModules(self, enabled_modules):
561         modules = self.infos["MODULES"]
562         files = {}
563         for module, information in modules.items():
564             information["enabled"] = module in enabled_modules
565             if information["enabled"]:
566                 for dependency in information["depends"]:
567                     if not dependency in modules:
568                         files[dependency] = files.get(dependency, 0) + 1
569         self.infos["MODULES"] = modules
570         self.infos["FILES"] = files
571
572     def __repr__(self):
573         return "<BProject:instance %d>%s" %(id(self), repr(self.infos))