Change SOURCES_PATH in BERTOS_PATH.
[bertos.git] / wizard / BProject.py
1 #!/usr/bin/env python
2 # encoding: utf-8
3 #
4 # This file is part of BeRTOS.
5 #
6 # Bertos is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
19 #
20 # As a special exception, you may use this file as part of a free software
21 # library without restriction.  Specifically, if other files instantiate
22 # templates or use macros or inline functions from this file, or you compile
23 # this file and link it with other files to produce an executable, this
24 # file does not by itself cause the resulting executable to be covered by
25 # the GNU General Public License.  This exception does not however
26 # invalidate any other reasons why the executable file might be covered by
27 # the GNU General Public License.
28 #
29 # Copyright 2008 Develer S.r.l. (http://www.develer.com/)
30 #
31 # $Id$
32 #
33 # Author: Lorenzo Berni <duplo@develer.com>
34 #
35
36 import os
37 import fnmatch
38 import copy
39 import pickle
40 import shutil
41 import copytree
42
43 import DefineException
44
45 from LoadException import VersionException, ToolchainException
46
47 import const
48
49 from bertos_utils import (
50                             # Utility functions
51                             isBertosDir, getTagSet, getInfos, updateConfigurationValues,
52                             loadConfigurationInfos, loadDefineLists, loadModuleDefinition,
53                             getCommentList, sub,
54
55                             # Project creation functions
56                             projectFileGenerator, versionFileGenerator, makefileGenerator,
57                             userMkGenerator, mkGenerator, loadPlugin, mergeSources,
58
59                             # Custom exceptions
60                             ParseError, SupportedException
61                         )
62
63 class BProject(object):
64     """
65     Simple class for store and retrieve project informations.
66     """
67
68     def __init__(self, project_file="", info_dict={}):
69         self.infos = {}
70         self._cached_queries = {}
71         self.edit = False
72         if project_file:
73             self.edit = True
74             self.loadBertosProject(project_file, info_dict)
75
76     #--- Load methods (methods that loads data into project) ------------------#
77
78     def loadBertosProject(self, project_file, info_dict):
79         project_dir = os.path.dirname(project_file)
80         project_data = pickle.loads(open(project_file, "r").read())
81         # If PROJECT_NAME is not defined it use the directory name as PROJECT_NAME
82         # NOTE: this can throw an Exception if the user has changed the directory containing the project
83         self.infos["PROJECT_NAME"] = project_data.get("PROJECT_NAME", os.path.basename(project_dir))
84         self.infos["PROJECT_PATH"] = os.path.dirname(project_file)
85         project_src_path = project_data.get("PROJECT_SRC_PATH", None)
86         project_src_path = os.path.join(project_dir, project_data.get("PROJECT_SRC_PATH", None))
87         if project_src_path:
88             self.infos["PROJECT_SRC_PATH"] = project_src_path
89             
90         else:
91             # In projects created with older versions of the Wizard this metadata doesn't exist
92             self.infos["PROJECT_SRC_PATH"] = os.path.join(self.infos["PROJECT_PATH"], self.infos["PROJECT_NAME"])
93
94         wizard_version = project_data.get("WIZARD_VERSION", 0)
95         if wizard_version < 1:
96             # Ignore the BERTOS_PATH inside the project file for older project
97             project_data["BERTOS_PATH"] = project_dir
98         else:
99             linked_sources_path = project_data["BERTOS_PATH"]
100             sources_abspath = os.path.abspath(os.path.join(project_dir, linked_sources_path))
101             project_data["BERTOS_PATH"] = sources_abspath
102         
103         self._loadBertosSourceStuff(project_data["BERTOS_PATH"], info_dict.get("BERTOS_PATH", None))
104         
105         self.infos["PRESET"] = project_data.get("PRESET", False)
106
107         # For those projects that don't have a VERSION file create a dummy one.
108         if not isBertosDir(project_dir):
109             version_file = open(os.path.join(const.DATA_DIR, "vtemplates/VERSION"), "r").read()
110             open(os.path.join(project_dir, "VERSION"), "w").write(version_file.replace("$version", "").strip())
111
112         self.loadSourceTree()
113         self._loadCpuStuff(project_data["CPU_NAME"], project_data["SELECTED_FREQ"])
114         self._loadToolchainStuff(project_data["TOOLCHAIN"], info_dict.get("TOOLCHAIN", None))
115         self.infos["OUTPUT"] = project_data["OUTPUT"]
116         self.loadModuleData(True)
117         self.setEnabledModules(project_data["ENABLED_MODULES"])
118
119     def _loadBertosSourceStuff(self, sources_path, forced_version=None):
120         if forced_version:
121             sources_path = forced_version
122         if os.path.exists(sources_path):
123             self.infos["BERTOS_PATH"] = sources_path
124         else:
125             raise VersionException(self)
126
127     def _loadCpuStuff(self, cpu_name, cpu_frequency):
128         self.infos["CPU_NAME"] = cpu_name
129         cpu_info = self.getCpuInfos()
130         for cpu in cpu_info:
131             if cpu["CPU_NAME"] == cpu_name:
132                 self.infos["CPU_INFOS"] = cpu
133                 break
134         tag_list = getTagSet(cpu_info)
135         # Create, fill and store the dict with the tags
136         tag_dict = {}
137         for element in tag_list:
138             tag_dict[element] = False
139         infos = self.info("CPU_INFOS")
140         for tag in tag_dict:
141             if tag in infos["CPU_TAGS"] + [infos["CPU_NAME"], infos["TOOLCHAIN"]]:
142                 tag_dict[tag] = True
143             else:
144                 tag_dict[tag] = False
145         self.infos["ALL_CPU_TAGS"] = tag_dict
146         self.infos["SELECTED_FREQ"] = cpu_frequency
147
148     def _loadToolchainStuff(self, toolchain, forced_toolchain=None):
149         toolchain = toolchain
150         if forced_toolchain:
151             toolchain = forced_toolchain
152         if os.path.exists(toolchain["path"]):
153             self.infos["TOOLCHAIN"] = toolchain
154         else:
155             raise ToolchainException(self)
156
157     def loadProjectFromPreset(self, preset):
158         """
159         Load a project from a preset.
160         NOTE: this is a stub.
161         """
162         project_file = os.path.join(preset, "project.bertos")
163         project_data = pickle.loads(open(project_file, "r").read())
164         self.loadSourceTree()
165         self._loadCpuStuff(project_data["CPU_NAME"], project_data["SELECTED_FREQ"])
166         self._loadToolchainStuff(project_data["TOOLCHAIN"])
167         # NOTE: this is a HACK!!!
168         # TODO: find a better way to reuse loadModuleData
169         preset_project_name = project_data.get("PROJECT_NAME", os.path.basename(preset))
170         preset_prj_src_path = os.path.join(preset, project_data.get("PROJECT_SRC_PATH", os.path.join(preset, preset_project_name)))
171
172         old_project_name = self.infos["PROJECT_NAME"]
173         old_project_path = self.infos["PROJECT_PATH"]
174         old_project_src_path = self.infos["PROJECT_SRC_PATH"]
175
176         self.infos["PROJECT_NAME"] = preset_project_name
177         self.infos["PROJECT_PATH"] = preset
178         self.infos["PROJECT_SRC_PATH"] = preset_prj_src_path
179
180         self.loadModuleData(True)
181         self.setEnabledModules(project_data["ENABLED_MODULES"])
182
183         self.infos["PROJECT_NAME"] = old_project_name
184         self.infos["PROJECT_PATH"] = old_project_path
185         self.infos["PROJECT_SRC_PATH"] = old_project_src_path
186
187         self.infos["PRESET_NAME"] = preset_project_name
188         self.infos["PRESET_PATH"] = preset
189         self.infos["PRESET_SRC_PATH"] = preset_prj_src_path
190
191     def loadProjectPresets(self):
192         """
193         Load the default presets (into the const.PREDEFINED_BOARDS_DIR).
194         """
195         # NOTE: this method does nothing (for now).
196         preset_path = os.path.join(self.infos["BERTOS_PATH"], const.PREDEFINED_BOARDS_DIR)
197         preset_tree = {}
198         if os.path.exists(preset_path):
199             preset_tree = self._loadProjectPresetTree(preset_path)
200         self.infos["PRESET_TREE"] = preset_tree
201
202     def _loadProjectPresetTree(self, path):
203         _tree = {}
204         _tree["info"] = self._loadPresetInfo(os.path.join(path, const.PREDEFINED_BOARD_SPEC_FILE))
205         _tree["info"]["filename"] = os.path.basename(path)
206         _tree["info"]["path"] = path
207         _tree["children"] = []
208         entries = set(os.listdir(path))
209         for entry in entries:
210             _path = os.path.join(path, entry)
211             if os.path.isdir(_path):
212                 sub_entries = set(os.listdir(_path))
213                 if const.PREDEFINED_BOARD_SPEC_FILE in sub_entries:
214                     _tree["children"].append(self._loadProjectPresetTree(_path))
215         # Add into the info dict the dir type (dir/project)
216         if _tree["children"]:
217             _tree["info"]["type"] = "dir"
218         else:
219             _tree["info"]["type"] = "project"
220         return _tree
221
222     def _loadPresetInfo(self, preset_spec_file):
223         D = {}
224         execfile(preset_spec_file, {}, D)
225         return D
226
227     def loadModuleData(self, edit=False):
228         module_info_dict = {}
229         list_info_dict = {}
230         configuration_info_dict = {}
231         file_dict = {}
232         for filename, path in self.findDefinitions("*.h") + self.findDefinitions("*.c") + self.findDefinitions("*.s") + self.findDefinitions("*.S"):
233             comment_list = getCommentList(open(path + "/" + filename, "r").read())
234             if len(comment_list) > 0:
235                 module_info = {}
236                 configuration_info = {}
237                 try:
238                     to_be_parsed, module_dict = loadModuleDefinition(comment_list[0])
239                 except ParseError, err:
240                     raise DefineException.ModuleDefineException(path, err.line_number, err.line)
241                 for module, information in module_dict.items():
242                     if "depends" not in information:
243                         information["depends"] = ()
244                     information["depends"] += (filename.split(".")[0],)
245                     information["category"] = os.path.basename(path)
246                     if "configuration" in information and len(information["configuration"]):
247                         configuration = module_dict[module]["configuration"]
248                         try:
249                             configuration_info[configuration] = loadConfigurationInfos(self.infos["BERTOS_PATH"] + "/" + configuration)
250                         except ParseError, err:
251                             raise DefineException.ConfigurationDefineException(self.infos["BERTOS_PATH"] + "/" + configuration, err.line_number, err.line)
252                         if edit:
253                             try:
254                                 path = self.infos["PROJECT_SRC_PATH"]
255                                 user_configuration = loadConfigurationInfos(configuration.replace("bertos", path))
256                                 configuration_info[configuration] = updateConfigurationValues(configuration_info[configuration], user_configuration)
257                             except ParseError, err:
258                                 raise DefineException.ConfigurationDefineException(configuration.replace("bertos", path))
259                 module_info_dict.update(module_dict)
260                 configuration_info_dict.update(configuration_info)
261                 if to_be_parsed:
262                     try:
263                         list_dict = loadDefineLists(comment_list[1:])
264                         list_info_dict.update(list_dict)
265                     except ParseError, err:
266                         raise DefineException.EnumDefineException(path, err.line_number, err.line)
267         for tag in self.infos["CPU_INFOS"]["CPU_TAGS"]:
268             for filename, path in self.findDefinitions("*_" + tag + ".h"):
269                 comment_list = getCommentList(open(path + "/" + filename, "r").read())
270                 list_info_dict.update(loadDefineLists(comment_list))
271         self.infos["MODULES"] = module_info_dict
272         self.infos["LISTS"] = list_info_dict
273         self.infos["CONFIGURATIONS"] = configuration_info_dict
274         self.infos["FILES"] = file_dict
275
276     def loadSourceTree(self):
277         """
278         Index BeRTOS source and load it in memory.
279         """
280         # Index only the BERTOS_PATH/bertos content
281         bertos_sources_dir = os.path.join(self.info("BERTOS_PATH"), "bertos")
282         file_dict = {}
283         if os.path.exists(bertos_sources_dir):
284             for element in os.walk(bertos_sources_dir):
285                 for f in element[2]:
286                     file_dict[f] = file_dict.get(f, []) + [element[0]]
287         self.infos["FILE_DICT"] = file_dict
288
289     def reloadCpuInfo(self):
290         for cpu_info in self.getCpuInfos():
291             if cpu_info["CPU_NAME"] == self.infos["CPU_NAME"]:
292                 self.infos["CPU_INFOS"] = cpu_info
293
294     #-------------------------------------------------------------------------#
295
296     def createBertosProject(self):
297         # NOTE: Temporary hack.
298         if self.edit:
299             self._editBertosProject()
300         else:
301             if not self.from_preset:
302                 self._newCustomBertosProject()
303             else:
304                 self._newBertosProjectFromPreset()
305
306     def _newBertosProject(self):
307         for directory in (self.maindir, self.srcdir, self.prjdir, self.cfgdir):
308             self._createDirectory(directory)
309         # Write the project file
310         self._writeProjectFile(os.path.join(self.maindir, "project.bertos"))
311         # VERSION file
312         self._writeVersionFile(os.path.join(self.maindir, "VERSION"))
313         # Destination makefile
314         self._writeMakefile(os.path.join(self.maindir, "Makefile"))
315         # Copy the sources
316         self._copySources(self.sources_dir, self.srcdir)
317         # Set properly the autoenabled parameters
318         self._setupAutoenabledParameters()
319         # Copy all the configuration files
320         self._writeCfgFiles(self.sources_dir, self.cfgdir)
321         # Destination wizard mk file
322         self._writeWizardMkFile(os.path.join(self.prjdir, os.path.basename(self.prjdir) + "_wiz.mk"))
323
324     def _newCustomBertosProject(self):
325         # Create/write/copy the common things
326         self._newBertosProject()
327         # Copy the clean hw files
328         self._createDirectory(self.hwdir)
329         # Copy all the hw files
330         self._writeHwFiles(self.sources_dir, self.hwdir)
331         # Destination user mk file
332         self._writeUserMkFile(os.path.join(self.prjdir, os.path.basename(self.prjdir) + ".mk"))
333         # Destination main.c file
334         self._writeMainFile(self.prjdir + "/main.c")
335         # Create project files for selected plugins
336         self._createProjectFiles()
337
338     def _newBertosProjectFromPreset(self):
339         # Create/write/copy the common things
340         self._newBertosProject()
341
342         # Copy all the files and dirs except cfg/hw/*_wiz.mk
343         self._writeCustomSrcFiles()
344
345         if self.infos["EMPTY_MAIN"]:
346             # Create and empty main.c file only if the user check the box
347             self._writeMainFile(self.prjdir + "/main.c")
348
349         # Create project files for selected plugins
350         self._createProjectFiles()
351
352     def _editBertosProject(self):
353         # Write the project file
354         self._writeProjectFile(os.path.join(self.maindir, "project.bertos"))
355         # VERSION file
356         self._writeVersionFile(os.path.join(self.maindir, "VERSION"))
357         # Destination makefile
358         self._writeMakefile(os.path.join(self.maindir, "Makefile"))
359         # Merge sources
360         self._mergeSources(self.sources_dir, self.srcdir, self.old_srcdir)
361         # Copy all the hw files
362         self._writeHwFiles(self.sources_dir, self.hwdir)
363         # Set properly the autoenabled parameters
364         self._setupAutoenabledParameters()
365         # Copy all the configuration files
366         self._writeCfgFiles(self.sources_dir, self.cfgdir)
367         # Destination wizard mk file
368         self._writeWizardMkFile(os.path.join(self.prjdir, os.path.basename(self.prjdir) + "_wiz.mk"))
369         # Create project files for selected plugins
370         self._createProjectFiles()
371
372     def _createProjectFiles(self):
373         # Files for selected plugins
374         relevants_files = {}
375         for plugin in self.infos["OUTPUT"]:
376             module = loadPlugin(plugin)
377             relevants_files[plugin] = module.createProject(self)
378         self.infos["RELEVANT_FILES"] = relevants_files
379
380     def _writeVersionFile(self, filename):
381         version_file = open(os.path.join(const.DATA_DIR, "vtemplates/VERSION"), "r").read()
382         open(filename, "w").write(versionFileGenerator(self, version_file))
383
384     def _writeProjectFile(self, filename):
385         f = open(filename, "w")
386         f.write(projectFileGenerator(self))
387         f.close()
388
389     def _writeMakefile(self, filename):
390         makefile = open(os.path.join(const.DATA_DIR, "mktemplates/Makefile"), "r").read()
391         makefile = makefileGenerator(self, makefile)
392         open(filename, "w").write(makefile)
393
394     def _writeUserMkFile(self, filename):
395         makefile = open(os.path.join(const.DATA_DIR, "mktemplates/template.mk"), "r").read()
396         # Deadly performances loss was here :(
397         makefile = userMkGenerator(self, makefile)
398         open(filename, "w").write(makefile)
399
400     def _writeWizardMkFile(self, filename):
401         makefile = open(os.path.join(const.DATA_DIR, "mktemplates/template_wiz.mk"), "r").read()
402         makefile = mkGenerator(self, makefile)
403         open(filename, "w").write(makefile)
404
405     def _writeMainFile(self, filename):
406         main = open(os.path.join(const.DATA_DIR, "srctemplates/main.c"), "r").read()
407         open(filename, "w").write(main)
408
409     def _writeHwFiles(self, source_dir, destination_dir):
410         for module, information in self.infos["MODULES"].items():
411             for hwfile in information["hw"]:
412                 string = open(source_dir + "/" + hwfile, "r").read()
413                 hwfile_path = destination_dir + "/" + os.path.basename(hwfile)
414                 if not self.edit or not os.path.exists(hwfile_path):
415                     # If not in editing mode it copies all the hw files. If in
416                     # editing mode it copies only the files that don't exist yet
417                     open(os.path.join(destination_dir,os.path.basename(hwfile)), "w").write(string)
418
419     def _writeCfgFiles(self, source_dir, destination_dir):
420         for configuration, information in self.infos["CONFIGURATIONS"].items():
421             string = open(source_dir + "/" + configuration, "r").read()
422             for start, parameter in information["paramlist"]:
423                 infos = information[parameter]
424                 value = infos["value"]
425                 if "unsigned" in infos["informations"] and infos["informations"]["unsigned"]:
426                     value += "U"
427                 if "long" in infos["informations"] and infos["informations"]["long"]:
428                     value += "L"
429                 string = sub(string, parameter, value)
430             f = open(os.path.join(destination_dir, os.path.basename(configuration)), "w")
431             f.write(string)
432             f.close()
433
434     def _writeCustomSrcFiles(self):
435         origin = self.infos["PRESET_SRC_PATH"]
436         # Files to be ignored (all project files, cfg dir, wizard mk file, all global ignored dirs)
437         project_related_stuff = (
438             "cfg",
439             self.infos["PROJECT_NAME"] + "_wiz.mk",
440             "project.bertos",
441             self.infos["PROJECT_NAME"] + ".project",
442             self.infos["PROJECT_NAME"] + ".workspace",
443         ) + const.IGNORE_LIST
444         for element in os.listdir(origin):
445             if element not in project_related_stuff:
446                 full_path = os.path.join(origin, element)
447                 if os.path.isdir(full_path):
448                     copytree.copytree(full_path, os.path.join(self.prjdir, element), ignore_list=const.IGNORE_LIST)
449                 else:
450                     shutil.copy(full_path, self.prjdir)
451
452     def _setupAutoenabledParameters(self):
453         for module, information in self.infos["MODULES"].items():
454             if "configuration" in information and information["configuration"] != "":
455                 configurations = self.infos["CONFIGURATIONS"]
456                 configuration = configurations[information["configuration"]]
457                 for start, parameter in configuration["paramlist"]:
458                     if "type" in configuration[parameter]["informations"] and configuration[parameter]["informations"]["type"] == "autoenabled":
459                         configuration[parameter]["value"] = "1" if information["enabled"] else "0"
460                 self.infos["CONFIGURATIONS"] = configurations
461
462     @property
463     def maindir(self):
464         return self.infos.get("PROJECT_PATH", None)
465
466     @property
467     def srcdir(self):
468         if self.maindir:
469             return os.path.join(self.maindir, "bertos")
470         else:
471             return None
472
473     @property
474     def prjdir(self):
475         return self.infos.get("PROJECT_SRC_PATH", None)
476
477     @property
478     def hwdir(self):
479         if self.prjdir:
480             return os.path.join(self.prjdir, "hw")
481         else:
482             return None
483
484     @property
485     def cfgdir(self):
486         if self.prjdir:
487             return os.path.join(self.prjdir, "cfg")
488         else:
489             return None
490
491     @property
492     def old_srcdir(self):
493         return self.infos.get("OLD_BERTOS_PATH", None)
494
495     @property
496     def sources_dir(self):
497         return self.infos.get("BERTOS_PATH", None)
498
499     @property
500     def from_preset(self):
501         return self.infos.get("PROJECT_FROM_PRESET", False)
502
503     def _createDirectory(self, directory):
504         if not directory:
505             return
506         if os.path.isdir(directory):
507             shutil.rmtree(directory, True)
508         os.makedirs(directory)
509
510     def _copySources(self, origin, destination):
511         # If not in editing mode it copies all the bertos sources in the /bertos subdirectory of the project
512         shutil.rmtree(destination, True)
513         copytree.copytree(origin + "/bertos", destination, ignore_list=const.IGNORE_LIST)
514
515     def _mergeSources(self, origin, destination, old_sources_dir):
516         if old_sources_dir:
517             mergeSources(destination, origin, old_sources_dir)
518
519     def setInfo(self, key, value):
520         """
521         Store the given value with the name key.
522         """
523         self.infos[key] = value
524
525     def info(self, key, default=None):
526         """
527         Retrieve the value associated with the name key.
528         """
529         if key in self.infos:
530             return copy.deepcopy(self.infos[key])
531         return default
532
533     def getCpuInfos(self):
534         cpuInfos = []
535         for definition in self.findDefinitions(const.CPU_DEFINITION):
536             cpuInfos.append(getInfos(definition))
537         return cpuInfos
538
539     def searchFiles(self, filename):
540         file_dict = self.infos["FILE_DICT"]
541         return [(filename, dirname) for dirname in file_dict.get(filename, [])]
542
543     def findDefinitions(self, ftype):
544         # Maintain a cache for every scanned BERTOS_PATH
545         definitions_dict = self._cached_queries.get(self.infos["BERTOS_PATH"], {})
546         definitions = definitions_dict.get(ftype, None)
547         if definitions is not None:
548             return definitions
549         file_dict = self.infos["FILE_DICT"]
550         definitions = []
551         for filename in file_dict:
552             if fnmatch.fnmatch(filename, ftype):
553                 definitions += [(filename, dirname) for dirname in file_dict.get(filename, [])]
554
555         # If no cache for the current BERTOS_PATH create an empty one
556         if not definitions_dict:
557             self._cached_queries[self.infos["BERTOS_PATH"]] = {}
558         # Fill the empty cache with the result
559         self._cached_queries[self.infos["BERTOS_PATH"]][ftype] = definitions
560         return definitions
561
562     def setEnabledModules(self, enabled_modules):
563         modules = self.infos["MODULES"]
564         files = {}
565         for module, information in modules.items():
566             information["enabled"] = module in enabled_modules
567             if information["enabled"]:
568                 for dependency in information["depends"]:
569                     if not dependency in modules:
570                         files[dependency] = files.get(dependency, 0) + 1
571         self.infos["MODULES"] = modules
572         self.infos["FILES"] = files
573
574     def __repr__(self):
575         return "<BProject:instance %d>%s" %(id(self), repr(self.infos))