Source Code for Module test_suite.test_suite_runner

  1  ############################################################################### 
  2  #                                                                             # 
  3  # Copyright (C) 2006-2014 Edward d'Auvergne                                   # 
  4  #                                                                             # 
  5  # This file is part of the program relax (http://www.nmr-relax.com).          # 
  6  #                                                                             # 
  7  # This program is free software: you can redistribute it and/or modify        # 
  8  # it under the terms of the GNU General Public License as published by        # 
  9  # the Free Software Foundation, either version 3 of the License, or           # 
 10  # (at your option) any later version.                                         # 
 11  #                                                                             # 
 12  # This program is distributed in the hope that it will be useful,             # 
 13  # but WITHOUT ANY WARRANTY; without even the implied warranty of              # 
 14  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               # 
 15  # GNU General Public License for more details.                                # 
 16  #                                                                             # 
 17  # You should have received a copy of the GNU General Public License           # 
 18  # along with this program.  If not, see <http://www.gnu.org/licenses/>.       # 
 19  #                                                                             # 
 20  ############################################################################### 
 21   
 22  # Dependency checks. 
 23  import dep_check 
 24   
 25  # Python module imports. 
 26  import os 
 27  import sys 
 28  if dep_check.wx_module: 
 29      import wx 
 30  import unittest 
 31   
 32  # Formatting. 
 33  from test_suite.formatting import summary_line 
 34   
 35  # Import the test suite categories. 
 36  if dep_check.wx_module: 
 37      from test_suite.gui_tests import GUI_test_runner 
 38  from test_suite.system_tests import System_test_runner 
 39  from test_suite.unit_tests.unit_test_runner import Unit_test_runner 
 40  from test_suite.verification_tests import Verification_test_runner 
 41   
 42  # relax module imports. 
 43  if dep_check.wx_module: 
 44      from gui import relax_gui 
 45      from gui import interpreter 
 46  from lib.text.sectioning import section, title 
 47  from test_suite.relax_test_runner import GuiTestRunner, RelaxTestRunner 
 48  from status import Status; status = Status() 
 49   
 50   
 51 -class Test_suite_runner: 
 52      """Class for running all components of the relax test suite. 
 53   
 54      This currently includes the following categories of tests: 
 55          - System/functional tests. 
 56          - Unit tests. 
 57          - GUI tests. 
 58          - Verification tests. 
 59      """ 
 60   
 61 -    def __init__(self, tests=[], from_gui=False, categories=['system', 'unit', 'gui', 'verification'], timing=False): 
 62          """Store the list of tests to preform. 
 63   
 64          The test list should be something like ['N_state_model.test_stereochem_analysis'].  The first part is the imported test case class, the second is the specific test. 
 65   
 66   
 67          @keyword tests:         The list of tests to preform.  If left at [], then all tests will be run. 
 68          @type tests:            list of str 
 69          @keyword from_gui:      A flag which indicates if the tests are being run from the GUI or not. 
 70          @type from_gui:         bool 
 71          @keyword categories:    The list of test categories to run, for example ['system', 'unit', 'gui', 'verification'] for all tests. 
 72          @type categories:       list of str 
 73          @keyword timing:        A flag which if True will enable timing of individual tests. 
 74          @type timing:           bool 
 75          """ 
 76   
 77          # Store the args. 
 78          self.tests = tests 
 79          self.from_gui = from_gui 
 80          self.categories = categories 
 81   
 82          # A list for skipped tests. 
 83          status.skip = [] 
 84   
 85          # Set up the test runner. 
 86          if from_gui: 
 87              self.runner = GuiTestRunner(stream=sys.stdout, timing=timing) 
 88          else: 
 89              self.runner = RelaxTestRunner(stream=sys.stdout, timing=timing) 
 90   
 91          # Let the tests handle the keyboard interrupt (for Python 2.7 and higher). 
 92          if hasattr(unittest, 'installHandler'): 
 93              unittest.installHandler() 
 94   
 95   
 96 -    def run_all_tests(self): 
 97          """Execute all of the test suite test types.""" 
 98   
 99          # Execute the system/functional tests. 
100          if 'system' in self.categories: 
101              status = self.run_system_tests(summary=False) 
102              if not status: 
103                  return 
104   
105          # Execute the unit tests. 
106          if 'unit' in self.categories: 
107              status = self.run_unit_tests(summary=False) 
108              if not status: 
109                  return 
110   
111          # Execute the GUI tests. 
112          if 'gui' in self.categories: 
113              status = self.run_gui_tests(summary=False) 
114              if not status: 
115                  return 
116   
117          # Execute the GUI tests. 
118          if 'verification' in self.categories: 
119              status = self.run_verification_tests(summary=False) 
120              if not status: 
121                  return 
122   
123          # Print out a summary of the test suite. 
124          self.summary() 
125   
126   
127   
128 -    def run_gui_tests(self, summary=True): 
129          """Execute the GUI tests. 
130   
131          @keyword summary:   A flag which if True will cause a summary to be printed. 
132          @type summary:      bool 
133          @return:            True if the tests were run, False if a KeyboardInterrupt occurred. 
134          @rtype:             bool 
135          """ 
136   
137          # Run the tests, catching the keyboard interrupt. 
138          try: 
139              # Print a header. 
140              title(file=sys.stdout, text='GUI tests') 
141   
142              # Run the tests. 
143              if dep_check.wx_module: 
144                  # Set up the GUI if needed (i.e. not in GUI mode already). 
145                  app = wx.GetApp() 
146                  if app == None: 
147                      # Initialise. 
148                      app = wx.App(redirect=False) 
149   
150                      # Build the GUI. 
151                      app.gui = relax_gui.Main(parent=None, id=-1, title="") 
152   
153                  # Execute the GUI tests. 
154                  gui_runner = GUI_test_runner() 
155                  self.runner.category = 'gui' 
156                  self.gui_result = gui_runner.run(self.tests, runner=self.runner) 
157   
158                  # Clean up for the GUI, if not in GUI mode. 
159                  if status.test_mode: 
160                      # Terminate the interpreter thread to allow the tests to cleanly exit. 
161                      interpreter_thread = interpreter.Interpreter() 
162                      interpreter_thread.exit() 
163   
164                      # Stop the GUI main loop. 
165                      app.ExitMainLoop() 
166   
167              # No wx module installed. 
168              else: 
169                  print("All GUI tests skipped due to the missing/broken wx module.\n") 
170                  self.gui_result = 'skip' 
171   
172              # Print out a summary of the test suite. 
173              if summary: 
174                  self.summary() 
175   
176          # Catch the keyboard interrupt. 
177          except KeyboardInterrupt: 
178              print("\nKeyboardInterrupt:  Terminating all tests.\n") 
179              return False 
180   
181          # All tests were run successfully. 
182          return True 
183   
184   
185 -    def run_system_tests(self, summary=True): 
186          """Execute the system/functional tests. 
187   
188          @keyword summary:   A flag which if True will cause a summary to be printed. 
189          @type summary:      bool 
190          @return:            True if the tests were run, False if a KeyboardInterrupt occurred. 
191          @rtype:             bool 
192          """ 
193   
194          # Run the tests, catching the keyboard interrupt. 
195          try: 
196              # Print a header. 
197              title(file=sys.stdout, text='System / functional tests') 
198   
199              # Run the tests. 
200              system_runner = System_test_runner() 
201              self.runner.category = 'system' 
202              self.system_result = system_runner.run(self.tests, runner=self.runner) 
203   
204              # Print out a summary of the test suite. 
205              if summary: 
206                  self.summary() 
207   
208          # Catch the keyboard interrupt. 
209          except KeyboardInterrupt: 
210              print("\nKeyboardInterrupt:  Terminating all tests.\n") 
211              return False 
212   
213          # All tests were run successfully. 
214          return True 
215   
216   
217 -    def run_unit_tests(self, summary=True): 
218          """Execute the unit tests. 
219   
220          @keyword summary:   A flag which if True will cause a summary to be printed. 
221          @type summary:      bool 
222          @return:            True if the tests were run, False if a KeyboardInterrupt occurred. 
223          @rtype:             bool 
224          """ 
225   
226          # Run the tests, catching the keyboard interrupt. 
227          try: 
228              # Print a header. 
229              title(file=sys.stdout, text='Unit tests') 
230   
231              # Run the tests. 
232              unit_runner = Unit_test_runner(root_path=status.install_path+os.sep+'test_suite'+os.sep+'unit_tests') 
233              self.runner.category = 'unit' 
234              self.unit_result = unit_runner.run(self.tests, runner=self.runner) 
235   
236              # Print out a summary of the test suite. 
237              if summary: 
238                  self.summary() 
239   
240          # Catch the keyboard interrupt. 
241          except KeyboardInterrupt: 
242              print("\nKeyboardInterrupt:  Terminating all tests.\n") 
243              return False 
244   
245          # All tests were run successfully. 
246          return True 
247   
248   
249 -    def run_verification_tests(self, summary=True): 
250          """Execute the software verification tests. 
251   
252          @keyword summary:   A flag which if True will cause a summary to be printed. 
253          @type summary:      bool 
254          """ 
255   
256          # Run the tests, catching the keyboard interrupt. 
257          try: 
258              # Print a header. 
259              title(file=sys.stdout, text='Software verification tests') 
260   
261              # Run the tests. 
262              verification_runner = Verification_test_runner() 
263              self.runner.category = 'verification' 
264              self.verification_result = verification_runner.run(self.tests, runner=self.runner) 
265   
266              # Print out a summary of the test suite. 
267              if summary: 
268                  self.summary() 
269   
270          # Catch the keyboard interrupt. 
271          except KeyboardInterrupt: 
272              print("\nKeyboardInterrupt:  Terminating all tests.\n") 
273              return False 
274   
275          # All tests were run successfully. 
276          return True 
277   
278   
279 -    def summary(self): 
280          """Print out a summary of the relax test suite.""" 
281   
282          # Title. 
283          title(file=sys.stdout, text="Summary of the relax test suite") 
284   
285          # The skipped tests. 
286          if status.skip_blacklisted_tests: 
287              self.summary_skipped() 
288   
289          # Subtitle. 
290          section(file=sys.stdout, text="Synopsis") 
291   
292          # System/functional test summary. 
293          if hasattr(self, 'system_result'): 
294              summary_line("System/functional tests", self.system_result) 
295   
296          # Unit test summary. 
297          if hasattr(self, 'unit_result'): 
298              summary_line("Unit tests", self.unit_result) 
299   
300          # GUI test summary. 
301          if hasattr(self, 'gui_result'): 
302              summary_line("GUI tests", self.gui_result) 
303   
304          # Verification test summary. 
305          if hasattr(self, 'verification_result'): 
306              summary_line("Software verification tests", self.verification_result) 
307   
308          # Synopsis. 
309          if hasattr(self, 'system_result') and hasattr(self, 'unit_result') and hasattr(self, 'gui_result') and hasattr(self, 'verification_result'): 
310              if self.gui_result == "skip": 
311                  test_status = self.system_result and self.unit_result and self.verification_result 
312              else: 
313                  test_status = self.system_result and self.unit_result and self.gui_result and self.verification_result 
314              summary_line("Synopsis", test_status) 
315   
316          # End. 
317          print('\n\n') 
318   
319   
320 -    def summary_skipped(self): 
321          """Print out information about skipped tests."""  
322   
323          # Counts. 
324          system_count = {} 
325          unit_count = {} 
326          gui_count = {} 
327          verification_count = {} 
328          for i in range(len(status.skipped_tests)): 
329              # Alias. 
330              test = status.skipped_tests[i] 
331   
332              # Skip all skipped tests whereby the module is set to None to indicate that the test skipping should not be reported. 
333              if test[1] == None: 
334                  continue 
335   
336              # Initialise in needed. 
337              if not test[1] in system_count: 
338                  system_count[test[1]] = 0 
339                  unit_count[test[1]] = 0 
340                  gui_count[test[1]] = 0 
341                  verification_count[test[1]] = 0 
342   
343              # A system test. 
344              if test[2] == 'system': 
345                  system_count[test[1]] += 1 
346   
347              # A unit test. 
348              if test[2] == 'unit': 
349                  unit_count[test[1]] += 1 
350   
351              # A GUI test. 
352              if test[2] == 'gui': 
353                  gui_count[test[1]] += 1 
354   
355              # A verification test. 
356              if test[2] == 'verification': 
357                  verification_count[test[1]] += 1 
358   
359          # The missing modules. 
360          missing_modules = sorted(system_count.keys()) 
361          section(file=sys.stdout, text="Optional packages/modules") 
362   
363          # Nothing missing. 
364          if not missing_modules: 
365              # Except for the wx module! 
366              if not dep_check.wx_module and hasattr(self, 'gui_result'): 
367                  print("All GUI tests skipped due to the missing wxPython module, no other tests skipped due to missing modules.\n") 
368   
369              # Normal printout. 
370              else: 
371                  print("No tests skipped due to missing modules.\n") 
372   
373              # The skip the table. 
374              return 
375   
376          # Header. 
377          print("Tests skipped due to missing optional packages/modules/software:\n") 
378          header = "%-33s" % "Module/package/software"  
379          if len(system_count): 
380              header = "%s %20s" % (header, "System test count") 
381          if len(unit_count): 
382              header = "%s %20s" % (header, "Unit test count") 
383          if len(gui_count): 
384              header = "%s %20s" % (header, "GUI test count") 
385          if len(verification_count): 
386              header = "%s %20s" % (header, "Verification test count") 
387          print('-'*len(header)) 
388          print(header) 
389          print('-'*len(header)) 
390   
391          # The table. 
392          for module in missing_modules: 
393              text = "%-33s" % module 
394              if len(system_count): 
395                  text = "%s %20s" % (text, system_count[module]) 
396              if len(unit_count): 
397                  text = "%s %20s" % (text, unit_count[module]) 
398              if len(gui_count): 
399                  text = "%s %20s" % (text, gui_count[module]) 
400              if len(verification_count): 
401                  text = "%s %20s" % (text, verification_count[module]) 
402              print(text) 
403   
404          # End the table. 
405          print('-'*len(header)) 
406          print("\n") 
407