Source Code for Module test_suite.test_suite_runner

  1  ############################################################################### 
  2  #                                                                             # 
  3  # Copyright (C) 2006-2014 Edward d'Auvergne                                   # 
  4  #                                                                             # 
  5  # This file is part of the program relax (http://www.nmr-relax.com).          # 
  6  #                                                                             # 
  7  # This program is free software: you can redistribute it and/or modify        # 
  8  # it under the terms of the GNU General Public License as published by        # 
  9  # the Free Software Foundation, either version 3 of the License, or           # 
 10  # (at your option) any later version.                                         # 
 11  #                                                                             # 
 12  # This program is distributed in the hope that it will be useful,             # 
 13  # but WITHOUT ANY WARRANTY; without even the implied warranty of              # 
 14  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               # 
 15  # GNU General Public License for more details.                                # 
 16  #                                                                             # 
 17  # You should have received a copy of the GNU General Public License           # 
 18  # along with this program.  If not, see <http://www.gnu.org/licenses/>.       # 
 19  #                                                                             # 
 20  ############################################################################### 
 21   
 22  # Dependency checks. 
 23  import dep_check 
 24   
 25  # Python module imports. 
 26  import os 
 27  import sys 
 28  if dep_check.wx_module: 
 29      import wx 
 30  import unittest 
 31   
 32  # Formatting. 
 33  from test_suite.formatting import summary_line 
 34   
 35  # Import the test suite categories. 
 36  if dep_check.wx_module: 
 37      from test_suite.gui_tests import GUI_test_runner 
 38  from test_suite.system_tests import System_test_runner 
 39  from test_suite.unit_tests.unit_test_runner import Unit_test_runner 
 40  from test_suite.verification_tests import Verification_test_runner 
 41   
 42  # relax module imports. 
 43  if dep_check.wx_module: 
 44      from gui import relax_gui 
 45      from gui import interpreter 
 46  from lib.text.sectioning import section, title 
 47  from test_suite.relax_test_runner import GuiTestRunner, RelaxTestRunner 
 48  from status import Status; status = Status() 
 49   
 50   
 51 -class Test_suite_runner: 
 52      """Class for running all components of the relax test suite. 
 53   
 54      This currently includes the following categories of tests: 
 55          - System/functional tests. 
 56          - Unit tests. 
 57          - GUI tests. 
 58          - Verification tests. 
 59      """ 
 60   
 61 -    def __init__(self, tests=[], from_gui=False, categories=['system', 'unit', 'gui', 'verification'], timing=False): 
 62          """Store the list of tests to preform. 
 63   
 64          The test list should be something like ['N_state_model.test_stereochem_analysis'].  The first part is the imported test case class, the second is the specific test. 
 65   
 66   
 67          @keyword tests:         The list of tests to preform.  If left at [], then all tests will be run. 
 68          @type tests:            list of str 
 69          @keyword from_gui:      A flag which indicates if the tests are being run from the GUI or not. 
 70          @type from_gui:         bool 
 71          @keyword categories:    The list of test categories to run, for example ['system', 'unit', 'gui', 'verification'] for all tests. 
 72          @type categories:       list of str 
 73          @keyword timing:        A flag which if True will enable timing of individual tests. 
 74          @type timing:           bool 
 75          """ 
 76   
 77          # Store the args. 
 78          self.tests = tests 
 79          self.from_gui = from_gui 
 80          self.categories = categories 
 81   
 82          # A list for skipped tests. 
 83          status.skip = [] 
 84   
 85          # Set up the test runner. 
 86          if from_gui: 
 87              self.runner = GuiTestRunner(stream=sys.stdout, timing=timing) 
 88          else: 
 89              self.runner = RelaxTestRunner(stream=sys.stdout, timing=timing) 
 90   
 91          # Let the tests handle the keyboard interrupt (for Python 2.7 and higher). 
 92          if hasattr(unittest, 'installHandler'): 
 93              unittest.installHandler() 
 94   
 95   
 96 -    def run_all_tests(self): 
 97          """Execute all of the test suite test types.""" 
 98   
 99          # Execute the system/functional tests. 
100          if 'system' in self.categories: 
101              status = self.run_system_tests(summary=False) 
102              if not status: 
103                  return 
104   
105          # Execute the unit tests. 
106          if 'unit' in self.categories: 
107              status = self.run_unit_tests(summary=False) 
108              if not status: 
109                  return 
110   
111          # Execute the GUI tests. 
112          if 'gui' in self.categories: 
113              status = self.run_gui_tests(summary=False) 
114              if not status: 
115                  return 
116   
117          # Execute the GUI tests. 
118          if 'verification' in self.categories: 
119              status = self.run_verification_tests(summary=False) 
120              if not status: 
121                  return 
122   
123          # Print out a summary of the test suite. 
124          self.summary() 
125   
126   
127   
128 -    def run_gui_tests(self, summary=True): 
129          """Execute the GUI tests. 
130   
131          @keyword summary:   A flag which if True will cause a summary to be printed. 
132          @type summary:      bool 
133          @return:            True if the tests were run, False if a KeyboardInterrupt occurred. 
134          @rtype:             bool 
135          """ 
136   
137          # Run the tests, catching the keyboard interrupt. 
138          try: 
139              # Print a header. 
140              title(file=sys.stdout, text='GUI tests') 
141   
142              # Run the tests. 
143              if dep_check.wx_module: 
144                  # Set up the GUI if needed (i.e. not in GUI mode already). 
145                  app = wx.GetApp() 
146                  if app == None: 
147                      # Initialise. 
148                      app = wx.App(redirect=False) 
149   
150                      # Build the GUI. 
151                      app.gui = relax_gui.Main(parent=None, id=-1, title="") 
152   
153                  # Execute the GUI tests. 
154                  gui_runner = GUI_test_runner() 
155                  self.runner.category = 'gui' 
156                  self.gui_result = gui_runner.run(self.tests, runner=self.runner) 
157   
158                  # Clean up for the GUI, if not in GUI mode. 
159                  if status.test_mode: 
160                      # Terminate the interpreter thread to allow the tests to cleanly exit. 
161                      interpreter_thread = interpreter.Interpreter() 
162                      interpreter_thread.exit() 
163   
164                      # Stop the GUI main loop. 
165                      app.ExitMainLoop() 
166   
167              # No wx module installed. 
168              else: 
169                  print("All GUI tests skipped due to the missing/broken wx module.\n") 
170                  self.gui_result = 'skip' 
171   
172              # Print out a summary of the test suite. 
173              if summary: 
174                  self.summary() 
175   
176          # Catch the keyboard interrupt. 
177          except KeyboardInterrupt: 
178              print("\nKeyboardInterrupt:  Terminating all tests.\n") 
179              return False 
180   
181          # All tests were run successfully. 
182          return True 
183   
184   
185 -    def run_system_tests(self, summary=True): 
186          """Execute the system/functional tests. 
187   
188          @keyword summary:   A flag which if True will cause a summary to be printed. 
189          @type summary:      bool 
190          @return:            True if the tests were run, False if a KeyboardInterrupt occurred. 
191          @rtype:             bool 
192          """ 
193   
194          # Run the tests, catching the keyboard interrupt. 
195          try: 
196              # Print a header. 
197              title(file=sys.stdout, text='System / functional tests') 
198   
199              # Run the tests. 
200              system_runner = System_test_runner() 
201              self.runner.category = 'system' 
202              self.system_result = system_runner.run(self.tests, runner=self.runner) 
203   
204              # Print out a summary of the test suite. 
205              if summary: 
206                  self.summary() 
207   
208          # Catch the keyboard interrupt. 
209          except KeyboardInterrupt: 
210              print("\nKeyboardInterrupt:  Terminating all tests.\n") 
211              return False 
212   
213          # All tests were run successfully. 
214          return True 
215   
216   
217 -    def run_unit_tests(self, summary=True): 
218          """Execute the unit tests. 
219   
220          @keyword summary:   A flag which if True will cause a summary to be printed. 
221          @type summary:      bool 
222          @return:            True if the tests were run, False if a KeyboardInterrupt occurred. 
223          @rtype:             bool 
224          """ 
225   
226          # Run the tests, catching the keyboard interrupt. 
227          try: 
228              # Print a header. 
229              title(file=sys.stdout, text='Unit tests') 
230   
231              # Run the tests. 
232              unit_runner = Unit_test_runner(root_path=status.install_path+os.sep+'test_suite'+os.sep+'unit_tests') 
233              self.runner.category = 'unit' 
234              self.unit_result = unit_runner.run(runner=self.runner) 
235   
236              # Print out a summary of the test suite. 
237              if summary: 
238                  self.summary() 
239   
240          # Catch the keyboard interrupt. 
241          except KeyboardInterrupt: 
242              print("\nKeyboardInterrupt:  Terminating all tests.\n") 
243              return False 
244   
245          # All tests were run successfully. 
246          return True 
247   
248   
249 -    def run_verification_tests(self, summary=True): 
250          """Execute the software verification tests. 
251   
252          @keyword summary:   A flag which if True will cause a summary to be printed. 
253          @type summary:      bool 
254          """ 
255   
256          # Run the tests, catching the keyboard interrupt. 
257          try: 
258              # Print a header. 
259              title(file=sys.stdout, text='Software verification tests') 
260   
261              # Run the tests. 
262              verification_runner = Verification_test_runner() 
263              self.runner.category = 'verification' 
264              self.verification_result = verification_runner.run(self.tests, runner=self.runner) 
265   
266              # Print out a summary of the test suite. 
267              if summary: 
268                  self.summary() 
269   
270          # Catch the keyboard interrupt. 
271          except KeyboardInterrupt: 
272              print("\nKeyboardInterrupt:  Terminating all tests.\n") 
273              return False 
274   
275          # All tests were run successfully. 
276          return True 
277   
278   
279 -    def summary(self): 
280          """Print out a summary of the relax test suite.""" 
281   
282          # Title. 
283          title(file=sys.stdout, text="Summary of the relax test suite") 
284   
285          # The skipped tests. 
286          self.summary_skipped() 
287   
288          # Subtitle. 
289          section(file=sys.stdout, text="Synopsis") 
290   
291          # System/functional test summary. 
292          if hasattr(self, 'system_result'): 
293              summary_line("System/functional tests", self.system_result) 
294   
295          # Unit test summary. 
296          if hasattr(self, 'unit_result'): 
297              summary_line("Unit tests", self.unit_result) 
298   
299          # GUI test summary. 
300          if hasattr(self, 'gui_result'): 
301              summary_line("GUI tests", self.gui_result) 
302   
303          # Verification test summary. 
304          if hasattr(self, 'verification_result'): 
305              summary_line("Software verification tests", self.verification_result) 
306   
307          # Synopsis. 
308          if hasattr(self, 'system_result') and hasattr(self, 'unit_result') and hasattr(self, 'gui_result') and hasattr(self, 'verification_result'): 
309              if self.gui_result == "skip": 
310                  status = self.system_result and self.unit_result and self.verification_result 
311              else: 
312                  status = self.system_result and self.unit_result and self.gui_result and self.verification_result 
313              summary_line("Synopsis", status) 
314   
315          # End. 
316          print('\n\n') 
317   
318   
319 -    def summary_skipped(self): 
320          """Print out information about skipped tests."""  
321   
322          # Counts. 
323          system_count = {} 
324          unit_count = {} 
325          gui_count = {} 
326          verification_count = {} 
327          for i in range(len(status.skipped_tests)): 
328              # Alias. 
329              test = status.skipped_tests[i] 
330   
331              # Skip all skipped tests whereby the module is set to None to indicate that the test skipping should not be reported. 
332              if test[1] == None: 
333                  continue 
334   
335              # Initialise in needed. 
336              if not test[1] in system_count: 
337                  system_count[test[1]] = 0 
338                  unit_count[test[1]] = 0 
339                  gui_count[test[1]] = 0 
340                  verification_count[test[1]] = 0 
341   
342              # A system test. 
343              if test[2] == 'system': 
344                  system_count[test[1]] += 1 
345   
346              # A unit test. 
347              if test[2] == 'unit': 
348                  unit_count[test[1]] += 1 
349   
350              # A GUI test. 
351              if test[2] == 'gui': 
352                  gui_count[test[1]] += 1 
353   
354              # A verification test. 
355              if test[2] == 'verification': 
356                  verification_count[test[1]] += 1 
357   
358          # The missing modules. 
359          missing_modules = sorted(system_count.keys()) 
360          section(file=sys.stdout, text="Optional packages/modules") 
361   
362          # Nothing missing. 
363          if not missing_modules: 
364              # Except for the wx module! 
365              if not dep_check.wx_module and hasattr(self, 'gui_result'): 
366                  print("All GUI tests skipped due to the missing wxPython module, no other tests skipped due to missing modules.\n") 
367   
368              # Normal printout. 
369              else: 
370                  print("No tests skipped due to missing modules.\n") 
371   
372              # The skip the table. 
373              return 
374   
375          # Header. 
376          print("Tests skipped due to missing packages/modules:\n") 
377          header = "%-30s" % "Module"  
378          if len(system_count): 
379              header = "%s %20s" % (header, "System test count") 
380          if len(unit_count): 
381              header = "%s %20s" % (header, "Unit test count") 
382          if len(gui_count): 
383              header = "%s %20s" % (header, "GUI test count") 
384          if len(verification_count): 
385              header = "%s %20s" % (header, "Verification test count") 
386          print('-'*len(header)) 
387          print(header) 
388          print('-'*len(header)) 
389   
390          # The table. 
391          for module in missing_modules: 
392              text = "%-30s" % module 
393              if len(system_count): 
394                  text = "%s %20s" % (text, system_count[module]) 
395              if len(unit_count): 
396                  text = "%s %20s" % (text, unit_count[module]) 
397              if len(gui_count): 
398                  text = "%s %20s" % (text, gui_count[module]) 
399              if len(verification_count): 
400                  text = "%s %20s" % (text, verification_count[module]) 
401              print(text) 
402   
403          # End the table. 
404          print('-'*len(header)) 
405          print("\n") 
406