test_suite.test_suite

54 """Class for running all components of the relax test suite. 55 56 This currently includes the following categories of tests: 57 - System/functional tests. 58 - Unit tests. 59 - GUI tests. 60 - Verification tests. 61 """ 62

63 - def __init__(self, tests=[], from_gui=False, categories=['system', 'unit', 'gui', 'verification'], timing=False, io_capture=True, list_tests=False):

64 """Store the list of tests to preform. 65 66 The test list should be something like ['N_state_model.test_stereochem_analysis']. The first part is the imported test case class, the second is the specific test. 67 68 69 @keyword tests: The list of tests to preform. If left at [], then all tests will be run. 70 @type tests: list of str 71 @keyword from_gui: A flag which indicates if the tests are being run from the GUI or not. 72 @type from_gui: bool 73 @keyword categories: The list of test categories to run, for example ['system', 'unit', 'gui', 'verification'] for all tests. 74 @type categories: list of str 75 @keyword timing: A flag which if True will enable timing of individual tests. 76 @type timing: bool 77 @keyword io_capture: A flag which if True will cause all IO to be captured and only printed out for failing or error tests. 78 @type io_capture: bool 79 @keyword list_tests: A flag which if True will cause the tests to be listed rather than executed. 80 @type list_tests: bool 81 """ 82 83 # Store the args. 84 self.tests = tests 85 self.from_gui = from_gui 86 self.categories = categories 87 self.list_tests = list_tests 88 89 # Set up the test runner. 90 if from_gui: 91 self.runner = GuiTestRunner(stream=sys.stdout, timing=timing, io_capture=io_capture) 92 else: 93 self.runner = RelaxTestRunner(stream=sys.stdout, timing=timing, io_capture=io_capture) 94 95 # Let the tests handle the keyboard interrupt (for Python 2.7 and higher). 96 if hasattr(unittest, 'installHandler'): 97 unittest.installHandler()

98 99

100 - def run_all_tests(self, reset=True):

101 """Execute all of the test suite test types. 102 103 @keyword reset: A flag which if True will reset the relax status objects for the tests. 104 @type reset: bool 105 @return: The combined status of the test categories. 106 @rtype: bool 107 """ 108 109 # Reset the list for skipped tests. 110 if reset: 111 status.skipped_tests = [] 112 113 # Store the status, so that all test categories are run. 114 test_status = True 115 116 # Execute the system/functional tests. 117 if 'system' in self.categories: 118 test_status = self.run_system_tests(summary=False, reset=False) and test_status 119 120 # Execute the unit tests. 121 if 'unit' in self.categories: 122 test_status = self.run_unit_tests(summary=False, reset=False) and test_status 123 124 # Execute the GUI tests. 125 if 'gui' in self.categories: 126 test_status = self.run_gui_tests(summary=False, reset=False) and test_status 127 128 # Execute the software verification tests. 129 if 'verification' in self.categories: 130 test_status = self.run_verification_tests(summary=False, reset=False) and test_status 131 132 # Print out a summary of the test suite and return the combined status. 133 if not self.list_tests: 134 self.summary() 135 return test_status

136 137

138 - def run_gui_tests(self, summary=True, reset=True):

139 """Execute the GUI tests. 140 141 @keyword summary: A flag which if True will cause a summary to be printed. 142 @type summary: bool 143 @keyword reset: A flag which if True will reset the relax status objects for the tests. 144 @type reset: bool 145 @return: True if the tests were run, False if a KeyboardInterrupt occurred. 146 @rtype: bool 147 """ 148 149 # Reset the list for skipped tests. 150 if reset: 151 status.skipped_tests = [] 152 153 # Run the tests, catching the keyboard interrupt. 154 try: 155 # Print a header. 156 title(file=sys.stdout, text='GUI tests') 157 158 # Run the tests. 159 if dep_check.wx_module: 160 # Set up the GUI if needed (i.e. not in GUI mode already). 161 app = wx.GetApp() 162 if app == None: 163 # Initialise. 164 app = wx.App(redirect=False) 165 166 # Build the GUI. 167 app.gui = relax_gui.Main(parent=None, id=-1, title="") 168 169 # Execute the GUI tests. 170 gui_runner = GUI_test_runner() 171 self.runner.category = 'gui' 172 self.gui_result = gui_runner.run(self.tests, runner=self.runner, list_tests=self.list_tests) 173 174 # Clean up for the GUI, if not in GUI mode. 175 if status.test_mode: 176 # Terminate the interpreter thread to allow the tests to cleanly exit. 177 interpreter_thread = interpreter.Interpreter() 178 interpreter_thread.exit() 179 180 # Stop the GUI main loop. 181 app.ExitMainLoop() 182 183 # No wx module installed. 184 else: 185 print("All GUI tests skipped due to the missing/broken wx module.\n") 186 self.gui_result = 'skip' 187 188 # Print out a summary of the test suite. 189 if summary and not self.list_tests: 190 self.summary() 191 192 # Catch the keyboard interrupt. 193 except KeyboardInterrupt: 194 print("\nKeyboardInterrupt: Terminating all tests.\n") 195 sys.exit(1) 196 197 # Catch GUI app set up errors. 198 except: 199 print("Failure in setting up the GUI tests.\n") 200 print_exc() 201 if summary and not self.list_tests: 202 self.summary() 203 self.gui_result = 'skip' 204 return False 205 206 # All tests were run successfully. 207 return self.gui_result

208 209

210 - def run_system_tests(self, summary=True, reset=True):

211 """Execute the system/functional tests. 212 213 @keyword summary: A flag which if True will cause a summary to be printed. 214 @type summary: bool 215 @keyword reset: A flag which if True will reset the relax status objects for the tests. 216 @type reset: bool 217 @return: True if the tests were run, False if a KeyboardInterrupt occurred. 218 @rtype: bool 219 """ 220 221 # Reset the list for skipped tests. 222 if reset: 223 status.skipped_tests = [] 224 225 # Run the tests, catching the keyboard interrupt. 226 try: 227 # Print a header. 228 title(file=sys.stdout, text='System / functional tests') 229 230 # Run the tests. 231 system_runner = System_test_runner() 232 self.runner.category = 'system' 233 self.system_result = system_runner.run(self.tests, runner=self.runner, list_tests=self.list_tests) 234 235 # Print out a summary of the test suite. 236 if summary and not self.list_tests: 237 self.summary() 238 239 # Catch the keyboard interrupt. 240 except KeyboardInterrupt: 241 print("\nKeyboardInterrupt: Terminating all tests.\n") 242 sys.exit(1) 243 244 # All tests were run successfully. 245 return self.system_result

246 247

248 - def run_unit_tests(self, summary=True, reset=True):

249 """Execute the unit tests. 250 251 @keyword summary: A flag which if True will cause a summary to be printed. 252 @type summary: bool 253 @keyword reset: A flag which if True will reset the relax status objects for the tests. 254 @type reset: bool 255 @return: True if the tests were run, False if a KeyboardInterrupt occurred. 256 @rtype: bool 257 """ 258 259 # Reset the list for skipped tests. 260 if reset: 261 status.skipped_tests = [] 262 263 # Run the tests, catching the keyboard interrupt. 264 try: 265 # Print a header. 266 title(file=sys.stdout, text='Unit tests') 267 268 # Run the tests. 269 unit_runner = Unit_test_runner(root_path=status.install_path+os.sep+'test_suite'+os.sep+'unit_tests') 270 self.runner.category = 'unit' 271 self.unit_result = unit_runner.run(self.tests, runner=self.runner, list_tests=self.list_tests) 272 273 # Print out a summary of the test suite. 274 if summary and not self.list_tests: 275 self.summary() 276 277 # Catch the keyboard interrupt. 278 except KeyboardInterrupt: 279 print("\nKeyboardInterrupt: Terminating all tests.\n") 280 sys.exit(1) 281 282 # All tests were run successfully. 283 return self.unit_result

284 285

286 - def run_verification_tests(self, summary=True, reset=True):

287 """Execute the software verification tests. 288 289 @keyword summary: A flag which if True will cause a summary to be printed. 290 @type summary: bool 291 @keyword reset: A flag which if True will reset the relax status objects for the tests. 292 @type reset: bool 293 """ 294 295 # Reset the list for skipped tests. 296 if reset: 297 status.skipped_tests = [] 298 299 # Run the tests, catching the keyboard interrupt. 300 try: 301 # Print a header. 302 title(file=sys.stdout, text='Software verification tests') 303 304 # Run the tests. 305 verification_runner = Verification_test_runner() 306 self.runner.category = 'verification' 307 self.verification_result = verification_runner.run(self.tests, runner=self.runner, list_tests=self.list_tests) 308 309 # Print out a summary of the test suite. 310 if summary and not self.list_tests: 311 self.summary() 312 313 # Catch the keyboard interrupt. 314 except KeyboardInterrupt: 315 print("\nKeyboardInterrupt: Terminating all tests.\n") 316 sys.exit(1) 317 318 # All tests were run successfully. 319 return self.verification_result

320 321

322 - def summary(self):

323 """Print out a summary of the relax test suite.""" 324 325 # Title. 326 title(file=sys.stdout, text="Summary of the relax test suite") 327 328 # The skipped tests. 329 if status.skip_blacklisted_tests: 330 self.summary_skipped() 331 332 # Subtitle. 333 section(file=sys.stdout, text="Synopsis") 334 335 # System/functional test summary. 336 if hasattr(self, 'system_result'): 337 summary_line("System/functional tests", self.system_result, width=status.text_width) 338 339 # Unit test summary. 340 if hasattr(self, 'unit_result'): 341 summary_line("Unit tests", self.unit_result, width=status.text_width) 342 343 # GUI test summary. 344 if hasattr(self, 'gui_result'): 345 summary_line("GUI tests", self.gui_result, width=status.text_width) 346 347 # Verification test summary. 348 if hasattr(self, 'verification_result'): 349 summary_line("Software verification tests", self.verification_result, width=status.text_width) 350 351 # Synopsis. 352 if hasattr(self, 'system_result') and hasattr(self, 'unit_result') and hasattr(self, 'gui_result') and hasattr(self, 'verification_result'): 353 if self.gui_result == "skip": 354 test_status = self.system_result and self.unit_result and self.verification_result 355 else: 356 test_status = self.system_result and self.unit_result and self.gui_result and self.verification_result 357 summary_line("Synopsis", test_status, width=status.text_width) 358 359 # End. 360 print('\n\n')

361 362

363 - def summary_skipped(self):

364 """Print out information about skipped tests.""" 365 366 # Counts. 367 system_count = {} 368 unit_count = {} 369 gui_count = {} 370 verification_count = {} 371 for i in range(len(status.skipped_tests)): 372 # Alias. 373 test = status.skipped_tests[i] 374 375 # Skip all skipped tests whereby the module is set to None to indicate that the test skipping should not be reported. 376 if test[1] == None: 377 continue 378 379 # Initialise in needed. 380 if not test[1] in system_count: 381 system_count[test[1]] = 0 382 unit_count[test[1]] = 0 383 gui_count[test[1]] = 0 384 verification_count[test[1]] = 0 385 386 # A system test. 387 if test[2] == 'system': 388 system_count[test[1]] += 1 389 390 # A unit test. 391 if test[2] == 'unit': 392 unit_count[test[1]] += 1 393 394 # A GUI test. 395 if test[2] == 'gui': 396 gui_count[test[1]] += 1 397 398 # A verification test. 399 if test[2] == 'verification': 400 verification_count[test[1]] += 1 401 402 # The missing modules. 403 missing_modules = sorted(system_count.keys()) 404 section(file=sys.stdout, text="Optional packages/modules") 405 406 # Nothing missing. 407 if not missing_modules: 408 # Except for the wx module! 409 if not dep_check.wx_module and hasattr(self, 'gui_result'): 410 print("All GUI tests skipped due to the missing wxPython module, no other tests skipped due to missing modules.\n") 411 412 # Normal printout. 413 else: 414 print("No tests skipped due to missing modules.\n") 415 416 # The skip the table. 417 return 418 419 # Header. 420 print("Tests skipped due to missing optional packages/modules/software:\n") 421 header = "%-33s" % "Module/package/software" 422 if len(system_count): 423 header = "%s %20s" % (header, "System test count") 424 if len(unit_count): 425 header = "%s %20s" % (header, "Unit test count") 426 if len(gui_count): 427 header = "%s %20s" % (header, "GUI test count") 428 if len(verification_count): 429 header = "%s %20s" % (header, "Verification test count") 430 print('-'*len(header)) 431 print(header) 432 print('-'*len(header)) 433 434 # The table. 435 for module in missing_modules: 436 text = "%-33s" % module 437 if len(system_count): 438 text = "%s %20s" % (text, system_count[module]) 439 if len(unit_count): 440 text = "%s %20s" % (text, unit_count[module]) 441 if len(gui_count): 442 text = "%s %20s" % (text, gui_count[module]) 443 if len(verification_count): 444 text = "%s %20s" % (text, verification_count[module]) 445 print(text) 446 447 # End the table. 448 print('-'*len(header)) 449 print("\n")

Source Code for Module test_suite.test_suite_runner