test_suite.test_suite

54 """Class for running all components of the relax test suite. 55 56 This currently includes the following categories of tests: 57 - System/functional tests. 58 - Unit tests. 59 - GUI tests. 60 - Verification tests. 61 """ 62

63 - def __init__(self, tests=[], from_gui=False, categories=['system', 'unit', 'gui', 'verification'], timing=False):

64 """Store the list of tests to preform. 65 66 The test list should be something like ['N_state_model.test_stereochem_analysis']. The first part is the imported test case class, the second is the specific test. 67 68 69 @keyword tests: The list of tests to preform. If left at [], then all tests will be run. 70 @type tests: list of str 71 @keyword from_gui: A flag which indicates if the tests are being run from the GUI or not. 72 @type from_gui: bool 73 @keyword categories: The list of test categories to run, for example ['system', 'unit', 'gui', 'verification'] for all tests. 74 @type categories: list of str 75 @keyword timing: A flag which if True will enable timing of individual tests. 76 @type timing: bool 77 """ 78 79 # Store the args. 80 self.tests = tests 81 self.from_gui = from_gui 82 self.categories = categories 83 84 # Set up the test runner. 85 if from_gui: 86 self.runner = GuiTestRunner(stream=sys.stdout, timing=timing) 87 else: 88 self.runner = RelaxTestRunner(stream=sys.stdout, timing=timing) 89 90 # Let the tests handle the keyboard interrupt (for Python 2.7 and higher). 91 if hasattr(unittest, 'installHandler'): 92 unittest.installHandler()

93 94

95 - def run_all_tests(self, reset=True):

96 """Execute all of the test suite test types. 97 98 @keyword reset: A flag which if True will reset the relax status objects for the tests. 99 @type reset: bool 100 @return: The combined status of the test categories. 101 @rtype: bool 102 """ 103 104 # Reset the list for skipped tests. 105 if reset: 106 status.skipped_tests = [] 107 108 # Store the status, so that all test categories are run. 109 test_status = True 110 111 # Execute the system/functional tests. 112 if 'system' in self.categories: 113 test_status = self.run_system_tests(summary=False, reset=False) and test_status 114 115 # Execute the unit tests. 116 if 'unit' in self.categories: 117 test_status = self.run_unit_tests(summary=False, reset=False) and test_status 118 119 # Execute the GUI tests. 120 if 'gui' in self.categories: 121 test_status = self.run_gui_tests(summary=False, reset=False) and test_status 122 123 # Execute the software verification tests. 124 if 'verification' in self.categories: 125 test_status = self.run_verification_tests(summary=False, reset=False) and test_status 126 127 # Print out a summary of the test suite and return the combined status. 128 self.summary() 129 return test_status

130 131

132 - def run_gui_tests(self, summary=True, reset=True):

133 """Execute the GUI tests. 134 135 @keyword summary: A flag which if True will cause a summary to be printed. 136 @type summary: bool 137 @keyword reset: A flag which if True will reset the relax status objects for the tests. 138 @type reset: bool 139 @return: True if the tests were run, False if a KeyboardInterrupt occurred. 140 @rtype: bool 141 """ 142 143 # Reset the list for skipped tests. 144 if reset: 145 status.skipped_tests = [] 146 147 # Run the tests, catching the keyboard interrupt. 148 try: 149 # Print a header. 150 title(file=sys.stdout, text='GUI tests') 151 152 # Run the tests. 153 if dep_check.wx_module: 154 # Set up the GUI if needed (i.e. not in GUI mode already). 155 app = wx.GetApp() 156 if app == None: 157 # Initialise. 158 app = wx.App(redirect=False) 159 160 # Build the GUI. 161 app.gui = relax_gui.Main(parent=None, id=-1, title="") 162 163 # Execute the GUI tests. 164 gui_runner = GUI_test_runner() 165 self.runner.category = 'gui' 166 self.gui_result = gui_runner.run(self.tests, runner=self.runner) 167 168 # Clean up for the GUI, if not in GUI mode. 169 if status.test_mode: 170 # Terminate the interpreter thread to allow the tests to cleanly exit. 171 interpreter_thread = interpreter.Interpreter() 172 interpreter_thread.exit() 173 174 # Stop the GUI main loop. 175 app.ExitMainLoop() 176 177 # No wx module installed. 178 else: 179 print("All GUI tests skipped due to the missing/broken wx module.\n") 180 self.gui_result = 'skip' 181 182 # Print out a summary of the test suite. 183 if summary: 184 self.summary() 185 186 # Catch the keyboard interrupt. 187 except KeyboardInterrupt: 188 print("\nKeyboardInterrupt: Terminating all tests.\n") 189 sys.exit(1) 190 191 # Catch GUI app set up errors. 192 except: 193 print("Failure in setting up the GUI tests.\n") 194 print_exc() 195 if summary: 196 self.summary() 197 self.gui_result = 'skip' 198 return False 199 200 # All tests were run successfully. 201 return self.gui_result

202 203

204 - def run_system_tests(self, summary=True, reset=True):

205 """Execute the system/functional tests. 206 207 @keyword summary: A flag which if True will cause a summary to be printed. 208 @type summary: bool 209 @keyword reset: A flag which if True will reset the relax status objects for the tests. 210 @type reset: bool 211 @return: True if the tests were run, False if a KeyboardInterrupt occurred. 212 @rtype: bool 213 """ 214 215 # Reset the list for skipped tests. 216 if reset: 217 status.skipped_tests = [] 218 219 # Run the tests, catching the keyboard interrupt. 220 try: 221 # Print a header. 222 title(file=sys.stdout, text='System / functional tests') 223 224 # Run the tests. 225 system_runner = System_test_runner() 226 self.runner.category = 'system' 227 self.system_result = system_runner.run(self.tests, runner=self.runner) 228 229 # Print out a summary of the test suite. 230 if summary: 231 self.summary() 232 233 # Catch the keyboard interrupt. 234 except KeyboardInterrupt: 235 print("\nKeyboardInterrupt: Terminating all tests.\n") 236 sys.exit(1) 237 238 # All tests were run successfully. 239 return self.system_result

240 241

242 - def run_unit_tests(self, summary=True, reset=True):

243 """Execute the unit tests. 244 245 @keyword summary: A flag which if True will cause a summary to be printed. 246 @type summary: bool 247 @keyword reset: A flag which if True will reset the relax status objects for the tests. 248 @type reset: bool 249 @return: True if the tests were run, False if a KeyboardInterrupt occurred. 250 @rtype: bool 251 """ 252 253 # Reset the list for skipped tests. 254 if reset: 255 status.skipped_tests = [] 256 257 # Run the tests, catching the keyboard interrupt. 258 try: 259 # Print a header. 260 title(file=sys.stdout, text='Unit tests') 261 262 # Run the tests. 263 unit_runner = Unit_test_runner(root_path=status.install_path+os.sep+'test_suite'+os.sep+'unit_tests') 264 self.runner.category = 'unit' 265 self.unit_result = unit_runner.run(self.tests, runner=self.runner) 266 267 # Print out a summary of the test suite. 268 if summary: 269 self.summary() 270 271 # Catch the keyboard interrupt. 272 except KeyboardInterrupt: 273 print("\nKeyboardInterrupt: Terminating all tests.\n") 274 sys.exit(1) 275 276 # All tests were run successfully. 277 return self.unit_result

278 279

280 - def run_verification_tests(self, summary=True, reset=True):

281 """Execute the software verification tests. 282 283 @keyword summary: A flag which if True will cause a summary to be printed. 284 @type summary: bool 285 @keyword reset: A flag which if True will reset the relax status objects for the tests. 286 @type reset: bool 287 """ 288 289 # Reset the list for skipped tests. 290 if reset: 291 status.skipped_tests = [] 292 293 # Run the tests, catching the keyboard interrupt. 294 try: 295 # Print a header. 296 title(file=sys.stdout, text='Software verification tests') 297 298 # Run the tests. 299 verification_runner = Verification_test_runner() 300 self.runner.category = 'verification' 301 self.verification_result = verification_runner.run(self.tests, runner=self.runner) 302 303 # Print out a summary of the test suite. 304 if summary: 305 self.summary() 306 307 # Catch the keyboard interrupt. 308 except KeyboardInterrupt: 309 print("\nKeyboardInterrupt: Terminating all tests.\n") 310 sys.exit(1) 311 312 # All tests were run successfully. 313 return self.verification_result

314 315

316 - def summary(self):

317 """Print out a summary of the relax test suite.""" 318 319 # Title. 320 title(file=sys.stdout, text="Summary of the relax test suite") 321 322 # The skipped tests. 323 if status.skip_blacklisted_tests: 324 self.summary_skipped() 325 326 # Subtitle. 327 section(file=sys.stdout, text="Synopsis") 328 329 # System/functional test summary. 330 if hasattr(self, 'system_result'): 331 summary_line("System/functional tests", self.system_result) 332 333 # Unit test summary. 334 if hasattr(self, 'unit_result'): 335 summary_line("Unit tests", self.unit_result) 336 337 # GUI test summary. 338 if hasattr(self, 'gui_result'): 339 summary_line("GUI tests", self.gui_result) 340 341 # Verification test summary. 342 if hasattr(self, 'verification_result'): 343 summary_line("Software verification tests", self.verification_result) 344 345 # Synopsis. 346 if hasattr(self, 'system_result') and hasattr(self, 'unit_result') and hasattr(self, 'gui_result') and hasattr(self, 'verification_result'): 347 if self.gui_result == "skip": 348 test_status = self.system_result and self.unit_result and self.verification_result 349 else: 350 test_status = self.system_result and self.unit_result and self.gui_result and self.verification_result 351 summary_line("Synopsis", test_status) 352 353 # End. 354 print('\n\n')

355 356

357 - def summary_skipped(self):

358 """Print out information about skipped tests.""" 359 360 # Counts. 361 system_count = {} 362 unit_count = {} 363 gui_count = {} 364 verification_count = {} 365 for i in range(len(status.skipped_tests)): 366 # Alias. 367 test = status.skipped_tests[i] 368 369 # Skip all skipped tests whereby the module is set to None to indicate that the test skipping should not be reported. 370 if test[1] == None: 371 continue 372 373 # Initialise in needed. 374 if not test[1] in system_count: 375 system_count[test[1]] = 0 376 unit_count[test[1]] = 0 377 gui_count[test[1]] = 0 378 verification_count[test[1]] = 0 379 380 # A system test. 381 if test[2] == 'system': 382 system_count[test[1]] += 1 383 384 # A unit test. 385 if test[2] == 'unit': 386 unit_count[test[1]] += 1 387 388 # A GUI test. 389 if test[2] == 'gui': 390 gui_count[test[1]] += 1 391 392 # A verification test. 393 if test[2] == 'verification': 394 verification_count[test[1]] += 1 395 396 # The missing modules. 397 missing_modules = sorted(system_count.keys()) 398 section(file=sys.stdout, text="Optional packages/modules") 399 400 # Nothing missing. 401 if not missing_modules: 402 # Except for the wx module! 403 if not dep_check.wx_module and hasattr(self, 'gui_result'): 404 print("All GUI tests skipped due to the missing wxPython module, no other tests skipped due to missing modules.\n") 405 406 # Normal printout. 407 else: 408 print("No tests skipped due to missing modules.\n") 409 410 # The skip the table. 411 return 412 413 # Header. 414 print("Tests skipped due to missing optional packages/modules/software:\n") 415 header = "%-33s" % "Module/package/software" 416 if len(system_count): 417 header = "%s %20s" % (header, "System test count") 418 if len(unit_count): 419 header = "%s %20s" % (header, "Unit test count") 420 if len(gui_count): 421 header = "%s %20s" % (header, "GUI test count") 422 if len(verification_count): 423 header = "%s %20s" % (header, "Verification test count") 424 print('-'*len(header)) 425 print(header) 426 print('-'*len(header)) 427 428 # The table. 429 for module in missing_modules: 430 text = "%-33s" % module 431 if len(system_count): 432 text = "%s %20s" % (text, system_count[module]) 433 if len(unit_count): 434 text = "%s %20s" % (text, unit_count[module]) 435 if len(gui_count): 436 text = "%s %20s" % (text, gui_count[module]) 437 if len(verification_count): 438 text = "%s %20s" % (text, verification_count[module]) 439 print(text) 440 441 # End the table. 442 print('-'*len(header)) 443 print("\n")

Source Code for Module test_suite.test_suite_runner