test_suite.test_suite

52 """Class for running all components of the relax test suite. 53 54 This currently includes the following categories of tests: 55 - System/functional tests. 56 - Unit tests. 57 - GUI tests. 58 - Verification tests. 59 """ 60

61 - def __init__(self, tests=[], from_gui=False, categories=['system', 'unit', 'gui', 'verification'], timing=False):

62 """Store the list of tests to preform. 63 64 The test list should be something like ['N_state_model.test_stereochem_analysis']. The first part is the imported test case class, the second is the specific test. 65 66 67 @keyword tests: The list of tests to preform. If left at [], then all tests will be run. 68 @type tests: list of str 69 @keyword from_gui: A flag which indicates if the tests are being run from the GUI or not. 70 @type from_gui: bool 71 @keyword categories: The list of test categories to run, for example ['system', 'unit', 'gui', 'verification'] for all tests. 72 @type categories: list of str 73 @keyword timing: A flag which if True will enable timing of individual tests. 74 @type timing: bool 75 """ 76 77 # Store the args. 78 self.tests = tests 79 self.from_gui = from_gui 80 self.categories = categories 81 82 # Set up the test runner. 83 if from_gui: 84 self.runner = GuiTestRunner(stream=sys.stdout, timing=timing) 85 else: 86 self.runner = RelaxTestRunner(stream=sys.stdout, timing=timing) 87 88 # Let the tests handle the keyboard interrupt (for Python 2.7 and higher). 89 if hasattr(unittest, 'installHandler'): 90 unittest.installHandler()

91 92

93 - def run_all_tests(self, reset=True):

94 """Execute all of the test suite test types. 95 96 @keyword reset: A flag which if True will reset the relax status objects for the tests. 97 @type reset: bool 98 """ 99 100 # Reset the list for skipped tests. 101 if reset: 102 status.skipped_tests = [] 103 104 # Execute the system/functional tests. 105 if 'system' in self.categories: 106 test_status = self.run_system_tests(summary=False, reset=False) 107 if not test_status: 108 return 109 110 # Execute the unit tests. 111 if 'unit' in self.categories: 112 test_status = self.run_unit_tests(summary=False, reset=False) 113 if not test_status: 114 return 115 116 # Execute the GUI tests. 117 if 'gui' in self.categories: 118 test_status = self.run_gui_tests(summary=False, reset=False) 119 if not test_status: 120 return 121 122 # Execute the GUI tests. 123 if 'verification' in self.categories: 124 test_status = self.run_verification_tests(summary=False, reset=False) 125 if not test_status: 126 return 127 128 # Print out a summary of the test suite. 129 self.summary()

130 131 132

133 - def run_gui_tests(self, summary=True, reset=True):

134 """Execute the GUI tests. 135 136 @keyword summary: A flag which if True will cause a summary to be printed. 137 @type summary: bool 138 @keyword reset: A flag which if True will reset the relax status objects for the tests. 139 @type reset: bool 140 @return: True if the tests were run, False if a KeyboardInterrupt occurred. 141 @rtype: bool 142 """ 143 144 # Reset the list for skipped tests. 145 if reset: 146 status.skipped_tests = [] 147 148 # Run the tests, catching the keyboard interrupt. 149 try: 150 # Print a header. 151 title(file=sys.stdout, text='GUI tests') 152 153 # Run the tests. 154 if dep_check.wx_module: 155 # Set up the GUI if needed (i.e. not in GUI mode already). 156 app = wx.GetApp() 157 if app == None: 158 # Initialise. 159 app = wx.App(redirect=False) 160 161 # Build the GUI. 162 app.gui = relax_gui.Main(parent=None, id=-1, title="") 163 164 # Execute the GUI tests. 165 gui_runner = GUI_test_runner() 166 self.runner.category = 'gui' 167 self.gui_result = gui_runner.run(self.tests, runner=self.runner) 168 169 # Clean up for the GUI, if not in GUI mode. 170 if status.test_mode: 171 # Terminate the interpreter thread to allow the tests to cleanly exit. 172 interpreter_thread = interpreter.Interpreter() 173 interpreter_thread.exit() 174 175 # Stop the GUI main loop. 176 app.ExitMainLoop() 177 178 # No wx module installed. 179 else: 180 print("All GUI tests skipped due to the missing/broken wx module.\n") 181 self.gui_result = 'skip' 182 183 # Print out a summary of the test suite. 184 if summary: 185 self.summary() 186 187 # Catch the keyboard interrupt. 188 except KeyboardInterrupt: 189 print("\nKeyboardInterrupt: Terminating all tests.\n") 190 return False 191 192 # All tests were run successfully. 193 return True

194 195

196 - def run_system_tests(self, summary=True, reset=True):

197 """Execute the system/functional tests. 198 199 @keyword summary: A flag which if True will cause a summary to be printed. 200 @type summary: bool 201 @keyword reset: A flag which if True will reset the relax status objects for the tests. 202 @type reset: bool 203 @return: True if the tests were run, False if a KeyboardInterrupt occurred. 204 @rtype: bool 205 """ 206 207 # Reset the list for skipped tests. 208 if reset: 209 status.skipped_tests = [] 210 211 # Run the tests, catching the keyboard interrupt. 212 try: 213 # Print a header. 214 title(file=sys.stdout, text='System / functional tests') 215 216 # Run the tests. 217 system_runner = System_test_runner() 218 self.runner.category = 'system' 219 self.system_result = system_runner.run(self.tests, runner=self.runner) 220 221 # Print out a summary of the test suite. 222 if summary: 223 self.summary() 224 225 # Catch the keyboard interrupt. 226 except KeyboardInterrupt: 227 print("\nKeyboardInterrupt: Terminating all tests.\n") 228 return False 229 230 # All tests were run successfully. 231 return True

232 233

234 - def run_unit_tests(self, summary=True, reset=True):

235 """Execute the unit tests. 236 237 @keyword summary: A flag which if True will cause a summary to be printed. 238 @type summary: bool 239 @keyword reset: A flag which if True will reset the relax status objects for the tests. 240 @type reset: bool 241 @return: True if the tests were run, False if a KeyboardInterrupt occurred. 242 @rtype: bool 243 """ 244 245 # Reset the list for skipped tests. 246 if reset: 247 status.skipped_tests = [] 248 249 # Run the tests, catching the keyboard interrupt. 250 try: 251 # Print a header. 252 title(file=sys.stdout, text='Unit tests') 253 254 # Run the tests. 255 unit_runner = Unit_test_runner(root_path=status.install_path+os.sep+'test_suite'+os.sep+'unit_tests') 256 self.runner.category = 'unit' 257 self.unit_result = unit_runner.run(self.tests, runner=self.runner) 258 259 # Print out a summary of the test suite. 260 if summary: 261 self.summary() 262 263 # Catch the keyboard interrupt. 264 except KeyboardInterrupt: 265 print("\nKeyboardInterrupt: Terminating all tests.\n") 266 return False 267 268 # All tests were run successfully. 269 return True

270 271

272 - def run_verification_tests(self, summary=True, reset=True):

273 """Execute the software verification tests. 274 275 @keyword summary: A flag which if True will cause a summary to be printed. 276 @type summary: bool 277 @keyword reset: A flag which if True will reset the relax status objects for the tests. 278 @type reset: bool 279 """ 280 281 # Reset the list for skipped tests. 282 if reset: 283 status.skipped_tests = [] 284 285 # Run the tests, catching the keyboard interrupt. 286 try: 287 # Print a header. 288 title(file=sys.stdout, text='Software verification tests') 289 290 # Run the tests. 291 verification_runner = Verification_test_runner() 292 self.runner.category = 'verification' 293 self.verification_result = verification_runner.run(self.tests, runner=self.runner) 294 295 # Print out a summary of the test suite. 296 if summary: 297 self.summary() 298 299 # Catch the keyboard interrupt. 300 except KeyboardInterrupt: 301 print("\nKeyboardInterrupt: Terminating all tests.\n") 302 return False 303 304 # All tests were run successfully. 305 return True

306 307

308 - def summary(self):

309 """Print out a summary of the relax test suite.""" 310 311 # Title. 312 title(file=sys.stdout, text="Summary of the relax test suite") 313 314 # The skipped tests. 315 if status.skip_blacklisted_tests: 316 self.summary_skipped() 317 318 # Subtitle. 319 section(file=sys.stdout, text="Synopsis") 320 321 # System/functional test summary. 322 if hasattr(self, 'system_result'): 323 summary_line("System/functional tests", self.system_result) 324 325 # Unit test summary. 326 if hasattr(self, 'unit_result'): 327 summary_line("Unit tests", self.unit_result) 328 329 # GUI test summary. 330 if hasattr(self, 'gui_result'): 331 summary_line("GUI tests", self.gui_result) 332 333 # Verification test summary. 334 if hasattr(self, 'verification_result'): 335 summary_line("Software verification tests", self.verification_result) 336 337 # Synopsis. 338 if hasattr(self, 'system_result') and hasattr(self, 'unit_result') and hasattr(self, 'gui_result') and hasattr(self, 'verification_result'): 339 if self.gui_result == "skip": 340 test_status = self.system_result and self.unit_result and self.verification_result 341 else: 342 test_status = self.system_result and self.unit_result and self.gui_result and self.verification_result 343 summary_line("Synopsis", test_status) 344 345 # End. 346 print('\n\n')

347 348

349 - def summary_skipped(self):

350 """Print out information about skipped tests.""" 351 352 # Counts. 353 system_count = {} 354 unit_count = {} 355 gui_count = {} 356 verification_count = {} 357 for i in range(len(status.skipped_tests)): 358 # Alias. 359 test = status.skipped_tests[i] 360 361 # Skip all skipped tests whereby the module is set to None to indicate that the test skipping should not be reported. 362 if test[1] == None: 363 continue 364 365 # Initialise in needed. 366 if not test[1] in system_count: 367 system_count[test[1]] = 0 368 unit_count[test[1]] = 0 369 gui_count[test[1]] = 0 370 verification_count[test[1]] = 0 371 372 # A system test. 373 if test[2] == 'system': 374 system_count[test[1]] += 1 375 376 # A unit test. 377 if test[2] == 'unit': 378 unit_count[test[1]] += 1 379 380 # A GUI test. 381 if test[2] == 'gui': 382 gui_count[test[1]] += 1 383 384 # A verification test. 385 if test[2] == 'verification': 386 verification_count[test[1]] += 1 387 388 # The missing modules. 389 missing_modules = sorted(system_count.keys()) 390 section(file=sys.stdout, text="Optional packages/modules") 391 392 # Nothing missing. 393 if not missing_modules: 394 # Except for the wx module! 395 if not dep_check.wx_module and hasattr(self, 'gui_result'): 396 print("All GUI tests skipped due to the missing wxPython module, no other tests skipped due to missing modules.\n") 397 398 # Normal printout. 399 else: 400 print("No tests skipped due to missing modules.\n") 401 402 # The skip the table. 403 return 404 405 # Header. 406 print("Tests skipped due to missing optional packages/modules/software:\n") 407 header = "%-33s" % "Module/package/software" 408 if len(system_count): 409 header = "%s %20s" % (header, "System test count") 410 if len(unit_count): 411 header = "%s %20s" % (header, "Unit test count") 412 if len(gui_count): 413 header = "%s %20s" % (header, "GUI test count") 414 if len(verification_count): 415 header = "%s %20s" % (header, "Verification test count") 416 print('-'*len(header)) 417 print(header) 418 print('-'*len(header)) 419 420 # The table. 421 for module in missing_modules: 422 text = "%-33s" % module 423 if len(system_count): 424 text = "%s %20s" % (text, system_count[module]) 425 if len(unit_count): 426 text = "%s %20s" % (text, unit_count[module]) 427 if len(gui_count): 428 text = "%s %20s" % (text, gui_count[module]) 429 if len(verification_count): 430 text = "%s %20s" % (text, verification_count[module]) 431 print(text) 432 433 # End the table. 434 print('-'*len(header)) 435 print("\n")

Source Code for Module test_suite.test_suite_runner