From b498db14ab8fd161dd5a1253326c5529ce5dbcc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pekka=20Kl=C3=A4rck?= Date: Wed, 7 Dec 2016 22:10:11 +0200 Subject: [PATCH] run_tests.py cleanup - Use external robotstatuschecker module instead of our vendored copy - Add docstring and use it in when showing usage on the cli - Use rebot_cli for generating logs/reports instead of running rebot on the system - General cleanup here and there --- requirements-dev.txt | 1 + test/resources/statuschecker.py | 190 -------------------------------- test/run_tests.py | 138 +++++++++++++---------- 3 files changed, 78 insertions(+), 251 deletions(-) delete mode 100755 test/resources/statuschecker.py diff --git a/requirements-dev.txt b/requirements-dev.txt index 936a01bfd..bde9d5ff7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1,2 @@ mockito +robotstatuschecker diff --git a/test/resources/statuschecker.py b/test/resources/statuschecker.py deleted file mode 100755 index 5af72b86c..000000000 --- a/test/resources/statuschecker.py +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2008-2012 Nokia Siemens Networks Oyj -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Robot Framework Test Status Checker - -Usage: statuschecker.py infile [outfile] - -This tool processes Robot Framework output XML files and checks that test case -statuses and messages are as expected. Main use case is post-processing output -files got when testing Robot Framework test libraries using Robot Framework -itself. - -If output file is not given, the input file is considered to be also output -file and it is edited in place. - -By default all test cases are expected to 'PASS' and have no message. Changing -the expected status to 'FAIL' is done by having word 'FAIL' (in uppercase) -somewhere in the test case documentation. Expected error message must then be -given after 'FAIL'. Error message can also be specified as a regular -expression by prefixing it with string 'REGEXP:'. Testing only the beginning -of the message is possible with 'STARTS:' prefix. - -This tool also allows testing the created log messages. They are specified -using a syntax 'LOG x.y:z LEVEL Actual message', which is described in detail -detail in the tool documentation. -""" - -import re - -from robot.result import ExecutionResult - - -def process_output(inpath, outpath=None): - result = ExecutionResult(inpath) - _process_suite(result.suite) - result.save(outpath) - return result.return_code - -def _process_suite(suite): - for subsuite in suite.suites: - _process_suite(subsuite) - for test in suite.tests: - _process_test(test) - -def _process_test(test): - exp = _Expected(test.doc) - _check_status(test, exp) - if test.status == 'PASS': - _check_logs(test, exp) - -def _check_status(test, exp): - if exp.status != test.status: - test.status = 'FAIL' - if exp.status == 'PASS': - test.message = ("Test was expected to PASS but it FAILED. " - "Error message:\n") + test.message - else: - test.message = ("Test was expected to FAIL but it PASSED. " - "Expected message:\n") + exp.message - elif not _message_matches(test.message, exp.message): - test.status = 'FAIL' - test.message = ("Wrong error message.\n\nExpected:\n%s\n\nActual:\n%s\n" - % (exp.message, test.message)) - elif test.status == 'FAIL': - test.status = 'PASS' - test.message = 'Original test failed as expected.' - -def _message_matches(actual, expected): - if actual == expected: - return True - if expected.startswith('REGEXP:'): - pattern = '^%s$' % expected.replace('REGEXP:', '', 1).strip() - if re.match(pattern, actual, re.DOTALL): - return True - if expected.startswith('STARTS:'): - start = expected.replace('STARTS:', '', 1).strip() - if actual.startswith(start): - return True - return False - -def _check_logs(test, exp): - for kw_indices, msg_index, level, message in exp.logs: - try: - kw = test.keywords[kw_indices[0]] - for index in kw_indices[1:]: - kw = kw.keywords[index] - except IndexError: - indices = '.'.join(str(i+1) for i in kw_indices) - test.status = 'FAIL' - test.message = ("Test '%s' does not have keyword with index '%s'" - % (test.name, indices)) - return - if len(kw.messages) <= msg_index: - if message != 'NONE': - test.status = 'FAIL' - test.message = ("Keyword '%s' should have had at least %d " - "messages" % (kw.name, msg_index+1)) - else: - if _check_log_level(level, test, kw, msg_index): - _check_log_message(message, test, kw, msg_index) - -def _check_log_level(expected, test, kw, index): - actual = kw.messages[index].level - if actual == expected: - return True - test.status = 'FAIL' - test.message = ("Wrong level for message %d of keyword '%s'.\n\n" - "Expected: %s\nActual: %s.\n%s" - % (index+1, kw.name, expected, - actual, kw.messages[index].message)) - return False - -def _check_log_message(expected, test, kw, index): - actual = kw.messages[index].message.strip() - if _message_matches(actual, expected): - return True - test.status = 'FAIL' - test.message = ("Wrong content for message %d of keyword '%s'.\n\n" - "Expected:\n%s\n\nActual:\n%s" - % (index+1, kw.name, expected, actual)) - return False - - -class _Expected: - - def __init__(self, doc): - self.status, self.message = self._get_status_and_message(doc) - self.logs = self._get_logs(doc) - - def _get_status_and_message(self, doc): - if 'FAIL' in doc: - return 'FAIL', doc.split('FAIL', 1)[1].split('LOG', 1)[0].strip() - return 'PASS', '' - - def _get_logs(self, doc): - logs = [] - for item in doc.split('LOG')[1:]: - index_str, msg_str = item.strip().split(' ', 1) - kw_indices, msg_index = self._get_indices(index_str) - level, message = self._get_log_message(msg_str) - logs.append((kw_indices, msg_index, level, message)) - return logs - - def _get_indices(self, index_str): - try: - kw_indices, msg_index = index_str.split(':') - except ValueError: - kw_indices, msg_index = index_str, '1' - kw_indices = [int(index) - 1 for index in kw_indices.split('.')] - return kw_indices, int(msg_index) - 1 - - def _get_log_message(self, msg_str): - try: - level, message = msg_str.split(' ', 1) - if level not in ['TRACE', 'DEBUG', 'INFO', 'WARN', 'FAIL']: - raise ValueError - except ValueError: - level, message = 'INFO', msg_str - return level, message - - -if __name__=='__main__': - import sys - import os - - if not 2 <= len(sys.argv) <= 3 or '--help' in sys.argv: - print(__doc__) - sys.exit(1) - infile = sys.argv[1] - outfile = sys.argv[2] if len(sys.argv) == 3 else None - print("Checking %s" % os.path.abspath(infile)) - rc = process_output(infile, outfile) - if outfile: - print("Output: %s" % os.path.abspath(outfile)) - if rc > 255: - rc = 255 - sys.exit(rc) diff --git a/test/run_tests.py b/test/run_tests.py index badc6e42f..cbc7c8469 100755 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -1,118 +1,134 @@ #!/usr/bin/env python +"""Script to run Selenium2Library acceptance tests. + +Tests are executed using Robot Framework and results verified automatically +afterwards using `robotstatuschecker` tool. The tool can be installed using +`pip install robotstatuschecker` and more information about it can be found +from https://github.com/robotframework/statuschecker/. Notice that initially +some tests fail. + +Usage: + + run_tests.py interpreter browser [options] + +Arguments: + + interpreter: Any Python interpreter supported by the library (e.g. `python`, + `jython`, `c:\\Python27\\python.exe`) + browser: Any browser supported by the library (e.g. `chrome`, `firefox`) + options: Additional command line options passed to Robot Framework + +Examples: + + run_tests.py python chrome + run_tests.py jython c:\\Python35\\python.exe --test "Click element" +""" + from __future__ import print_function import os import sys -from subprocess import Popen, call +from subprocess import call, Popen, STDOUT from tempfile import TemporaryFile +from robot import rebot_cli +try: + import robotstatuschecker +except ImportError: + sys.exit('Required `robotstatuschecker` not installed.\n' + 'Install it with `pip install robotstatuschecker`.') + import env from run_unit_tests import run_unit_tests -ROBOT_ARGS = [ - '--doc', 'SeleniumSPacceptanceSPtestsSPwithSP%(browser)s', - '--outputdir', '%(outdir)s', - '--variable', 'browser:%(browser)s', - '--variable', 'pyversion:%(pyVersion)s', - '--escape', 'space:SP', - '--report', 'none', - '--log', 'none', - #'--suite', 'Acceptance.Keywords.Textfields', +ROBOT_OPTIONS = [ + '--doc', 'Selenium2Library acceptance tests with {browser}', + '--outputdir', '{outdir}', + '--variable', 'BROWSER:{browser}', + '--variable', 'PY_VERSION:{py_version}', + '--report', 'NONE', + '--log', 'NONE', '--loglevel', 'DEBUG', - '--pythonpath', '%(pythonpath)s', - '--noncritical', 'known_issue_-_%(pyVersion)s', - '--noncritical', 'known_issue_-_%(browser)s', + '--pythonpath', '{pythonpath}', ] -REBOT_ARGS = [ - '--outputdir', '%(outdir)s', - '--name', '%(browser)sSPAcceptanceSPTests', - '--escape', 'space:SP', +REBOT_OPTIONS = [ + '--outputdir', '{outdir}', '--critical', 'regression', '--noncritical', 'inprogress', - '--noncritical', 'known_issue_-_%(pyVersion)s', - '--noncritical', 'known_issue_-_%(browser)s', + '--noncritical', 'known_issue_-_*', ] -ARG_VALUES = {'outdir': env.RESULTS_DIR, - 'pythonpath': ':'.join((env.SRC_DIR, env.TEST_LIBS_DIR))} +OPTION_VALUES = {'outdir': env.RESULTS_DIR, + 'pythonpath': ':'.join((env.SRC_DIR, env.TEST_LIBS_DIR))} -def acceptance_tests(interpreter, browser, args): - ARG_VALUES['browser'] = browser.replace('*', '') - ARG_VALUES['pyVersion'] = interpreter + sys.version[:3] +def acceptance_tests(interpreter, browser, options): + OPTION_VALUES['browser'] = browser.replace('*', '') + OPTION_VALUES['py_version'] = interpreter + sys.version[:3] start_http_server() - runner = {'python': 'pybot', 'jython': 'jybot', 'ipy': 'ipybot'}[interpreter] - if os.sep == '\\': - runner += '.bat' - execute_tests(runner, args) + if not os.path.exists(env.RESULTS_DIR): + os.mkdir(env.RESULTS_DIR) + execute_tests(interpreter, options) stop_http_server() - return process_output(args) + return process_output(options) def start_http_server(): - server_output = TemporaryFile() Popen(['python', env.HTTP_SERVER_FILE, 'start'], - stdout=server_output, stderr=server_output) + stdout=TemporaryFile(), stderr=STDOUT) -def execute_tests(runner, args): - if not os.path.exists(env.RESULTS_DIR): - os.mkdir(env.RESULTS_DIR) - command = [runner] + [arg % ARG_VALUES for arg in ROBOT_ARGS] + args + [env.ACCEPTANCE_TEST_DIR] +def execute_tests(interpeter, options): + runner = [interpeter, '-m', 'robot.run'] + options = [opt.format(**OPTION_VALUES) for opt in ROBOT_OPTIONS] + options + command = runner + options + [env.ACCEPTANCE_TEST_DIR] print() print('Starting test execution with command:\n' + ' '.join(command)) syslog = os.path.join(env.RESULTS_DIR, 'syslog.txt') - call(command, shell=os.sep=='\\', env=dict(os.environ, ROBOT_SYSLOG_FILE=syslog)) + # Running tests as an external process, not using `robot_cli`, to allow + # using different interpreter that is used for running this script. + call(command, env=dict(os.environ, ROBOT_SYSLOG_FILE=syslog)) def stop_http_server(): call(['python', env.HTTP_SERVER_FILE, 'stop']) -def process_output(args): - print() - call(['python', os.path.join(env.RESOURCES_DIR, 'statuschecker.py'), - os.path.join(env.RESULTS_DIR, 'output.xml')]) - rebot = 'rebot' if os.sep == '/' else 'rebot.bat' - rebot_cmd = [rebot] + [ arg % ARG_VALUES for arg in REBOT_ARGS ] + args + \ - [os.path.join(ARG_VALUES['outdir'], 'output.xml') ] +def process_output(options): + print('Verifying results...') + output = os.path.join(env.RESULTS_DIR, 'output.xml') + robotstatuschecker.process_output(output, verbose=False) + options = [opt.format(**OPTION_VALUES) for opt in REBOT_OPTIONS] + options + try: + rebot_cli(options + [output]) + except SystemExit as exit: + rc = exit.code print() - print('Starting output processing with command:\n' + ' '.join(rebot_cmd)) - rc = call(rebot_cmd, env=os.environ) if rc == 0: - print('All critical tests passed') + print('All critical tests passed.') else: - print('%d critical test%s failed' % (rc, 's' if rc != 1 else '')) + print('%d critical test%s failed.' % (rc, 's' if rc != 1 else '')) return rc -def _help(): - print('Usage: python run_tests.py python|jython browser [options]') - print() - print('See README.txt for details.') - sys.exit(255) - - def _run_unit_tests(): print('Running unit tests') failures = run_unit_tests() if failures != 0: - print('\n%d unit tests failed - not running acceptance tests!' % failures) - else: - print('All unit tests passed') + print('\n%Unit tests failed! Not running acceptance tests!') return failures if __name__ == '__main__': - if not len(sys.argv) > 2: - _help() + if len(sys.argv) < 3 or '--help' in sys.argv: + sys.exit(__doc__) unit_failures = _run_unit_tests() if unit_failures: sys.exit(unit_failures) interpreter = sys.argv[1] browser = sys.argv[2].lower() - args = sys.argv[3:] + options = sys.argv[3:] if browser != 'unit': - rc = acceptance_tests(interpreter, browser, args) + rc = acceptance_tests(interpreter, browser, options) sys.exit(rc)