diff --git a/test/env.py b/test/env.py index 43187d647..0c2218932 100755 --- a/test/env.py +++ b/test/env.py @@ -10,8 +10,6 @@ HTTP_SERVER_FILE = os.path.join(RESOURCES_DIR, "testserver", "testserver.py") SRC_DIR = os.path.join(ROOT_DIR, "..", "src") -BROWSER = os.environ.get("BROWSER", "firefox") - TRAVIS = os.environ.get("TRAVIS", None) TRAVIS_JOB_NUMBER = os.environ.get("TRAVIS_JOB_NUMBER") SAUCE_USERNAME = os.environ.get("SAUCE_USERNAME") diff --git a/test/resources/statuschecker.py b/test/resources/statuschecker.py deleted file mode 100755 index 623fc07ce..000000000 --- a/test/resources/statuschecker.py +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2008-2012 Nokia Siemens Networks Oyj -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Robot Framework Test Status Checker - -Usage: statuschecker.py infile [outfile] - -This tool processes Robot Framework output XML files and checks that test case -statuses and messages are as expected. Main use case is post-processing output -files got when testing Robot Framework test libraries using Robot Framework -itself. - -If output file is not given, the input file is considered to be also output -file and it is edited in place. - -By default all test cases are expected to 'PASS' and have no message. Changing -the expected status to 'FAIL' is done by having word 'FAIL' (in uppercase) -somewhere in the test case documentation. Expected error message must then be -given after 'FAIL'. Error message can also be specified as a regular -expression by prefixing it with string 'REGEXP:'. Testing only the beginning -of the message is possible with 'STARTS:' prefix. - -This tool also allows testing the created log messages. They are specified -using a syntax 'LOG x.y:z LEVEL Actual message', which is described in detail -detail in the tool documentation. -""" - -import re - -from robot.result import ExecutionResult - - -def process_output(inpath, outpath=None): - result = ExecutionResult(inpath) - _process_suite(result.suite) - result.save(outpath) - return result.return_code - - -def _process_suite(suite): - for subsuite in suite.suites: - _process_suite(subsuite) - for test in suite.tests: - _process_test(test) - - -def _process_test(test): - exp = _Expected(test.doc) - _check_status(test, exp) - if test.status == 'PASS': - _check_logs(test, exp) - - -def _check_status(test, exp): - if exp.status != test.status: - test.status = 'FAIL' - if exp.status == 'PASS': - test.message = ("Test was expected to PASS but it FAILED. " - "Error message:\n") + test.message - else: - test.message = ("Test was expected to FAIL but it PASSED. " - "Expected message:\n") + exp.message - elif not _message_matches(test.message, exp.message): - test.status = 'FAIL' - test.message = ("Wrong error message.\n\nExpected:\n%s\n\nActual:\n%s\n" - % (exp.message, test.message)) - elif test.status == 'FAIL': - test.status = 'PASS' - test.message = 'Original test failed as expected.' - - -def _message_matches(actual, expected): - if actual == expected: - return True - if expected.startswith('REGEXP:'): - pattern = '^%s$' % expected.replace('REGEXP:', '', 1).strip() - if re.match(pattern, actual, re.DOTALL): - return True - if expected.startswith('STARTS:'): - start = expected.replace('STARTS:', '', 1).strip() - if actual.startswith(start): - return True - return False - - -def _check_logs(test, exp): - for kw_indices, msg_index, level, message in exp.logs: - try: - kw = test.keywords[kw_indices[0]] - for index in kw_indices[1:]: - kw = kw.keywords[index] - except IndexError: - indices = '.'.join(str(i+1) for i in kw_indices) - test.status = 'FAIL' - test.message = ("Test '%s' does not have keyword with index '%s'" - % (test.name, indices)) - return - if len(kw.messages) <= msg_index: - if message != 'NONE': - test.status = 'FAIL' - test.message = ("Keyword '%s' should have had at least %d " - "messages" % (kw.name, msg_index+1)) - else: - if _check_log_level(level, test, kw, msg_index): - _check_log_message(message, test, kw, msg_index) - - -def _check_log_level(expected, test, kw, index): - actual = kw.messages[index].level - if actual == expected: - return True - test.status = 'FAIL' - test.message = ("Wrong level for message %d of keyword '%s'.\n\n" - "Expected: %s\nActual: %s.\n%s" - % (index+1, kw.name, expected, - actual, kw.messages[index].message)) - return False - - -def _check_log_message(expected, test, kw, index): - actual = kw.messages[index].message.strip() - if _message_matches(actual, expected): - return True - test.status = 'FAIL' - test.message = ("Wrong content for message %d of keyword '%s'.\n\n" - "Expected:\n%s\n\nActual:\n%s" - % (index+1, kw.name, expected, actual)) - return False - - -class _Expected: - - def __init__(self, doc): - self.status, self.message = self._get_status_and_message(doc) - self.logs = self._get_logs(doc) - - def _get_status_and_message(self, doc): - if 'FAIL' in doc: - return 'FAIL', doc.split('FAIL', 1)[1].split('LOG', 1)[0].strip() - return 'PASS', '' - - def _get_logs(self, doc): - logs = [] - for item in doc.split('LOG')[1:]: - index_str, msg_str = item.strip().split(' ', 1) - kw_indices, msg_index = self._get_indices(index_str) - level, message = self._get_log_message(msg_str) - logs.append((kw_indices, msg_index, level, message)) - return logs - - def _get_indices(self, index_str): - try: - kw_indices, msg_index = index_str.split(':') - except ValueError: - kw_indices, msg_index = index_str, '1' - kw_indices = [int(index) - 1 for index in kw_indices.split('.')] - return kw_indices, int(msg_index) - 1 - - def _get_log_message(self, msg_str): - try: - level, message = msg_str.split(' ', 1) - if level not in ['TRACE', 'DEBUG', 'INFO', 'WARN', 'FAIL']: - raise ValueError - except ValueError: - level, message = 'INFO', msg_str - return level, message - - -if __name__ == '__main__': - import sys - import os - - if not 2 <= len(sys.argv) <= 3 or '--help' in sys.argv: - print(__doc__) - sys.exit(1) - infile = sys.argv[1] - outfile = sys.argv[2] if len(sys.argv) == 3 else None - print("Checking %s" % os.path.abspath(infile)) - rc = process_output(infile, outfile) - if outfile: - print("Output: %s" % os.path.abspath(outfile)) - if rc > 255: - rc = 255 - sys.exit(rc) diff --git a/test/run_tests.py b/test/run_tests.py index be875117c..72eb989c9 100755 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -1,142 +1,163 @@ #!/usr/bin/env python +"""Script to run Selenium2Library acceptance tests. + +Tests are executed using Robot Framework and results verified automatically +afterwards using `robotstatuschecker` tool. The tool can be installed using +`pip install robotstatuschecker` and more information about it can be found +from https://github.com/robotframework/statuschecker/. Notice that initially +some tests fail. + +Usage: + + run_tests.py interpreter browser [options] + +Arguments: + + interpreter: Any Python interpreter supported by the library (e.g. `python`, + `jython`, `c:\\Python27\\python.exe`) + browser: Any browser supported by the library (e.g. `chrome`, `firefox`) + options: Additional command line options passed to Robot Framework + +Examples: + + run_tests.py python chrome + run_tests.py jython c:\\Python35\\python.exe --test "Click element" +""" + from __future__ import print_function +from contextlib import contextmanager import os import sys -from subprocess import Popen, call -from tempfile import TemporaryFile +import subprocess + +from robot import rebot_cli +try: + import robotstatuschecker +except ImportError: + sys.exit('Required `robotstatuschecker` not installed.\n' + 'Install it with `pip install robotstatuschecker`.') import env from run_unit_tests import run_unit_tests -REBOT_ARGS = [ - '--outputdir', '%(outdir)s', - '--name', '%(browser)sSPAcceptanceSPTests', - '--escape', 'space:SP', - '--noncritical', 'known_issue_%(browser)s', -] - -ROBOT_ARGS = [ - '--doc', 'SeleniumSPacceptanceSPtestsSPwithSP%(browser)s', - '--outputdir', '%(outdir)s', - '--variable', 'browser:%(browser)s', - '--variable', 'pyversion:%(pyVersion)s', - '--escape', 'space:SP', +ROBOT_OPTIONS = [ + '--doc', 'Selenium2Library acceptance tests with {browser}', + '--outputdir', env.RESULTS_DIR, + '--variable', 'BROWSER:{browser}', + '--variable', 'PY_VERSION:{py_version}', + '--report', 'NONE', + '--log', 'NONE', '--loglevel', 'DEBUG', - '--pythonpath', '%(pythonpath)s', - '--noncritical', 'known_issue_%(browser)s', + '--pythonpath', os.pathsep.join([env.SRC_DIR, env.TEST_LIBS_DIR]), +] +REBOT_OPTIONS = [ + '--outputdir', env.RESULTS_DIR, + '--critical', 'regression', + '--noncritical', 'inprogress', + '--noncritical', 'known issue {browser}', ] - -ARG_VALUES = { - 'outdir': env.RESULTS_DIR, - 'pythonpath': ':'.join((env.SRC_DIR, env.TEST_LIBS_DIR)) -} - - -def acceptance_tests(interpreter, browser, args): - ARG_VALUES['browser'] = browser.replace('*', '') - ARG_VALUES['pyVersion'] = interpreter + sys.version[:3] - ARG_VALUES['sauceUserName'] = env.SAUCE_USERNAME - ARG_VALUES['sauceAccessKey'] = env.SAUCE_ACCESS_KEY - ARG_VALUES['travisJobNumber'] = env.TRAVIS_JOB_NUMBER - - if env.TRAVIS: - ROBOT_ARGS.extend(['--variable', 'SAUCE_USERNAME:%(sauceUserName)s']) - ROBOT_ARGS.extend(['--variable', 'SAUCE_ACCESS_KEY:%(sauceAccessKey)s']) - if env.BROWSER != "firefox": - ROBOT_ARGS.extend( - ['--variable', - 'DESIRED_CAPABILITIES:build:%(travisJobNumber)s-%(browser)s,tunnel-identifier:%(travisJobNumber)s' - ] - ) - ROBOT_ARGS.extend( - [ - '--variable', - 'REMOTE_URL:http://%(sauceUserName)s:%(sauceAccessKey)s@ondemand.saucelabs.com:80/wd/hub' - ] - ) - start_http_server() - runner = {'python': 'pybot', 'jython': 'jybot', 'ipy': 'ipybot'}[interpreter] - if os.sep == '\\': - runner += '.bat' - try: - return execute_tests(runner, args) - finally: - stop_http_server() -def start_http_server(): - server_output = TemporaryFile() - Popen(['python', env.HTTP_SERVER_FILE, 'start'], - stdout=server_output, stderr=server_output) +def unit_tests(): + print('Running unit tests') + failures = run_unit_tests() + if failures: + print('\nUnit tests failed! Not running acceptance tests.') + sys.exit(failures) -def execute_tests(runner, args): +def acceptance_tests(interpreter, browser, options): if not os.path.exists(env.RESULTS_DIR): os.mkdir(env.RESULTS_DIR) - command = [runner] + [arg % ARG_VALUES for arg in ROBOT_ARGS] + args + [env.ACCEPTANCE_TEST_DIR] - # replace sensitive information - commands = ' '.join(command) - if env.BROWSER != "firefox": - for hidden in (env.SAUCE_USERNAME, env.SAUCE_ACCESS_KEY): - commands = commands.replace(hidden, "*" * len(hidden)) - print('Starting test execution with command:\n' + commands + "\n") - syslog = os.path.join(env.RESULTS_DIR, 'syslog.txt') - return call(command, shell=os.sep=='\\', env=dict(os.environ, ROBOT_SYSLOG_FILE=syslog)) + with http_server(): + execute_tests(interpreter, browser, options) + failures = process_output(browser, options) + if failures: + print('\n{} critical test{} failed.' + .format(failures, 's' if failures != 1 else '')) + else: + print('\nAll critical tests passed.') + sys.exit(failures) -def stop_http_server(): - call(['python', env.HTTP_SERVER_FILE, 'stop']) +@contextmanager +def http_server(): + serverlog = open(os.path.join(env.RESULTS_DIR, 'serverlog.txt'), 'w') + process = subprocess.Popen(['python', env.HTTP_SERVER_FILE, 'start'], + stdout=serverlog, stderr=subprocess.STDOUT) + try: + yield + finally: + subprocess.call(['python', env.HTTP_SERVER_FILE, 'stop']) + process.wait() + serverlog.close() -def process_output(args): - print() - call(['python', os.path.join(env.RESOURCES_DIR, 'statuschecker.py'), - os.path.join(env.RESULTS_DIR, 'output.xml')]) - rebot = 'rebot' if os.sep == '/' else 'rebot.bat' - rebot_cmd = [rebot] + [arg % ARG_VALUES for arg in REBOT_ARGS] + args + \ - [os.path.join(ARG_VALUES['outdir'], 'output.xml')] - print('Starting output processing with command:\n' + ' '.join(rebot_cmd)) - rc = call(rebot_cmd, env=os.environ) - if rc == 0: - print('All critical tests passed') +def execute_tests(interpreter, browser, cli_options): + runner = [interpreter, '-m', 'robot.run'] + options = [opt.format(browser=browser, + py_version=interpreter + sys.version[:3]) + for opt in ROBOT_OPTIONS] + if env.TRAVIS: + options.extend(get_travis_conf(browser)) + command = runner + options + cli_options + [env.ACCEPTANCE_TEST_DIR] + log_start(command) + syslog = os.path.join(env.RESULTS_DIR, 'syslog.txt') + # Running tests as an external process, not using `robot_cli`, to allow + # using different interpreter that is used for running this script. + subprocess.call(command, env=dict(os.environ, ROBOT_SYSLOG_FILE=syslog)) + + +def get_travis_conf(browser): + conf = [ + '--variable', 'SAUCE_USERNAME:{}'.format(env.SAUCE_USERNAME), + '--variable', 'SAUCE_ACCESS_KEY:{}'.format(env.SAUCE_ACCESS_KEY) + ] + if browser == 'firefox': + conf.extend([ + '--variable', + 'REMOTE_URL:http://{}:{}@ondemand.saucelabs.com:80/wd/hub'.format( + env.SAUCE_USERNAME, env.SAUCE_ACCESS_KEY + ) + ]) else: - print('%d critical test%s failed' % (rc, 's' if rc != 1 else '')) - return rc - - -def _exit(rc): - sys.exit(rc) + conf.extend([ + '--variable', + 'DESIRED_CAPABILITIES:build:{0}-{1},tunnel-identifier:{0}'.format( + env.TRAVIS_JOB_NUMBER, browser + ) + ]) + return conf -def _help(): - print('Usage: python run_tests.py python|jython browser [options]') +def log_start(command_list): + command = subprocess.list2cmdline(command_list) + for hidden in [env.SAUCE_USERNAME, env.SAUCE_ACCESS_KEY]: + if hidden: + command = command.replace(hidden, '*' * len(hidden)) print() - print('See README.txt for details.') - sys.exit(255) + print('Starting test execution with command:') + print(command) -def _run_unit_tests(): - print('Running unit tests') - failures = run_unit_tests() - if failures != 0: - print('\n%d unit tests failed - not running acceptance tests!' % failures) - else: - print('All unit tests passed') - return failures +def process_output(browser, cli_options): + print('Verifying results...') + output = os.path.join(env.RESULTS_DIR, 'output.xml') + robotstatuschecker.process_output(output, verbose=False) + options = [opt.format(browser=browser) for opt in REBOT_OPTIONS] + try: + rebot_cli(options + cli_options + [output]) + except SystemExit as exit: + return exit.code if __name__ == '__main__': - if not len(sys.argv) > 2: - _help() - unit_failures = _run_unit_tests() - if unit_failures: - sys.exit(unit_failures) - interpreter = sys.argv[1] - browser = sys.argv[2].lower() - args = sys.argv[3:] - if browser != 'unit': - rc = acceptance_tests(interpreter, browser, args) - sys.exit(rc) + if len(sys.argv) < 3 or '--help' in sys.argv: + sys.exit(__doc__) + unit_tests() + acceptance_tests(interpreter=sys.argv[1], + browser=sys.argv[2].lower(), + options=sys.argv[3:])