diff --git a/platform/platform_tests/downloader_test.cpp b/platform/platform_tests/downloader_test.cpp index 6886c44b7c..cc86fa175f 100644 --- a/platform/platform_tests/downloader_test.cpp +++ b/platform/platform_tests/downloader_test.cpp @@ -20,13 +20,13 @@ #include -#define TEST_URL1 "http://scout.alex.bio:34568/unit_tests/1.txt" -#define TEST_URL_404 "http://scout.alex.bio:34568/unit_tests/notexisting_unittest" -#define TEST_URL_PERMANENT "http://scout.alex.bio:34568/unit_tests/permanent" +#define TEST_URL1 "http://localhost:34568/unit_tests/1.txt" +#define TEST_URL_404 "http://localhost:34568/unit_tests/notexisting_unittest" +#define TEST_URL_PERMANENT "http://localhost:34568/unit_tests/permanent" #define TEST_URL_INVALID_HOST "http://not-valid-host123532.ath.cx" -#define TEST_URL_BIG_FILE "http://scout.alex.bio:34568/unit_tests/47kb.file" +#define TEST_URL_BIG_FILE "http://localhost:34568/unit_tests/47kb.file" #define TEST_URL_HTTPS "https://github.com" -#define TEST_URL_POST "http://scout.alex.bio:34568/unit_tests/post.php" +#define TEST_URL_POST "http://localhost:34568/unit_tests/post.php" using namespace downloader; diff --git a/platform/platform_tests/platform_tests.pro b/platform/platform_tests/platform_tests.pro index 27500702b4..d484f92855 100644 --- a/platform/platform_tests/platform_tests.pro +++ b/platform/platform_tests/platform_tests.pro @@ -29,7 +29,7 @@ win32*|linux* { SOURCES += \ ../../testing/testingmain.cpp \ apk_test.cpp \ -# downloader_test.cpp \ + downloader_test.cpp \ jansson_test.cpp \ language_test.cpp \ location_test.cpp \ diff --git a/run_desktop_tests.py b/run_desktop_tests.py index 0201bc75c0..45f75c7e67 100755 --- a/run_desktop_tests.py +++ b/run_desktop_tests.py @@ -17,17 +17,24 @@ be found, i.e. the tests that were specified in the skip list, but do not exist. """ from __future__ import print_function + +import getopt from os import listdir from os.path import isfile, join +import os +import socket import subprocess import sys -import getopt +import testserver +import urllib2 tests_path = "" -workspace_path = "./omim-build-release" +workspace_path = "./omim-build-release/out/release" skiplist = [] +logfile = "testlog.log" +PORT = 34568 def print_pretty(result, tests): if len(tests) == 0: @@ -50,48 +57,63 @@ Possbile options: -e --exclude: list of tests to exclude, comma separated, no spaces allowed +-o --output : resulting log file. Default testlog.log + Example -./run_desktop_tests.py -f /Users/Jenkins/Home/jobs/Multiplatform/workspace/omim-build-release -e drape_tests,some_other_tests +./run_desktop_tests.py -f /Users/Jenkins/Home/jobs/Multiplatform/workspace/omim-build-release/out/release -e drape_tests,some_other_tests -o my_log_file.log """) def set_global_vars(): try: - opts, args = getopt.getopt( - sys.argv[1:], - "he:f:", - ["help", "exclude=", "folder="]) + opts, args = getopt.getopt(sys.argv[1:], "he:f:o:", + ["help", "exclude=", "folder=", "output="]) except getopt.GetoptError as err: print(str(err)) usage() sys.exit(2) - for o, a in opts: - if o in ("-h", "--help"): + for option, argument in opts: + if option in ("-h", "--help"): usage() sys.exit() - elif o in ("-e", "--exclude"): - exclude_tests = a.split(",") + if option in ("-o", "--output"): + global logfile + logfile = argument + elif option in ("-e", "--exclude"): + exclude_tests = argument.split(",") for exclude_test in exclude_tests: global skiplist skiplist.append(exclude_test) - elif o in ("-f", "--folder"): + elif option in ("-f", "--folder"): global workspace_path - workspace_path = a + workspace_path = argument else: assert False, "unhandled option" +def start_server(): + server = testserver.TestServer() + server.start_serving() + + +def stop_server(): + try: + urllib2.urlopen('http://localhost:{port}/kill'.format(port=PORT), timeout=5) + except (urllib2.URLError, socket.timeout): + print("Failed to stop the server...") + + def run_tests(): - tests_path = "{workspace_path}/out/release".format( - workspace_path=workspace_path) + tests_path = "{workspace_path}".format(workspace_path=workspace_path) failed = [] passed = [] skipped = [] + server = None for file in listdir(tests_path): if not file.endswith("_tests"): @@ -101,13 +123,19 @@ def run_tests(): skipped.append(file) continue - process = subprocess.Popen( - tests_path + - "/" + - file, + if file == "platform_tests": + start_server() + + process = subprocess.Popen("{tests_path}/{file} 2>> {logfile}". + format(tests_path=tests_path, file=file, logfile=logfile), shell=True, stdout=subprocess.PIPE) + process.wait() + + if file == "platform_tests": + stop_server() + if process.returncode > 0: failed.append(file) else: @@ -116,9 +144,18 @@ def run_tests(): return {"failed": failed, "passed": passed, "skipped": skipped} +def rm_log_file(): + try: + os.remove(logfile) + except OSError: + pass + + def main(): set_global_vars() + rm_log_file() + results = run_tests() print_pretty("failed", results["failed"]) diff --git a/testing/testingmain.cpp b/testing/testingmain.cpp index d47779b01e..8c2943cd27 100644 --- a/testing/testingmain.cpp +++ b/testing/testingmain.cpp @@ -1,14 +1,15 @@ -#include "testing/testregister.hpp" #include "testing/testing.hpp" +#include "testing/testregister.hpp" #include "base/logging.hpp" -#include "base/string_utils.hpp" #include "base/regexp.hpp" +#include "base/string_utils.hpp" +#include "base/timer.hpp" #include "std/iostream.hpp" +#include "std/target_os.hpp" #include "std/string.hpp" #include "std/vector.hpp" -#include "std/target_os.hpp" #ifdef OMIM_UNIT_TEST_WITH_QT_EVENT_LOOP @@ -94,8 +95,10 @@ int main(int argc, char * argv[]) try { + my::HighResTimer timer(true); // Run the test. pTest->m_Fn(); + uint64_t const elapsed = timer.ElapsedNano(); if (g_bLastTestOK) { @@ -108,6 +111,7 @@ int main(int argc, char * argv[]) testResults[iTest] = false; ++numFailedTests; } + LOG(LINFO, ("Test took", elapsed, "ns")); } catch (TestFailureException const & ) diff --git a/testlog_to_xml_converter.py b/testlog_to_xml_converter.py new file mode 100755 index 0000000000..f1077c3fb6 --- /dev/null +++ b/testlog_to_xml_converter.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python + +''' +This script generates jUnit-style xml files from the log written by our tests. +This xml file is used in Jenkins to show the state of the test execution. + +Created on May 13, 2015 + +@author: t.danshin +''' + +from __future__ import print_function +import getopt +import sys +import xml.etree.ElementTree as ElementTree + + + +class TestInfo: + + + def __init__(self, test_name): + self.test_suite, name = test_name.split("::", 1) + + self.test_suite = self.test_suite[0: -4] + name = name.replace("::", ".") + + self.test_name = name + self.test_comment = None + self.test_result = None + self.test_duration = 0.0 + + + + def append_comment(self, comment): + if self.test_comment is None: + self.test_comment = comment + else: + self.test_comment += "\n" + comment + + + def __repr__(self): + local_comment = "" + if self.test_comment is not None: + local_comment = self.test_comment + return "{}::{}: {} -> {}\n".format(self.test_suite, self.test_name, local_comment, self.test_result) + + + def xml(self): + d = ElementTree.Element("testcase", {"name":self.test_name, "classname":self.test_suite, "time":str(self.test_duration)}) + if self.test_comment is not None: + b = ElementTree.SubElement(d, "system-err") + b.text = self.test_comment + + if self.test_result == "FAILED": + ElementTree.SubElement(d, "failure") + return d + + def set_duration(self, nanos): + self.test_duration = float(nanos) / 1000000000 + + +class Parser: + + def __init__(self, logfile, xml_file): + self.logfile = logfile if logfile else "testlog.log" + self.xml_file = xml_file if xml_file else "test_results.xml" + + self.root = ElementTree.Element("testsuite") + + + def parse_log_file(self): + + with open(self.logfile) as f: + test_info = None + + for line in f.readlines(): + line = line.rstrip().decode('utf-8') + + if line.startswith("Running"): + test_info = TestInfo(line[len("Running "):]) + + elif line.startswith("Test took"): + if test_info is not None: + test_info.set_duration(line[len("Test took "):-3]) + self.root.append(test_info.xml()) + + test_info = None + + elif line == "OK" or line == "FAILED": + if test_info is not None: + test_info.test_result = line + + else: + if test_info is not None: + test_info.append_comment(line) + + + def write_xml_file(self): + print(">>> Self xml file: {}".format(self.xml_file)) + + ElementTree.ElementTree(self.root).write(self.xml_file, encoding="UTF-8", xml_declaration=True) + + +def usage(): + print(""" +Possbile options: + +-h --help : print this help + +-i --input : the path of the original log file to parse + +-o --output: the path of the output xml file + + +Example + +./testlog_to_xml_converter.py -i testlog.log -o /Users/t.danshin/Desktop/testxml.xml + +""") + + +def read_cl_options(): + try: + opts, args = getopt.getopt(sys.argv[1:], "hi:o:", + ["help", "input=", "output="]) + except getopt.GetoptError as err: + print(str(err)) + usage() + sys.exit(2) + + ret = {} + + for option, argument in opts: + if option in ("-h", "--help"): + usage() + sys.exit() + elif option in ("-i", "--input"): + ret["logfile"] = argument + elif option in ("-o", "--output"): + ret["xml_file"] = argument + else: + assert False, "unhandled option" + + return ret + + +def main(): + options = read_cl_options() + try: + parser = Parser(options["logfile"], options["xml_file"]) + except: + usage() + exit(2) + parser.parse_log_file() + parser.write_xml_file() + + print("Finished writing the xUnit-style xml file") + + +if __name__ == '__main__': + main() diff --git a/testserver.py b/testserver.py new file mode 100644 index 0000000000..fb3cf1443a --- /dev/null +++ b/testserver.py @@ -0,0 +1,253 @@ +""" +This is a simple web-server that does very few things. It is necessary for +the downloader tests. Currently it works only for a subset of tests, it +doesn't yet work for chunked downloads. + +Here is the logic behind the initialization: +Because several instances of the test can run simultaneously on the Build +machine, we have to take this into account and not start another server if +one is already running. However, there is a chance that a server will not +terminate correctly, and will still hold the port, so we will not be able +to initialize another server. + +So, before initalizing a new server we "ping" it. If it replies with "pong", +we know that a working instance is already running, and we do not start a new +one. + +If it doesn't reply with pong, it might mean that it is either not running +at all, or it is running, but dead. In this case, we try to init a server, +and if we catch an exception, we kill all other processes that have the +word "testserver.py" in the name, but whose ids are not our id, and then +we init the server once again. If that fails, there is nothing we can do +about it, so we don't catch any exceptions there. + +Another thing to note is that you cannot stop the server from the same thread +as it was started from, and the only way I found possible to kill it is from a +timer. + +""" + +from __future__ import print_function + +from BaseHTTPServer import BaseHTTPRequestHandler +from BaseHTTPServer import HTTPServer +import cgi +from numpy.distutils.exec_command import exec_command +import os +import re +import socket +from subprocess import Popen, PIPE +import sys +import thread +from threading import Timer +import threading +import types +import urllib2 + + +PORT = 34568 +LIFESPAN = 180.0 # timeout for the self destruction timer - how much time + # passes between the last request and the server killing + # itself +PING_TIMEOUT = 5 # Nubmer of seconds to wait for ping response + + +class InternalServer(HTTPServer): + + + + def kill_me(self): + print("The server's life has come to an end") + self.shutdown() + + + def reset_selfdestruct_timer(self): + if self.self_destruct_timer is not None: + self.self_destruct_timer.cancel() + + self.self_destruct_timer = Timer(LIFESPAN, self.kill_me) + self.self_destruct_timer.start() + + + def __init__(self, server_address, RequestHandlerClass, + bind_and_activate=True): + self.self_destruct_timer = None + self.clients = 1 + HTTPServer.__init__(self, server_address, RequestHandlerClass, + bind_and_activate=bind_and_activate) + self.reset_selfdestruct_timer() + + + def suicide(self): + self.clients -= 1 + if self.clients == 0: + if self.self_destruct_timer is not None: + self.self_destruct_timer.cancel() + + quick_and_painless_timer = Timer(0.1, self.kill_me) + quick_and_painless_timer.start() + + +class TestServer: + + + def __init__(self): + self.server = None + html = str() + try: + print("Pinging the server...") + response = urllib2.urlopen('http://localhost:{port}/ping'.format(port=PORT), timeout=PING_TIMEOUT); + html = response.read() + except (urllib2.URLError, socket.timeout): + print("The server does not currently serve...") + + if html != "pong": + print("html != pong") + try: + self.init_server() + except socket.error: + print("Killing siblings") + self.kill_siblings() + self.init_server() + + + def init_server(self): + self.server = InternalServer(('localhost', PORT), PostHandler) + + + def start_serving(self): + if self.server is not None: + thread = threading.Thread(target=self.server.serve_forever) + thread.deamon = True + thread.start() + + + def exec_command(self, command): + print(command) + p = Popen(command.split(" "), shell=True, stdout=PIPE, stderr=PIPE) + output, err = p.communicate() + p.wait() + return output[0] + + + def kill_siblings(self): + output = gen_to_list(re.sub("\s{1,}", " ", x.strip()) for x in exec_command("ps -w")[1].split("\n")) + + my_pid = str(os.getpid()) + + my_name = map(lambda x: x.split(" ")[4], # the name of python script + filter(lambda x: x.startswith(my_pid), output))[0] + + siblings = filter(lambda x: x != my_pid, + map(lambda x: x.split(" ")[0], + filter(lambda x: my_name in x, output))) + + if len(siblings) > 0: + command = "kill {}".format(" ".join(siblings)) + exec_command(command) + else: + print("The process has no siblings") + + +def gen_to_list(generator): + l = [] + for i in generator: + l.append(i) + return l + + +class PostHandler(BaseHTTPRequestHandler): + + def do_POST(self): + self.server.reset_selfdestruct_timer() + print("URL is: " + self.path) + ctype, pdict = cgi.parse_header(self.headers.getheader('content-type')) + if ctype == 'multipart/form-data': + self.send_response(500) + self.end_headers() + + elif ctype == 'application/x-www-form-urlencoded': + + length = int(self.headers.getheader('content-length')) + data = self.rfile.read(length) + + self.send_response(200) + self.send_header("Content-Length", length + 1) + self.end_headers() + + self.wfile.write(data + "\n") + + + def do_GET(self): + self.server.reset_selfdestruct_timer() + switch = {"/unit_tests/1.txt": self.test1, + "/unit_tests/notexisting_unittest": self.test_404, + "/unit_tests/permanent" : self.test_301, + "/unit_tests/47kb.file" : self.test_47_kb, + "/ping" : self.pong, + "/kill": self.kill, + } + switch[self.path]() + return + + + def pong(self): + self.server.clients += 1 + self.send_200() + self.wfile.write("pong") + + + def test1(self): + message = "Test1" + + the_range = self.headers.get('Range') + if the_range is not None: + print("The range is not none") + print(the_range) + meaningful_string = the_range[6:] + first, last = meaningful_string.split("-") + message = message[int(first): int(last)] + print ("The message is: " + message) + + self.send_response(200) + self.send_header("Content-Length", len(message)) + self.end_headers() + self.wfile.write(message) + + + def send_200(self): + self.send_response(200) + self.end_headers() + + + def test_404(self): + self.send_response(404) + self.end_headers() + + + def test_301(self): + self.send_response(301) + self.send_header("Location", "google.com") + self.end_headers() + + + def test_47_kb(self): + self.send_response(200) + self.send_header("Content-Length", 47 * 1024) + self.end_headers() + for i in range(1, 47 * 1024): + self.wfile.write(255) + + + def kill(self): + message = "Bye..." + self.send_response(200) + self.send_header("Content-Length", len(message)) + self.end_headers() + self.wfile.write(message) + self.server.suicide() + + +if __name__ == '__main__': + server = TestServer() + server.start_serving()