Merge pull request #22003 from spxtr/test-history

hack/jenkins/job-configs/kubernetes-test-history.yaml

+- job:
+    name: kubernetes-test-summary
+    description: 'Create a daily test summary and upload to GCS. Test owner: spxtr.'
+    triggers:
+        # Run every night at midnight at a random minute.
+        - timed: 'H 0 * * *'
+    scm:
+        - git:
+            url: https://www.github.com/kubernetes/kubernetes
+            branches:
+                - master
+            browser: githubweb
+            browser-url: https://github.com/kubernetes/kubernetes
+            skip-tag: true
+    builders:
+        - shell: |
+            cd hack/jenkins/test-history
+            ./gen_history http://jenkins-master:8080/
+    publishers:
+        - email-ext:
+            recipients: spxtr@google.com

hack/jenkins/test-history/gen_history

+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Compiles a static HTML site containing the last day's worth of test results.
+# Pass the URL of Jenkins into $1
+
+set -o errexit
+set -o nounset
+
+readonly jenkins="$1"
+readonly datestr=$(date +"%Y-%m-%d")
+
+# Create JSON report
+time python gen_json.py "${jenkins}" kubernetes
+
+# Create static HTML report out of the JSON
+python gen_html.py > static/tests.html
+python gen_html.py kubernetes-e2e > static/tests-e2e.html
+python gen_html.py kubernetes-soak > static/tests-soak.html
+python gen_html.py kubernetes-e2e-gce > static/tests-e2e-gce.html
+python gen_html.py kubernetes-e2e-gke > static/tests-e2e-gke.html
+python gen_html.py kubernetes-upgrade > static/tests-upgrade.html
+
+# Fill in the last updated time into the template.
+cat index_template.html | sed -e "s/TIME/Last updated: ${datestr}/" > static/index.html
+
+# Upload to GCS
+readonly bucket="kubernetes-test-history"
+readonly gcs_acl="public-read"
+gsutil -q cp -az "${gcs_acl}" "tests.json" "gs://${bucket}/logs/${datestr}.json"
+gsutil -q cp -ra "${gcs_acl}" "static" "gs://${bucket}/"

hack/jenkins/test-history/gen_html.py

+#!/usr/bin/env python
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Creates an HTML report for all jobs starting with a given prefix.
+
+Reads the JSON from tests.json, and prints the HTML to stdout.
+
+This code is pretty nasty, but gets the job done.
+
+It would be really spiffy if this used an HTML template system, but for now
+we're old-fashioned. We could also generate these with JS, directly from the
+JSON. That would allow custom filtering and stuff like that.
+"""
+
+from __future__ import print_function
+
+import json
+import string
+import sys
+
+def gen_tests(data, prefix):
+    """Creates the HTML for all test cases.
+
+    Args:
+        data: Parsed JSON data that was created by gen_json.py.
+        prefix: Considers Jenkins jobs that start with this.
+
+    Returns:
+        The HTML as a list of elements along with the number of passing,
+        unstable, failing, and skipped tests.
+    """
+    html = ['<ul class="test">']
+    total_okay = 0
+    total_unstable = 0
+    total_failed = 0
+    total_skipped = 0
+    for test in sorted(data, key=string.lower):
+        test_html = ['<ul class="suite">']
+        has_test = False
+        has_failed = False
+        has_unstable = False
+        for suite in sorted(data[test]):
+            if not suite.startswith(prefix):
+                continue
+            has_test = True
+            num_failed = 0
+            num_builds = 0
+            total_time = 0
+            for build in data[test][suite]:
+                num_builds += 1
+                if build['failed']:
+                    num_failed += 1
+                total_time += build['time']
+            avg_time = total_time / num_builds
+            unit = 's'
+            if avg_time > 60:
+                avg_time /= 60
+                unit = 'm'
+            if num_failed == num_builds:
+                has_failed = True
+                status = 'failed'
+            elif num_failed > 0:
+                has_unstable = True
+                status = 'unstable'
+            else:
+                status = 'okay'
+            test_html.append('<li class="suite">')
+            test_html.append('<span class="{}">{}/{}</span>'.format(status, str(num_builds - num_failed), str(num_builds)))
+            test_html.append('<span class="time">{}</span>'.format(str(int(avg_time)) + unit))
+            test_html.append(suite)
+            test_html.append('</li>')
+        if has_failed:
+            status = 'failed'
+            total_failed += 1
+        elif has_unstable:
+            status = 'unstable'
+            total_unstable += 1
+        elif has_test:
+            status = 'okay'
+            total_okay += 1
+        else:
+            status = 'skipped'
+            total_skipped += 1
+        html.append('<li class="test {}">{}'.format(status, test))
+        html.extend(test_html)
+        html.append('</ul>')
+        html.append('</li>')
+    html.append('</ul>')
+    return html, total_okay, total_unstable, total_failed, total_skipped
+
+def gen_html(data, prefix):
+    """Creates the HTML for the entire page.
+
+    Args: Same as gen_tests.
+    Returns: Just the list of HTML elements.
+    """
+    tests_html, okay, unstable, failed, skipped = gen_tests(data, prefix)
+    html = ['<html>', '<head>']
+    html.append('<link rel="stylesheet" type="text/css" href="style.css" />')
+    html.append('<script src="script.js"></script>')
+    html.append('</head>')
+    html.append('<body>')
+    if len(prefix) > 0:
+        html.append('<div id="header">Suites starting with {}:'.format(prefix))
+    else:
+        html.append('<div id="header">All suites:')
+    html.append('<span class="total okay" onclick="toggle(\'okay\');">{}</span>'.format(str(okay)))
+    html.append('<span class="total unstable" onclick="toggle(\'unstable\');">{}</span>'.format(str(unstable)))
+    html.append('<span class="total failed" onclick="toggle(\'failed\');">{}</span>'.format(str(failed)))
+    html.append('<span class="total skipped" onclick="toggle(\'skipped\');">{}</span>'.format(str(skipped)))
+    html.append('</div>')
+    html.extend(tests_html)
+    html.append('</body>')
+    html.append('</html>')
+    return html
+
+if __name__ == '__main__':
+    prefix = ''
+    if len(sys.argv) == 2:
+        prefix = sys.argv[1]
+    with open('tests.json', 'r') as f:
+        print('\n'.join(gen_html(json.load(f), prefix)))

hack/jenkins/test-history/gen_json.py

+#!/usr/bin/env python
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generates a JSON file containing test history for the last day.
+
+Writes the JSON out to tests.json.
+"""
+
+from __future__ import print_function
+
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+import urllib2
+import xml.etree.ElementTree as ET
+import zlib
+
+
+def get_json(url):
+    """Does an HTTP GET to url and parses the JSON response. None on failure."""
+    try:
+        content = urllib2.urlopen(url).read().decode('utf-8')
+        return json.loads(content)
+    except urllib2.HTTPError:
+        return None
+
+def get_jobs(server):
+    """Generates all job names running on the server."""
+    jenkins_json = get_json('{}/api/json'.format(server))
+    if not jenkins_json:
+        return
+    for job in jenkins_json['jobs']:
+        yield job['name']
+
+def get_builds(server, job):
+    """Generates all build numbers for a given job."""
+    job_json = get_json('{}/job/{}/api/json'.format(server, job))
+    if not job_json:
+        return
+    for build in job_json['builds']:
+        yield build['number']
+
+def get_build_info(server, job, build):
+    """Returns building status along with timestamp for a given build."""
+    path = '{}/job/{}/{}/api/json'.format(server, job, str(build))
+    build_json = get_json(path)
+    if not build_json:
+        return
+    return build_json['building'], build_json['timestamp']
+
+def gcs_ls(path):
+    """Lists objects under a path on gcs."""
+    try:
+        result = subprocess.check_output(
+            ['gsutil', 'ls', path],
+            stderr=open(os.devnull, 'w'))
+    except subprocess.CalledProcessError:
+        result = b''
+    for subpath in result.decode('utf-8').split():
+        yield subpath
+
+def gcs_ls_build(job, build):
+    """Lists all files under a given job and build path."""
+    url = 'gs://kubernetes-jenkins/logs/{}/{}'.format(job, str(build))
+    for path in gcs_ls(url):
+        yield path
+
+def gcs_ls_artifacts(job, build):
+    """Lists all artifacts for a build."""
+    for path in gcs_ls_build(job, build):
+        if path.endswith('artifacts/'):
+            for artifact in gcs_ls(path):
+                yield artifact
+
+def gcs_ls_junit_paths(job, build):
+    """Lists the paths of JUnit XML files for a build."""
+    for path in gcs_ls_artifacts(job, build):
+        if re.match('.*/junit.*\.xml$', path):
+            yield path
+
+def gcs_get_tests(path):
+    """Generates test data out of the provided JUnit path.
+
+    Returns None if there's an issue parsing the XML.
+    Yields name, time, failed, skipped for each test.
+    """
+    try:
+        data = subprocess.check_output(
+            ['gsutil', 'cat', path], stderr=open(os.devnull, 'w'))
+    except subprocess.CalledProcessError:
+        return
+
+    try:
+        data = zlib.decompress(data, zlib.MAX_WBITS | 16)
+    except zlib.error:
+        # Don't fail if it's not gzipped.
+        pass
+    data = data.decode('utf-8')
+
+    try:
+        root = ET.fromstring(data)
+    except ET.ParseError:
+        return
+
+    for child in root:
+        name = child.attrib['name']
+        time = float(child.attrib['time'])
+        failed = False
+        skipped = False
+        for param in child:
+            if param.tag == 'skipped':
+                skipped = True
+            elif param.tag == 'failure':
+                failed = True
+        yield name, time, failed, skipped
+
+def get_tests_from_junit_path(path):
+    """Generates all tests in a JUnit GCS path."""
+    for test in gcs_get_tests(path):
+        if not test:
+            continue
+        yield test
+
+def get_tests_from_build(job, build):
+    """Generates all tests for a build."""
+    for junit_path in gcs_ls_junit_paths(job, build):
+        for test in get_tests_from_junit_path(junit_path):
+            yield test
+
+def get_daily_builds(server, prefix):
+    """Generates all (job, build) pairs for the last day."""
+    now = time.time()
+    for job in get_jobs(server):
+        if not job.startswith(prefix):
+            continue
+        for build in reversed(sorted(get_builds(server, job))):
+            building, timestamp = get_build_info(server, job, build)
+            # Skip if it's still building.
+            if building:
+                continue
+            # Quit once we've walked back over a day.
+            if now - timestamp / 1000 > 60*60*24:
+                break
+            yield job, build
+
+def get_tests(server, prefix):
+    """Returns a dictionary of tests to be JSON encoded."""
+    tests = {}
+    for job, build in get_daily_builds(server, prefix):
+        print('{}/{}'.format(job, str(build)))
+        for name, duration, failed, skipped in get_tests_from_build(job, build):
+            if name not in tests:
+                tests[name] = {}
+            if skipped:
+                continue
+            if job not in tests[name]:
+                tests[name][job] = []
+            tests[name][job].append({
+                'build': build,
+                'failed': failed,
+                'time': duration
+            })
+    return tests
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print('Usage: {} <server> <prefix>'.format(sys.argv[0]))
+        sys.exit(1)
+    server, prefix = sys.argv[1:]
+    print('Finding tests prefixed with {} at server {}'.format(prefix, server))
+    tests = get_tests(server, prefix)
+    with open('tests.json', 'w') as f:
+        json.dump(tests, f, sort_keys=True)

hack/jenkins/test-history/index_template.html

+<html>
+<body>
+<h2>All tests starting with:</h2>
+<ul>
+<li><a href="tests.html">kubernetes</a></li>
+<li><a href="tests-e2e.html">kubernetes-e2e</a></li>
+<li><a href="tests-e2e-gce.html">kubernetes-e2e-gce</a></li>
+<li><a href="tests-e2e-gke.html">kubernetes-e2e-gke</a></li>
+<li><a href="tests-upgrade.html">kubernetes-upgrade</a></li>
+<li><a href="tests-soak.html">kubernetes-soak</a></li>
+</ul>
+TIME
+</body>
+</html>
+

hack/jenkins/test-history/static/script.js

+function toggle(cls) {
+	var els = document.getElementsByClassName(cls);
+	var show = false
+	for (var i = 0; i < els.length; i++) {
+		if (els[i].className == 'test ' + cls) {
+			if (els[i].style.display == 'none') {
+				els[i].style.display = 'block';
+				show = true;
+			} else {
+				els[i].style.display = 'none';
+				show = false;
+			}
+		}
+	}
+	// UGLY HACK
+	document.getElementsByClassName('total ' + cls)[0].style.color = show ? '#000000' : '#888888';
+}
+
+function defaultToggles() {
+	toggle('okay');
+	toggle('skipped');
+}
+window.onload = defaultToggles;

hack/jenkins/test-history/static/style.css

+body {
+    margin: 0;
+    padding: 0;
+    font-family: monospace;
+}
+
+#header {
+    width: 100%;
+    font-size: x-large;
+    font-weight: bold;
+    padding: 10px;
+}
+
+#header span{
+    display: inline-block;
+    width: 60px;
+    padding-left: 30px;
+    border-radius: 10px;
+    margin: 5px;
+    cursor: pointer;
+    user-select: none;
+    -webkit-user-select: none;
+    -moz-user-select: none;
+    -ms-user-select: none;
+    user-select: none;
+}
+
+span.total.okay {
+    background: linear-gradient(to left, #e3e3e3 80%, #009900 20%);
+}
+
+span.total.unstable {
+    background: linear-gradient(to left, #e3e3e3 80%, #EE9500 20%);
+}
+
+span.total.failed {
+    background: linear-gradient(to left, #e3e3e3 80%, #AA0000 20%);
+}
+
+span.total.skipped {
+    background: linear-gradient(to left, #e3e3e3 80%, #999999 20%);
+}
+
+ul.test {
+    list-style-type: none;
+    padding: 0 0 5px 5px;
+    width: 100%;
+}
+
+li.test {
+    margin: 0 0 10px 0;
+    border-top-left-radius: 10px;
+    border-bottom-left-radius: 10px;
+    padding: 5px;
+    font-size: large;
+    font-weight: bold;
+    padding-left: 4%;
+}
+
+li.test.okay {
+    background: linear-gradient(to left, #e3e3e3 97%, #009900 3%);
+}
+
+li.test.unstable {
+    background: linear-gradient(to left, #e3e3e3 97%, #EE9500 3%);
+}
+
+li.test.failed {
+    background: linear-gradient(to left, #e3e3e3 97%, #AA0000 3%);
+}
+
+li.test.skipped {
+    background: linear-gradient(to left, #e3e3e3 97%, #999999 3%);
+}
+
+ul.suite {
+    list-style: none;
+    margin: 2px 0 0 0;
+    padding-left: 20px;
+}
+
+li.suite {
+    margin: 0 0 2px 0;
+    font-size: normal;
+    font-weight: normal;
+}
+
+li.suite span {
+    float: left;
+    width: 70px;
+    font-weight: bold;
+}
+
+li.suite span.time {
+    width: 50px;
+    font-weight: normal;
+}
+
+li.suite span.okay {
+    color: green;
+}
+
+li.suite span.unstable {
+    color: orange;
+}
+
+li.suite span.failed {
+    color: red;
+}