Add blocking suites to daily summary

hack/jenkins/test-history/gen_history

@@ -24,10 +24,12 @@ readonly jenkins="$1"
 readonly datestr=$(date +"%Y-%m-%d")
 
 # Create JSON report
-time python gen_json.py "${jenkins}" kubernetes
+time python gen_json.py \
+  "--server=${jenkins}" \
+  "--match=^kubernetes|kubernetes-build|kubelet-gce-e2e-ci"
 
 # Create static HTML reports out of the JSON
-python gen_html.py --suites --prefixes ,e2e,soak,e2e-gce,e2e-gke,upgrade --output-dir static --input tests.json
+python gen_html.py --output-dir=static --input=tests.json
 
 # Upload to GCS
 readonly bucket="kubernetes-test-history"

hack/jenkins/test-history/gen_html_test.py

@@ -25,46 +25,112 @@ import unittest
 import gen_html
 
 TEST_DATA = {
-  "test1":
-      {"kubernetes-release": [{"build": 3, "failed": False, "time": 3.52},
-                              {"build": 4, "failed": True, "time": 63.21}],
-       "kubernetes-debug": [{"build": 5, "failed": False, "time": 7.56},
-                            {"build": 6, "failed": False, "time": 8.43}],
+  'test1':
+      {'kubernetes-release': [{'build': 3, 'failed': False, 'time': 3.52},
+                              {'build': 4, 'failed': True, 'time': 63.21}],
+       'kubernetes-debug': [{'build': 5, 'failed': False, 'time': 7.56},
+                            {'build': 6, 'failed': False, 'time': 8.43}],
       },
-  "test2":
-      {"kubernetes-debug": [{"build": 6, "failed": True, "time": 3.53}]},
+  'test2':
+      {'kubernetes-debug': [{'build': 6, 'failed': True, 'time': 3.53}]},
 }
 
 class GenHtmlTest(unittest.TestCase):
-    def gen_html(self, *args):
+    """Unit tests for gen_html.py."""
+    # pylint: disable=invalid-name
+
+    def testHtmlHeader_NoScript(self):
+        result = '\n'.join(gen_html.html_header('', False))
+        self.assertNotIn('<script', result)
+
+    def testHtmlHeader_NoTitle(self):
+        def Test(title):
+            result = '\n'.join(gen_html.html_header(title, False))
+            self.assertNotIn('<title', result)
+        Test('')
+        Test(None)
+
+    def testHtmlHeader_Title(self):
+        lines = gen_html.html_header('foo', False)
+        for item in lines:
+          if '<title' in item:
+            self.assertIn('foo', item)
+            break
+        else:
+          self.fail('No foo in: %s' % '\n'.join(lines))
+
+    def testHtmlHeader_Script(self):
+        lines = gen_html.html_header('', True)
+        for item in lines:
+          if '<script' in item:
+            break
+        else:
+          self.fail('No script in: %s' % '\n'.join(lines))
+
+    @staticmethod
+    def gen_html(*args):
+        """Call gen_html with TEST_DATA."""
         return gen_html.gen_html(TEST_DATA, *args)[0]
 
     def testGenHtml(self):
+        """Test that the expected tests and jobs are in the results."""
         html = self.gen_html('')
-        self.assertIn("test1", html)
-        self.assertIn("test2", html)
-        self.assertIn("release", html)
-        self.assertIn("debug", html)
+        self.assertIn('test1', html)
+        self.assertIn('test2', html)
+        self.assertIn('release', html)
+        self.assertIn('debug', html)
 
     def testGenHtmlFilter(self):
+        """Test that filtering to just the release jobs works."""
         html = self.gen_html('release')
-        self.assertIn("release", html)
+        self.assertIn('release', html)
         self.assertIn('skipped">\ntest2', html)
-        self.assertNotIn("debug", html)
+        self.assertNotIn('debug', html)
 
     def testGenHtmlFilterExact(self):
+        """Test that filtering to an exact name works."""
         html = self.gen_html('release', True)
+        self.assertIn('release', html)
         self.assertNotIn('debug', html)
 
+    def testGetOptions(self):
+        """Test argument parsing works correctly."""
+
+        def check(args, expected_output_dir, expected_input):
+            """Check that args is parsed correctly."""
+            options = gen_html.get_options(args)
+            self.assertEquals(expected_output_dir, options.output_dir)
+            self.assertEquals(expected_input, options.input)
+
+
+        check(['--output-dir=foo', '--input=bar'], 'foo', 'bar')
+        check(['--output-dir', 'foo', '--input', 'bar'], 'foo', 'bar')
+        check(['--input=bar', '--output-dir=foo'], 'foo', 'bar')
+
+    def testGetOptions_Missing(self):
+        """Test missing arguments raise an exception."""
+        def check(args):
+            """Check that args raise an exception."""
+            with self.assertRaises(SystemExit):
+                gen_html.get_options(args)
+
+        check([])
+        check(['--output-dir=foo'])
+        check(['--input=bar'])
+
     def testMain(self):
+        """Test main() creates pages."""
         temp_dir = tempfile.mkdtemp(prefix='kube-test-hist-')
         try:
             tests_json = os.path.join(temp_dir, 'tests.json')
-            with open(tests_json, 'w') as f:
-                json.dump(TEST_DATA, f)
-            gen_html.main(['--suites', '--prefixes', ',rel,deb',
-                           '--output-dir', temp_dir, '--input', tests_json])
-            for page in ('index', 'suite-kubernetes-debug', 'tests', 'tests-rel', 'tests-deb'):
+            with open(tests_json, 'w') as buf:
+                json.dump(TEST_DATA, buf)
+            gen_html.main(tests_json, temp_dir)
+            for page in (
+                    'index',
+                    'tests-kubernetes',
+                    'suite-kubernetes-release',
+                    'suite-kubernetes-debug'):
                 self.assertTrue(os.path.exists('%s/%s.html' % (temp_dir, page)))
         finally:
             shutil.rmtree(temp_dir)

hack/jenkins/test-history/gen_json.py

@@ -21,6 +21,7 @@ Writes the JSON out to tests.json.
 
 from __future__ import print_function
 
+import argparse
 import json
 import os
 import re
@@ -28,7 +29,7 @@ import subprocess
 import sys
 import time
 import urllib2
-import xml.etree.ElementTree as ET
+from xml.etree import ElementTree
 import zlib
 
 
@@ -40,6 +41,7 @@ def get_json(url):
     except urllib2.HTTPError:
         return None
 
+
 def get_jobs(server):
     """Generates all job names running on the server."""
     jenkins_json = get_json('{}/api/json'.format(server))
@@ -56,14 +58,16 @@ def get_builds(server, job):
     for build in job_json['builds']:
         yield build['number']
 
+
 def get_build_info(server, job, build):
     """Returns building status along with timestamp for a given build."""
     path = '{}/job/{}/{}/api/json'.format(server, job, str(build))
     build_json = get_json(path)
     if not build_json:
-        return
+        return True, 0
     return build_json['building'], build_json['timestamp']
 
+
 def gcs_ls(path):
     """Lists objects under a path on gcs."""
     try:
@@ -81,6 +85,7 @@ def gcs_ls_build(job, build):
     for path in gcs_ls(url):
         yield path
 
+
 def gcs_ls_artifacts(job, build):
     """Lists all artifacts for a build."""
     for path in gcs_ls_build(job, build):
@@ -88,12 +93,14 @@ def gcs_ls_artifacts(job, build):
             for artifact in gcs_ls(path):
                 yield artifact
 
+
 def gcs_ls_junit_paths(job, build):
     """Lists the paths of JUnit XML files for a build."""
     for path in gcs_ls_artifacts(job, build):
-        if re.match('.*/junit.*\.xml$', path):
+        if re.match(r'.*/junit.*\.xml$', path):
             yield path
 
+
 def gcs_get_tests(path):
     """Generates test data out of the provided JUnit path.
 
@@ -113,13 +120,13 @@ def gcs_get_tests(path):
         pass
 
     try:
-        root = ET.fromstring(data)
-    except ET.ParseError:
+        root = ElementTree.fromstring(data)
+    except ElementTree.ParseError:
         return
 
     for child in root:
         name = child.attrib['name']
-        time = float(child.attrib['time'])
+        ctime = float(child.attrib['time'])
         failed = False
         skipped = False
         for param in child:
@@ -127,7 +134,8 @@ def gcs_get_tests(path):
                 skipped = True
             elif param.tag == 'failure':
                 failed = True
-        yield name, time, failed, skipped
+        yield name, ctime, failed, skipped
+
 
 def get_tests_from_junit_path(path):
     """Generates all tests in a JUnit GCS path."""
@@ -136,17 +144,19 @@ def get_tests_from_junit_path(path):
             continue
         yield test
 
+
 def get_tests_from_build(job, build):
     """Generates all tests for a build."""
     for junit_path in gcs_ls_junit_paths(job, build):
         for test in get_tests_from_junit_path(junit_path):
             yield test
 
-def get_daily_builds(server, prefix):
+
+def get_daily_builds(server, matcher):
     """Generates all (job, build) pairs for the last day."""
     now = time.time()
     for job in get_jobs(server):
-        if not job.startswith(prefix):
+        if not matcher(job):
             continue
         for build in reversed(sorted(get_builds(server, job))):
             building, timestamp = get_build_info(server, job, build)
@@ -158,10 +168,11 @@ def get_daily_builds(server, prefix):
                 break
             yield job, build
 
-def get_tests(server, prefix):
+
+def get_tests(server, matcher):
     """Returns a dictionary of tests to be JSON encoded."""
     tests = {}
-    for job, build in get_daily_builds(server, prefix):
+    for job, build in get_daily_builds(server, matcher):
         print('{}/{}'.format(job, str(build)))
         for name, duration, failed, skipped in get_tests_from_build(job, build):
             if name not in tests:
@@ -177,12 +188,33 @@ def get_tests(server, prefix):
             })
     return tests
 
+
+def main(server, match):
+    """Collect test info in matching jobs."""
+    print('Finding tests in jobs matching {} at server {}'.format(
+        match, server))
+    matcher = re.compile(match)
+    tests = get_tests(server, matcher)
+    with open('tests.json', 'w') as buf:
+        json.dump(tests, buf, sort_keys=True)
+
+
+def get_options(argv):
+    """Process command line arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--server',
+        help='hostname of jenkins server',
+        required=True,
+    )
+    parser.add_argument(
+        '--match',
+        help='filter to job names matching this re',
+        required=True,
+    )
+    return parser.parse_args(argv)
+
+
 if __name__ == '__main__':
-    if len(sys.argv) != 3:
-        print('Usage: {} <server> <prefix>'.format(sys.argv[0]))
-        sys.exit(1)
-    server, prefix = sys.argv[1:]
-    print('Finding tests prefixed with {} at server {}'.format(prefix, server))
-    tests = get_tests(server, prefix)
-    with open('tests.json', 'w') as f:
-        json.dump(tests, f, sort_keys=True)
+    OPTIONS = get_options(sys.argv[1:])
+    main(OPTIONS.server, OPTIONS.match)

hack/jenkins/test-history/gen_json_test.py

+#!/usr/bin/env python
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for gen_json."""
+
+import unittest
+
+import gen_json
+
+
+class GenJsonTest(unittest.TestCase):
+    """Unit tests for gen_json.py."""
+    # pylint: disable=invalid-name
+
+    def testGetOptions(self):
+        """Test argument parsing works correctly."""
+        def check(args, expected_server, expected_match):
+            """Check that all args are parsed as expected."""
+            options = gen_json.get_options(args)
+            self.assertEquals(expected_server, options.server)
+            self.assertEquals(expected_match, options.match)
+
+
+        check(['--server=foo', '--match=bar'], 'foo', 'bar')
+        check(['--server', 'foo', '--match', 'bar'], 'foo', 'bar')
+        check(['--match=bar', '--server=foo'], 'foo', 'bar')
+
+    def testGetOptions_Missing(self):
+        """Test missing arguments raise an exception."""
+        def check(args):
+            """Check that missing args raise an exception."""
+            with self.assertRaises(SystemExit):
+                gen_json.get_options(args)
+
+        check([])
+        check(['--server=foo'])
+        check(['--match=bar'])
+
+
+
+if __name__ == '__main__':
+    unittest.main()