Link build results to testgrid (fixes #272).

gubernator/filters.py

@@ -187,6 +187,13 @@ def do_select(seq, pred):
     return filter(pred, seq)
 
 
+def do_tg_url(testgrid_query, test_name=''):
+    if test_name:
+        regex = '^Overall$|' + re.escape(test_name)
+        testgrid_query += '&include-filter-by-regex=%s' % urllib.quote(regex)
+    return 'https://k8s-testgrid.appspot.com/%s' % testgrid_query
+
+
 do_basename = os.path.basename
 do_dirname = os.path.dirname
 do_quote_plus = urllib.quote_plus

gubernator/filters_test.py

@@ -16,6 +16,7 @@
 
 import re
 import unittest
+import urllib
 
 import filters
 
@@ -101,6 +102,15 @@ def expect(payload, expected, user=''):
         expect({'attn': {'foo': 'Needs Rebase'}}, 'Needs Rebase', user='foo')
         expect({'attn': {'foo': 'Needs Rebase'}, 'labels': {'lgtm'}}, 'LGTM', user='foo')
 
+    def test_tg_url(self):
+        self.assertEqual(
+            filters.do_tg_url('a#b'),
+            'https://k8s-testgrid.appspot.com/a#b')
+        self.assertEqual(
+            filters.do_tg_url('a#b', '[low] test'),
+            'https://k8s-testgrid.appspot.com/a#b&include-filter-by-regex=%s' %
+            urllib.quote('^Overall$|\\[low\\]\\ test'))
+
 
 if __name__ == '__main__':
     unittest.main()

gubernator/log_parser.py

@@ -59,7 +59,7 @@ def log_html(lines, matched_lines, hilight_words, skip_fmt):
             skip_string = ('<span class="skip">'
                 '<a href="javascript:show_skipped(\'%s\')" onclick="this.style.display=\'none\'">'
                 '%s</a></span>'
-                '<span class="skipped" id=%s style="display:none; float: left;">') % (skip_id,
+                '<span class="skipped" id=%s style="display:none;">') % (skip_id,
                 skip_fmt(skip_amount), skip_id)
             lines[previous_end] = "%s%s" % (skip_string, lines[previous_end])
             output.extend(lines[previous_end:match-CONTEXT])

gubernator/main_test.py

@@ -21,7 +21,6 @@
     $ nosetests --with-gae --gae-lib-root ~/google_appengine/
 """
 
-import os
 import unittest
 
 import webtest
@@ -31,7 +30,6 @@
 import main
 import gcs_async
 import gcs_async_test
-import view_pr
 
 write = gcs_async_test.write
 
@@ -81,115 +79,11 @@ def setUp(self):
         self.init_stubs()
         init_build(self.BUILD_DIR)
 
-    def get_build_page(self):
-        return app.get('/build' + self.BUILD_DIR)
-
     def test_index(self):
         """Test that the index works."""
         response = app.get('/')
         self.assertIn('kubernetes-e2e-gce', response)
 
-    def test_missing(self):
-        """Test that a missing build gives a 404."""
-        response = app.get('/build' + self.BUILD_DIR.replace('1234', '1235'),
-                           status=404)
-        self.assertIn('1235', response)
-
-    def test_missing_started(self):
-        """Test that a missing started.json still renders a proper page."""
-        build_dir = '/kubernetes-jenkins/logs/job-with-no-started/1234/'
-        init_build(build_dir, started=False)
-        response = app.get('/build' + build_dir)
-        self.assertIn('Build Result: SUCCESS', response)
-        self.assertIn('job-with-no-started', response)
-        self.assertNotIn('Started', response)  # no start timestamp
-        self.assertNotIn('github.com', response)  # no version => no src links
-
-    def test_missing_finished(self):
-        """Test that a missing finished.json still renders a proper page."""
-        build_dir = '/kubernetes-jenkins/logs/job-still-running/1234/'
-        init_build(build_dir, finished=False)
-        response = app.get('/build' + build_dir)
-        self.assertIn('Build Result: Not Finished', response)
-        self.assertIn('job-still-running', response)
-        self.assertIn('Started', response)
-
-    def test_build(self):
-        """Test that the build page works in the happy case."""
-        response = self.get_build_page()
-        self.assertIn('2014-07-28', response)  # started
-        self.assertIn('16m40s', response)      # build duration
-        self.assertIn('Third', response)       # test name
-        self.assertIn('1m36s', response)       # test duration
-        self.assertIn('Build Result: SUCCESS', response)
-        self.assertIn('Error Goes Here', response)
-        self.assertIn('test.go#L123">', response)  # stacktrace link works
-
-    def test_build_no_failures(self):
-        """Test that builds with no Junit artifacts work."""
-        gcs.delete(self.BUILD_DIR + 'artifacts/junit_01.xml')
-        response = self.get_build_page()
-        self.assertIn('No Test Failures', response)
-
-    def test_build_show_log(self):
-        """Test that builds that failed with no failures show the build log."""
-        gcs.delete(self.BUILD_DIR + 'artifacts/junit_01.xml')
-        write(self.BUILD_DIR + 'finished.json',
-              {'result': 'FAILURE', 'timestamp': 1406536800})
-
-        # Unable to fetch build-log.txt, still works.
-        response = self.get_build_page()
-        self.assertNotIn('Error lines', response)
-
-        self.testbed.init_memcache_stub()  # clear cached result
-        write(self.BUILD_DIR + 'build-log.txt',
-              u'ERROR: test \u039A\n\n\n\n\n\n\n\n\nblah'.encode('utf8'))
-        response = self.get_build_page()
-        self.assertIn('Error lines', response)
-        self.assertIn('No Test Failures', response)
-        self.assertIn('ERROR</span>: test', response)
-        self.assertNotIn('blah', response)
-
-    def test_build_failure_no_text(self):
-        # Some failures don't have any associated text.
-        write(self.BUILD_DIR + 'artifacts/junit_01.xml', '''
-            <testsuites>
-                <testsuite tests="1" failures="1" time="3.274" name="k8s.io/test/integration">
-                    <testcase classname="integration" name="TestUnschedulableNodes" time="0.210">
-                        <failure message="Failed" type=""/>
-                    </testcase>
-                </testsuite>
-            </testsuites>''')
-        response = self.get_build_page()
-        self.assertIn('TestUnschedulableNodes', response)
-        self.assertIn('junit_01.xml', response)
-
-    def test_build_pr_link(self):
-        ''' The build page for a PR build links to the PR results.'''
-        build_dir = '/%s/123/e2e/567/' % view_pr.PR_PREFIX
-        init_build(build_dir)
-        response = app.get('/build' + build_dir)
-        self.assertIn('PR #123', response)
-        self.assertIn('href="/pr/123"', response)
-
-    def test_cache(self):
-        """Test that caching works at some level."""
-        response = self.get_build_page()
-        gcs.delete(self.BUILD_DIR + 'started.json')
-        gcs.delete(self.BUILD_DIR + 'finished.json')
-        response2 = self.get_build_page()
-        self.assertEqual(str(response), str(response2))
-
-    def test_build_list(self):
-        """Test that the job page shows a list of builds."""
-        response = app.get('/builds' + os.path.dirname(self.BUILD_DIR[:-1]))
-        self.assertIn('/1234/">1234</a>', response)
-
-    def test_job_list(self):
-        """Test that the job list shows our job."""
-        response = app.get('/jobs/kubernetes-jenkins/logs')
-        self.assertIn('somejob/">somejob</a>', response)
-
     def test_nodelog_missing_files(self):
         """Test that a missing all files gives a 404."""
         build_dir = self.BUILD_DIR + 'nodelog?pod=abc'
@@ -309,57 +203,3 @@ def test_timestamp_no_apiserver(self):
                     '01-01T01:01:01.005Z last line')
         response = app.get('/build' + nodelog_url)
         self.assertIn(expected, response)
-
-
-class PRTest(TestBase):
-    BUILDS = {
-        'build': [('12', {'version': 'bb', 'timestamp': 1467147654}, None),
-                  ('11', {'version': 'bb', 'timestamp': 1467146654}, {'result': 'PASSED'}),
-                  ('10', {'version': 'aa', 'timestamp': 1467136654}, {'result': 'FAILED'})],
-        'e2e': [('47', {'version': 'bb', 'timestamp': '1467147654'}, {'result': '[UNSET]'}),
-                ('46', {'version': 'aa', 'timestamp': '1467136700'}, {'result': '[UNSET]'})]
-    }
-
-    def setUp(self):
-        self.init_stubs()
-
-    def init_pr_directory(self):
-        gcs_async_test.install_handler(self.testbed.get_stub('urlfetch'),
-            {'123/': ['build', 'e2e'],
-             '123/build/': ['11', '10', '12'],  # out of order
-             '123/e2e/': ['47', '46']})
-
-        for job, builds in self.BUILDS.iteritems():
-            for build, started, finished in builds:
-                path = '/%s/123/%s/%s/' % (view_pr.PR_PREFIX, job, build)
-                if started:
-                    write(path + 'started.json', started)
-                if finished:
-                    write(path + 'finished.json', finished)
-
-    def test_pr_builds(self):
-        self.init_pr_directory()
-        builds = view_pr.pr_builds('123')
-        self.assertEqual(builds, self.BUILDS)
-
-    def test_pr_handler(self):
-        self.init_pr_directory()
-        response = app.get('/pr/123')
-        self.assertIn('e2e/47', response)
-        self.assertIn('PASSED', response)
-        self.assertIn('colspan="3"', response)  # header
-        self.assertIn('github.com/kubernetes/kubernetes/pull/123', response)
-        self.assertIn('28 20:44', response)
-
-    def test_pr_handler_missing(self):
-        gcs_async_test.install_handler(self.testbed.get_stub('urlfetch'),
-            {'124/': []})
-        response = app.get('/pr/124')
-        self.assertIn('No Results', response)
-
-    def test_pr_build_log_redirect(self):
-        path = '123/some-job/55/build-log.txt'
-        response = app.get('/pr/' + path)
-        self.assertEqual(response.status_code, 302)
-        self.assertIn('https://storage.googleapis.com', response.location)
-        self.assertIn(path, response.location)

gubernator/pb_glance.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+A tiny, minimal protobuf2 parser that's able to extract enough information
+to be useful.
+'''
+
+import cStringIO as StringIO
+
+
+def parse_protobuf(data, schema=None):
+    '''
+    Do a simple parse of a protobuf2 given minimal type information.
+
+    Args:
+        data: a string containing the encoded protocol buffer.
+        schema: a dict containing information about each field number.
+            The keys are field numbers, and the values represent:
+                - str: the name of the field
+                - dict: schema to recursively decode an embedded message.
+                        May contain a 'name' key to name the field.
+    Returns:
+        dict: mapping from fields to values. The fields may be strings instead of
+            numbers if schema named them, and the value will *always* be
+            a list of values observed for that key.
+    '''
+    if schema is None:
+        schema = {}
+
+    buf = StringIO.StringIO(data)
+
+    def read_varint():
+        out = 0
+        shift = 0
+        c = 0x80
+        while c & 0x80:
+            c = ord(buf.read(1))
+            out = out | ((c & 0x7f) << shift)
+            shift += 7
+        return out
+
+    values = {}
+
+    while buf.tell() < len(data):
+        key = read_varint()
+        wire_type = key & 0b111
+        field_number = key >> 3
+        field_name = field_number
+        if wire_type == 0:
+            value = read_varint()
+        elif wire_type == 1:  # 64-bit
+            value = buf.read(8)
+        elif wire_type == 2:  # length-delim
+            length = read_varint()
+            value = buf.read(length)
+            if isinstance(schema.get(field_number), basestring):
+                field_name = schema[field_number]
+            elif field_number in schema:
+                # yes, I'm using dynamic features of a dynamic language.
+                # pylint: disable=redefined-variable-type
+                value = parse_protobuf(value, schema[field_number])
+                field_name = schema[field_number].get('name', field_name)
+        elif wire_type == 5:  # 32-bit
+            value = buf.read(4)
+        else:
+            raise ValueError('unhandled wire type %d' % wire_type)
+        values.setdefault(field_name, []).append(value)
+
+    return values

gubernator/pb_glance_test.py

@@ -0,0 +1,56 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import pb_glance
+
+
+def tostr(data):
+    if isinstance(data, list):
+        return ''.join(c if isinstance(c, str) else chr(c) for c in data)
+    return data
+
+
+class PBGlanceTest(unittest.TestCase):
+    def expect(self, data, expected, types=None):
+        result = pb_glance.parse_protobuf(tostr(data), types)
+        self.assertEqual(result, expected)
+
+    def test_basic(self):
+        self.expect(
+            [0, 1,                  # varint
+             0, 0x96, 1,            # multi-byte varint
+             (1<<3)|1, 'abcdefgh',  # 64-bit
+             (2<<3)|2, 5, 'value',  # length-delimited (string)
+             (3<<3)|5, 'abcd',      # 32-bit
+            ],
+            {
+                0: [1, 150],
+                1: ['abcdefgh'],
+                2: ['value'],
+                3: ['abcd'],
+            })
+
+    def test_embedded(self):
+        self.expect([2, 2, 3<<3, 1], {0: [{3: [1]}]}, {0: {}})
+
+    def test_field_names(self):
+        self.expect([2, 2, 'hi'], {'greeting': ['hi']}, {0: 'greeting'})
+
+    def test_embedded_names(self):
+        self.expect(
+            [2, 4, (3<<3)|2, 2, 'hi'],
+            {'msg': [{'greeting': ['hi']}]},
+            {0: {'name': 'msg', 3: 'greeting'}})

gubernator/templates/build.html

@@ -25,6 +25,9 @@ <h1>{% if pr %}<a href="/pr/{{pr}}">PR #{{pr}}</a> {% endif %}<a href="/builds{{
 	<p>Build Result: {{finished['result'] if finished else 'Not Finished'}}
 	<p><a href="https://console.cloud.google.com/storage/browser{{build_dir}}">artifacts</a>
 	<a href="https://storage.googleapis.com{{build_dir}}/build-log.txt">build-log.txt</a>
+	% if testgrid_query
+		<p><a href="{{testgrid_query|tg_url}}">Testgrid history for this job</a>
+	% endif
 	</div>
 	<div id="failures">
 	% if failures
@@ -41,6 +44,9 @@ <h3><a class="anchor" id="{{name|slugify}}" href="#{{name|slugify}}">{{name}}<sp
 				% else
 					<p>Filter through <a href="/build{{build_dir}}/nodelog?junit={{filename|basename}}&wrap=on">log files</a>
 				% endif
+				% if testgrid_query
+					| View <a href="{{testgrid_query|tg_url(name)}}">test history</a> on testgrid
+				% endif
 			% else
 				<span class="inset-filename">from <a href="https://storage.googleapis.com{{filename}}">{{filename|basename}}</a></span>
 			% endif