diff --git a/app/bin/build_and_test.sh b/app/bin/build_and_test.sh index a5afb6a3ee..da1af910ed 100755 --- a/app/bin/build_and_test.sh +++ b/app/bin/build_and_test.sh @@ -10,7 +10,7 @@ if [ ! -f "pubspec.yaml" -a ! -f "app.yaml" ]; then fi rm -rf build -pub get +pub get --no-packages-dir pub run test pub build cp web/*.dart build/web/ diff --git a/app/pubspec.yaml b/app/pubspec.yaml index d32d0e8229..15a6837914 100644 --- a/app/pubspec.yaml +++ b/app/pubspec.yaml @@ -4,7 +4,7 @@ homepage: https://github.com/flutter/cocoon environment: sdk: '>=1.13.0 <2.0.0' dependencies: - angular2: 2.0.0-beta.17 + angular2: 2.0.0-beta.21 args: ^0.13.4 browser: ^0.10.0 charted: ^0.4.0 diff --git a/commands/get_status.go b/commands/get_status.go index ad34b4c8d6..74906ba5d7 100644 --- a/commands/get_status.go +++ b/commands/get_status.go @@ -35,11 +35,11 @@ func GetStatus(c *db.Cocoon, inputJSON []byte) (interface{}, error) { var statuses []*BuildStatus for _, checklist := range checklists { - // Need to define another error variable to not "shadow" the other one, Go figure! - stages, errr := c.QueryTasksGroupedByStage(checklist.Key) + var stages []*db.Stage + stages, err = c.QueryTasksGroupedByStage(checklist.Key) - if errr != nil { - return nil, errr + if err != nil { + return nil, err } statuses = append(statuses, &BuildStatus{ diff --git a/commands/refresh_chromebot_status.go b/commands/refresh_chromebot_status.go index 427d0e6788..0a75ec9325 100644 --- a/commands/refresh_chromebot_status.go +++ b/commands/refresh_chromebot_status.go @@ -8,9 +8,6 @@ import ( "cocoon/db" "encoding/json" "fmt" - "io/ioutil" - - "google.golang.org/appengine/urlfetch" ) // RefreshChromebotStatusResult contains chromebot status results. @@ -113,15 +110,8 @@ func fetchChromebotBuildStatuses(cocoon *db.Cocoon, builderName string) ([]*Chro } func fetchJSON(cocoon *db.Cocoon, url string) (interface{}, error) { - httpClient := urlfetch.Client(cocoon.Ctx) - response, err := httpClient.Get(url) - if err != nil { - return nil, err - } - - defer response.Body.Close() + body, err := cocoon.FetchURL(url) - body, err := ioutil.ReadAll(response.Body) if err != nil { return nil, err } diff --git a/commands/refresh_github_commits.go b/commands/refresh_github_commits.go index 5aeac7b6b3..87c4dc98e1 100644 --- a/commands/refresh_github_commits.go +++ b/commands/refresh_github_commits.go @@ -8,13 +8,12 @@ import ( "cocoon/db" "encoding/json" "fmt" - "io/ioutil" "golang.org/x/net/context" "google.golang.org/appengine/datastore" "google.golang.org/appengine/log" - "google.golang.org/appengine/urlfetch" + yaml "gopkg.in/yaml.v2" ) // RefreshGithubCommitsResult pulls down the latest GitHub commit data and @@ -32,21 +31,7 @@ type CommitSyncResult struct { // RefreshGithubCommits returns the information about the latest GitHub commits. func RefreshGithubCommits(cocoon *db.Cocoon, inputJSON []byte) (interface{}, error) { - httpClient := urlfetch.Client(cocoon.Ctx) - - // Fetch data from GitHub - githubResp, err := httpClient.Get("https://api.github.com/repos/flutter/flutter/commits") - - if err != nil { - return nil, err - } - - if githubResp.StatusCode != 200 { - return nil, fmt.Errorf("GitHub API responded with a non-200 HTTP status: %v", githubResp.StatusCode) - } - - defer githubResp.Body.Close() - commitData, err := ioutil.ReadAll(githubResp.Body) + commitData, err := cocoon.FetchURL("https://api.github.com/repos/flutter/flutter/commits") if err != nil { return nil, err @@ -56,7 +41,7 @@ func RefreshGithubCommits(cocoon *db.Cocoon, inputJSON []byte) (interface{}, err err = json.Unmarshal(commitData, &commits) if err != nil { - return nil, err + return nil, fmt.Errorf("%v: %v", err, string(commitData)) } if len(commits) > 0 { @@ -98,7 +83,13 @@ func RefreshGithubCommits(cocoon *db.Cocoon, inputJSON []byte) (interface{}, err return err } - tasks := createTaskList(nowMillisSinceEpoch, checklistKey) + var tasks []*db.Task + tasks, err = createTaskList(cocoon, nowMillisSinceEpoch, checklistKey, commit.Sha) + + if err != nil { + return err + } + for _, task := range tasks { _, err = txc.PutTask(nil, task) if err != nil { @@ -127,8 +118,7 @@ func RefreshGithubCommits(cocoon *db.Cocoon, inputJSON []byte) (interface{}, err return RefreshGithubCommitsResult{commitResults}, nil } -// TODO(yjbanov): the task list should be stored in the flutter/flutter repo. -func createTaskList(createTimestamp int64, checklistKey *datastore.Key) []*db.Task { +func createTaskList(cocoon *db.Cocoon, createTimestamp int64, checklistKey *datastore.Key, commit string) ([]*db.Task, error) { var makeTask = func(stageName string, name string, requiredCapabilities []string) *db.Task { return &db.Task{ ChecklistKey: checklistKey, @@ -142,31 +132,42 @@ func createTaskList(createTimestamp int64, checklistKey *datastore.Key) []*db.Ta } } - return []*db.Task{ + // These built-in tasks are not listed in the manifest. + tasks := []*db.Task{ makeTask("travis", "travis", []string{"can-update-travis"}), makeTask("chromebot", "mac_bot", []string{"can-update-chromebots"}), makeTask("chromebot", "linux_bot", []string{"can-update-chromebots"}), + } - makeTask("devicelab", "complex_layout_scroll_perf__timeline_summary", []string{"has-android-device"}), - makeTask("devicelab", "flutter_gallery__start_up", []string{"has-android-device"}), - makeTask("devicelab", "complex_layout__start_up", []string{"has-android-device"}), - makeTask("devicelab", "flutter_gallery__transition_perf", []string{"has-android-device"}), - makeTask("devicelab", "mega_gallery__refresh_time", []string{"has-android-device"}), - - makeTask("devicelab", "flutter_gallery__build", []string{"has-android-device"}), - makeTask("devicelab", "complex_layout__build", []string{"has-android-device"}), - makeTask("devicelab", "basic_material_app__size", []string{"has-android-device"}), + url := fmt.Sprintf("https://raw.githubusercontent.com/flutter/flutter/%v/dev/devicelab/manifest.yaml", commit) + manifestYaml, err := cocoon.FetchURL(url) - makeTask("devicelab", "analyzer_cli__analysis_time", []string{"has-android-device"}), - makeTask("devicelab", "analyzer_server__analysis_time", []string{"has-android-device"}), + if err != nil { + // There is no guarantee that every commit will have a manifest file + log.Warningf(cocoon.Ctx, "Error fetching CI manifest at %v. Error: %v", url, err) + return tasks, nil + } - makeTask("devicelab", "hot_mode_dev_cycle__benchmark", []string{"has-android-device"}), + var manifest Manifest + yaml.Unmarshal(manifestYaml, &manifest) - // iOS - makeTask("devicelab_ios", "complex_layout_scroll_perf_ios__timeline_summary", []string{"has-ios-device"}), - makeTask("devicelab_ios", "flutter_gallery_ios__start_up", []string{"has-ios-device"}), - makeTask("devicelab_ios", "complex_layout_ios__start_up", []string{"has-ios-device"}), - makeTask("devicelab_ios", "flutter_gallery_ios__transition_perf", []string{"has-ios-device"}), + for name, info := range manifest.Tasks { + tasks = append(tasks, makeTask(info.Stage, name, info.RequiredAgentCapabilities)) } + + return tasks, nil +} + +// Manifest contains CI tasks. +type Manifest struct { + Tasks map[string]ManifestTask +} + +// ManifestTask contains information about a CI task. +type ManifestTask struct { + Name string + Description string + Stage string + RequiredAgentCapabilities []string } diff --git a/commands/refresh_travis_status.go b/commands/refresh_travis_status.go index b40676d1c4..305fbf0866 100644 --- a/commands/refresh_travis_status.go +++ b/commands/refresh_travis_status.go @@ -7,9 +7,6 @@ package commands import ( "cocoon/db" "encoding/json" - "io/ioutil" - - "google.golang.org/appengine/urlfetch" ) // RefreshTravisStatusResult pulls down the latest Travis builds and updates @@ -40,15 +37,7 @@ func RefreshTravisStatus(cocoon *db.Cocoon, inputJSON []byte) (interface{}, erro } // Fetch data from Travis - httpClient := urlfetch.Client(cocoon.Ctx) - travisResp, err := httpClient.Get("https://api.travis-ci.org/repos/flutter/flutter/builds") - if err != nil { - return nil, err - } - - defer travisResp.Body.Close() - - buildData, err := ioutil.ReadAll(travisResp.Body) + buildData, err := cocoon.FetchURL("https://api.travis-ci.org/repos/flutter/flutter/builds") if err != nil { return nil, err } diff --git a/db/db.go b/db/db.go index 61f981fd00..f5e906f844 100644 --- a/db/db.go +++ b/db/db.go @@ -8,6 +8,7 @@ import ( "crypto/rand" "fmt" "io" + "io/ioutil" "sort" "time" @@ -16,6 +17,7 @@ import ( "golang.org/x/net/context" "google.golang.org/appengine/datastore" + "google.golang.org/appengine/urlfetch" ) // NewCocoon creates a new Cocoon. @@ -580,3 +582,27 @@ func NowMillis() int64 { func (t *Task) AgeInMillis() int64 { return NowMillis() - t.CreateTimestamp } + +// FetchURL performs an HTTP GET request on the given URL and returns data if +// response is HTTP 200. +func (c *Cocoon) FetchURL(url string) ([]byte, error) { + httpClient := urlfetch.Client(c.Ctx) + response, err := httpClient.Get(url) + + if err != nil { + return nil, err + } + + if response.StatusCode != 200 { + return nil, fmt.Errorf("HTTP GET %v responded with a non-200 HTTP status: %v", url, response.StatusCode) + } + + defer response.Body.Close() + commitData, err := ioutil.ReadAll(response.Body) + + if err != nil { + return nil, err + } + + return commitData, err +}