From 773095425cf5d1f767924f6dbc99ff5b859cd94f Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Tue, 7 Sep 2021 21:29:31 -0500 Subject: [PATCH] [Heartbeat] Add httpcommon options to ZipURL (#27699) Support httpcommon options for zip_url sources for browser based monitors. This adds custom SSL, proxy, and timeout options. Fixes #27597 (cherry picked from commit acb4fd58e39462f579633d46b0c99e6d1886d09a) --- .../docs/monitors/monitor-browser.asciidoc | 6 +- monitors.d/plaintodos.yml | 12 ++ .../monitors/browser/source/zipurl.go | 21 +- .../monitors/browser/source/zipurl_test.go | 186 ++++++++++++------ 4 files changed, 165 insertions(+), 60 deletions(-) create mode 100644 monitors.d/plaintodos.yml diff --git a/heartbeat/docs/monitors/monitor-browser.asciidoc b/heartbeat/docs/monitors/monitor-browser.asciidoc index 24ea6dc7bebe..aa1a2ee1cebb 100644 --- a/heartbeat/docs/monitors/monitor-browser.asciidoc +++ b/heartbeat/docs/monitors/monitor-browser.asciidoc @@ -65,6 +65,7 @@ Under `zip_url`, specify these options: located in the repository. *`username`*:: The username for authenticating with the zip endpoint. This setting is optional. *`password`*:: The password for authenticating with the zip endpoint. This setting is optional. +*`ssl`*:: SSL options applied to downloading the zip, not the browser. See <> for more details. If `username` and `password` are provided, they will be sent as HTTP Basic Authentication headers to the remote zip endpoint. @@ -83,9 +84,11 @@ Example configuration: folder: "examples/todos" username: "" password: "" + # ssl options apply to downloading the zip, not the browser + #ssl: + # certificate_authorities: ['/etc/ca.crt'] ------------------------------------------------------------------------------- - [float] [[monitor-source-local]] ===== `Local directory` @@ -198,7 +201,6 @@ Example configuration: *`tags`*:: run only journeys with the given tag(s), or globs *`match`*:: run only journeys with a name or tags that matches the configured glob - [float] [[monitor-browser-synthetics-args]] ==== `synthetics_args` diff --git a/monitors.d/plaintodos.yml b/monitors.d/plaintodos.yml new file mode 100644 index 000000000000..5927ab74a0e0 --- /dev/null +++ b/monitors.d/plaintodos.yml @@ -0,0 +1,12 @@ +- name: Todos + id: todos + type: browser + enabled: true + schedule: "@every 3m" + tags: todos-app + params: + url: "https://elastic.github.io/synthetics-demo/" + source: + zip_url: + url: "https://github.com/elastic/synthetics-demo/archive/refs/heads/main.zip" + folder: "todos/synthetics-tests" diff --git a/x-pack/heartbeat/monitors/browser/source/zipurl.go b/x-pack/heartbeat/monitors/browser/source/zipurl.go index 400bf2589101..9dc9c8ab6339 100644 --- a/x-pack/heartbeat/monitors/browser/source/zipurl.go +++ b/x-pack/heartbeat/monitors/browser/source/zipurl.go @@ -14,6 +14,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/elastic/beats/v7/libbeat/common/transport/httpcommon" ) type ZipURLSource struct { @@ -23,13 +25,25 @@ type ZipURLSource struct { Password string `config:"password" json:"password"` Retries int `config:"retries" default:"3" json:"retries"` BaseSource - // Etag from last successful fetch - etag string TargetDirectory string `config:"target_directory" json:"target_directory"` + + // Etag from last successful fetch + etag string + + Transport httpcommon.HTTPTransportSettings `config:",inline" yaml:",inline"` + + httpClient *http.Client } var ErrNoEtag = fmt.Errorf("No ETag header in zip file response. Heartbeat requires an etag to efficiently cache downloaded code") +func (z *ZipURLSource) Validate() (err error) { + if z.httpClient == nil { + z.httpClient, _ = z.Transport.Client() + } + return err +} + func (z *ZipURLSource) Fetch() error { changed, err := checkIfChanged(z) if err != nil { @@ -181,6 +195,7 @@ func retryingZipRequest(method string, z *ZipURLSource) (resp *http.Response, er } func zipRequest(method string, z *ZipURLSource) (*http.Response, error) { + req, err := http.NewRequest(method, z.URL, nil) if err != nil { return nil, fmt.Errorf("could not issue request to: %s %w", z.URL, err) @@ -188,7 +203,7 @@ func zipRequest(method string, z *ZipURLSource) (*http.Response, error) { if z.Username != "" && z.Password != "" { req.SetBasicAuth(z.Username, z.Password) } - return http.DefaultClient.Do(req) + return z.httpClient.Do(req) } func download(z *ZipURLSource, tf *os.File) (etag string, err error) { diff --git a/x-pack/heartbeat/monitors/browser/source/zipurl_test.go b/x-pack/heartbeat/monitors/browser/source/zipurl_test.go index 2283fcf443f7..9c1606846422 100644 --- a/x-pack/heartbeat/monitors/browser/source/zipurl_test.go +++ b/x-pack/heartbeat/monitors/browser/source/zipurl_test.go @@ -5,9 +5,9 @@ package source import ( - "context" "fmt" "net/http" + "net/http/httptest" "os" "path" "path/filepath" @@ -15,59 +15,121 @@ import ( "testing" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/source/fixtures" ) -func TestZipUrlFetchNoAuth(t *testing.T) { - address, teardown := setupTests() - defer teardown() - - zus := &ZipURLSource{ - URL: fmt.Sprintf("http://%s/fixtures/todos.zip", address), - Folder: "/", - Retries: 3, +func TestSimpleCases(t *testing.T) { + type testCase struct { + name string + cfg common.MapStr + tlsServer bool + wantFetchErr bool + } + testCases := []testCase{ + { + "basics", + common.MapStr{ + "folder": "/", + "retries": 3, + }, + false, + false, + }, + { + "targetdir", + common.MapStr{ + "folder": "/", + "retries": 3, + "target_directory": "/tmp/synthetics/blah", + }, + false, + false, + }, + { + "auth success", + common.MapStr{ + "folder": "/", + "retries": 3, + "username": "testuser", + "password": "testpass", + }, + false, + false, + }, + { + "auth failure", + common.MapStr{ + "folder": "/", + "retries": 3, + "username": "testuser", + "password": "badpass", + }, + false, + true, + }, + { + "ssl ignore cert errors", + common.MapStr{ + "folder": "/", + "retries": 3, + "ssl": common.MapStr{ + "enabled": "true", + "verification_mode": "none", + }, + }, + true, + false, + }, + { + "bad ssl", + common.MapStr{ + "folder": "/", + "retries": 3, + "ssl": common.MapStr{ + "enabled": "true", + "certificate_authorities": []string{}, + }, + }, + true, + true, + }, } - fetchAndCheckDir(t, zus) -} -func TestZipUrlFetchWithAuth(t *testing.T) { - address, teardown := setupTests() - defer teardown() + for _, tc := range testCases { + url, teardown := setupTests(tc.tlsServer) + defer teardown() + t.Run(tc.name, func(t *testing.T) { + tc.cfg["url"] = fmt.Sprintf("%s/fixtures/todos.zip", url) + zus, err := dummyZus(tc.cfg) + require.NoError(t, err) - zus := &ZipURLSource{ - URL: fmt.Sprintf("http://%s/fixtures/todos.zip", address), - Folder: "/", - Retries: 3, - Username: "testuser", - Password: "testpass", - } - fetchAndCheckDir(t, zus) -} + require.NotNil(t, zus.httpClient) -func TestZipUrlTargetDirectory(t *testing.T) { - address, teardown := setupTests() - defer teardown() + if tc.wantFetchErr == true { + err := zus.Fetch() + require.Error(t, err) + return + } - zus := &ZipURLSource{ - URL: fmt.Sprintf("http://%s/fixtures/todos.zip", address), - Folder: "/", - Retries: 3, - TargetDirectory: "/tmp/synthetics/blah", + fetchAndCheckDir(t, zus) + }) } - fetchAndCheckDir(t, zus) } func TestZipUrlWithSameEtag(t *testing.T) { - address, teardown := setupTests() + address, teardown := setupTests(false) defer teardown() - zus := ZipURLSource{ - URL: fmt.Sprintf("http://%s/fixtures/todos.zip", address), - Folder: "/", - Retries: 3, - } - err := zus.Fetch() + zus, err := dummyZus(common.MapStr{ + "url": fmt.Sprintf("%s/fixtures/todos.zip", address), + "folder": "/", + "retries": 3, + }) + require.NoError(t, err) + err = zus.Fetch() defer zus.Close() require.NoError(t, err) @@ -80,32 +142,33 @@ func TestZipUrlWithSameEtag(t *testing.T) { } func TestZipUrlWithBadUrl(t *testing.T) { - _, teardown := setupTests() + _, teardown := setupTests(false) defer teardown() - zus := ZipURLSource{ - URL: "http://notahost.notadomaintoehutoeuhn", - Folder: "/", - Retries: 2, - } - err := zus.Fetch() + zus, err := dummyZus(common.MapStr{ + "url": "http://notahost.notadomaintoehutoeuhn", + "folder": "/", + "retries": 2, + }) + require.NoError(t, err) + err = zus.Fetch() defer zus.Close() require.Error(t, err) } -func setupTests() (addr string, teardown func()) { +func setupTests(tls bool) (addr string, teardown func()) { // go offline, so we dont invoke npm install for unit tests GoOffline() - srv := createServer() - address := srv.Addr + srv := createServer(tls) + address := srv.URL return address, func() { GoOnline() - srv.Shutdown(context.Background()) + srv.Close() } } -func createServer() (addr *http.Server) { +func createServer(tls bool) (addr *httptest.Server) { _, filename, _, _ := runtime.Caller(0) fixturesPath := path.Join(filepath.Dir(filename), "fixtures") fileServer := http.FileServer(http.Dir(fixturesPath)) @@ -121,10 +184,12 @@ func createServer() (addr *http.Server) { http.StripPrefix("/fixtures", fileServer).ServeHTTP(resp, req) }) - srv := &http.Server{Addr: "localhost:1234", Handler: mux} - go func() { - srv.ListenAndServe() - }() + var srv *httptest.Server + if tls { + srv = httptest.NewTLSServer(mux) + } else { + srv = httptest.NewServer(mux) + } return srv } @@ -140,3 +205,14 @@ func fetchAndCheckDir(t *testing.T, zip *ZipURLSource) { _, err = os.Stat(zip.TargetDirectory) require.True(t, os.IsNotExist(err), "TargetDirectory %s should have been deleted", zip.TargetDirectory) } + +func dummyZus(conf map[string]interface{}) (*ZipURLSource, error) { + zus := &ZipURLSource{} + y, _ := yaml.Marshal(conf) + c, err := common.NewConfigWithYAML(y, string(y)) + if err != nil { + return nil, err + } + err = c.Unpack(zus) + return zus, err +}