Skip to content

Commit

Permalink
Add describe to logs output
Browse files Browse the repository at this point in the history
In debug situations when all logs are loaded, it is also helpful to get an
understanding of the respective pods by executing a `kubectl describe`.

Add output of `kubectl ... describe ...` to the logs retrieval data.

Add deployment YAML output of all pods, too.
HeavyWombat committed Oct 8, 2019

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 8652309 commit 4f99b01
Showing 5 changed files with 350 additions and 138 deletions.
19 changes: 12 additions & 7 deletions go.mod
Original file line number Diff line number Diff line change
@@ -16,10 +16,8 @@ require (
github.com/homeport/dyff v0.10.3
github.com/homeport/ytbx v1.1.2
github.com/imdario/mergo v0.3.7 // indirect
github.com/json-iterator/go v1.1.7 // indirect
github.com/lucasb-eyer/go-colorful v1.0.2
github.com/mitchellh/go-homedir v1.1.0
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d
github.com/onsi/ginkgo v1.10.2
github.com/onsi/gomega v1.7.0
@@ -31,9 +29,16 @@ require (
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/src-d/go-git.v4 v4.13.1
gopkg.in/yaml.v2 v2.2.4
k8s.io/api v0.0.0-20190704095032-f4ca3d3bdf1d
k8s.io/apimachinery v0.0.0-20190704094733-8f6ac2502e51
k8s.io/client-go v0.0.0-20190704100234-640d9f240853
k8s.io/utils v0.0.0-20190712204705-3dccf664f023 // indirect
sigs.k8s.io/yaml v1.1.0 // indirect
k8s.io/api v0.0.0-20191005115622-2e41325d9e4b
k8s.io/apimachinery v0.0.0-20191006235458-f9f2f3f8ab02
k8s.io/cli-runtime v0.0.0-20191005121332-4d28aef60981
k8s.io/client-go v0.0.0-20191006235818-c918cd02a1a3
k8s.io/kubectl v0.0.0-20191007002032-340a90f4c38f
)

replace (
golang.org/x/crypto => github.com/golang/crypto v0.0.0-20191002192127-34f69633bfdc
golang.org/x/net => github.com/golang/net v0.0.0-20191007182048-72f939374954
golang.org/x/sync => github.com/golang/sync v0.0.0-20190911185100-cd5d95a43a6e
golang.org/x/sys => github.com/golang/sys v0.0.0-20191008105621-543471e840be
)
221 changes: 175 additions & 46 deletions go.sum

Large diffs are not rendered by default.

25 changes: 25 additions & 0 deletions pkg/havener/common.go
Original file line number Diff line number Diff line change
@@ -145,3 +145,28 @@ func isSystemNamespace(namespace string) bool {

return false
}

func clusterName() (string, error) {
data, err := ioutil.ReadFile(getKubeConfig())
if err != nil {
return "", err
}

var cfg map[string]interface{}
if err := yaml.Unmarshal(data, &cfg); err != nil {
return "", err
}

for _, entry := range cfg["clusters"].([]interface{}) {
switch entry.(type) {
case map[interface{}]interface{}:
for key, value := range entry.(map[interface{}]interface{}) {
if key == "name" {
return value.(string), nil
}
}
}
}

return "", fmt.Errorf("unable to determine cluster name based on Kubernetes configuration")
}
2 changes: 1 addition & 1 deletion pkg/havener/havener.go
Original file line number Diff line number Diff line change
@@ -95,7 +95,7 @@ func NewHavener() (*Hvnr, error) {
return nil, wrap.Error(err, "unable to get access to cluster")
}

clusterName, err := ClusterName()
clusterName, err := clusterName()
if err != nil {
return nil, wrap.Error(err, "unable to get cluster name")
}
221 changes: 137 additions & 84 deletions pkg/havener/logs.go
Original file line number Diff line number Diff line change
@@ -22,20 +22,25 @@ package havener

import (
"archive/tar"
"bytes"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/gonvenience/wrap"
yaml "gopkg.in/yaml.v2"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/cli-runtime/pkg/printers"
"k8s.io/client-go/rest"
"k8s.io/kubectl/pkg/describe"
"k8s.io/kubectl/pkg/describe/versioned"
)

// LogDirName is the subdirectory name where retrieved logs are stored
@@ -88,45 +93,9 @@ func createDirectory(path string) error {
return nil
}

// ClusterName returns the name of the (first) cluster defined in the Kubernetes configuration file.
func ClusterName() (string, error) {
data, err := ioutil.ReadFile(getKubeConfig())
if err != nil {
return "", err
}

var cfg map[string]interface{}
if err := yaml.Unmarshal(data, &cfg); err != nil {
return "", err
}

for _, entry := range cfg["clusters"].([]interface{}) {
switch entry.(type) {
case map[interface{}]interface{}:
for key, value := range entry.(map[interface{}]interface{}) {
if key == "name" {
return value.(string), nil
}
}
}
}

return "", fmt.Errorf("unable to determine cluster name based on Kubernetes configuration")
}

// RetrieveLogs downloads log and configuration files from some well known location of all the pods
// of all the namespaces and stored them in the local file system.
func (h *Hvnr) RetrieveLogs(parallelDownloads int, target string, includeConfigFiles bool) error {
clusterName, err := ClusterName()
if err != nil {
return err
}

namespaces, err := ListNamespaces(h.client)
if err != nil {
return err
}

if absolute, err := filepath.Abs(target); err == nil {
target = absolute
}
@@ -144,6 +113,7 @@ func (h *Hvnr) RetrieveLogs(parallelDownloads int, target string, includeConfigF
wg.Add(parallelDownloads)
for i := 0; i < parallelDownloads; i++ {
go func() {
defer wg.Done()
for task := range tasks {
switch task.assignment {
case "known-logs":
@@ -175,51 +145,82 @@ func (h *Hvnr) RetrieveLogs(parallelDownloads int, target string, includeConfigF
errors,
h.retrieveContainerLogs(task.pod, task.baseDir)...,
)

case "describe-pods":
if err := h.describePod(task.pod, task.baseDir); err != nil {
errors = append(errors, err)
}

case "store-yaml":
if err := h.saveDeploymentYAML(task.pod, task.baseDir); err != nil {
errors = append(errors, err)
}
}
}

wg.Done()
}()
}

for _, namespace := range namespaces {
listResult, err := h.client.CoreV1().Pods(namespace).List(metav1.ListOptions{})
if err != nil {
errors = append(errors, err)
continue
}
pods, err := h.ListPods()
if err != nil {
return err
}

sort.Slice(pods, func(i, j int) bool {
return pods[i].CreationTimestamp.
After(pods[j].CreationTimestamp.Time)
})

var clusterName = h.ClusterName()
for idx := range pods {
pod := pods[idx]
baseDir := filepath.Join(
target,
LogDirName,
clusterName,
namespace,
pod.Namespace,
)

for p := range listResult.Items {
// In any case, download the container logs
// Create an empty directory for the pod details
if err := createDirectory(filepath.Join(baseDir, pod.Name)); err != nil {
return err
}

// Store the describe output of the pod
tasks <- &task{
assignment: "describe-pods",
pod: pod,
baseDir: baseDir,
}

// Store the deployment YAML of the pod
tasks <- &task{
assignment: "store-yaml",
pod: pod,
baseDir: baseDir,
}

if pod.Status.Phase == corev1.PodRunning {
// Download the container logs
tasks <- &task{
assignment: "container-logs",
pod: &listResult.Items[p],
pod: pod,
baseDir: baseDir,
}

if listResult.Items[p].Status.Phase == corev1.PodRunning {
// For running pods, download known log files
// For running pods, download known log files
tasks <- &task{
assignment: "known-logs",
pod: pod,
baseDir: baseDir,
}

// For running pods, download configuration file
if includeConfigFiles {
tasks <- &task{
assignment: "known-logs",
pod: &listResult.Items[p],
assignment: "config-files",
pod: pod,
baseDir: baseDir,
}

// For running pods, download configuration file
if includeConfigFiles {
tasks <- &task{
assignment: "config-files",
pod: &listResult.Items[p],
baseDir: baseDir,
}
}
}
}
}
@@ -330,41 +331,93 @@ func untar(inputStream io.Reader, targetPath string) error {
func (h *Hvnr) retrieveContainerLogs(pod *corev1.Pod, baseDir string) []error {
errors := []error{}

for _, container := range pod.Spec.Containers {
streamToFile := func(req *rest.Request, filename string) error {
readCloser, err := req.Stream()
if err != nil {
return err
}

defer readCloser.Close()

file, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR, os.FileMode(0644))
if err != nil {
return err
}

if _, err := io.Copy(file, readCloser); err != nil {
return err
}

return nil
}

for _, container := range append(pod.Spec.InitContainers, pod.Spec.Containers...) {
req := h.client.CoreV1().RESTClient().
Get().
Namespace(pod.GetNamespace()).
Name(pod.Name).
Resource("pods").
SubResource("log").
Param("container", container.Name)
Param("container", container.Name).
Param("timestamps", strconv.FormatBool(true))

readCloser, err := req.Stream()
if err != nil {
filename := filepath.Join(baseDir, pod.Name, container.Name+".log")
if err := streamToFile(req, filename); err != nil {
errors = append(errors, err)
continue
}
}

defer readCloser.Close()
return errors
}

targetDir := filepath.Join(baseDir, pod.Name)
if err := createDirectory(targetDir); err != nil {
errors = append(errors, err)
continue
}
func (h *Hvnr) describePod(pod *corev1.Pod, baseDir string) error {
if describer, ok := versioned.DescriberFor(schema.GroupKind{Group: corev1.GroupName, Kind: "Pod"}, h.restconfig); ok {
output, err := describer.Describe(
pod.Namespace,
pod.Name,
describe.DescriberSettings{
ShowEvents: true,
},
)

target := filepath.Join(targetDir, container.Name+".log")
file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(0644))
if err != nil {
errors = append(errors, err)
continue
return err
}

if _, err := io.Copy(file, readCloser); err != nil {
errors = append(errors, err)
continue
}
return ioutil.WriteFile(
filepath.Join(baseDir, pod.Name, "pod-describe.output"),
[]byte(output),
0644,
)
}

return errors
return nil
}

func (h *Hvnr) saveDeploymentYAML(pod *corev1.Pod, baseDir string) error {
// Whatever GroupVersionKind really is, but if it is empty the printer will
// refuse to work, so set `Kind` and `Version` with reasonable defaults
// knowing that this will only be pods.
if pod.GetObjectKind().GroupVersionKind().Empty() {
pod.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{
Kind: "Pod",
Version: "v1",
})
}

var (
printer printers.YAMLPrinter
buf bytes.Buffer
)

if err := printer.PrintObj(pod, &buf); err != nil {
return err
}

return ioutil.WriteFile(
filepath.Join(baseDir, pod.Name, "pod.yaml"),
buf.Bytes(),
0644,
)
}

0 comments on commit 4f99b01

Please sign in to comment.