diff --git a/src/data/nav.yml b/src/data/nav.yml index 40236b006..32aa7f163 100644 --- a/src/data/nav.yml +++ b/src/data/nav.yml @@ -14,6 +14,19 @@ url: '/collect-data/get-started-nerdgraph-api-explorer' - title: Monitor your network devices with New Relic url: '/collect-data/network-performance-monitoring' + - title: Monitor your Kubernetes cluster with New Relic and Pixie + url: '/collect-data/pixie' + pages: + - title: Set up your TinyHat environment + url: '/collect-data/pixie/set-up-env' + - title: Explore your cluster + url: '/collect-data/pixie/explore-cluster' + - title: Instrument your cluster + url: '/collect-data/pixie/instrument-cluster' + - title: Debug missing images in your service + url: '/collect-data/pixie/scenario-1' + - title: Debug latency in the TinyHat admin + url: '/collect-data/pixie/scenario-2' - title: Automate workflows icon: nr-automation url: '/automate-workflows' diff --git a/src/images/ab-test/gateway-service-traffic.png b/src/images/ab-test/gateway-service-traffic.png new file mode 100644 index 000000000..288c0c91d Binary files /dev/null and b/src/images/ab-test/gateway-service-traffic.png differ diff --git a/src/images/pixie/400-response-details.png b/src/images/pixie/400-response-details.png new file mode 100644 index 000000000..a2f5454c4 Binary files /dev/null and b/src/images/pixie/400-response-details.png differ diff --git a/src/images/pixie/admin-delay.png b/src/images/pixie/admin-delay.png new file mode 100644 index 000000000..24af6a180 Binary files /dev/null and b/src/images/pixie/admin-delay.png differ diff --git a/src/images/pixie/admin.png b/src/images/pixie/admin.png new file mode 100644 index 000000000..eba8063be Binary files /dev/null and b/src/images/pixie/admin.png differ diff --git a/src/images/pixie/broken-image.png b/src/images/pixie/broken-image.png new file mode 100644 index 000000000..773777bf4 Binary files /dev/null and b/src/images/pixie/broken-image.png differ diff --git a/src/images/pixie/cat-ears-bob.png b/src/images/pixie/cat-ears-bob.png new file mode 100644 index 000000000..5d3543d44 Binary files /dev/null and b/src/images/pixie/cat-ears-bob.png differ diff --git a/src/images/pixie/cluster-script.png b/src/images/pixie/cluster-script.png new file mode 100644 index 000000000..b51bed349 Binary files /dev/null and b/src/images/pixie/cluster-script.png differ diff --git a/src/images/pixie/default-namespace.png b/src/images/pixie/default-namespace.png new file mode 100644 index 000000000..bf8ae2644 Binary files /dev/null and b/src/images/pixie/default-namespace.png differ diff --git a/src/images/pixie/default-service-map.png b/src/images/pixie/default-service-map.png new file mode 100644 index 000000000..248a8c9cd Binary files /dev/null and b/src/images/pixie/default-service-map.png differ diff --git a/src/images/pixie/error-rates.png b/src/images/pixie/error-rates.png new file mode 100644 index 000000000..25511cfa2 Binary files /dev/null and b/src/images/pixie/error-rates.png differ diff --git a/src/images/pixie/fetch-400s.png b/src/images/pixie/fetch-400s.png new file mode 100644 index 000000000..d386f8a6c Binary files /dev/null and b/src/images/pixie/fetch-400s.png differ diff --git a/src/images/pixie/flamegraph-default-namespace.png b/src/images/pixie/flamegraph-default-namespace.png new file mode 100644 index 000000000..7eae713e0 Binary files /dev/null and b/src/images/pixie/flamegraph-default-namespace.png differ diff --git a/src/images/pixie/flamegraph-script.png b/src/images/pixie/flamegraph-script.png new file mode 100644 index 000000000..32131f6ec Binary files /dev/null and b/src/images/pixie/flamegraph-script.png differ diff --git a/src/images/pixie/flamegraph-services.png b/src/images/pixie/flamegraph-services.png new file mode 100644 index 000000000..6fd299074 Binary files /dev/null and b/src/images/pixie/flamegraph-services.png differ diff --git a/src/images/pixie/frontend-latency-spikes.png b/src/images/pixie/frontend-latency-spikes.png new file mode 100644 index 000000000..54def827b Binary files /dev/null and b/src/images/pixie/frontend-latency-spikes.png differ diff --git a/src/images/pixie/frontend-service-latency.png b/src/images/pixie/frontend-service-latency.png new file mode 100644 index 000000000..12ed13ab7 Binary files /dev/null and b/src/images/pixie/frontend-service-latency.png differ diff --git a/src/images/pixie/frontend.png b/src/images/pixie/frontend.png new file mode 100644 index 000000000..bd81a30cf Binary files /dev/null and b/src/images/pixie/frontend.png differ diff --git a/src/images/pixie/gateway-service-stats.png b/src/images/pixie/gateway-service-stats.png new file mode 100644 index 000000000..0985c5beb Binary files /dev/null and b/src/images/pixie/gateway-service-stats.png differ diff --git a/src/images/pixie/high-latency.png b/src/images/pixie/high-latency.png new file mode 100644 index 000000000..707d4a8a1 Binary files /dev/null and b/src/images/pixie/high-latency.png differ diff --git a/src/images/pixie/live-debugging.png b/src/images/pixie/live-debugging.png new file mode 100644 index 000000000..d6814be4b Binary files /dev/null and b/src/images/pixie/live-debugging.png differ diff --git a/src/images/pixie/no-hat.png b/src/images/pixie/no-hat.png new file mode 100644 index 000000000..4b2c98f18 Binary files /dev/null and b/src/images/pixie/no-hat.png differ diff --git a/src/images/pixie/nr-kubernetes.png b/src/images/pixie/nr-kubernetes.png new file mode 100644 index 000000000..db5f1e928 Binary files /dev/null and b/src/images/pixie/nr-kubernetes.png differ diff --git a/src/images/pixie/order-of-operations.png b/src/images/pixie/order-of-operations.png new file mode 100644 index 000000000..c63f749bb Binary files /dev/null and b/src/images/pixie/order-of-operations.png differ diff --git a/src/images/pixie/query.png b/src/images/pixie/query.png new file mode 100644 index 000000000..b5aa26e00 Binary files /dev/null and b/src/images/pixie/query.png differ diff --git a/src/images/pixie/sample-of-slow-requests.png b/src/images/pixie/sample-of-slow-requests.png new file mode 100644 index 000000000..74bd88ccf Binary files /dev/null and b/src/images/pixie/sample-of-slow-requests.png differ diff --git a/src/images/pixie/scenario-1-network-response.png b/src/images/pixie/scenario-1-network-response.png new file mode 100644 index 000000000..1ab6203a9 Binary files /dev/null and b/src/images/pixie/scenario-1-network-response.png differ diff --git a/src/images/pixie/scenario-2-twitter.png b/src/images/pixie/scenario-2-twitter.png new file mode 100644 index 000000000..f59a9ede9 Binary files /dev/null and b/src/images/pixie/scenario-2-twitter.png differ diff --git a/src/images/pixie/scripts.png b/src/images/pixie/scripts.png new file mode 100644 index 000000000..3734d779f Binary files /dev/null and b/src/images/pixie/scripts.png differ diff --git a/src/images/pixie/service-script-selector.png b/src/images/pixie/service-script-selector.png new file mode 100644 index 000000000..8db8e2ed0 Binary files /dev/null and b/src/images/pixie/service-script-selector.png differ diff --git a/src/images/pixie/service-script.png b/src/images/pixie/service-script.png new file mode 100644 index 000000000..8e2b6b9ad Binary files /dev/null and b/src/images/pixie/service-script.png differ diff --git a/src/images/pixie/traffic.png b/src/images/pixie/traffic.png new file mode 100644 index 000000000..8e2e9f26b Binary files /dev/null and b/src/images/pixie/traffic.png differ diff --git a/src/images/pixie/two-functions.png b/src/images/pixie/two-functions.png new file mode 100644 index 000000000..0f8f7d267 Binary files /dev/null and b/src/images/pixie/two-functions.png differ diff --git a/src/markdown-pages/collect-data/pixie-lab/explore-cluster.mdx b/src/markdown-pages/collect-data/pixie-lab/explore-cluster.mdx new file mode 100644 index 000000000..8c97396aa --- /dev/null +++ b/src/markdown-pages/collect-data/pixie-lab/explore-cluster.mdx @@ -0,0 +1,107 @@ +--- +path: '/collect-data/pixie/explore-cluster' +duration: '5 min' +title: 'Explore your cluster' +template: 'GuideTemplate' +description: 'Explore your cluster' +procIdx: 2 +--- + + + +This procedure is part of a lab that teaches you how to monitor your Kubernetes cluster with Pixie. + +Each procedure in the lab builds upon the last, so make sure you've completed the last procedure, [_Set up your TinyHat environment_](/collect-data/pixie/set-up-env), before starting this one. + + + +Your website, TinyHat.me, runs in Kubernetes. While Pixie helps you monitor your cluster and find the source of problems that arise within it, it's helpful to know about your cluster's structure and the components that make it up before learning how to use Pixie. + + + + + +View your pods: + +<> + +```bash +kubectl get pods +[output] NAME READY STATUS RESTARTS AGE +[output] add-service-7bd4df79d8-pvtz6 1/1 Running 0 2m31s +[output] admin-service-6877db7bcc-t86qj 1/1 Running 0 2m31s +[output] fetch-service-768d9f9988-8k8jp 1/1 Running 2 2m31s +[output] frontend-service-d6d79cc6d-q496f 1/1 Running 0 2m31s +[output] gateway-service-5bc8bcc679-stxbn 1/1 Running 0 2m31s +[output] manipulation-service-96c8fd7d4-64gfr 1/1 Running 0 2m31s +[output] moderate-service-859498d88b-nsg5q 1/1 Running 0 2m31s +[output] mysql-6d65cdf67f-59dfg 1/1 Running 0 2m31s +[output] upload-service-7c8d8b648f-x6kq5 1/1 Running 0 2m31s +``` + + + + + +Every pod should be running by now, but if one isn't, give it few more minutes. + + + +Here, you see nine containerized services, each with their own pod. Instead of going into detail about these services now, you'll learn about them in context as you walk through this lab's scenarios. + + + + + +View your services next: + +<> + +```bash +kubectl get services +[output] NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +[output] add-service ClusterIP 10.106.237.140 80/TCP 3m22s +[output] admin-service ClusterIP 10.101.252.142 80/TCP 3m22s +[output] fetch-service ClusterIP 10.105.255.200 80/TCP 3m22s +[output] frontend-service LoadBalancer 10.110.241.254 10.110.241.254 80:30183/TCP 3m22s +[output] gateway-service LoadBalancer 10.100.146.125 10.100.146.125 80:30605/TCP 3m22s +[output] kubernetes ClusterIP 10.96.0.1 443/TCP 3m27s +[output] manipulation-service ClusterIP 10.105.191.228 80/TCP 3m22s +[output] moderate-service ClusterIP 10.97.33.66 80/TCP 3m22s +[output] mysql ClusterIP 10.111.4.242 3306/TCP 3m22s +[output] upload-service ClusterIP 10.98.220.212 80/TCP 3m22s +``` + + + +Here, you see your Kubernetes services. Notice that most of your app services have a corresponding Kubernetes service. One of these services is particularly important: `frontend-service`. This is a `LoadBalancer` that you connect to when you view your site. + + + + + +Speaking of viewing your site, do that now by opening the `EXTERNAL_IP` that you see for `frontend-service` in your browser: + +![TinyHat.me](../../../images/pixie/frontend.png) + + + + + +Select a number of hats to display and a hat type. Then, select **Hat Me**: + +![TinyHat.me showing a picture of Bob Ross with cat ears](../../../images/pixie/cat-ears-bob.png) + +Now Bob Ross is wearing the hat(s) that you selected. Cool, right? + + + + + +With a better understanding of your cluster's resources and your website, you're better prepared to install Pixie and use it to monitor and debug your application when problems arise. + + + +This procedure is part of a course that teaches you how to monitor your Kubernetes cluster with Pixie. Now that you've explored your cluster, [instrument it with Pixie](/collect-data/pixie/instrument-cluster). + + diff --git a/src/markdown-pages/collect-data/pixie-lab/index.mdx b/src/markdown-pages/collect-data/pixie-lab/index.mdx new file mode 100644 index 000000000..647d699e4 --- /dev/null +++ b/src/markdown-pages/collect-data/pixie-lab/index.mdx @@ -0,0 +1,41 @@ +--- +path: '/collect-data/pixie' +title: 'Monitor your Kubernetes cluster with New Relic and Pixie' +template: 'LabOverviewTemplate' +description: 'Monitor your Kubernetes cluster with New Relic and Pixie' +--- + + + +You’re the developer of TinyHat.me, a website and API for putting virtual hats on virtual heads that runs in kubernetes. But TinyHat has big problems. It’s time to instrument your cluster with [Pixie](https://docs.newrelic.com/docs/auto-telemetry-pixie/get-started-auto-telemetry-pixie/) and use it to find the source of your users’ frustrations. + + + +
+ +
+ +## Objectives + +- Instrument your kubernetes cluster with New Relic + Pixie +- Use PxL scripts in the Live Debugger to gather insights into your cluster’s behaviors +- Find the root cause of two bugs in your software + +
+ +
+ +## Requirements + +- A free [New Relic account](https://newrelic.com/signup?utm_source=developer-site) +- [Minikube](https://v1-18.docs.kubernetes.io/docs/tasks/tools/install-minikube/) +- [Helm 3](https://helm.sh/docs/intro/install/) +- [Hyperkit](https://minikube.sigs.k8s.io/docs/drivers/hyperkit/) (for Mac) or [KVM2](https://minikube.sigs.k8s.io/docs/drivers/kvm2/) (Linux) minikube driver + +
+ +
+ +
diff --git a/src/markdown-pages/collect-data/pixie-lab/instrument.mdx b/src/markdown-pages/collect-data/pixie-lab/instrument.mdx new file mode 100644 index 000000000..34970d4c3 --- /dev/null +++ b/src/markdown-pages/collect-data/pixie-lab/instrument.mdx @@ -0,0 +1,152 @@ +--- +path: '/collect-data/pixie/instrument-cluster' +duration: '15 min' +title: 'Instrument your cluster' +template: 'GuideTemplate' +description: 'Instrument your cluster' +procIdx: 3 +--- + + + +This procedure is part of a lab that teaches you how to monitor your Kubernetes cluster with Pixie. + +Each procedure in the lab builds upon the last, so make sure you've completed the last procedure, [_Explore your cluster_](/collect-data/pixie/explore-cluster), before starting this one. + + + +As the developer of TinyHat.me, you need to have visibility into your cluster. You need to know how healthy your application is. You need to know when things go wrong. But you've put it off for so long because instrumenting Kubernetes is hard and time-consuming. This is one of the things that makes Pixie so valuable. + +Pixie is an open source, state of the art Kubernetes monitoring solution that provides: + +- **Automatic** and **instant** baseline observability of your cluster +- Actionable, **code-level** insights of your applications + +With Pixie's auto-telemetry, you'll [instrument your cluster](https://docs.newrelic.com/docs/auto-telemetry-pixie/install-auto-telemetry-pixie/) in minutes to get dynamic data such as protocol traces, resource metrics, and app metrics from your cluster—[all without an agent](https://docs.pixielabs.ai/about-pixie/pixie-ebpf/)! + + + +If you haven't signed into [New Relic](one.newrelic.com), do that now so you're ready to install Pixie into your cluster. + + + + + + + +Open [New Relic](https://one.newrelic.com). On the right side of the upper navigation bar, click **Add more data**: + +![Arrow pointing to the add more data button](../../../images/pixie/add-more-data.png) + + + + + +Click **Guided install**: + +![Arrow pointing to the guided install button](../../../images/pixie/guided-install.png) + +This walks you through the installation process. + + + + + +Click **Kubernetes** to let New Relic guide you through instrumenting your Kubernetes cluster: + +![Arrow pointing to the Kubernetes button](../../../images/pixie/kubernetes-install.png) + + + + + +Click **Begin installation**: + +![Arrow pointing to the begin installation button](../../../images/pixie/begin-installation.png) + + + + + +Select your account, name your cluster "tiny-hat", and click **Continue**: + +![Box highlighting the account and cluster name fields](../../../images/pixie/account-and-cluster.png) + +This specifies that TinyHat.me, and all its services, should live in a New Relic cluster called "tiny-hat" in the account you selected. + + + + + +Leave the default choices on the next screen. These provide a range of observability features for your cluster. Notably, the default options include "Instant service-level insights, full-body requests, and application profiles through Pixie" which you focus on in this lab. + +If you see the option, read and accept the **Terms of Service and Privacy Notice for Pixie** and click **Continue**: + +![Box highlighting Pixie instrumentation](../../../images/pixie/pixie-instrumentation.png) + +In the next screen, you see a command for installing our Kubernetes integration into your cluster. + + + + + +Click **Copy command**: + +![Arrow pointing to the copy command button](../../../images/pixie/helm.png) + +Now you're ready to install Pixie into your cluster. + + + + + +Switch back to your terminal and paste the Helm command. + + + + + +While you're installing Pixie, switch back to New Relic and click **Continue** to progress your guided installation to the final step. Here, you see a message that says "Listening for data": + +![New Relic page showing listening for data](../../../images/pixie/listening.png) + +In a few minutes, Helm will have fully installed the necessary charts. You should see a message with the name, deployed date, namespace, and more: + +<> + +```bash copyable=false +[output] NAME: newrelic-bundle +[output] LAST DEPLOYED: Thu Sep 23 13:50:24 2021 +[output] NAMESPACE: newrelic +[output] STATUS: deployed +[output] REVISION: 1 +[output] TEST SUITE: None +``` + + + +Soon after that, the New Relic page updates to tell you that we're receiving data from your cluster: + +![New Relic page showing a successful install](../../../images/pixie/successful-install.png) + + + + + +Click **Kubernetes cluster explorer** to see your nodes, pods, deployments and a host of other data about your cluster, all in one view: + +![New Relic cluster explorer](../../../images/pixie/cluster-explorer.png) + +Remember that you didn't have to install language-specific agents or specify hardly any data about your cluster during installation. Pixie was able to do figure it all out in minutes! On top of all the data you see in the cluster explorer, click a pod or a node to dig deeper and see the granular data that Pixie was able to access near-instantly: + +![Pod data collected by Pixie](../../../images/pixie/pod-data.png) + + + + + + + +This procedure is part of a lab that teaches you how to monitor your Kubernetes cluster with Pixie. Now that you've instrumented your cluster, [use Pixie to debug your application](/collect-data/pixie/scenario-1). + + diff --git a/src/markdown-pages/collect-data/pixie-lab/scenario-1.mdx b/src/markdown-pages/collect-data/pixie-lab/scenario-1.mdx new file mode 100644 index 000000000..8dda4e43d --- /dev/null +++ b/src/markdown-pages/collect-data/pixie-lab/scenario-1.mdx @@ -0,0 +1,298 @@ +--- +path: '/collect-data/pixie/scenario-1' +duration: '15 min' +title: 'Debug missing images in your service' +template: 'GuideTemplate' +description: "Figure out why users can't render a particular hat in TinyHat.me" +procIdx: 4 +--- + +test + + + +This procedure is part of a lab that teaches you how to monitor your Kubernetes cluster with Pixie. + +Each procedure in the lab builds upon the last, so make sure you've completed the last procedure, [_Instrument your cluster_](/collect-data/pixie/instrument-cluster), before starting this one. + + + +Until now, you've been working with application services that don't have bugs (we hope). You've been able to access TinyHat.me and render Bob Ross with one or more silly hats without a hitch. But it's time to deploy some new code to your cluster. + +Change to the `scenario-1` branch and set up your environment: + +<> + +```bash +git checkout scenario-1 +./setup.sh +[output] Switched to branch 'scenario-1' +[output] Please wait while we update your lab environment. +[output] deployment.apps/fetch-service configured +[output] Done! +``` + + + +Uh oh! You look on social media and see some confused customers: + +

+A Twitter thread showing some users can't access a hat +

+ +What's wrong with TinyHat.me? Use Pixie to find out. + +## Reproduce the issue + +You've been notified by your users that they can't see a particular hat on TinyHat.me. Before you start debugging your code, reproduce the issue for yourself. + + + + + +Look up your frontend's external IP address: + +<> + +```bash +kubectl get services +[output] NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +[output] add-service ClusterIP 10.109.114.34 80/TCP 20m +[output] admin-service ClusterIP 10.110.29.145 80/TCP 20m +[output] fetch-service ClusterIP 10.104.224.242 80/TCP 20m +[output] frontend-service LoadBalancer 10.102.82.89 {green}10.102.82.89 {normal}80:32161/TCP 20m +[output] gateway-service LoadBalancer 10.101.237.225 10.101.237.225 80:32469/TCP 20m +[output] kubernetes ClusterIP 10.96.0.1 443/TCP 20m +[output] manipulation-service ClusterIP 10.107.23.237 80/TCP 20m +[output] moderate-service ClusterIP 10.105.207.153 80/TCP 20m +[output] mysql ClusterIP 10.97.194.23 3306/TCP 20m +[output] upload-service ClusterIP 10.108.113.235 80/TCP 20m +``` + + + + + + + +Paste the IP in your browser: + +![TinyHat.me](../../../images/pixie/frontend.png) + +This looks the same as it did before, except that there's a new hat style, called PIXIE. + + + + + +Select a number of hats, the PIXIE style, and **Hat me**: + +![TinyHat.me](../../../images/pixie/no-hat.png) + +Oops! Where did Bob go? Instead of rendering Bob Ross with your hat selection, the frontend served no image at all. + + + +The number of hats you choose to display has no effect on the result. + + + + + + + +Observe a network request in your browser's developer tools: + +![The error response from your service](../../../images/pixie/scenario-1-network-response.png) + +Here, the response from the image request says, "This hat style does not exist! If you want this style - try submitting it." + +You know that the PIXIE hat style most certainly does exist because you chose it from the selector. But for some reason, the application can't render the image. + + + + + +For good measure, try to render a different hat style: + +![Your site successfully renders Bob Ross with cat ears](../../../images/pixie/cat-ears-bob.png) + +It worked! Your users were right. There's something wrong with your application. + + + + + +## Solve the mystery with Pixie + +The bad news is that you've confirmed there's an error in your application. The good news is that you recently instrumented your cluster with Pixie! Go to New Relic and sign into your account, if you haven't already. + + + + + +From the New Relic homepage, go to **Kubernetes**: + +![New Relic site with an arrow pointing to Kubernetes](../../../images/pixie/nr-kubernetes.png) + + + + + +Choose your **tiny-hat** cluster: + +![Arrow pointing to the tiny-hat cluster](../../../images/pixie/tiny-hat-cluster.png) + + + + + +Then click **Live Debugging with Pixie**: + +![Arrow pointing to the live-debugger](../../../images/pixie/live-debugging.png) + +This is Pixie's live, code-level debugger: + +![Pixie's live debugger](../../../images/pixie/cluster-script.png) + +You use it to drill down and learn more about the services in your cluster. + + + +When you go to the live debugger, you may see an error saying your cluster is disconnected. This is normal, as it takes a little while for New Relic to start seeing your Pixie data. Wait a few more minutes and refresh the debugger. + + + + + + + +Notice the **script** dropdown menu at the top of the debugger: + +![Pixie's script selector](../../../images/pixie/scripts.png) + +Pixie's live debugger renders data based on open source scripts written in PxL, [a proprietary scripting language](https://docs.pixielabs.ai/tutorials/pxl-scripts/write-pxl-scripts). The default script is `px/cluster`, which shows cluster-level information including: + +- A service graph +- Nodes +- Namespaces +- Services +- Pods + + + + + +Scroll down to see the error rates for your services: + +![A box showing high error rates for your services](../../../images/pixie/error-rates.png) + +Yikes! You have three services returning a high percentage of errors: + +- fetch-service +- frontend-service +- gateway-service + + + +If your data doesn't show any requests, switch back to your frontend, and try to render the PIXIE hat again a few times. + + + +You know that the website lives at `frontend-service`. You can reasonably rule this out as the culprit because you know it renders other hats just fine. That leaves two potential problem services: + +- `gateway-service` +- `fetch-service` + + + + + +To decide which service to look at first, scroll back up to the service graph: + +![Pixie service graph highlighting the erroring services](../../../images/pixie/order-of-operations.png) + +Here, you see that the `frontend-service` requests data from `gateway-service`. In turn, `gateway-service` requests data from `fetch-service`. So, following that order of operations, focus first on the `gateway-service`. + + + + + +Double click the **gateway-service** in your service graph to learn more about it: + +![Pixie's service view](../../../images/pixie/service-script.png) + +Notice that Pixie's live debugger has seamlessly replaced cluster data with service data by changing the script: + +![A box showing the service script is selected](../../../images/pixie/service-script-selector.png) + +You're now using the `px/service` script filtered down to `default/gateway-service`. In this service, you see a graph with the errors, but not much about where those errors are coming from. + + + + + +Click the script selector to switch to the `px/service_stats` script. Filter the **svc** to `default/gateway-service`: + +![Gateway service stats](../../../images/pixie/gateway-service-stats.png) + +This gives a better picture of the errors in your service. + +Scroll down to the **Incoming Traffic** and **Outgoing Traffic** tables: + +![Gateway service traffic](../../../images/pixie/traffic.png) + +The gateway service is returning the same amount of errors on inbound requests as it's receiving from outbound requests to the fetch service. This is a good indicator that you need to look at what's happening upstream. + +Switch to the `px/http_data_filtered` script, targeting `default/fetch-service` and requests with a 400 response status code: + +![400 responses in the fetch service](../../../images/pixie/fetch-400s.png) + + + +If you don't see any rows, switch back to your frontend, and try to render the PIXIE hat again a few times. + + + +Click on the row to learn more about requests to the fetch service that result in errors: + +![400 response details](../../../images/pixie/400-response-details.png) + +Here, you see that the request path looks like `/fetch?style=PIXIE&number=1`. This looks right, because the hat style you chose is called `PIXIE`. So if the fetch service is still returning 400s, something wrong is happening when it tries to find the hat. + +Switch to the `px/mysql_data` script and add a source filter for `default/fetch-service`: + +Many of these queries are returning no results. Click on one and look at the `req_body` to see the query: + +![Fetch PIXIE query](../../../images/pixie/query.png) + +<> + +```sql +SELECT * FROM main.images WHERE BINARY description='pixie' AND approve='true' +``` + + + +There's the problem! The `BINARY` type cast effectively makes the `WHERE` condition case sensitive. Since the hat's style is called `PIXIE`, this condition fails to find it. Now that you know, you can fix this query in your fetch service. + +## Summary + +To recap, you observed an error in your application and used Pixie in New Relic to: + +1. Understand your services' relationships +2. Review the error percentages for each of your services +3. Look at individual response bodies +4. Find a semantic error in a query within one of those services + +And you didn't even need to individually install agents in any of your services. Pixie was able to deliver all the information you needed! + + + + + + + +This lesson is part of a lab that teaches you how to monitor your Kubernetes cluster with Pixie. Next, try to [figure out why some APIs have high latency](/collect-data/pixie/scenario-2). + + \ No newline at end of file diff --git a/src/markdown-pages/collect-data/pixie-lab/scenario-2.mdx b/src/markdown-pages/collect-data/pixie-lab/scenario-2.mdx new file mode 100644 index 000000000..0fbe65971 --- /dev/null +++ b/src/markdown-pages/collect-data/pixie-lab/scenario-2.mdx @@ -0,0 +1,270 @@ +--- +path: '/collect-data/pixie/scenario-2' +duration: '45 min' +title: 'Debug latency in the TinyHat admin' +template: 'GuideTemplate' +description: 'Debug latency in the TinyHat admin' +procIdx: 5 +--- + + + +This procedure is part of a lab that teaches you how to monitor your Kubernetes cluster with Pixie. + +Each procedure in the lab builds upon the last, so make sure you've completed the last procedure, [_Debug missing images in your service_](/collect-data/pixie/scenario-1), before starting this one. + + + +After fixing your last TinyHat.me bug, you and your team are feeling confident. It's time to push your next release to production. + +Change to the `scenario-2` branch and set up your environment: + +<> + +```bash +git checkout scenario-2 +./setup.sh +[output] Switched to branch 'scenario-2' +[output] Please wait while we update your lab environment. +[output] deployment.apps/admin-service configured +[output] deployment.apps/fetch-service configured +[output] deployment.apps/frontend-service configured +[output] Done! +``` + + + +Now that you've released your new code, it's time to check Twitter for some user feedback: + +

+A Twitter thread showing some users can't access a hat +

+ +Oh no! The admin page seems to be slow. + +## Reproduce the issue + +You've been notified by your users that the TinyHat admin page is running slowly. Reproduce the issue for yourself. + + + + + +Look up your frontend's external IP address: + +<> + +```bash +kubectl get services +[output] NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +[output] add-service ClusterIP 10.109.114.34 80/TCP 20m +[output] admin-service ClusterIP 10.110.29.145 80/TCP 20m +[output] fetch-service ClusterIP 10.104.224.242 80/TCP 20m +[output] frontend-service LoadBalancer 10.102.82.89 {green}10.102.82.89 {normal}80:32161/TCP 20m +[output] gateway-service LoadBalancer 10.101.237.225 10.101.237.225 80:32469/TCP 20m +[output] kubernetes ClusterIP 10.96.0.1 443/TCP 20m +[output] manipulation-service ClusterIP 10.107.23.237 80/TCP 20m +[output] moderate-service ClusterIP 10.105.207.153 80/TCP 20m +[output] mysql ClusterIP 10.97.194.23 3306/TCP 20m +[output] upload-service ClusterIP 10.108.113.235 80/TCP 20m +``` + + + + + + + +Go to the admin page of your site: + +<> + +``` +/admin?password=ilovecats +``` + + + +![Admin](../../../images/pixie/admin.png) + +The page should load in between 5 and 10 seconds: + +![Browser console showing slow response times](../../../images/pixie/admin-delay.png) + +Once again, your users were right. Why is your admin site taking so long to load? Use Pixie to find out! + + + + + +## Hunt for the slowness with Pixie + +Five seconds is too long for your admin page to load. You can use Pixie to discover the source of the delay. + + + + + +From the New Relic homepage, go to **Kubernetes**: + +![New Relic site with an arrow pointing to Kubernetes](../../../images/pixie/nr-kubernetes.png) + + + + + +Choose your **tiny-hat** cluster: + +![Arrow pointing to the tiny-hat cluster](../../../images/pixie/tiny-hat-cluster.png) + + + + + + +Then click **Live Debugging with Pixie**: + +![Arrow pointing to the live-debugger](../../../images/pixie/live-debugging.png) + + + + + +Once again, the default script is `px/cluster`: + +![Pixie's cluster script](../../../images/pixie/scripts.png) + + + + + +Scroll down, and filter to the `default` namespace: + +![Filter to the default namespace](../../../images/pixie/default-namespace.png) + +This shows the same information as the cluster script, but only for the `default` namespace. + + + +If you don't see any data, switch back to your admin tab and refresh. + + + + + + + +Observe the service map for this namespace: + +![Service map for the default namespace](../../../images/pixie/default-service-map.png) + +Here, you see the first exernal request comes to the frontend service. The frontend service makes an internal call to the gateway service, which then calls the admin service. Use your knowledge of this traffic flow to debug your high latency issue. + + + + + +Scroll down to the **Service List** and notice some of the high latency in your services: + +![High service latency](../../../images/pixie/high-latency.png) + + + + + +Because you know the first service to receive requests is the frontend service, click on its name in the table: + +![Frontend service latency](../../../images/pixie/frontend-service-latency.png) + +Here, you see spikes in latency over time: + +![Frontend latency over time](../../../images/pixie/frontend-latency-spikes.png) + +But this only shows you the symptom of an underlying problem. To figure out why your frontend service is slow, you need to dig a little deeper. + + + + + +Scroll down to see the **Sample Of Slow Requests** table: + +![Sample of slow requests table](../../../images/pixie/sample-of-slow-requests.png) + +Here, you see that the frontend's requests to `/api/admin` are slow. That's a promising lead! + +Now that you know that the admin service is the cause of your app latency, it's time to investigate that service. One good place to look when you want to know what's causing slowness in your application is a performance flamegraph for that service. + +Pixie's performance flamegraph shows the frequency that every function appears in stack traces across your CPUs. Pixie collects this data with a feature called [continuous application profiling](https://docs.pixielabs.ai/tutorials/pixie-101/profiler/) where it samples stack traces, aggregates those samples, and then groups the aggregates in the flamegraph. + + + + + +Change to the `px/perf_flamegraph` script to see it in action: + +![Flamegraph script](../../../images/pixie/flamegraph-script.png) + +This graph is complex and, by default, shows more information than you need. + + + + + +Filter down to the `default` namespace: + +![Filter your flamegraph to the default namespace](../../../images/pixie/flamegraph-default-namespace.png) + +That's better! The graph is now filtered to show two specific services: + +- `manipulation-service` +- `admin-service` + +![Flamegraph services](../../../images/pixie/flamegraph-services.png) + +Focus on the admin service because that's what recieves requests from the frontend. + + + + + +Notice the two function cells from your admin service's main module: + +![Main module functions in your flamegraph](../../../images/pixie/two-functions.png) + +Remember, the width of the cell represents the frequency that the cell's function appears in Pixie's samples. Effectively, this represents how long the application spends in that function. + +In this graph, you can see that `main.createSampleImage` is wider than `main.setupSQLQuery`, implying that the admin service spends more time creating sample images than it does setting up its SQL query. You can also infer from the name of the function and the width of the `manipulation-service` in the graph that `main.createSampleImage` makes a call to that service. + +Based on your findings, you hypothesize that every time the frontend service calls `/api/admin`, the `main.createSampleImage` function gets executed, which sends a request to the manipulation service. + +In a real world situation, you could confirm this hypothesis by looking at your code. You could then solve the problem by caching sample images so that they're not created on every request. + + + + + + +## Summary + +To recap, you observed latency in your application's administrative backend and used Pixie in New Relic to: + +1. Understand your services' relationships +2. Narrow down the sources of high latency +3. Discover the function that's causing long response times + +Throughout this lab, you saw just a few ways that Pixie can help you monitor your software and debug issues when they come up. + +## Tear down + +Now that you're done with this lab, tear down your cluster: + +```bash +minikube delete +``` + +## Homework + +Well done! Now that you've gotten your feet wet using Pixie and New Relic to monitor your Kubernetes clusters, here are some things you can do on your own as you prepare to apply these learnings in real-life: + +- Read [Auto-telemetry with Pixie for instant Kubernetes observability](https://docs.newrelic.com/docs/auto-telemetry-pixie/get-started-auto-telemetry-pixie/) to learn more about auto-telemetry with New Relic + Pixie +- Read [Pixie's documentation](https://docs.pixielabs.ai/) to learn more about the inner workings of Pixie \ No newline at end of file diff --git a/src/markdown-pages/collect-data/pixie-lab/set-up-env.mdx b/src/markdown-pages/collect-data/pixie-lab/set-up-env.mdx new file mode 100644 index 000000000..6a66cac29 --- /dev/null +++ b/src/markdown-pages/collect-data/pixie-lab/set-up-env.mdx @@ -0,0 +1,143 @@ +--- +path: '/collect-data/pixie/set-up-env' +duration: '10 min' +title: 'Set up your TinyHat environment' +template: 'GuideTemplate' +description: 'Before you begin, set up a local minikube cluster that runs your TinyHat application.' +procIdx: 1 +--- + + + +This procedure is part of a lab that teaches you how to monitor your Kubernetes cluster with Pixie. If you haven't already, check out the [lab introduction](/collect-data/pixie). + + + +Before you learn how you can use Pixie to monitor your Kubernetes clusters, you need to set one up. We've designed a cluster for you to use in this lab. + + + + + +Clone `pixie-lab-materials` from GitHub: + +<> + +```bash +git clone https://github.com/newrelic-experimental/pixie-lab-materials +``` + + + +This repository contains all the code you need to complete this lab. + + + + + +Start a new minikube cluster: + +<> + +```bash +minikube start --driver=hyperkit --cni=flannel --cpus=4 --memory=8000 +[output] 😄 minikube v1.13.1 on Darwin 11.6 +[output] ✨ Using the hyperkit driver based on user configuration +[output] 👍 Starting control plane node minikube in cluster minikube +[output] 🔥 Creating hyperkit VM (CPUs=4, Memory=8000MB, Disk=20000MB) ... +[output] 🐳 Preparing Kubernetes v1.19.2 on Docker 19.03.12 ... +[output] 🔗 Configuring Flannel (Container Networking Interface) ... +[output] 🔎 Verifying Kubernetes components... +[output] 🌟 Enabled addons: default-storageclass, storage-provisioner +[output] 🏄 Done! kubectl is now configured to use "minikube" by default +``` + + + +This uses the pixie-supported hyperkit driver and configures your network memory and CPU with limits that will work well with this lab. + + + +If you're using linux, use the `kvm2` driver instead. If you're missing any dependencies, see the [requirements](/collect-data/pixie/#requirements) in the lab overview for installation links. + + + + + + + +From the root directory and the main branch of your `pixie-lab-materials` repo, apply your manifests: + +<> + +```bash +cd pixie-lab-materials +git checkout main +kubectl apply -f kube +[output] Already on 'main' +[output] Your branch is up to date with 'origin/main'. +[output] deployment.apps/add-service created +[output] service/add-service created +[output] deployment.apps/admin-service created +[output] service/admin-service created +[output] deployment.apps/fetch-service created +[output] service/fetch-service created +[output] deployment.apps/frontend-service created +[output] service/frontend-service created +[output] deployment.apps/gateway-service created +[output] service/gateway-service created +[output] deployment.apps/manipulation-service created +[output] service/manipulation-service created +[output] deployment.apps/moderate-service created +[output] service/moderate-service created +[output] service/mysql created +[output] deployment.apps/mysql created +[output] persistentvolume/mysql-pv-volume created +[output] persistentvolumeclaim/mysql-pv-claim created +[output] configmap/mysql-initdb-config created +[output] deployment.apps/upload-service created +[output] service/upload-service created +``` + + + +This deploys your TinyHat application into your minikube cluster. + + + + + +In a new terminal window, open a minikube tunnel: + +<> + +```bash +minikube tunnel +``` + + + +This exposes the load balancer services in your cluster so you can access them. + + + +You may have to provide your user's password to allow minikube to work. + + + + + + + +You should now have two terminal windows: + +- One that contains your tunnel. This needs to remain open as long as you use your website. +- One for running the rest of the commands in this lab + +In the next procedure, you familiarize yourself with your cluster and website. + + + +This procedure is part of a lab that teaches you how to monitor your Kubernetes cluster with Pixie. Now that you've set up your environment, [explore your cluster](/collect-data/pixie/explore-cluster). + + diff --git a/src/templates/LabOverviewTemplate.js b/src/templates/LabOverviewTemplate.js index 7aca24272..56bfb3c64 100644 --- a/src/templates/LabOverviewTemplate.js +++ b/src/templates/LabOverviewTemplate.js @@ -6,7 +6,8 @@ import PageLayout from '../components/PageLayout'; import MDXContainer from '../components/MDXContainer'; import GuideListing from '../components/GuideListing/GuideListing'; import GuideTile from '../components/GuideTile/GuideTile'; -import * as styles from './LabOverviewTemplate.module.scss'; +import * as overviewStyles from './OverviewTemplate.module.scss'; +import * as labOverviewStyles from './LabOverviewTemplate.module.scss'; import DevSiteSeo from '../components/DevSiteSeo'; @@ -14,6 +15,7 @@ const LabOverviewTemplate = ({ data, location }) => { const { mdx, guides } = data; const { frontmatter, body } = mdx; const { title, description } = frontmatter; + console.log(guides?.nodes) function sortProcedures(a, b) { if (a.frontmatter.procIdx > b.frontmatter.procIdx) { @@ -36,20 +38,19 @@ const LabOverviewTemplate = ({ data, location }) => { {body} {!!guides?.nodes.length && ( <> -

Procedures

- - +

Procedures

+ + {guides?.nodes .sort(sortProcedures) .map(({ frontmatter }, index) => (