feat: add basic implementation of ddsketch (#1)

This adds a basic implementation of the [`sketches-go`](https://github.com/DataDog/sketches-go) library needed for data streams to work.

.doctor.exs

@@ -0,0 +1,18 @@
+%Doctor.Config{
+  ignore_modules: [
+    ~r/Enumerable/,
+    ~r/Datadog.Sketch.Protobuf/
+  ],
+  ignore_paths: [],
+  min_module_doc_coverage: 40,
+  min_module_spec_coverage: 0,
+  min_overall_doc_coverage: 50,
+  min_overall_spec_coverage: 0,
+  moduledoc_required: true,
+  exception_moduledoc_required: true,
+  raise: true,
+  reporter: Doctor.Reporters.Full,
+  struct_type_spec_required: true,
+  umbrella: false,
+  failed: false
+}

.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true

.formatter.exs

@@ -0,0 +1,4 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+]

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @stordco/core-platform

.github/workflows/ci.yml

@@ -0,0 +1,115 @@
+---
+name: CI
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+  push:
+    branches:
+      - main
+
+jobs:
+  Credo:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Elixir
+        uses: stordco/actions-elixir/setup@v1
+        with:
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          hex-token: ${{ secrets.HEX_API_KEY }}
+
+      - name: Credo
+        run: mix credo
+
+  Dialyzer:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Elixir
+        uses: stordco/actions-elixir/setup@v1
+        with:
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          hex-token: ${{ secrets.HEX_API_KEY }}
+
+      - name: Dialyzer
+        run: mix dialyzer
+
+  Doctor:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Elixir
+        uses: stordco/actions-elixir/setup@v1
+        with:
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          hex-token: ${{ secrets.HEX_API_KEY }}
+
+      - name: Doctor
+        run: mix doctor
+
+  Format:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Elixir
+        uses: stordco/actions-elixir/setup@v1
+        with:
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          hex-token: ${{ secrets.HEX_API_KEY }}
+
+      - name: Format
+        run: mix format --check-formatted
+
+  Test:
+    name: Test (Elixir ${{ matrix.versions.elixir }} OTP ${{ matrix.versions.otp }})
+    runs-on: ubuntu-20.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        versions:
+          - elixir: 1.11
+            otp: 23
+          - elixir: 1.12
+            otp: 24
+          - elixir: 1.13
+            otp: 25
+          - elixir: 1.14
+            otp: 25
+
+    env:
+      MIX_ENV: test
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Elixir
+        uses: stordco/actions-elixir/setup@v1
+        with:
+          elixir-version: ${{ matrix.versions.elixir }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          hex-token: ${{ secrets.HEX_API_KEY }}
+          otp-version: ${{ matrix.versions.otp }}
+
+      - name: Compile
+        run: mix compile --warnings-as-errors
+
+      - name: Test
+        run: mix test --cover

.github/workflows/pr.yml

@@ -0,0 +1,30 @@
+---
+name: PR
+
+on:
+  pull_request:
+    types:
+      - edited
+      - opened
+      - reopened
+      - synchronize
+
+jobs:
+  title:
+    name: Check Title
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Check
+        uses: stordco/[email protected]
+        with:
+          regex: '^(feat!|fix!|feat|fix|chore):\s.*'
+          hint: |
+            You can pass the following formats:
+
+            feat: some title of the PR
+            fix: some title of the PR
+            chore: update some action

.github/workflows/publish.yml

@@ -0,0 +1,29 @@
+---
+name: Publish
+
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  Hex:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Elixir
+        uses: stordco/actions-elixir/setup@v1
+        with:
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          hex-token: ${{ secrets.HEX_API_KEY }}
+
+      - name: Compile
+        run: mix compile --docs
+
+      - name: Publish
+        run: mix hex.publish --yes
+        env:
+          HEX_API_KEY: ${{ secrets.HEX_API_KEY }}

.github/workflows/release.yml

@@ -0,0 +1,20 @@
+---
+name: Release
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  please:
+    name: Please
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: google-github-actions/release-please-action@v3
+        with:
+          release-type: elixir
+          package-name: dd_data_streams
+          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          extra-files: README.md

.gitignore

@@ -0,0 +1,67 @@
+# Created by https://www.toptal.com/developers/gitignore/api/elixir,phoenix,vim,macosx
+# Edit at https://www.toptal.com/developers/gitignore?templates=elixir,phoenix,vim,macosx
+
+### Elixir ###
+/_build
+/cover
+/deps
+/doc
+/.fetch
+erl_crash.dump
+*.ez
+*.beam
+/config/*.secret.exs
+.elixir_ls/
+
+### Elixir Patch ###
+
+#!! ERROR: macosx is undefined. Use list command to see defined gitignore types !!#
+
+### Phoenix ###
+# gitignore template for Phoenix projects
+# website: http://www.phoenixframework.org/
+#
+# Recommended template: Elixir.gitignore
+
+# Temporary files
+/tmp
+
+# Static artifacts
+/node_modules
+/assets/node_modules
+
+# Since we are building assets from web/static,
+# we ignore priv/static. You may want to comment
+# this depending on your deployment strategy.
+/priv/static/
+
+# Installer-related files
+/installer/_build
+/installer/tmp
+/installer/doc
+/installer/deps
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+# End of https://www.toptal.com/developers/gitignore/api/elixir,phoenix,vim,macosx
+
+.vscode

.tool-versions

@@ -0,0 +1,2 @@
+elixir 1.14.3
+erlang 25.2.3

CHANGELOG.md

@@ -0,0 +1 @@
+# Changelog

README.md

@@ -1,2 +1,69 @@
-# data-streams-ex
-DataDog data streams library for Elixir
+# Data Streams Ex
+
+This is a port of the [data-streams-go](https://github.com/DataDog/data-streams-go) library to Elixir.
+
+## Introduction
+
+This product is meant to measure end to end latency in async pipelines. It's in an Alpha phase. We currently support instrumentations of async pipelines using Kafka. Integrations with other systems will follow soon.
+
+## Glossary
+
+- Data stream: A set of services connected together via *queues*
+- Pathway: A single branch of connected services
+- Queue: A connection between two services
+- Edge latency: Latency of a queue between two services
+- Latency from origin: Latency from the first tracked service, down to the current service
+- Checkpoint: records at what time a specific operation on a payload occurred (eg: The payload was sent to Kafka). The product can then measure latency between checkpoints.
+
+The product can measure edge latency, and latency from origin, for a set of checkpoints connected together via queues.
+To do so, we propagate timestamps, and a hash of the path that messages took with the payload.
+
+## Installation
+
+Just add [`dd_data_streams`](https://hexdocs.pm/dd_data_streams) to your `mix.exs` file like so:
+
+<!-- {x-release-please-start-version} -->
+```elixir
+def deps do
+  [
+    {:dd_data_streams, "~> 0.1.0"}
+  ]
+end
+```
+<!-- {x-release-please-end} -->
+
+Documentation is automatically generated and published to [HexDocs](https://hexdocs.pm/dd_data_streams).
+
+## Elixir instrumentation
+
+**Prerequisites**
+- Datadog Agent 7.34.0+
+- latest version of [the data streams library](https://github.com/stordco/data-streams-ex)
+
+You will need to configure the pipeline with the trace agent URL and enable it to start on application start. This can be done via your `config/` files:
+
+```elixir
+config :dd_data_streams, :pipeline,
+  enabled?: true,
+  host: "localhost",
+  port: 8126
+```
+
+The host and port should point to your Datadog agent.
+
+The instrumentation relies on creating checkpoints at various points in your data stream services with specific tags, recording the pathway that messages take along the way. For a complete picture of your services, you will also need to configure some metadata about the current service running. You can do this in your `config/` files as well:
+
+```elixir
+config :dd_data_streams, :metadata,
+  service: "my-service",
+  env: "production",
+  primary_tag: "datacenter:d1" # You can leave this blank if you don't have a primary tag.
+```
+
+We recommend you keep these tags matching all your other instrumentation, like Open Telemetry and `:telemetry`, to ensure Datadog can aggregate data accurately.
+
+### Kafka
+
+To instrument your data stream services that use Kafka queues, you can use the `Datadog.DataStreams.Integrations.Kafka` module.
+
+> **NOTE**: TODO

benchmarks/datadog/sketch/store.exs

@@ -0,0 +1,16 @@
+inputs = %{
+  "small" => 1..100 |> Enum.to_list() |> Enum.shuffle(),
+  "medium" => 1..10_000 |> Enum.to_list() |> Enum.shuffle(),
+  "large" => 1..1_000_000 |> Enum.to_list() |> Enum.shuffle()
+}
+
+Benchee.run(
+  %{
+    "dense" => fn list ->
+      Enum.reduce(list, Datadog.Sketch.Store.Dense.new(), fn i, store ->
+        Datadog.Sketch.Store.Dense.add(store, i)
+      end)
+    end,
+  },
+  inputs: inputs
+)