Skip to content

Commit 9fe16a1

Browse files
aojeaaojea
committed
first commit
1 parent 59e4ba5 commit 9fe16a1

18 files changed

+2784
-2
lines changed

‎.github/workflows/e2e.yml

+165
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
name: e2e
2+
3+
on:
4+
push:
5+
branches:
6+
- 'main'
7+
tags:
8+
- 'v*'
9+
pull_request:
10+
branches: [ main ]
11+
workflow_dispatch:
12+
13+
env:
14+
GO_VERSION: "1.22.0"
15+
K8S_VERSION: "v1.29.2"
16+
KIND_VERSION: "v0.22.0"
17+
REGISTRY: ghcr.io
18+
IMAGE_NAME: registry.k8s.io/kube-netpol
19+
KIND_CLUSTER_NAME: kind
20+
21+
permissions: write-all
22+
23+
jobs:
24+
build:
25+
name: build
26+
runs-on: ubuntu-latest
27+
steps:
28+
- name: Set up Go
29+
uses: actions/setup-go@v2
30+
with:
31+
go-version: ${{ env.GO_VERSION }}
32+
id: go
33+
34+
- name: Check out code
35+
uses: actions/checkout@v2
36+
37+
- name: Build
38+
run: |
39+
docker build -t registry.k8s.io/kube-netpol:test -f Dockerfile .
40+
mkdir _output
41+
docker save registry.k8s.io/kube-netpol:test > _output/kube-netpol-image.tar
42+
43+
- uses: actions/upload-artifact@v2
44+
with:
45+
name: test-image
46+
path: _output/kube-netpol-image.tar
47+
48+
e2e:
49+
name: e2e
50+
runs-on: ubuntu-22.04
51+
timeout-minutes: 100
52+
needs:
53+
- build
54+
strategy:
55+
fail-fast: false
56+
matrix:
57+
# TODO add "dual", waiting on KEP https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/3705-cloud-node-ips
58+
ipFamily: ["ipv4", "ipv6"]
59+
env:
60+
JOB_NAME: "kube-netpol-${{ matrix.ipFamily }}"
61+
IP_FAMILY: ${{ matrix.ipFamily }}
62+
steps:
63+
- name: Check out code
64+
uses: actions/checkout@v2
65+
66+
- name: Enable ipv4 and ipv6 forwarding
67+
run: |
68+
sudo sysctl -w net.ipv6.conf.all.forwarding=1
69+
sudo sysctl -w net.ipv4.ip_forward=1
70+
71+
- name: Set up environment (download dependencies)
72+
run: |
73+
TMP_DIR=$(mktemp -d)
74+
# Test binaries
75+
curl -L https://dl.k8s.io/${{ env.K8S_VERSION }}/kubernetes-test-linux-amd64.tar.gz -o ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz
76+
tar xvzf ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz \
77+
--directory ${TMP_DIR} \
78+
--strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test
79+
# kubectl
80+
curl -L https://dl.k8s.io/${{ env.K8S_VERSION }}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl
81+
# kind
82+
curl -Lo ${TMP_DIR}/kind https://kind.sigs.k8s.io/dl/${{ env.KIND_VERSION }}/kind-linux-amd64
83+
# Install
84+
sudo cp ${TMP_DIR}/ginkgo /usr/local/bin/ginkgo
85+
sudo cp ${TMP_DIR}/e2e.test /usr/local/bin/e2e.test
86+
sudo cp ${TMP_DIR}/kubectl /usr/local/bin/kubectl
87+
sudo cp ${TMP_DIR}/kind /usr/local/bin/kind
88+
sudo chmod +x /usr/local/bin/*
89+
90+
- name: Create multi node cluster
91+
run: |
92+
# output_dir
93+
mkdir -p _artifacts
94+
# create cluster
95+
cat <<EOF | /usr/local/bin/kind create cluster \
96+
--name ${{ env.KIND_CLUSTER_NAME}} \
97+
--image kindest/node:${{ env.K8S_VERSION }} \
98+
-v7 --wait 1m --retain --config=-
99+
kind: Cluster
100+
apiVersion: kind.x-k8s.io/v1alpha4
101+
networking:
102+
ipFamily: ${IP_FAMILY}
103+
nodes:
104+
- role: control-plane
105+
- role: worker
106+
- role: worker
107+
EOF
108+
# dump the kubeconfig for later
109+
/usr/local/bin/kind get kubeconfig --name ${{ env.KIND_CLUSTER_NAME}} > _artifacts/kubeconfig.conf
110+
111+
- uses: actions/download-artifact@v2
112+
with:
113+
name: test-image
114+
115+
- name: Install kube-netpol
116+
run: |
117+
# preload kube-netpol image
118+
docker load --input kube-netpol-image.tar
119+
/usr/local/bin/kind load docker-image registry.k8s.io/kube-netpol:test --name ${{ env.KIND_CLUSTER_NAME}}
120+
sed -i s#registry.k8s.io/kube-netpol.*#registry.k8s.io/kube-netpol:test# install.yaml
121+
/usr/local/bin/kubectl apply -f ./install.yaml
122+
123+
- name: Get Cluster status
124+
run: |
125+
# wait network is ready
126+
sleep 5
127+
/usr/local/bin/kubectl get nodes -o wide
128+
/usr/local/bin/kubectl get pods -A
129+
/usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns
130+
/usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l app=kube-network-policies
131+
132+
- name: Run tests
133+
run: |
134+
export KUBERNETES_CONFORMANCE_TEST='y'
135+
export E2E_REPORT_DIR=${PWD}/_artifacts
136+
137+
# Run tests
138+
/usr/local/bin/ginkgo --nodes=25 \
139+
--focus="Netpol" \
140+
/usr/local/bin/e2e.test \
141+
-- \
142+
--kubeconfig=${PWD}/_artifacts/kubeconfig.conf \
143+
--provider=local \
144+
--dump-logs-on-failure=false \
145+
--report-dir=${E2E_REPORT_DIR} \
146+
--disable-log-dump=true
147+
148+
- name: Upload Junit Reports
149+
if: always()
150+
uses: actions/upload-artifact@v2
151+
with:
152+
name: kind-junit-${{ env.JOB_NAME }}-${{ github.run_id }}
153+
path: './_artifacts/*.xml'
154+
155+
- name: Export logs
156+
if: always()
157+
run: |
158+
/usr/local/bin/kind export logs --name ${KIND_CLUSTER_NAME} --loglevel=debug ./_artifacts/logs
159+
160+
- name: Upload logs
161+
if: always()
162+
uses: actions/upload-artifact@v2
163+
with:
164+
name: kind-logs-${{ env.JOB_NAME }}-${{ github.run_id }}
165+
path: ./_artifacts/logs

‎Dockerfile

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
ARG GOARCH="amd64"
2+
FROM golang:1.22 AS builder
3+
WORKDIR /src
4+
COPY . .
5+
# build
6+
RUN go mod download
7+
RUN CGO_ENABLED=0 go build -o /go/bin/netpol ./cmd
8+
# STEP 2: Build small image
9+
FROM registry.k8s.io/build-image/distroless-iptables:v0.5.2
10+
COPY --from=builder --chown=root:root /go/bin/netpol /bin/netpol
11+
CMD ["/bin/netpol"]

‎README.md

+38-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,42 @@
1-
# kube-network-policies
1+
# Kubernetes network policies
2+
3+
Network policies are hard to implement efficiently and in large clusters this is translated to performance and scalability problems.
4+
5+
Most of the existing implementations use the same approach of processing the APIs and transforming them in the corresponding dataplane implementation: iptables, nftables, ebpf or ovs, ...
6+
7+
This project takes a different approach. It uses the NFQUEUE functionality implemented in netfilter to process the first packet of each connection in userspace and emit a verdict. The advantage is that the dataplane implementation does not need to represent all the complex logic, allowing it to scale better. The disadvantage is that we need to pass each new connection packet through userspace.
8+
9+
There are some performance improvements that can be applied, such as to restrict in the dataplane the packets that are sent to userspace to the ones that have network policies only, so only
10+
the Pods affected by network policies will hit the first byte performance.
11+
12+
## Metrics
13+
14+
Prometheus metrics are exposed on the address defined by the flag
15+
16+
```
17+
-metrics-bind-address string
18+
The IP address and port for the metrics server to serve on (default ":9080")
19+
```
20+
21+
Current implemented metrics are:
22+
23+
* packet_process_time: Time it has taken to process each packet (microseconds)
24+
* packet_process_duration_microseconds: A summary of the packet processing durations in microseconds
25+
* packet_count: Number of packets
26+
* nfqueue_queue_total: The number of packets currently queued and waiting to be processed by the application
27+
* nfqueue_queue_dropped: Number of packets that had to be dropped by the kernel because too many packets are already waiting for user space to send back the mandatory accept/drop verdicts
28+
* nfqueue_user_dropped: Number of packets that were dropped within the netlink subsystem. Such drops usually happen when the corresponding socket buffer is full; that is, user space is not able to read messages fast enough
29+
* nfqueue_packet_id: ID of the most recent packet queued
30+
31+
## Testing
32+
33+
See [.docs/testing/README.md]
34+
35+
## References
36+
37+
* https://home.regit.org/netfilter-en/using-nfqueue-and-libnetfilter_queue/
38+
* https://netfilter.org/projects/libnetfilter_queue/doxygen/html/
239

3-
Kubernetes network policies
440

541
## Community, discussion, contribution, and support
642

‎cmd/main.go

+101
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"flag"
6+
"fmt"
7+
"net"
8+
"net/http"
9+
"os"
10+
"os/signal"
11+
"time"
12+
13+
"github.com/prometheus/client_golang/prometheus/promhttp"
14+
"golang.org/x/sys/unix"
15+
"sigs.k8s.io/kube-network-policies/pkg/networkpolicy"
16+
17+
"k8s.io/client-go/informers"
18+
"k8s.io/client-go/kubernetes"
19+
"k8s.io/client-go/rest"
20+
"k8s.io/klog/v2"
21+
)
22+
23+
var (
24+
failOpen bool
25+
queueID int
26+
metricsBindAddress string
27+
)
28+
29+
func init() {
30+
flag.BoolVar(&failOpen, "fail-open", false, "If set, don't drop packets if the controller is not running")
31+
flag.IntVar(&queueID, "nfqueue-id", 100, "Number of the nfqueue used")
32+
flag.StringVar(&metricsBindAddress, "metrics-bind-address", ":9080", "The IP address and port for the metrics server to serve on")
33+
34+
flag.Usage = func() {
35+
fmt.Fprint(os.Stderr, "Usage: kube-netpol [options]\n\n")
36+
flag.PrintDefaults()
37+
}
38+
}
39+
40+
func main() {
41+
// enable logging
42+
klog.InitFlags(nil)
43+
flag.Parse()
44+
//
45+
if _, _, err := net.SplitHostPort(metricsBindAddress); err != nil {
46+
klog.Fatalf("error parsing metrics bind address %s : %v", metricsBindAddress, err)
47+
}
48+
49+
cfg := networkpolicy.Config{
50+
FailOpen: failOpen,
51+
QueueID: queueID,
52+
}
53+
// creates the in-cluster config
54+
config, err := rest.InClusterConfig()
55+
if err != nil {
56+
panic(err.Error())
57+
}
58+
// creates the clientset
59+
clientset, err := kubernetes.NewForConfig(config)
60+
if err != nil {
61+
panic(err.Error())
62+
}
63+
64+
// trap Ctrl+C and call cancel on the context
65+
ctx := context.Background()
66+
ctx, cancel := context.WithCancel(ctx)
67+
68+
// Enable signal handler
69+
signalCh := make(chan os.Signal, 2)
70+
defer func() {
71+
close(signalCh)
72+
cancel()
73+
}()
74+
signal.Notify(signalCh, os.Interrupt, unix.SIGINT)
75+
76+
informersFactory := informers.NewSharedInformerFactory(clientset, 0)
77+
78+
http.Handle("/metrics", promhttp.Handler())
79+
go http.ListenAndServe(metricsBindAddress, nil)
80+
81+
networkPolicyController := networkpolicy.NewController(
82+
clientset,
83+
informersFactory.Networking().V1().NetworkPolicies(),
84+
informersFactory.Core().V1().Namespaces(),
85+
informersFactory.Core().V1().Pods(),
86+
cfg,
87+
)
88+
go networkPolicyController.Run(ctx)
89+
90+
informersFactory.Start(ctx.Done())
91+
92+
select {
93+
case <-signalCh:
94+
klog.Infof("Exiting: received signal")
95+
cancel()
96+
case <-ctx.Done():
97+
}
98+
99+
// grace period to cleanup resources
100+
time.Sleep(5 * time.Second)
101+
}

0 commit comments

Comments
 (0)