This repository has been archived by the owner on Oct 29, 2021. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 142
/
nsm-init.go
297 lines (270 loc) · 11.3 KB
/
nsm-init.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"flag"
"fmt"
"os"
"path"
"strings"
"time"
"github.com/ligato/networkservicemesh/pkg/nsm/apis/common"
"github.com/ligato/networkservicemesh/pkg/nsm/apis/netmesh"
"github.com/ligato/networkservicemesh/pkg/nsm/apis/nsmconnect"
"github.com/ligato/networkservicemesh/pkg/tools"
"github.com/ligato/networkservicemesh/plugins/nsmserver"
"github.com/sirupsen/logrus"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
)
const (
// clientConnectionTimeout defines time the client waits for establishing connection with the server
clientConnectionTimeout = time.Second * 60
// clientConnectionTimeout defines retry interval for establishing connection with the server
clientConnectionRetry = time.Second * 2
// location of network namespace for a process
netnsfile = "/proc/self/ns/net"
// MaxSymLink is maximum length of Symbolic Link
MaxSymLink = 8192
)
var (
clientSocketPath = path.Join(nsmserver.SocketBaseDir, nsmserver.ServerSock)
clientSocketUserPath = flag.String("nsm-socket", "", "Location of NSM process client access socket")
configMapName = flag.String("configmap-name", "", "Name of a ConfigMap with requested configuration.")
kubeconfig = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Either this or master needs to be set if the provisioner is being run out of cluster.")
)
type networkService struct {
Name string `json:"name" yaml:"name"`
ServiceInterface []*common.Interface `json:"serviceInterface" yaml:"serviceInterface"`
}
func checkClientConfigMap(name, namespace string, k8s kubernetes.Interface) (*v1.ConfigMap, error) {
return k8s.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{})
}
func buildClient() (*kubernetes.Clientset, error) {
var config *rest.Config
var err error
kubeconfigEnv := os.Getenv("KUBECONFIG")
if kubeconfigEnv != "" {
kubeconfig = &kubeconfigEnv
}
if *kubeconfig != "" {
config, err = clientcmd.BuildConfigFromFlags("", *kubeconfig)
} else {
config, err = rest.InClusterConfig()
}
if err != nil {
return nil, err
}
k8s, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
return k8s, nil
}
func parseConfigMap(cm *v1.ConfigMap) ([]*networkService, error) {
nSs := make([]*networkService, 0)
rawData, ok := cm.Data["networkService"]
if !ok {
return nil, fmt.Errorf("missing required key 'networkService:'")
}
sr := strings.NewReader(rawData)
decoder := yaml.NewYAMLOrJSONDecoder(sr, 512)
if err := decoder.Decode(&nSs); err != nil {
logrus.Errorf("decoding %+v failed with error: %v", rawData, err)
return nil, err
}
return nSs, nil
}
func main() {
flag.Parse()
flag.Set("logtostderr", "true")
// Building kubernetes client
k8s, err := buildClient()
if err != nil {
logrus.Errorf("nsm client: fail to build kubernetes client with error: %+v, exiting...", err)
os.Exit(1)
}
// Checking presence of client's ConfigMap, if it is not provided, then init container gracefully exits
if *configMapName == "" {
logrus.Info("nsm client: no client's configmap name was provided, exiting...")
os.Exit(0)
}
name := *configMapName
// POD's namespace is taken from env variable, which must be either defined via downward api for in-cluster case
// or explicitely exported for out-of-cluster case.
namespace := os.Getenv("INIT_NAMESPACE")
if namespace == "" {
logrus.Error("nsm client: cannot detect namespace, make sure INIT_NAMESPACE variable is set via downward api, exiting...")
os.Exit(1)
}
podName := os.Getenv("HOSTNAME")
// podUID is used as a unique identifier for nsm init process, it will stay the same throughout life of
// pod and will guarantee idempotency of possible repeated requests to NSM
pod, err := k8s.CoreV1().Pods(namespace).Get(podName, metav1.GetOptions{})
if err != nil {
logrus.Errorf("nsm client: failure to get pod %s/%s with error: %+v, exiting...", namespace, podName, err)
os.Exit(1)
}
podUID := string(pod.GetUID())
configMap, err := checkClientConfigMap(name, namespace, k8s)
if err != nil {
logrus.Errorf("nsm client: failure to access client's configmap at %s/%s with error: %+v, exiting...", namespace, name, err)
os.Exit(1)
}
// Attempting to extract Client's config from the config map and store it in networkService slice
networkServices, err := parseConfigMap(configMap)
if err != nil {
logrus.Errorf("nsm client: failure to parse client's configmap %s/%s with error: %+v, exiting...", namespace, name, err)
os.Exit(1)
}
// Checking number of NetworkServices extracted from the config map and if it is 0 then
// print log message and gracefully exit
if len(networkServices) == 0 {
logrus.Infof("nsm client: no NetworkServices were discovered in client's configmap %s/%s, exiting...", namespace, name)
os.Exit(0)
}
// Checking if nsm client socket exists and of not crash init container
clientSocket := clientSocketPath
if clientSocketUserPath != nil {
clientSocket = *clientSocketUserPath
}
if _, err := os.Stat(clientSocket); err != nil {
logrus.Errorf("nsm client: failure to access nsm socket at %s with error: %+v, exiting...", clientSocket, err)
os.Exit(1)
}
conn, err := tools.SocketOperationCheck(clientSocket)
if err != nil {
logrus.Fatalf("nsm client: failure to communicate with the socket %s with error: %+v", clientSocket, err)
}
defer conn.Close()
logrus.Infof("nsm client: connection to nsm server on socket: %s succeeded.", clientSocket)
// Init related activities start here
nsmClient := nsmconnect.NewClientConnectionClient(conn)
// Getting list of available NetworkServices on local NSM server
availablaNetworkServices, err := getNetworkServices(nsmClient)
if err != nil {
logrus.Fatalf("nsm client: failed to get a list of NetworkServices from NSM with error: %+v, exiting...", err)
os.Exit(1)
}
if len(availablaNetworkServices) == 0 {
// Since local NSM has no any NetworkServices, then there is nothing to configure for the client
logrus.Info("nsm client: Local NSM does not have any NetworkServices, exiting...")
os.Exit(0)
}
logrus.Info("nsm client: list of discovered network services:")
for _, s := range availablaNetworkServices {
logrus.Infof(" network service: %s/%s", s.Metadata.Namespace, s.Metadata.Name)
for _, c := range s.Channel {
logrus.Infof(" Channel: %s/%s", c.Metadata.Namespace, c.Metadata.Name)
for _, i := range c.Interface {
logrus.Infof(" Interface type: %s preference: %s", i.GetType(), i.GetPreference())
}
}
}
logrus.Infof("nsm client: %d NetworkServices discovered from Local NSM.", len(availablaNetworkServices))
// For NSM to program container's dataplane, container's linux namespace must be sent to NSM
linuxNS, err := tools.GetCurrentNS()
if err != nil {
logrus.Fatalf("nsm client: failed to get a linux namespace for pod %s/%s with error: %+v, exiting...", namespace, podName, err)
os.Exit(1)
}
// Loop through all network services in the config map and request connection to each one. In case of a single
// connection request failure, the initialization process is consider as failed.
// TODO (sbezverk) Discuss if a non-mandatory atrribute should be added to NetworkService config.
for _, ns := range networkServices {
cReq := nsmconnect.ConnectionRequest{
RequestId: podUID,
Metadata: &common.Metadata{
Name: podName,
Namespace: namespace,
},
NetworkServiceName: ns.Name,
LinuxNamespace: linuxNS,
Interface: ns.ServiceInterface,
}
logrus.Infof("Connection request: %+v number of interfaces: %d", cReq, len(cReq.Interface))
connParams, err := requestConnection(nsmClient, &cReq)
if err != nil {
logrus.Fatalf("nsm client: failed to request connection for Network Service %s with error: %+v, exiting...", ns.Name, err)
os.Exit(1)
}
logrus.Infof("nsm client: connection to Network Service %s suceeded, connection parameters: %+v, exiting...", ns.Name, connParams)
}
// Init related activities ends here
logrus.Info("nsm client: initialization is completed successfully, exiting...")
}
func getNetworkServices(nsmClient nsmconnect.ClientConnectionClient) ([]*netmesh.NetworkService, error) {
ctx, cancel := context.WithTimeout(context.Background(), clientConnectionTimeout)
defer cancel()
ticker := time.NewTicker(clientConnectionRetry)
defer ticker.Stop()
var err error
for {
select {
case <-ctx.Done():
return nil, fmt.Errorf("nsm client: Discovery request did not succeed within %d seconds, last known error: %+v", clientConnectionTimeout, err)
case <-ticker.C:
resp, err := nsmClient.RequestDiscovery(ctx, &nsmconnect.DiscoveryRequest{})
if err == nil {
return resp.NetworkService, nil
}
logrus.Infof("nsm client: Discovery request failed with: %+v, re-attempting in %d seconds", err, clientConnectionRetry)
}
}
}
func requestConnection(nsmClient nsmconnect.ClientConnectionClient, cReq *nsmconnect.ConnectionRequest) (*nsmconnect.ConnectionParameters, error) {
ctx, cancel := context.WithTimeout(context.Background(), clientConnectionTimeout)
defer cancel()
ticker := time.NewTicker(clientConnectionRetry)
defer ticker.Stop()
var err error
for {
select {
case <-ctx.Done():
return nil, fmt.Errorf("nsm client: Request Connection to NSM timedout (%d)seconds with error: %+v", clientConnectionTimeout, err)
case <-ticker.C:
cResp, err := nsmClient.RequestConnection(ctx, cReq)
if err == nil && cResp.Accepted {
return cResp.ConnectionParameters, nil
}
switch status.Convert(err).Code() {
case codes.Aborted:
// Aborted inidcates an unrecoverable issue, retries are not needed
fallthrough
case codes.NotFound:
// NotFound indicates that requested Network Service does not exist, retries are not needed
return nil, fmt.Errorf("nsm client: Request Connection to NSM has failed with error: %+v", err)
case codes.AlreadyExists:
// AlreadyExists inidcates not completed dataplane programming, will retry until connection timeout expires
// or success returned
logrus.Infof("nsm client: NSM inidcates already existing non-completed Connection Request, retrying in %d seconds",
clientConnectionRetry)
default:
logrus.Infof("nsm client: Request Connection to NSM has failed with error: %+v, retrying in %d seconds", err, clientConnectionRetry)
}
// There was no error, but NSM did not set Accepted as true, possibly unaccounted error condition or a bug
if cResp != nil {
logrus.Infof("nsm client: NSM failed Connection Request with an admission error: %s, check NSM log for more details. Failed request ID: %s",
cResp.AdmissionError, cReq.RequestId)
}
}
}
}