Skip to content

Commit

Permalink
Add NodeStage to internal CSI implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
fierlion committed Sep 13, 2023
1 parent 21bd233 commit 06652c6
Show file tree
Hide file tree
Showing 893 changed files with 428,154 additions and 176 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ _bin/
*.orig
/agent/version/_version.go
/ecs-init/version/version.go
/ecs-agent/daemonimages/csidriver/tarfiles
.agignore
*.sublime-*
.DS_Store
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ container-health-check-image:

# all .go files in the agent, excluding vendor/, model/ and testutils/ directories, and all *_test.go and *_mocks.go files
GOFILES:=$(shell go list -f '{{$$p := .}}{{range $$f := .GoFiles}}{{$$p.Dir}}/{{$$f}} {{end}}' ./agent/... ./ecs-agent/... \
| grep -v /testutils/ | grep -v _test\.go$ | grep -v _mocks\.go$ | grep -v /model)
| grep -v /testutils/ | grep -v _test\.go$ | grep -v _mocks\.go$ | grep -v /model | grep -v /daemonimages )

.PHONY: gocyclo
gocyclo:
Expand Down
7 changes: 4 additions & 3 deletions agent/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
Expand Down Expand Up @@ -198,8 +198,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
Expand All @@ -221,6 +221,7 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
Expand Down Expand Up @@ -473,8 +474,8 @@ gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
Expand Down
8 changes: 8 additions & 0 deletions ecs-agent/daemonimages/csidriver/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Build csi-driver from the minimal eks base to add required mount utils
# TODO update Dockerfile with arch to build arm and windows

FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-csi-ebs:latest.2 AS linux-amazon
COPY csi-driver /bin/aws-ebs-csi-driver
ENTRYPOINT ["/bin/aws-ebs-csi-driver"]


87 changes: 87 additions & 0 deletions ecs-agent/daemonimages/csidriver/driver/constants.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package driver

// constants of keys in VolumeContext
const (
// VolumeAttributePartition represents key for partition config in VolumeContext
// this represents the partition number on a device used to mount
VolumeAttributePartition = "partition"
)

// constants for fstypes
const (
// FSTypeExt2 represents the ext2 filesystem type
FSTypeExt2 = "ext2"
// FSTypeExt3 represents the ext3 filesystem type
FSTypeExt3 = "ext3"
// FSTypeExt4 represents the ext4 filesystem type
FSTypeExt4 = "ext4"
// FSTypeXfs represents the xfs filesystem type
FSTypeXfs = "xfs"
// FSTypeNtfs represents the ntfs filesystem type
FSTypeNtfs = "ntfs"
)

// constants of disk partition suffix
const (
diskPartitionSuffix = ""
nvmeDiskPartitionSuffix = "p"
)

// constants of keys in volume parameters
const (
// BlockSizeKey configures the block size when formatting a volume
BlockSizeKey = "blocksize"

// INodeSizeKey configures the inode size when formatting a volume
INodeSizeKey = "inodesize"

// BytesPerINodeKey configures the `bytes-per-inode` when formatting a volume
BytesPerINodeKey = "bytesperinode"

// NumberOfINodesKey configures the `number-of-inodes` when formatting a volume
NumberOfINodesKey = "numberofinodes"
)

type fileSystemConfig struct {
NotSupportedParams map[string]struct{}
}

// constants of keys in PublishContext
const (
// devicePathKey represents key for device path in PublishContext
// devicePath is the device path where the volume is attached to
DevicePathKey = "devicePath"
)

var (
FileSystemConfigs = map[string]fileSystemConfig{
FSTypeExt2: {
NotSupportedParams: map[string]struct{}{},
},
FSTypeExt3: {
NotSupportedParams: map[string]struct{}{},
},
FSTypeExt4: {
NotSupportedParams: map[string]struct{}{},
},
FSTypeXfs: {
NotSupportedParams: map[string]struct{}{
BytesPerINodeKey: {},
NumberOfINodesKey: {},
},
},
FSTypeNtfs: {
NotSupportedParams: map[string]struct{}{
BlockSizeKey: {},
INodeSizeKey: {},
BytesPerINodeKey: {},
NumberOfINodesKey: {},
},
},
}
)

func (fsConfig fileSystemConfig) isParameterSupported(paramName string) bool {
_, notSupported := fsConfig.NotSupportedParams[paramName]
return !notSupported
}
55 changes: 55 additions & 0 deletions ecs-agent/daemonimages/csidriver/driver/controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package driver

import (
"strconv"

csi "github.com/container-storage-interface/spec/lib/go/csi"
"k8s.io/klog/v2"
)

var (
// volumeCaps represents how the volume could be accessed.
// It is SINGLE_NODE_WRITER since EBS volume could only be
// attached to a single node at any given time.
volumeCaps = []csi.VolumeCapability_AccessMode{
{
Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
},
}
)

func isValidVolumeCapabilities(volCaps []*csi.VolumeCapability) bool {
// hasSupport := func(cap *csi.VolumeCapability) bool {
// for _, c := range volumeCaps {
// if c.GetMode() == cap.AccessMode.GetMode() {
// return true
// }
// }
// return false
// }
//
foundAll := true
// for _, c := range volCaps {
// if !hasSupport(c) {
// foundAll = false
// }
// }
return foundAll
}

func isValidVolumeContext(volContext map[string]string) bool {
//There could be multiple volume attributes in the volumeContext map
//Validate here case by case
if partition, ok := volContext[VolumeAttributePartition]; ok {
partitionInt, err := strconv.ParseInt(partition, 10, 64)
if err != nil {
klog.ErrorS(err, "failed to parse partition as int", "partition", partition)
return false
}
if partitionInt < 0 {
klog.ErrorS(err, "invalid partition config", "partition", partition)
return false
}
}
return true
}
72 changes: 72 additions & 0 deletions ecs-agent/daemonimages/csidriver/driver/internal/inflight.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
// http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package internal

import (
"sync"

"k8s.io/klog/v2"
)

// Idempotent is the interface required to manage in flight requests.
type Idempotent interface {
// The CSI data types are generated using a protobuf.
// The generated structures are guaranteed to implement the Stringer interface.
// Example: https://github.com/container-storage-interface/spec/blob/master/lib/go/csi/csi.pb.go#L3508
// We can use the generated string as the key of our internal inflight database of requests.
String() string
}

const (
VolumeOperationAlreadyExistsErrorMsg = "An operation with the given Volume %s already exists"
)

// InFlight is a struct used to manage in flight requests per volumeId.
type InFlight struct {
mux *sync.Mutex
inFlight map[string]bool
}

// NewInFlight instanciates a InFlight structures.
func NewInFlight() *InFlight {
return &InFlight{
mux: &sync.Mutex{},
inFlight: make(map[string]bool),
}
}

// Insert inserts the entry to the current list of inflight request key is volumeId for node and req hash for controller .
// Returns false when the key already exists.
func (db *InFlight) Insert(key string) bool {
db.mux.Lock()
defer db.mux.Unlock()

_, ok := db.inFlight[key]
if ok {
return false
}

db.inFlight[key] = true
return true
}

// Delete removes the entry from the inFlight entries map.
// It doesn't return anything, and will do nothing if the specified key doesn't exist.
func (db *InFlight) Delete(key string) {
db.mux.Lock()
defer db.mux.Unlock()

delete(db.inFlight, key)
klog.V(4).InfoS("Node Service: volume operation finished", "key", key)
}
51 changes: 48 additions & 3 deletions ecs-agent/daemonimages/csidriver/driver/mount.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,39 @@
package driver

import (
"github.com/aws/amazon-ecs-agent/ecs-agent/daemonimages/csidriver/mounter"
"os"
"path/filepath"

"github.com/aws/amazon-ecs-agent/ecs-agent/daemonimages/csidriver/mounter"
mountutils "k8s.io/mount-utils"
)

// Mounter defines an interface for many volume related options. As of now, only
// 'PathExists' is added to determine if a file path exists on the node.
// Mounter is the interface implemented by NodeMounter.
// A mix & match of functions defined in upstream libraries. (FormatAndMount
// from struct SafeFormatAndMount, PathExists from an old edition of
// mount.Interface). Define it explicitly so that it can be mocked and to
// insulate from oft-changing upstream interfaces/structs
type Mounter interface {
mountutils.Interface

FormatAndMountSensitiveWithFormatOptions(source string, target string, fstype string, options []string, sensitiveOptions []string, formatOptions []string) error
IsCorruptedMnt(err error) bool
GetDeviceNameFromMount(mountPath string) (string, int, error)
MakeFile(path string) error
MakeDir(path string) error
PathExists(path string) (bool, error)
NeedResize(devicePath string, deviceMountPath string) (bool, error)
Unpublish(path string) error
Unstage(path string) error
NewResizeFs() (Resizefs, error)
}

type Resizefs interface {
Resize(devicePath, deviceMountPath string) (bool, error)
}

// NodeMounter implements Mounter.
// A superstruct of SafeFormatAndMount.
type NodeMounter struct {
*mountutils.SafeFormatAndMount
}
Expand All @@ -38,3 +59,27 @@ func newNodeMounter() (Mounter, error) {
}
return &NodeMounter{safeMounter}, nil
}

// DeviceIdentifier is for mocking os io functions used for the driver to
// identify an EBS volume's corresponding device (in Linux, the path under
// /dev; in Windows, the volume number) so that it can mount it. For volumes
// already mounted, see GetDeviceNameFromMount.
// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html#identify-nvme-ebs-device
type DeviceIdentifier interface {
Lstat(name string) (os.FileInfo, error)
EvalSymlinks(path string) (string, error)
}

type nodeDeviceIdentifier struct{}

func newNodeDeviceIdentifier() DeviceIdentifier {
return &nodeDeviceIdentifier{}
}

func (i *nodeDeviceIdentifier) Lstat(name string) (os.FileInfo, error) {
return os.Lstat(name)
}

func (i *nodeDeviceIdentifier) EvalSymlinks(path string) (string, error) {
return filepath.EvalSymlinks(path)
}
Loading

0 comments on commit 06652c6

Please sign in to comment.