From bf35930f140293d03443c8c14669ee83925e6b3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yi=C4=9Fithan=20Karabulut?= Date: Tue, 25 Jun 2024 19:03:09 +0300 Subject: [PATCH] refactoring file/folder structure. some changes were made. --- Makefile | 2 +- main.go => cmd/main.go | 13 +++-- server.go => fileserver/fileserver.go | 63 ++++++++++--------------- fileserver/message.go | 17 +++++++ store/pathkey.go | 23 +++++++++ store.go => store/store.go | 68 +++------------------------ store_test.go => store/store_test.go | 2 +- store/transform_func.go | 39 +++++++++++++++ 8 files changed, 120 insertions(+), 107 deletions(-) rename main.go => cmd/main.go (72%) rename server.go => fileserver/fileserver.go (85%) create mode 100644 fileserver/message.go create mode 100644 store/pathkey.go rename store.go => store/store.go (62%) rename store_test.go => store/store_test.go (99%) create mode 100644 store/transform_func.go diff --git a/Makefile b/Makefile index 8dc86bc..b7e38bf 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ build: - @go build -o bin/fs + @go build -o bin/fs cmd/main.go run: build @./bin/fs diff --git a/main.go b/cmd/main.go similarity index 72% rename from main.go rename to cmd/main.go index 318e5fc..2373c35 100644 --- a/main.go +++ b/cmd/main.go @@ -6,24 +6,26 @@ import ( "log" "time" + "github.com/yigithankarabulut/distributed-file-storage/fileserver" "github.com/yigithankarabulut/distributed-file-storage/p2p" + "github.com/yigithankarabulut/distributed-file-storage/store" ) -func makeServer(listenAddr string, nodes ...string) *FileServer { +func makeServer(listenAddr string, nodes ...string) *fileserver.FileServer { tcpTransport := p2p.NewTCPTransport( p2p.WithListenAddr(listenAddr), p2p.WithHandshakeFunc(p2p.NOPHandshakeFunc), p2p.WithDecoder(&p2p.DefaultDecoder{}), ) - fileServerOpts := FileServerOpts{ + fileServerOpts := fileserver.ServerOpts{ StorageRoot: listenAddr + "_network", - PathTransformFunc: CASPathTransformFunc, + PathTransformFunc: store.CASPathTransformFunc, Transport: tcpTransport, BootstrapNodes: nodes, } - s := NewFileServer(fileServerOpts) + s := fileserver.NewFileServer(fileServerOpts) tcpTransport.OnPeer = s.OnPeer @@ -59,5 +61,6 @@ func main() { } fmt.Println(string(b)) - } + +// 8:04. creating crypto file. diff --git a/server.go b/fileserver/fileserver.go similarity index 85% rename from server.go rename to fileserver/fileserver.go index cc6a09f..87ddb08 100644 --- a/server.go +++ b/fileserver/fileserver.go @@ -1,4 +1,4 @@ -package main +package fileserver import ( "bytes" @@ -11,55 +11,40 @@ import ( "time" "github.com/yigithankarabulut/distributed-file-storage/p2p" + "github.com/yigithankarabulut/distributed-file-storage/store" ) -// Message is a struct that contains the payload of the message. -type Message struct { - Payload any -} - -// MessageStoreFile is a struct that contains the key and the size of the file. -type MessageStoreFile struct { - Key string - Size int64 -} - -// MessageGetFile is a struct that contains the key of the file. -type MessageGetFile struct { - Key string -} - -// FileServerOpts is a struct that contains the configuration for the file server. -type FileServerOpts struct { +// ServerOpts is a struct that contains the configuration for the file server. +type ServerOpts struct { ListenAddr string StorageRoot string - PathTransformFunc PathTransformFunc + PathTransformFunc store.PathTransformFunc Transport p2p.Transport BootstrapNodes []string } // FileServer is a struct that contains the configuration for the file server. type FileServer struct { - FileServerOpts + ServerOpts peerLock sync.Mutex peers map[string]p2p.Peer - store *Store + store *store.Store doneChan chan struct{} } // NewFileServer creates a new file server instance with the given options. -func NewFileServer(opts FileServerOpts) *FileServer { - store := NewStore( - WithRoot(opts.StorageRoot), - WithPathTransformFunc(opts.PathTransformFunc), +func NewFileServer(opts ServerOpts) *FileServer { + s := store.NewStore( + store.WithRoot(opts.StorageRoot), + store.WithPathTransformFunc(opts.PathTransformFunc), ) return &FileServer{ - FileServerOpts: opts, - store: store, - doneChan: make(chan struct{}), - peers: make(map[string]p2p.Peer), + ServerOpts: opts, + store: s, + doneChan: make(chan struct{}), + peers: make(map[string]p2p.Peer), } } @@ -175,15 +160,15 @@ func (s *FileServer) Store(key string, r io.Reader) error { return nil } -func (s *FileServer) stream(msg *Message) error { - peers := make([]io.Writer, 0, len(s.peers)) - for _, peer := range s.peers { - peers = append(peers, peer) - } - - mw := io.MultiWriter(peers...) - return gob.NewEncoder(mw).Encode(msg) -} +// func (s *FileServer) stream(msg *Message) error { +// peers := make([]io.Writer, 0, len(s.peers)) +// for _, peer := range s.peers { +// peers = append(peers, peer) +// } +// +// mw := io.MultiWriter(peers...) +// return gob.NewEncoder(mw).Encode(msg) +// } func (s *FileServer) broadcast(msg *Message) error { buf := new(bytes.Buffer) diff --git a/fileserver/message.go b/fileserver/message.go new file mode 100644 index 0000000..a584c85 --- /dev/null +++ b/fileserver/message.go @@ -0,0 +1,17 @@ +package fileserver + +// Message is a struct that contains the payload of the message. +type Message struct { + Payload any +} + +// MessageStoreFile is a struct that contains the key and the size of the file. +type MessageStoreFile struct { + Key string + Size int64 +} + +// MessageGetFile is a struct that contains the key of the file. +type MessageGetFile struct { + Key string +} diff --git a/store/pathkey.go b/store/pathkey.go new file mode 100644 index 0000000..b9b0003 --- /dev/null +++ b/store/pathkey.go @@ -0,0 +1,23 @@ +package store + +import "strings" + +// PathKey is a struct that contains the pathname and the original key. +type PathKey struct { + PathName string + FileName string +} + +// FirstPathName returns the first path name of the PathKey. +func (p PathKey) FirstPathName() string { + paths := strings.Split(p.PathName, "/") + if len(paths) == 0 { + return "" + } + return paths[0] +} + +// FullPath returns the filename of the PathKey. +func (p PathKey) FullPath() string { + return p.PathName + "/" + p.FileName +} diff --git a/store.go b/store/store.go similarity index 62% rename from store.go rename to store/store.go index 17c86f2..f288438 100644 --- a/store.go +++ b/store/store.go @@ -1,70 +1,16 @@ -package main +package store import ( - "crypto/sha1" //nolint:gosec - "encoding/hex" + //nolint:gosec "errors" "fmt" "io" "log" "os" - "strings" ) const defaultRootFolderName = "store" -// CASPathTransformFunc is a function that transforms a key into a CAS path. -// CAS stands for Content Addressable Storage. -func CASPathTransformFunc(key string) PathKey { - hash := sha1.Sum([]byte(key)) //nolint:gosec - hashStr := hex.EncodeToString(hash[:]) - - blockSize := 5 - sliceLen := len(hashStr) / blockSize - paths := make([]string, sliceLen) - - for i := 0; i < sliceLen; i++ { - from, to := i*blockSize, (i*blockSize)+blockSize - paths[i] = hashStr[from:to] - } - - return PathKey{ - PathName: strings.Join(paths, "/"), - FileName: hashStr, - } -} - -// PathTransformFunc is a function that transforms a path. -type PathTransformFunc func(string) PathKey - -// DefaultPathTransformFunc is the default path transform function. -var DefaultPathTransformFunc = func(key string) PathKey { - return PathKey{ - PathName: key, - FileName: key, - } -} - -// PathKey is a struct that contains the pathname and the original key. -type PathKey struct { - PathName string - FileName string -} - -// FirstPathName returns the first path name of the PathKey. -func (p PathKey) FirstPathName() string { - paths := strings.Split(p.PathName, "/") - if len(paths) == 0 { - return "" - } - return paths[0] -} - -// FullPath returns the filename of the PathKey. -func (p PathKey) FullPath() string { - return p.PathName + "/" + p.FileName -} - // Store is an interface that can be implemented to store. type Store struct { // Root is the root directory of the store, containing all the folders/files of the system. @@ -72,25 +18,25 @@ type Store struct { PathTransformFunc PathTransformFunc } -// StoreOption is a functional option for configuring a Store. -type StoreOption func(*Store) +// Option is a functional option for configuring a Store. +type Option func(*Store) // WithRoot is a functional option for setting the root of the Store. -func WithRoot(root string) StoreOption { +func WithRoot(root string) Option { return func(s *Store) { s.Root = root } } // WithPathTransformFunc is a functional option for setting the path transform function of the Store. -func WithPathTransformFunc(f PathTransformFunc) StoreOption { +func WithPathTransformFunc(f PathTransformFunc) Option { return func(s *Store) { s.PathTransformFunc = f } } // NewStore creates a new Store with the given options. -func NewStore(opts ...StoreOption) *Store { +func NewStore(opts ...Option) *Store { s := &Store{ PathTransformFunc: DefaultPathTransformFunc, Root: defaultRootFolderName, diff --git a/store_test.go b/store/store_test.go similarity index 99% rename from store_test.go rename to store/store_test.go index 6c565a2..b52440e 100644 --- a/store_test.go +++ b/store/store_test.go @@ -1,4 +1,4 @@ -package main +package store import ( "bytes" diff --git a/store/transform_func.go b/store/transform_func.go new file mode 100644 index 0000000..91c0393 --- /dev/null +++ b/store/transform_func.go @@ -0,0 +1,39 @@ +package store + +import ( + "crypto/sha1" //nolint:gosec + "encoding/hex" + "strings" +) + +// PathTransformFunc is a function that transforms a path. +type PathTransformFunc func(string) PathKey + +// CASPathTransformFunc is a function that transforms a key into a CAS path. +// CAS stands for Content Addressable Storage. +func CASPathTransformFunc(key string) PathKey { + hash := sha1.Sum([]byte(key)) //nolint:gosec + hashStr := hex.EncodeToString(hash[:]) + + blockSize := 5 + sliceLen := len(hashStr) / blockSize + paths := make([]string, sliceLen) + + for i := 0; i < sliceLen; i++ { + from, to := i*blockSize, (i*blockSize)+blockSize + paths[i] = hashStr[from:to] + } + + return PathKey{ + PathName: strings.Join(paths, "/"), + FileName: hashStr, + } +} + +// DefaultPathTransformFunc is the default path transform function. +var DefaultPathTransformFunc = func(key string) PathKey { + return PathKey{ + PathName: key, + FileName: key, + } +}