Skip to content

Commit

Permalink
refactoring file/folder structure. some changes were made.
Browse files Browse the repository at this point in the history
  • Loading branch information
yigithankarabulut committed Jun 25, 2024
1 parent 95ad4e1 commit bf35930
Show file tree
Hide file tree
Showing 8 changed files with 120 additions and 107 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
build:
@go build -o bin/fs
@go build -o bin/fs cmd/main.go

run: build
@./bin/fs
Expand Down
13 changes: 8 additions & 5 deletions main.go → cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,26 @@ import (
"log"
"time"

"github.com/yigithankarabulut/distributed-file-storage/fileserver"
"github.com/yigithankarabulut/distributed-file-storage/p2p"
"github.com/yigithankarabulut/distributed-file-storage/store"
)

func makeServer(listenAddr string, nodes ...string) *FileServer {
func makeServer(listenAddr string, nodes ...string) *fileserver.FileServer {
tcpTransport := p2p.NewTCPTransport(
p2p.WithListenAddr(listenAddr),
p2p.WithHandshakeFunc(p2p.NOPHandshakeFunc),
p2p.WithDecoder(&p2p.DefaultDecoder{}),
)

fileServerOpts := FileServerOpts{
fileServerOpts := fileserver.ServerOpts{
StorageRoot: listenAddr + "_network",
PathTransformFunc: CASPathTransformFunc,
PathTransformFunc: store.CASPathTransformFunc,
Transport: tcpTransport,
BootstrapNodes: nodes,
}

s := NewFileServer(fileServerOpts)
s := fileserver.NewFileServer(fileServerOpts)

tcpTransport.OnPeer = s.OnPeer

Expand Down Expand Up @@ -59,5 +61,6 @@ func main() {
}

fmt.Println(string(b))

}

// 8:04. creating crypto file.
63 changes: 24 additions & 39 deletions server.go → fileserver/fileserver.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package main
package fileserver

import (
"bytes"
Expand All @@ -11,55 +11,40 @@ import (
"time"

"github.com/yigithankarabulut/distributed-file-storage/p2p"
"github.com/yigithankarabulut/distributed-file-storage/store"
)

// Message is a struct that contains the payload of the message.
type Message struct {
Payload any
}

// MessageStoreFile is a struct that contains the key and the size of the file.
type MessageStoreFile struct {
Key string
Size int64
}

// MessageGetFile is a struct that contains the key of the file.
type MessageGetFile struct {
Key string
}

// FileServerOpts is a struct that contains the configuration for the file server.
type FileServerOpts struct {
// ServerOpts is a struct that contains the configuration for the file server.
type ServerOpts struct {
ListenAddr string
StorageRoot string
PathTransformFunc PathTransformFunc
PathTransformFunc store.PathTransformFunc
Transport p2p.Transport
BootstrapNodes []string
}

// FileServer is a struct that contains the configuration for the file server.
type FileServer struct {
FileServerOpts
ServerOpts

peerLock sync.Mutex
peers map[string]p2p.Peer

store *Store
store *store.Store
doneChan chan struct{}
}

// NewFileServer creates a new file server instance with the given options.
func NewFileServer(opts FileServerOpts) *FileServer {
store := NewStore(
WithRoot(opts.StorageRoot),
WithPathTransformFunc(opts.PathTransformFunc),
func NewFileServer(opts ServerOpts) *FileServer {
s := store.NewStore(
store.WithRoot(opts.StorageRoot),
store.WithPathTransformFunc(opts.PathTransformFunc),
)
return &FileServer{
FileServerOpts: opts,
store: store,
doneChan: make(chan struct{}),
peers: make(map[string]p2p.Peer),
ServerOpts: opts,
store: s,
doneChan: make(chan struct{}),
peers: make(map[string]p2p.Peer),
}
}

Expand Down Expand Up @@ -175,15 +160,15 @@ func (s *FileServer) Store(key string, r io.Reader) error {
return nil
}

func (s *FileServer) stream(msg *Message) error {
peers := make([]io.Writer, 0, len(s.peers))
for _, peer := range s.peers {
peers = append(peers, peer)
}

mw := io.MultiWriter(peers...)
return gob.NewEncoder(mw).Encode(msg)
}
// func (s *FileServer) stream(msg *Message) error {
// peers := make([]io.Writer, 0, len(s.peers))
// for _, peer := range s.peers {
// peers = append(peers, peer)
// }
//
// mw := io.MultiWriter(peers...)
// return gob.NewEncoder(mw).Encode(msg)
// }

func (s *FileServer) broadcast(msg *Message) error {
buf := new(bytes.Buffer)
Expand Down
17 changes: 17 additions & 0 deletions fileserver/message.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package fileserver

// Message is a struct that contains the payload of the message.
type Message struct {
Payload any
}

// MessageStoreFile is a struct that contains the key and the size of the file.
type MessageStoreFile struct {
Key string
Size int64
}

// MessageGetFile is a struct that contains the key of the file.
type MessageGetFile struct {
Key string
}
23 changes: 23 additions & 0 deletions store/pathkey.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package store

import "strings"

// PathKey is a struct that contains the pathname and the original key.
type PathKey struct {
PathName string
FileName string
}

// FirstPathName returns the first path name of the PathKey.
func (p PathKey) FirstPathName() string {
paths := strings.Split(p.PathName, "/")
if len(paths) == 0 {
return ""
}
return paths[0]
}

// FullPath returns the filename of the PathKey.
func (p PathKey) FullPath() string {
return p.PathName + "/" + p.FileName
}
68 changes: 7 additions & 61 deletions store.go → store/store.go
Original file line number Diff line number Diff line change
@@ -1,96 +1,42 @@
package main
package store

import (
"crypto/sha1" //nolint:gosec
"encoding/hex"
//nolint:gosec
"errors"
"fmt"
"io"
"log"
"os"
"strings"
)

const defaultRootFolderName = "store"

// CASPathTransformFunc is a function that transforms a key into a CAS path.
// CAS stands for Content Addressable Storage.
func CASPathTransformFunc(key string) PathKey {
hash := sha1.Sum([]byte(key)) //nolint:gosec
hashStr := hex.EncodeToString(hash[:])

blockSize := 5
sliceLen := len(hashStr) / blockSize
paths := make([]string, sliceLen)

for i := 0; i < sliceLen; i++ {
from, to := i*blockSize, (i*blockSize)+blockSize
paths[i] = hashStr[from:to]
}

return PathKey{
PathName: strings.Join(paths, "/"),
FileName: hashStr,
}
}

// PathTransformFunc is a function that transforms a path.
type PathTransformFunc func(string) PathKey

// DefaultPathTransformFunc is the default path transform function.
var DefaultPathTransformFunc = func(key string) PathKey {
return PathKey{
PathName: key,
FileName: key,
}
}

// PathKey is a struct that contains the pathname and the original key.
type PathKey struct {
PathName string
FileName string
}

// FirstPathName returns the first path name of the PathKey.
func (p PathKey) FirstPathName() string {
paths := strings.Split(p.PathName, "/")
if len(paths) == 0 {
return ""
}
return paths[0]
}

// FullPath returns the filename of the PathKey.
func (p PathKey) FullPath() string {
return p.PathName + "/" + p.FileName
}

// Store is an interface that can be implemented to store.
type Store struct {
// Root is the root directory of the store, containing all the folders/files of the system.
Root string
PathTransformFunc PathTransformFunc
}

// StoreOption is a functional option for configuring a Store.
type StoreOption func(*Store)
// Option is a functional option for configuring a Store.
type Option func(*Store)

// WithRoot is a functional option for setting the root of the Store.
func WithRoot(root string) StoreOption {
func WithRoot(root string) Option {
return func(s *Store) {
s.Root = root
}
}

// WithPathTransformFunc is a functional option for setting the path transform function of the Store.
func WithPathTransformFunc(f PathTransformFunc) StoreOption {
func WithPathTransformFunc(f PathTransformFunc) Option {
return func(s *Store) {
s.PathTransformFunc = f
}
}

// NewStore creates a new Store with the given options.
func NewStore(opts ...StoreOption) *Store {
func NewStore(opts ...Option) *Store {
s := &Store{
PathTransformFunc: DefaultPathTransformFunc,
Root: defaultRootFolderName,
Expand Down
2 changes: 1 addition & 1 deletion store_test.go → store/store_test.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package main
package store

import (
"bytes"
Expand Down
39 changes: 39 additions & 0 deletions store/transform_func.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package store

import (
"crypto/sha1" //nolint:gosec
"encoding/hex"
"strings"
)

// PathTransformFunc is a function that transforms a path.
type PathTransformFunc func(string) PathKey

// CASPathTransformFunc is a function that transforms a key into a CAS path.
// CAS stands for Content Addressable Storage.
func CASPathTransformFunc(key string) PathKey {
hash := sha1.Sum([]byte(key)) //nolint:gosec
hashStr := hex.EncodeToString(hash[:])

blockSize := 5
sliceLen := len(hashStr) / blockSize
paths := make([]string, sliceLen)

for i := 0; i < sliceLen; i++ {
from, to := i*blockSize, (i*blockSize)+blockSize
paths[i] = hashStr[from:to]
}

return PathKey{
PathName: strings.Join(paths, "/"),
FileName: hashStr,
}
}

// DefaultPathTransformFunc is the default path transform function.
var DefaultPathTransformFunc = func(key string) PathKey {
return PathKey{
PathName: key,
FileName: key,
}
}

0 comments on commit bf35930

Please sign in to comment.