add make file

Makefile

@@ -0,0 +1,42 @@
+PROJECT_NAME := "fsEngine"
+PKG := "github.com/fanap-infra/fsEngine"
+PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/)
+GO_FILES := $(shell find . -name '*.go' | grep -v /vendor/ | grep -v _test.go)
+
+.PHONY: all dep build clean test coverage coverhtml lint-test unit-test
+
+all: build
+
+lint-test:dep
+	@golangci-lint run
+
+fmt-test:
+	@golangci-lint run -p format
+
+fmt:
+	@go fmt ./...
+
+test: export EXEC_MODE = TEST
+
+test: unit-test race-test
+
+unit-test: dep
+	@go test -count=1 -p=1 -short ./...
+
+race-test: dep
+	@go test -race -count=1 -p=1 -short ./...
+
+# msan: dep
+# 	@go test -msan -short ${PKG_LIST}
+
+# coverage:
+# 	./tools/coverage.sh;
+
+# coverhtml:
+# 	./tools/coverage.sh html;
+
+dep:
+	@go mod init; \
+	go mod tidy; \
+	go mod download; \
+	go mod verify;

crud.go

@@ -58,5 +58,23 @@ func (fse *FSEngine) RemoveVirtualFile(id uint32) error {
 	if ok {
 		return fmt.Errorf("virtual file id : %d is opened", id)
 	}
-	return nil
+
+	fileInfo, err := fse.header.GetFileData(id)
+	if err != nil {
+		return err
+	}
+	blm, err := blockAllocationMap.Open(fse.log, fse, fse.maxNumberOfBlocks, fileInfo.GetLastBlock(),
+		fileInfo.GetRMapBlocks())
+	if err != nil {
+		return err
+	}
+	blocks := blm.ToArray()
+	// fse.log.Infov("blm length",
+	//	"fse blocks length", len(fse.header.GetBlocksNumber().ToArray()), "blocks length", len(blocks))
+	for _, bIndex := range blocks {
+		fse.header.UnsetBlockAsAllocated(bIndex)
+	}
+	// fse.log.Infov("blm length",
+	//	"fse blocks length", len(fse.blockAllocationMap.ToArray()))
+	return fse.header.RemoveVirtualFile(id)
 }

crud_test.go

@@ -21,7 +21,8 @@ func TestVirtualFile_Remove(t *testing.T) {
 	assert.Equal(t, nil, err)
 	_ = utils.DeleteFile(homePath + fsPathTest)
 	_ = utils.DeleteFile(homePath + headerPathTest)
-	fse, err := CreateFileSystem(homePath+fsPathTest, fileSizeTest, blockSizeTest, log.GetScope("test"))
+	eventListener := EventsListener{t: t}
+	fse, err := CreateFileSystem(homePath+fsPathTest, fileSizeTest, blockSizeTest, &eventListener, log.GetScope("test"))
 	assert.Equal(t, nil, err)
 	assert.Equal(t, true, utils.FileExists(homePath+fsPathTest))
 	assert.Equal(t, true, utils.FileExists(homePath+headerPathTest))
@@ -85,7 +86,9 @@ func TestVirtualFile_Open(t *testing.T) {
 	assert.Equal(t, nil, err)
 	_ = utils.DeleteFile(homePath + fsPathTest)
 	_ = utils.DeleteFile(homePath + headerPathTest)
-	fse, err := CreateFileSystem(homePath+fsPathTest, fileSizeTest, blockSizeTest, log.GetScope("test"))
+	var eventListener EventsListener
+	fse, err := CreateFileSystem(homePath+fsPathTest, fileSizeTest, blockSizeTest, &eventListener,
+		log.GetScope("test"))
 	assert.Equal(t, nil, err)
 	assert.Equal(t, true, utils.FileExists(homePath+fsPathTest))
 	assert.Equal(t, true, utils.FileExists(homePath+headerPathTest))
@@ -123,3 +126,68 @@ func TestVirtualFile_Open(t *testing.T) {
 	_ = utils.DeleteFile(homePath + fsPathTest)
 	_ = utils.DeleteFile(homePath + headerPathTest)
 }
+
+func TestVirtualFile_RemoveUnsetBlocks(t *testing.T) {
+	homePath, err := os.UserHomeDir()
+	assert.Equal(t, nil, err)
+	_ = utils.DeleteFile(homePath + fsPathTest)
+	_ = utils.DeleteFile(homePath + headerPathTest)
+	eventListener := EventsListener{t: t}
+	fse, err := CreateFileSystem(homePath+fsPathTest, fileSizeTest, blockSizeTest, &eventListener, log.GetScope("test"))
+	assert.Equal(t, nil, err)
+	assert.Equal(t, true, utils.FileExists(homePath+fsPathTest))
+	assert.Equal(t, true, utils.FileExists(homePath+headerPathTest))
+	var testIDs []uint32
+	var testNames []string
+	blocksIndexes := make([][]uint32, 0)
+	MaxByteArraySize := int(blockSizeTest * 0.5)
+	VFSize := int(3.5 * blockSizeTest)
+	TestSize := 5
+	MaxID := 1000
+	var vfs []*virtualFile.VirtualFile
+	for i := 0; i < TestSize; i++ {
+		tmp := uint32(rand.Intn(MaxID))
+		if utils.ItemExists(testIDs, tmp) {
+			i = i - 1
+			continue
+		}
+		testIDs = append(testIDs, tmp)
+		testNames = append(testNames, "test"+strconv.Itoa(i))
+		vf, err := fse.NewVirtualFile(testIDs[i], testNames[i])
+		assert.Equal(t, nil, err)
+		vfs = append(vfs, vf)
+	}
+
+	for _, vf := range vfs {
+		size := 0
+		for {
+			token := make([]byte, uint32(rand.Intn(MaxByteArraySize)))
+			m, err := rand.Read(token)
+			assert.Equal(t, nil, err)
+			size = size + m
+			n, err := vf.Write(token)
+			assert.Equal(t, nil, err)
+			assert.Equal(t, m, n)
+
+			if size > VFSize {
+				break
+			}
+		}
+		blocksIndexes = append(blocksIndexes, fse.header.GetBLMArray())
+		err = vf.Close()
+		assert.Equal(t, nil, err)
+	}
+
+	for i := 0; i < len(testIDs); i++ {
+		err := fse.RemoveVirtualFile(testIDs[i])
+		assert.Equal(t, nil, err)
+		for j := 0; j < len(blocksIndexes[i]); j++ {
+			assert.Equal(t, false, fse.header.IsBlockAllocated(blocksIndexes[i][j]))
+		}
+	}
+
+	err = fse.Close()
+	assert.Equal(t, nil, err)
+	_ = utils.DeleteFile(homePath + fsPathTest)
+	_ = utils.DeleteFile(homePath + headerPathTest)
+}

events.go

@@ -1,6 +1,5 @@
 package fsEngine
 
 type Events interface {
-	DeleteFile(fileID uint32)
-	DeleteFileByArchiver(archiverFile string)
+	VirtualFileDeleted(fileID uint32, message string)
 }

file.go

@@ -40,7 +40,7 @@ type FSEngine struct {
 	crudMutex  sync.Mutex
 	Cache      *lru.Cache
 	// fileIndexIsFlip bool
-	EventsHandler Events
+	eventsHandler Events
 	Quit          chan struct{}
 }
 

internal/Header/blm.go

@@ -64,3 +64,15 @@ func (hfs *HFileSystem) FindNextFreeBlockAndAllocate() uint32 {
 func (hfs *HFileSystem) SetBlockAsAllocated(blockIndex uint32) error {
 	return hfs.blockAllocationMap.SetBlockAsAllocated(blockIndex)
 }
+
+func (hfs *HFileSystem) UnsetBlockAsAllocated(blockIndex uint32) {
+	hfs.blockAllocationMap.UnsetBlockAsAllocated(blockIndex)
+}
+
+func (hfs *HFileSystem) GetBLMArray() []uint32 {
+	return hfs.blockAllocationMap.ToArray()
+}
+
+func (hfs *HFileSystem) IsBlockAllocated(blockIndex uint32) bool {
+	return hfs.blockAllocationMap.IsBlockAllocated(blockIndex)
+}

internal/Header/fileIndex.go

@@ -3,6 +3,8 @@ package Header_
 import (
 	"fmt"
 	"hash/crc32"
+
+	"github.com/fanap-infra/fsEngine/internal/fileIndex"
 )
 
 func (hfs *HFileSystem) generateFileIndex() ([]byte, error) {
@@ -102,3 +104,11 @@ func (hfs *HFileSystem) parseFileIndex() error {
 func (hfs *HFileSystem) UpdateBAM(fileID uint32, data []byte) error {
 	return hfs.fileIndex.UpdateBAM(fileID, data)
 }
+
+func (hfs *HFileSystem) UpdateFileIndexes(fileID uint32, firstBlock uint32, lastBlock uint32) error {
+	return hfs.fileIndex.UpdateFileIndexes(fileID, firstBlock, lastBlock)
+}
+
+func (hfs *HFileSystem) FindOldestFile() (*fileIndex.File, error) {
+	return hfs.fileIndex.FindOldestFile()
+}

internal/fileIndex/fileIndex.go

@@ -3,6 +3,11 @@ package fileIndex
 import (
 	"fmt"
 	"sync"
+	"time"
+
+	"github.com/fanap-infra/log"
+
+	"github.com/golang/protobuf/ptypes"
 )
 
 const (
@@ -20,11 +25,44 @@ func (i *FileIndex) AddFile(fileId uint32, name string) error {
 	if i.checkFileExist(fileId) {
 		return fmt.Errorf("file id %v has been added before", fileId)
 	}
+	createdTime, err := ptypes.TimestampProto(time.Now().Local())
+	if err != nil {
+		return err
+	}
 	i.table.NumberFiles++
-	i.table.Files[fileId] = &File{Id: fileId, FirstBlock: 0, LastBlock: 0, Name: name, RMapBlocks: make([]byte, 0)}
+	i.table.Files[fileId] = &File{
+		Id: fileId, FirstBlock: 0, LastBlock: 0,
+		Name: name, RMapBlocks: make([]byte, 0),
+		CreatedTime: createdTime,
+	}
 	return nil
 }
 
+func (i *FileIndex) FindOldestFile() (*File, error) {
+	i.rwMux.Lock()
+	defer i.rwMux.Unlock()
+	oldestTime := time.Now().Local()
+	var foundedFile *File
+	foundedFile = nil
+	if len(i.table.Files) == 0 {
+		return nil, fmt.Errorf("there is no file")
+	}
+	for _, file := range i.table.Files {
+		createdTime, err := ptypes.Timestamp(file.CreatedTime)
+		if err != nil {
+			log.Errorv("can not parse file created time", "id", file.Id,
+				"err", err.Error())
+			continue
+		}
+		if oldestTime.After(createdTime) {
+			foundedFile = file
+			oldestTime = createdTime
+		}
+	}
+
+	return foundedFile, nil
+}
+
 func (i *FileIndex) checkFileExist(fileId uint32) bool {
 	_, isExist := i.table.Files[fileId]
 	return isExist
@@ -82,3 +120,14 @@ func (i *FileIndex) UpdateBAM(fileId uint32, bam []byte) error {
 	i.table.Files[fileId].RMapBlocks = bam
 	return nil
 }
+
+func (i *FileIndex) UpdateFileIndexes(fileId uint32, firstBlock uint32, lastBlock uint32) error {
+	i.rwMux.Lock()
+	defer i.rwMux.Unlock()
+	if !i.checkFileExist(fileId) {
+		return fmt.Errorf("file id %v does not exist", fileId)
+	}
+	i.table.Files[fileId].FirstBlock = firstBlock
+	i.table.Files[fileId].LastBlock = lastBlock
+	return nil
+}