refactor: storage

hasher/hasher.go

@@ -2,11 +2,26 @@ package hasher
 
 import (
 	"crypto/md5"
+	"crypto/sha1"
 	"encoding/hex"
 )
 
 func ToMD5(in string) string {
+	return ToMD5Bytes([]byte(in))
+}
+
+func ToMD5Bytes(in []byte) string {
 	h := md5.New()
-	_, _ = h.Write([]byte(in))
+	_, _ = h.Write(in)
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+func ToSha1(in string) string {
+	return ToSha1Bytes([]byte(in))
+}
+
+func ToSha1Bytes(in []byte) string {
+	h := sha1.New()
+	_, _ = h.Write(in)
 	return hex.EncodeToString(h.Sum(nil))
 }

searcher/default_searcher.go

@@ -228,7 +228,7 @@ func (p *DefaultSearcher) saveRemoteURLData(ctx *plugin.PluginContext, urls []st
 			continue
 		}
 		logger := logutil.GetLogger(context.Background()).With(zap.String("url", url))
-		key := hasher.ToMD5(url)
+		key := hasher.ToSha1(url)
 		if ok, _ := store.IsDataExist(ctx.GetContext(), key); ok {
 			rs[url] = key
 			continue

store/disk_storage.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"path/filepath"
 	"yamdc/hasher"
+
+	"github.com/google/uuid"
 )
 
 type diskStorage struct {
@@ -17,11 +19,9 @@ func NewDiskStorage(dir string) IStorage {
 }
 
 func (s *diskStorage) generateStorePath(key string) string {
-	save := hasher.ToMD5(key)
+	save := hasher.ToSha1(key)
 	p1 := save[:2]
-	p2 := save[2:4]
-	p3 := save[4:6]
-	return filepath.Join(s.dir, p1, p2, p3, save)
+	return filepath.Join(s.dir, p1, save)
 }
 
 func (s *diskStorage) GetData(ctx context.Context, key string) ([]byte, error) {
@@ -34,9 +34,14 @@ func (s *diskStorage) PutData(ctx context.Context, key string, value []byte) err
 	if err := os.MkdirAll(dir, 0755); err != nil {
 		return fmt.Errorf("create dir failed, err:%w", err)
 	}
-	if err := os.WriteFile(p, value, 0644); err != nil {
+	tempPath := filepath.Join(dir, "tmp."+uuid.NewString())
+	defer os.Remove(tempPath) //处理完, 删除临时文件
+	if err := os.WriteFile(tempPath, value, 0644); err != nil {
 		return fmt.Errorf("write data failed, err:%w", err)
 	}
+	if err := os.Rename(tempPath, p); err != nil {
+		return fmt.Errorf("rename failed, err:%w", err)
+	}
 	return nil
 }
 

store/storage.go

@@ -2,8 +2,7 @@ package store
 
 import (
 	"context"
-
-	"github.com/google/uuid"
+	"yamdc/hasher"
 )
 
 type DataRewriteFunc func(ctx context.Context, data []byte) ([]byte, error)
@@ -29,7 +28,10 @@ func PutData(ctx context.Context, key string, value []byte) error {
 }
 
 func AnonymousPutData(ctx context.Context, value []byte) (string, error) {
-	key := uuid.NewString()
+	key := hasher.ToSha1Bytes(value)
+	if ok, _ := IsDataExist(ctx, key); ok {
+		return key, nil
+	}
 	if err := PutData(ctx, key, value); err != nil {
 		return "", err
 	}