diff --git a/go.mod b/go.mod index 38e047e3..1ac33f08 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01 github.com/klauspost/reedsolomon v1.9.9 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 - github.com/minio/highwayhash v1.0.1 + github.com/minio/highwayhash v1.0.2 github.com/ncw/swift/v2 v2.0.1 github.com/pkg/sftp v1.11.0 github.com/pkg/xattr v0.4.1 @@ -34,6 +34,7 @@ require ( github.com/calebcase/tmpfile v1.0.3 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dnaeon/go-vcr v1.2.0 // indirect + github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8 // indirect github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 // indirect github.com/godbus/dbus v4.1.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect diff --git a/go.sum b/go.sum index 649989cc..8ef12fbb 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,8 @@ github.com/gilbertchen/goamz v0.0.0-20170712012135-eada9f4e8cc2 h1:VDPwi3huqeJBt github.com/gilbertchen/goamz v0.0.0-20170712012135-eada9f4e8cc2/go.mod h1:AoxJeh8meXUrSWBLiq9BJvYMd9RAAGgEUU0gSkNedRY= github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c h1:0SR0aXvil/eQReU0olxp/j04B+Y/47fjDMotIxaAgKo= github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:HDsXH7AAfDsfYYX0te4zsNbnwVvZ2RtLEOCjN4y84jw= +github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8 h1:ijgl4Y+OKCIFiCPk/Rf9tb6PrarVqitu5TynpyCmRK0= +github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8/go.mod h1:0lQcVva56+L1PuUFXLOsJ6arJQaU0baIH8q+IegeBhg= github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508 h1:SqTyk5KkNXp7zTdTttIZSDcTrL5uau4K/2OpKvgBZVI= github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508/go.mod h1:w/pisxUZezf2XzU9Ewjphcf6q1mZtOzKPHhJiuc8cag= github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01 h1:LqwS9qL6SrDkp0g0iwUkETrDdtB9gTKaIbSn9imUq5o= @@ -165,6 +167,8 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/highwayhash v1.0.1 h1:dZ6IIu8Z14VlC0VpfKofAhCy74wu/Qb5gcn52yWoz/0= github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= +github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104 h1:ULR/QWMgcgRiZLUjSSJMU+fW+RDMstRdmnDWj9Q+AsA= github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104/go.mod h1:wqKykBG2QzQDJEzvRkcS8x6MiSJkF52hXZsXcjaB3ls= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/src/duplicacy_chunk.go b/src/duplicacy_chunk.go index 8ed3a740..b76376e6 100644 --- a/src/duplicacy_chunk.go +++ b/src/duplicacy_chunk.go @@ -24,6 +24,12 @@ import ( "github.com/bkaradzic/go-lz4" "github.com/minio/highwayhash" "github.com/klauspost/reedsolomon" + + // This is a fork of github.com/minio/highwayhash at 1.0.1 that computes incorrect hash on + // arm64 machines. We need this fork to be able to read the chunks created by Duplicacy + // CLI 3.0.1 which unfortunately relies on incorrect hashes to determine if each shard is valid. + wronghighwayhash "github.com/gilbertchen/highwayhash" + ) // A chunk needs to acquire a new buffer and return the old one for every encrypt/decrypt operation, therefore @@ -426,6 +432,8 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err recoveryNeeded := false hashKey := make([]byte, 32) availableShards := 0 + wrongHashDetected := false + for i := 0; i < dataShards + parityShards; i++ { start := dataOffset + i * shardSize if start + shardSize > len(encryptedBuffer.Bytes()) { @@ -441,7 +449,24 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err if err != nil { return err } - if bytes.Compare(hasher.Sum(nil), encryptedBuffer.Bytes()[hashOffset + i * 32: hashOffset + (i + 1) * 32]) != 0 { + + matched := bytes.Compare(hasher.Sum(nil), encryptedBuffer.Bytes()[hashOffset + i * 32: hashOffset + (i + 1) * 32]) == 0 + + if !matched && runtime.GOARCH == "arm64" { + hasher, err := wronghighwayhash.New(hashKey) + if err == nil { + _, err = hasher.Write(encryptedBuffer.Bytes()[start: start + shardSize]) + if err == nil { + matched = bytes.Compare(hasher.Sum(nil), encryptedBuffer.Bytes()[hashOffset + i * 32: hashOffset + (i + 1) * 32]) == 0 + if matched && !wrongHashDetected { + LOG_WARN("CHUNK_ERASURECODE", "Hash for shard %d was calculated with a wrong version of highwayhash", i) + wrongHashDetected = true + } + } + } + } + + if !matched { if i < dataShards { recoveryNeeded = true }