-
Notifications
You must be signed in to change notification settings - Fork 1.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add Key() to LayeredMap and Snapshotter #337
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,20 +17,28 @@ limitations under the License. | |
package snapshot | ||
|
||
import ( | ||
"bytes" | ||
"encoding/json" | ||
"fmt" | ||
"path/filepath" | ||
"strings" | ||
|
||
"github.com/GoogleContainerTools/kaniko/pkg/util" | ||
) | ||
|
||
type LayeredMap struct { | ||
layers []map[string]string | ||
whiteouts []map[string]string | ||
added []map[string]string | ||
hasher func(string) (string, error) | ||
// cacheHasher doesn't include mtime in it's hash so that filesystem cache keys are stable | ||
cacheHasher func(string) (string, error) | ||
} | ||
|
||
func NewLayeredMap(h func(string) (string, error)) *LayeredMap { | ||
func NewLayeredMap(h func(string) (string, error), c func(string) (string, error)) *LayeredMap { | ||
l := LayeredMap{ | ||
hasher: h, | ||
hasher: h, | ||
cacheHasher: c, | ||
} | ||
l.layers = []map[string]string{} | ||
return &l | ||
|
@@ -39,8 +47,18 @@ func NewLayeredMap(h func(string) (string, error)) *LayeredMap { | |
func (l *LayeredMap) Snapshot() { | ||
l.whiteouts = append(l.whiteouts, map[string]string{}) | ||
l.layers = append(l.layers, map[string]string{}) | ||
l.added = append(l.added, map[string]string{}) | ||
} | ||
|
||
// Key returns a hash for added files | ||
func (l *LayeredMap) Key() (string, error) { | ||
c := bytes.NewBuffer([]byte{}) | ||
enc := json.NewEncoder(c) | ||
enc.Encode(l.added) | ||
return util.SHA256(c) | ||
} | ||
|
||
// GetFlattenedPathsForWhiteOut returns all paths in the current FS | ||
func (l *LayeredMap) GetFlattenedPathsForWhiteOut() map[string]struct{} { | ||
paths := map[string]struct{}{} | ||
for _, l := range l.layers { | ||
|
@@ -85,11 +103,18 @@ func (l *LayeredMap) MaybeAddWhiteout(s string) (bool, error) { | |
|
||
// Add will add the specified file s to the layered map. | ||
func (l *LayeredMap) Add(s string) error { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is the name of the file or contents? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. sorry, is that what you were asking? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes. i was just wondering if we shd be hashing the file contents as well. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. we call the hasher function on line 106:
and that function will read the file contents, modtime, and some other things as well There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. hmm.. After reading the PR description again, i think the cacheHasher does it. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yah, it does! |
||
// Use hash function and add to layers | ||
newV, err := l.hasher(s) | ||
if err != nil { | ||
return fmt.Errorf("Error creating hash for %s: %s", s, err) | ||
return fmt.Errorf("Error creating hash for %s: %v", s, err) | ||
} | ||
l.layers[len(l.layers)-1][s] = newV | ||
// Use cache hash function and add to added | ||
cacheV, err := l.cacheHasher(s) | ||
if err != nil { | ||
return fmt.Errorf("Error creating cache hash for %s: %v", s, err) | ||
} | ||
l.added[len(l.added)-1][s] = cacheV | ||
return nil | ||
} | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,79 @@ | ||
/* | ||
Copyright 2018 Google LLC | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
package snapshot | ||
|
||
import ( | ||
"testing" | ||
) | ||
|
||
func Test_CacheKey(t *testing.T) { | ||
tests := []struct { | ||
name string | ||
map1 map[string]string | ||
map2 map[string]string | ||
equal bool | ||
}{ | ||
{ | ||
name: "maps are the same", | ||
map1: map[string]string{ | ||
"a": "apple", | ||
"b": "bat", | ||
"c": "cat", | ||
"d": "dog", | ||
"e": "egg", | ||
}, | ||
map2: map[string]string{ | ||
"c": "cat", | ||
"d": "dog", | ||
"b": "bat", | ||
"a": "apple", | ||
"e": "egg", | ||
}, | ||
equal: true, | ||
}, | ||
{ | ||
name: "maps are different", | ||
map1: map[string]string{ | ||
"a": "apple", | ||
"b": "bat", | ||
"c": "cat", | ||
}, | ||
map2: map[string]string{ | ||
"c": "", | ||
"b": "bat", | ||
"a": "apple", | ||
}, | ||
equal: false, | ||
}, | ||
} | ||
for _, test := range tests { | ||
t.Run(test.name, func(t *testing.T) { | ||
lm1 := LayeredMap{added: []map[string]string{test.map1}} | ||
lm2 := LayeredMap{added: []map[string]string{test.map2}} | ||
k1, err := lm1.Key() | ||
if err != nil { | ||
t.Fatalf("error getting key for map 1: %v", err) | ||
} | ||
k2, err := lm2.Key() | ||
if err != nil { | ||
t.Fatalf("error getting key for map 2: %v", err) | ||
} | ||
if test.equal != (k1 == k2) { | ||
t.Fatalf("unexpected result: \nExpected\n%s\nActual\n%s\n", k1, k2) | ||
} | ||
}) | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -18,6 +18,7 @@ package util | |
|
||
import ( | ||
"crypto/md5" | ||
"crypto/sha256" | ||
"encoding/hex" | ||
"io" | ||
"os" | ||
|
@@ -72,6 +73,36 @@ func Hasher() func(string) (string, error) { | |
return hasher | ||
} | ||
|
||
// CacheHasher takes into account everything the regular hasher does except for mtime | ||
func CacheHasher() func(string) (string, error) { | ||
hasher := func(p string) (string, error) { | ||
h := md5.New() | ||
fi, err := os.Lstat(p) | ||
if err != nil { | ||
return "", err | ||
} | ||
h.Write([]byte(fi.Mode().String())) | ||
|
||
h.Write([]byte(strconv.FormatUint(uint64(fi.Sys().(*syscall.Stat_t).Uid), 36))) | ||
h.Write([]byte(",")) | ||
h.Write([]byte(strconv.FormatUint(uint64(fi.Sys().(*syscall.Stat_t).Gid), 36))) | ||
|
||
if fi.Mode().IsRegular() { | ||
f, err := os.Open(p) | ||
if err != nil { | ||
return "", err | ||
} | ||
defer f.Close() | ||
if _, err := io.Copy(h, f); err != nil { | ||
return "", err | ||
} | ||
} | ||
|
||
return hex.EncodeToString(h.Sum(nil)), nil | ||
} | ||
return hasher | ||
} | ||
|
||
// MtimeHasher returns a hash function, which only looks at mtime to determine if a file has changed. | ||
// Note that the mtime can lag, so it's possible that a file will have changed but the mtime may look the same. | ||
func MtimeHasher() func(string) (string, error) { | ||
|
@@ -86,3 +117,13 @@ func MtimeHasher() func(string) (string, error) { | |
} | ||
return hasher | ||
} | ||
|
||
// SHA256 returns the shasum of the contents of r | ||
func SHA256(r io.Reader) (string, error) { | ||
hasher := sha256.New() | ||
_, err := io.Copy(hasher, r) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. return error if its not nil. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. done |
||
if err != nil { | ||
return "", err | ||
} | ||
return hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))), nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It's not clear to me why we're json encoding anything. Is the idea to get a stable hash for a map[string]string?
I'm not sure if the sorting of a map when encoding it will be stable...
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yah, the idea was to get a stable hash, I thought it would work because of this from the docs on json.Marshal:, which is used by the encoder
My plan was to get a hash for the config file in the same way.