diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml index 0547d88ac91..f9226243036 100644 --- a/.github/workflows/kvrocks.yaml +++ b/.github/workflows/kvrocks.yaml @@ -143,7 +143,6 @@ jobs: - name: Ubuntu GCC os: ubuntu-22.04 compiler: gcc - enable_grocksdb: -tags=enable_grocksdb - name: SonarCloud with Coverage os: ubuntu-22.04 compiler: gcc @@ -327,7 +326,6 @@ jobs: export LSAN_OPTIONS="suppressions=$(realpath ./tests/lsan-suppressions)" export TSAN_OPTIONS="suppressions=$(realpath ./tests/tsan-suppressions)" export PATH=$PATH:$HOME/local/bin/ - export CGO_ENABLED=1 GOCASE_RUN_ARGS="" if [[ -n "${{ matrix.with_openssl }}" ]] && [[ "${{ matrix.os }}" == ubuntu* ]]; then git clone https://github.com/jsha/minica @@ -338,7 +336,7 @@ jobs: cp minica.pem tests/gocase/tls/cert/ca.crt GOCASE_RUN_ARGS="-tlsEnable" fi - ./x.py test go build -parallel 2 $GOCASE_RUN_ARGS ${{ matrix.ignore_when_tsan}} ${{ matrix.ignore_when_asan}} ${{ matrix.ignore_when_ubsan}} ${{ matrix.enable_grocksdb }} + ./x.py test go build -parallel 2 $GOCASE_RUN_ARGS ${{ matrix.ignore_when_tsan}} ${{ matrix.ignore_when_asan}} ${{ matrix.ignore_when_ubsan}} - name: Install redis-py run: pip3 install redis==5.2.0 @@ -491,7 +489,7 @@ jobs: - name: Setup openSUSE if: ${{ startsWith(matrix.image, 'opensuse') }} run: | - zypper install -y gcc11 gcc11-c++ make wget git autoconf automake python3 python3-pip curl tar gzip cmake go zlib-devel + zypper install -y gcc11 gcc11-c++ make wget git autoconf automake python3 python3-pip curl tar gzip cmake go update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-11 100 update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-11 100 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 @@ -503,7 +501,7 @@ jobs: run: | dnf install -y epel-release dnf config-manager --set-enabled powertools - dnf install -y git gcc-toolset-12 autoconf automake libtool python3 python3-pip openssl-devel which cmake zlib-devel gcc-toolset-12-libstdc++-devel + dnf install -y git gcc-toolset-12 autoconf automake libtool python3 python3-pip openssl-devel which cmake source /opt/rh/gcc-toolset-12/enable update-alternatives --install /usr/bin/gcc gcc /opt/rh/gcc-toolset-12/root/usr/bin/gcc 100 update-alternatives --install /usr/bin/g++ g++ /opt/rh/gcc-toolset-12/root/usr/bin/g++ 100 @@ -516,7 +514,7 @@ jobs: run: | dnf install -y epel-release dnf config-manager --set-enabled crb - dnf install -y git gcc-toolset-12 autoconf automake libtool python3 python3-pip openssl-devel which cmake zlib-devel + dnf install -y git gcc-toolset-12 autoconf automake libtool python3 python3-pip openssl-devel which cmake source /opt/rh/gcc-toolset-12/enable update-alternatives --install /usr/bin/gcc gcc /opt/rh/gcc-toolset-12/root/usr/bin/gcc 100 update-alternatives --install /usr/bin/g++ g++ /opt/rh/gcc-toolset-12/root/usr/bin/g++ 100 @@ -535,7 +533,7 @@ jobs: if: ${{ startsWith(matrix.image, 'alpine') }} run: | apk update - apk add bash cmake curl git python3 wget make gcc g++ autoconf linux-headers py3-pip py3-redis zlib-dev + apk add bash cmake curl git python3 wget make gcc g++ autoconf linux-headers py3-pip py3-redis echo "NPROC=$(nproc)" >> $GITHUB_ENV - name: Cache redis @@ -582,7 +580,6 @@ jobs: - name: Run Go Integration Cases run: | export PATH=$PATH:$HOME/local/bin/ - export CGO_ENABLED=1 GOCASE_RUN_ARGS="" ./x.py test go build -parallel 2 $GOCASE_RUN_ARGS diff --git a/tests/gocase/go.mod b/tests/gocase/go.mod index 37ca0f11256..fd580f129b0 100644 --- a/tests/gocase/go.mod +++ b/tests/gocase/go.mod @@ -3,7 +3,6 @@ module github.com/apache/kvrocks/tests/gocase go 1.23.0 require ( - github.com/linxGnu/grocksdb v1.9.9 github.com/redis/go-redis/v9 v9.7.3 github.com/shirou/gopsutil/v4 v4.25.2 github.com/stretchr/testify v1.10.0 diff --git a/tests/gocase/go.sum b/tests/gocase/go.sum index 1dc47e066c0..62d2d97e2a7 100644 --- a/tests/gocase/go.sum +++ b/tests/gocase/go.sum @@ -15,8 +15,6 @@ github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/linxGnu/grocksdb v1.9.9 h1:CzSS/vHLtVIdxdrjvqWR/sm93u/0eB6UcoO14YqObnw= -github.com/linxGnu/grocksdb v1.9.9/go.mod h1:C3CNe9UYc9hlEM2pC82AqiGS3LRW537u9LFV4wIZuHk= github.com/lufia/plan9stats v0.0.0-20250303091104-876f3ea5145d h1:fjMbDVUGsMQiVZnSQsmouYJvMdwsGiDipOZoN66v844= github.com/lufia/plan9stats v0.0.0-20250303091104-876f3ea5145d/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/tests/gocase/unit/sst/sst_load_test.go b/tests/gocase/unit/sst/sst_load_test.go deleted file mode 100644 index d2c20bc1168..00000000000 --- a/tests/gocase/unit/sst/sst_load_test.go +++ /dev/null @@ -1,609 +0,0 @@ -//go:build enable_grocksdb - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package sst - -import ( - "context" - "encoding/binary" - "fmt" - "log" - "math/rand" - "os" - "path/filepath" - "sort" - "testing" - "time" - - "github.com/linxGnu/grocksdb" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/apache/kvrocks/tests/gocase/util" -) - -const ( - DefaultKvrocksNamespace = "__namespace" - metaDataEncodingMask uint8 = 0x82 - versionCounterBits = 11 -) - -type Metadata struct { - Flags uint8 - Expire uint64 - Version uint64 - Size uint64 -} - -type SSTResponse struct { - filesLoaded int64 `redis:"files_loaded"` -} - -func NewMetadata() *Metadata { - src := rand.NewSource(time.Now().UnixNano()) - r := rand.New(src) - - timestamp := uint64(time.Now().UnixMicro()) - counter := uint64(r.Int63()) - version := (timestamp << versionCounterBits) + (counter % (1 << versionCounterBits)) - - return &Metadata{ - Flags: metaDataEncodingMask, - Version: version, - } -} - -func (m *Metadata) Encode() []byte { - buf := make([]byte, 25) // 1 + 8 + 8 + 8 bytes - buf[0] = m.Flags - binary.BigEndian.PutUint64(buf[1:], m.Expire) - binary.BigEndian.PutUint64(buf[9:], m.Version) - binary.BigEndian.PutUint64(buf[17:], m.Size) - return buf -} - -func encodeInternalKey(namespace, key, field string, version uint64) []byte { - nsLen := len(namespace) - keyLen := len(key) - fieldLen := len(field) - - // Pre-calculate total size: 1 byte for ns size + ns + 4 bytes for key size + key + 8 bytes for version + field - out := make([]byte, 1+nsLen+4+keyLen+8+fieldLen) - - out[0] = uint8(nsLen) - copy(out[1:], namespace) - binary.BigEndian.PutUint32(out[1+nsLen:], uint32(keyLen)) - copy(out[5+nsLen:], key) - binary.BigEndian.PutUint64(out[5+nsLen+keyLen:], version) - copy(out[13+nsLen+keyLen:], field) - - return out -} - -func encodeRedisHashKey(namespace, userKey string) []byte { - totalLen := 1 + len(namespace) + len(userKey) - buf := make([]byte, totalLen) - buf[0] = uint8(len(namespace)) - copy(buf[1:], namespace) - copy(buf[1+len(namespace):], userKey) - return buf -} - -func createSSTFile(filename string, data map[string]string) error { - envOpts := grocksdb.NewDefaultEnvOptions() - sstWriterOpts := grocksdb.NewDefaultOptions() - sstWriterOpts.SetCompression(grocksdb.CompressionType(2)) - sstWriter := grocksdb.NewSSTFileWriter(envOpts, sstWriterOpts) - defer sstWriter.Destroy() - - err := sstWriter.Open(filename) - if err != nil { - return fmt.Errorf("failed to open SST file writer: %v", err) - } - - // Get all keys and sort them - keys := make([]string, 0, len(data)) - for k := range data { - keys = append(keys, k) - } - sort.Strings(keys) - - // Add keys in sorted order - for _, k := range keys { - err = sstWriter.Add([]byte(k), []byte(data[k])) - if err != nil { - log.Printf("Error adding key %s to SST: %v", k, err) - } - } - - err = sstWriter.Finish() - if err != nil { - return fmt.Errorf("failed to finish SST file: %v", err) - } - return nil -} - -func makeTempDir() (string, error) { - return os.MkdirTemp("", "sst_test_*") -} - -func toInt64(val interface{}) (int64, error) { - switch v := val.(type) { - case int64: - return v, nil - case int: - return int64(v), nil - case float64: - return int64(v), nil - default: - return 0, fmt.Errorf("value is not a number, got %T", val) - } -} - -func ExtractSSTResponse(result interface{}) (*SSTResponse, error) { - resultMap, ok := result.(map[interface{}]interface{}) - if !ok { - return nil, fmt.Errorf("expected map[interface{}]interface{}, got %T", result) - } - response := &SSTResponse{} - for field, target := range map[string]*int64{ - "files_loaded": &response.filesLoaded, - } { - if val, ok := resultMap[field]; ok { - converted, err := toInt64(val) - if err != nil { - return nil, fmt.Errorf("%s: %v", field, err) - } - *target = converted - } - } - return response, nil -} - -func TestSSTLoad(t *testing.T) { - configOptions := []util.ConfigOptions{ - { - Name: "resp3-enabled", - Options: []string{"yes"}, - ConfigType: util.YesNo, - }, - } - configsMatrix, err := util.GenerateConfigsMatrix(configOptions) - require.NoError(t, err) - for _, configs := range configsMatrix { - testSSTLoad(t, configs) - } -} - -var testSSTLoad = func(t *testing.T, configs util.KvrocksServerConfigs) { - srv := util.StartServer(t, configs) - defer srv.Close() - ctx := context.Background() - rdb := srv.NewClient() - defer func() { require.NoError(t, rdb.Close()) }() - - t.Run("Test load cmd with no folder", func(t *testing.T) { - r := rdb.Do(ctx, "sst", "load") - assert.Error(t, r.Err()) - }) - - t.Run("Test wrong subcommand", func(t *testing.T) { - dir, err := makeTempDir() - require.NoError(t, err) - defer os.RemoveAll(dir) - r := rdb.Do(ctx, "sst", "wrong-sub-command", dir) - assert.Error(t, r.Err()) - }) - - t.Run("Test wrong load option", func(t *testing.T) { - dir, err := makeTempDir() - require.NoError(t, err) - defer os.RemoveAll(dir) - r := rdb.Do(ctx, "sst", "load", dir, "wrong-load-option") - assert.Error(t, r.Err()) - }) - - t.Run("Test empty folder", func(t *testing.T) { - dir, err := makeTempDir() - require.NoError(t, err) - defer os.RemoveAll(dir) - r := rdb.Do(ctx, "sst", "load", dir) - assert.NoError(t, r.Err()) - resp, err := ExtractSSTResponse(r.Val()) - assert.NoError(t, err) - assert.Equal(t, int64(0), resp.filesLoaded) - }) - - t.Run("Test load redis hash keys", func(t *testing.T) { - dir, err := makeTempDir() - require.NoError(t, err) - defer os.RemoveAll(dir) - - namespace := DefaultKvrocksNamespace - data := map[string][]map[string]string{ - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - } - keys := make(map[string]string, len(data)) - metaKeys := make(map[string]string, len(data)) - for hashK := range data { - meta := NewMetadata() - fields := data[hashK] - for _, field := range fields { - for fieldK, fieldV := range field { - internalKey := encodeInternalKey(namespace, hashK, fieldK, meta.Version) - keys[string(internalKey)] = fieldV - } - } - hashKey := encodeRedisHashKey(namespace, hashK) - meta.Size++ - metaKeys[string(hashKey)] = string(meta.Encode()) - } - - err = createSSTFile(filepath.Join(dir, "kvrocks_default.sst"), keys) - assert.NoError(t, err) - err = createSSTFile(filepath.Join(dir, "kvrocks_metadata.sst"), metaKeys) - assert.NoError(t, err) - - r := rdb.Do(ctx, "sst", "load", dir) - assert.NoError(t, r.Err()) - resp, err := ExtractSSTResponse(r.Val()) - assert.NoError(t, err) - assert.Equal(t, int64(2), resp.filesLoaded) - - //verify files didn't get moved - _, err = os.Stat(filepath.Join(dir, "kvrocks_default.sst")) - assert.NoError(t, err) - _, err = os.Stat(filepath.Join(dir, "kvrocks_metadata.sst")) - assert.NoError(t, err) - - for hashK, fields := range data { - for _, field := range fields { - for fieldK, expectedVal := range field { - val := rdb.HGet(ctx, hashK, fieldK) - assert.NoError(t, val.Err()) - assert.Equal(t, expectedVal, val.Val(), "Hash field value mismatch for key:%s field:%s", hashK, fieldK) - } - } - } - }) - - t.Run("Test load and update redis hash keys", func(t *testing.T) { - dir, err := makeTempDir() - require.NoError(t, err) - defer os.RemoveAll(dir) - - namespace := DefaultKvrocksNamespace - data := map[string][]map[string]string{ - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - } - keys := make(map[string]string, len(data)) - metaKeys := make(map[string]string, len(data)) - for hashK := range data { - meta := NewMetadata() - fields := data[hashK] - for _, field := range fields { - for fieldK, fieldV := range field { - internalKey := encodeInternalKey(namespace, hashK, fieldK, meta.Version) - keys[string(internalKey)] = fieldV - } - } - hashKey := encodeRedisHashKey(namespace, hashK) - meta.Size++ - metaKeys[string(hashKey)] = string(meta.Encode()) - } - - err = createSSTFile(filepath.Join(dir, "kvrocks_default.sst"), keys) - assert.NoError(t, err) - err = createSSTFile(filepath.Join(dir, "kvrocks_metadata.sst"), metaKeys) - assert.NoError(t, err) - - r := rdb.Do(ctx, "sst", "load", dir) - assert.NoError(t, r.Err()) - resp, err := ExtractSSTResponse(r.Val()) - assert.NoError(t, err) - assert.Equal(t, int64(2), resp.filesLoaded) - - for hashK, fields := range data { - for _, field := range fields { - for fieldK, expectedVal := range field { - val := rdb.HGet(ctx, hashK, fieldK) - assert.NoError(t, val.Err()) - assert.Equal(t, expectedVal, val.Val(), "Hash field value mismatch for key:%s field:%s", hashK, fieldK) - } - } - } - // update the keys fields via redis interface - for hashK := range data { - fields := data[hashK] - for _, field := range fields { - for fieldK := range field { - newVal := "__avoid_collisions__" + util.RandString(1, 10, util.Alpha) - r := rdb.HSet(ctx, hashK, fieldK, newVal) - assert.NoError(t, r.Err()) - field[fieldK] = newVal - } - } - } - // validate the updates made via redis interface - for hashK, fields := range data { - for _, field := range fields { - for fieldK, expectedVal := range field { - val := rdb.HGet(ctx, hashK, fieldK) - assert.NoError(t, val.Err()) - assert.Equal(t, expectedVal, val.Val(), "Hash field value mismatch for key:%s field:%s", hashK, fieldK) - } - } - } - // update the values and sst load - for hashK := range data { - fields := data[hashK] - for _, field := range fields { - for fieldK := range field { - newVal := "__avoid_collisions__" + util.RandString(1, 10, util.Alpha) - field[fieldK] = newVal - } - } - } - - keys = make(map[string]string, len(data)) - metaKeys = make(map[string]string, len(data)) - for hashK := range data { - meta := NewMetadata() - fields := data[hashK] - for _, field := range fields { - for fieldK, fieldV := range field { - internalKey := encodeInternalKey(namespace, hashK, fieldK, meta.Version) - keys[string(internalKey)] = fieldV - } - } - hashKey := encodeRedisHashKey(namespace, hashK) - meta.Size++ - metaKeys[string(hashKey)] = string(meta.Encode()) - } - - err = createSSTFile(filepath.Join(dir, "kvrocks_default.sst"), keys) - assert.NoError(t, err) - err = createSSTFile(filepath.Join(dir, "kvrocks_metadata.sst"), metaKeys) - assert.NoError(t, err) - - r = rdb.Do(ctx, "sst", "load", dir) - assert.NoError(t, r.Err()) - resp, err = ExtractSSTResponse(r.Val()) - assert.NoError(t, err) - assert.Equal(t, int64(2), resp.filesLoaded) - - for hashK, fields := range data { - for _, field := range fields { - for fieldK, expectedVal := range field { - val := rdb.HGet(ctx, hashK, fieldK) - assert.NoError(t, val.Err()) - assert.Equal(t, expectedVal, val.Val(), "Hash field value mismatch for key:%s field:%s", hashK, fieldK) - } - } - } - }) - - t.Run("Test load redis hash keys with move option", func(t *testing.T) { - dir, err := makeTempDir() - require.NoError(t, err) - defer os.RemoveAll(dir) - - namespace := DefaultKvrocksNamespace - data := map[string][]map[string]string{ - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - } - keys := make(map[string]string, len(data)) - metaKeys := make(map[string]string, len(data)) - for hashK := range data { - meta := NewMetadata() - fields := data[hashK] - for _, field := range fields { - for fieldK, fieldV := range field { - internalKey := encodeInternalKey(namespace, hashK, fieldK, meta.Version) - keys[string(internalKey)] = fieldV - } - } - hashKey := encodeRedisHashKey(namespace, hashK) - meta.Size++ - metaKeys[string(hashKey)] = string(meta.Encode()) - } - - err = createSSTFile(filepath.Join(dir, "kvrocks_default.sst"), keys) - assert.NoError(t, err) - err = createSSTFile(filepath.Join(dir, "kvrocks_metadata.sst"), metaKeys) - assert.NoError(t, err) - - r := rdb.Do(ctx, "sst", "load", dir, "movefiles", "yes") - assert.NoError(t, r.Err()) - resp, err := ExtractSSTResponse(r.Val()) - assert.NoError(t, err) - assert.Equal(t, int64(2), resp.filesLoaded) - - //verify files did get moved - _, err = os.Stat(filepath.Join(dir, "kvrocks_default.sst")) - assert.True(t, os.IsNotExist(err)) - _, err = os.Stat(filepath.Join(dir, "kvrocks_metadata.sst")) - assert.True(t, os.IsNotExist(err)) - - for hashK, fields := range data { - for _, field := range fields { - for fieldK, expectedVal := range field { - val := rdb.HGet(ctx, hashK, fieldK) - assert.NoError(t, val.Err()) - assert.Equal(t, expectedVal, val.Val(), "Hash field value mismatch for key:%s field:%s", hashK, fieldK) - } - } - } - }) - - t.Run("Test load redis hash keys with no metadata entries", func(t *testing.T) { - dir, err := makeTempDir() - require.NoError(t, err) - defer os.RemoveAll(dir) - - namespace := DefaultKvrocksNamespace - data := map[string][]map[string]string{ - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - } - keys := make(map[string]string, len(data)) - metaKeys := make(map[string]string, len(data)) - for hashK := range data { - meta := NewMetadata() - fields := data[hashK] - for _, field := range fields { - for fieldK, fieldV := range field { - internalKey := encodeInternalKey(namespace, hashK, fieldK, meta.Version) - keys[string(internalKey)] = fieldV - } - } - hashKey := encodeRedisHashKey(namespace, hashK) - meta.Size++ - metaKeys[string(hashKey)] = string(meta.Encode()) - } - - err = createSSTFile(filepath.Join(dir, "kvrocks_default.sst"), keys) - assert.NoError(t, err) - - r := rdb.Do(ctx, "sst", "load", dir) - assert.NoError(t, r.Err()) - resp, err := ExtractSSTResponse(r.Val()) - assert.NoError(t, err) - assert.Equal(t, int64(1), resp.filesLoaded) - - for hashK, fields := range data { - for _, field := range fields { - for fieldK := range field { - val := rdb.HGet(ctx, hashK, fieldK) - assert.Error(t, val.Err()) - } - } - } - }) - - t.Run("Test load redis hash keys with expiration", func(t *testing.T) { - dir, err := makeTempDir() - require.NoError(t, err) - defer os.RemoveAll(dir) - - namespace := DefaultKvrocksNamespace - data := map[string][]map[string]string{ - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - "__avoid_collisions__" + util.RandString(1, 10, util.Alpha): { - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - {"__avoid_collisions__" + util.RandString(1, 10, util.Alpha): "__avoid_collisions__" + util.RandString(1, 10, util.Alpha)}, - }, - } - keys := make(map[string]string, len(data)) - metaKeys := make(map[string]string, len(data)) - for hashK := range data { - meta := NewMetadata() - meta.Expire = uint64(time.Now().Add(5 * time.Second).UnixMilli()) - fields := data[hashK] - for _, field := range fields { - for fieldK, fieldV := range field { - internalKey := encodeInternalKey(namespace, hashK, fieldK, meta.Version) - keys[string(internalKey)] = fieldV - } - } - hashKey := encodeRedisHashKey(namespace, hashK) - meta.Size++ - metaKeys[string(hashKey)] = string(meta.Encode()) - } - - err = createSSTFile(filepath.Join(dir, "kvrocks_default.sst"), keys) - assert.NoError(t, err) - err = createSSTFile(filepath.Join(dir, "kvrocks_metadata.sst"), metaKeys) - assert.NoError(t, err) - - r := rdb.Do(ctx, "sst", "load", dir) - assert.NoError(t, r.Err()) - resp, err := ExtractSSTResponse(r.Val()) - assert.NoError(t, err) - assert.Equal(t, int64(2), resp.filesLoaded) - - // verify keys have expiration - for hashK := range data { - expireDuration := rdb.ExpireTime(ctx, hashK) - assert.NotEmpty(t, expireDuration.Val().Milliseconds()) - } - //verify keys have expired - time.Sleep(5 * time.Second) - for hashK, fields := range data { - for _, field := range fields { - for fieldK := range field { - val := rdb.HGet(ctx, hashK, fieldK) - assert.Error(t, val.Err()) - } - } - } - }) -} diff --git a/x.py b/x.py index 787eecade41..deb3b452390 100755 --- a/x.py +++ b/x.py @@ -223,21 +223,6 @@ def clang_tidy(dir: str, jobs: Optional[int], clang_tidy_path: str, run_clang_ti run(run_command, *options, *regexes, verbose=True, cwd=basedir) -def env_with_cgo_flags(build_dir: Path): - rocksdb_inc = build_dir.joinpath('_deps/rocksdb-src/include') - rocksdb_lib = build_dir.joinpath('_deps/rocksdb-build') - zlib_lib = build_dir.joinpath('_deps/zstd-src/lib') - lz4_lib = build_dir.joinpath('_deps/lz4-src/lib') - snappy_lib = build_dir.joinpath('_deps/snappy-build') - - env = os.environ.copy() - env.update({ - "CGO_CFLAGS": f"-I{rocksdb_inc} {env.get('CGO_CFLAGS', '')}", - "CGO_LDFLAGS": f"-L{rocksdb_lib} -L{zlib_lib} -L{lz4_lib} -L{snappy_lib} {env.get('CGO_LDFLAGS', '')}", - }) - return env - - def golangci_lint(golangci_lint_path: str) -> None: def get_gopath() -> Tuple[Path, Path]: go = find_command('go', msg='go is required for testing') @@ -337,7 +322,7 @@ def test_go(dir: str, cli_path: str, rest: List[str]) -> None: *rest ] - run(go, *args, cwd=str(basedir), verbose=True, env=env_with_cgo_flags(Path(dir).absolute())) + run(go, *args, cwd=str(basedir), verbose=True) if __name__ == '__main__':