Skip to content

Commit 6b120b8

Browse files
sguiheuxsguiheux
authored
feat(cdn): add route to be able to resync a backend with database (#5829)
1 parent c39725e commit 6b120b8

File tree

13 files changed

+301
-0
lines changed

13 files changed

+301
-0
lines changed

‎engine/cdn/cdn_router.go

+2
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,6 @@ func (s *Service) initRouter(ctx context.Context) {
5252
r.Handle("/admin/database/encryption/{entity}", nil, r.GET(s.getAdminDatabaseEncryptedTuplesByEntity))
5353
r.Handle("/admin/database/encryption/{entity}/roll/{pk}", nil, r.POST(s.postAdminDatabaseRollEncryptedEntityByPrimaryKey))
5454

55+
r.Handle("/admin/backend/{id}/resync/{type}", nil, r.POST(s.postAdminResyncBackendWithDatabaseHandler))
56+
5557
}

‎engine/cdn/storage/cds/cds.go

+5
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/ovh/cds/engine/gorpmapper"
1414
"github.com/ovh/cds/sdk"
1515
"github.com/ovh/cds/sdk/cdsclient"
16+
"github.com/rockbears/log"
1617
)
1718

1819
type CDS struct {
@@ -130,3 +131,7 @@ func (c *CDS) Read(_ sdk.CDNItemUnit, r io.Reader, w io.Writer) error {
130131
func (c *CDS) Write(_ sdk.CDNItemUnit, _ io.Reader, _ io.Writer) error {
131132
return nil
132133
}
134+
135+
func (c *CDS) ResyncWithDatabase(ctx context.Context, _ gorp.SqlExecutor, _ sdk.CDNItemType, _ bool) {
136+
log.Error(ctx, "Resynchronization with database not implemented for CDS storage unit")
137+
}

‎engine/cdn/storage/dao.go

+13
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,19 @@ func HasItemUnitsByUnitAndHashLocator(db gorp.SqlExecutor, unitID string, hashLo
174174
return len(ids) > 0, sdk.WithStack(err)
175175
}
176176

177+
func HashItemUnitByApiRefHash(db gorp.SqlExecutor, apiRefHash string, unitID string) (bool, error) {
178+
query := `
179+
SELECT count(sui.id) FROM storage_unit_item sui
180+
JOIN item on item.id = sui.item_id
181+
WHERE item.api_ref_hash = $1 AND unit_id = $2
182+
`
183+
nb, err := db.SelectInt(query, apiRefHash, unitID)
184+
if err != nil {
185+
return false, sdk.WithStack(err)
186+
}
187+
return nb > 0, nil
188+
}
189+
177190
func LoadItemUnitByID(ctx context.Context, m *gorpmapper.Mapper, db gorp.SqlExecutor, id string, opts ...gorpmapper.GetOptionFunc) (*sdk.CDNItemUnit, error) {
178191
query := gorpmapper.NewQuery("SELECT * FROM storage_unit_item WHERE id = $1 AND to_delete = false").Args(id)
179192
return getItemUnit(ctx, m, db, query, opts...)

‎engine/cdn/storage/local/buffer.go

+39
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ import (
44
"context"
55
"fmt"
66
"os"
7+
"path/filepath"
8+
9+
"github.com/go-gorp/gorp"
10+
"github.com/rockbears/log"
711

812
"github.com/ovh/cds/engine/cdn/storage"
913
"github.com/ovh/cds/engine/cdn/storage/encryption"
@@ -58,3 +62,38 @@ func (b *Buffer) Size(_ sdk.CDNItemUnit) (int64, error) {
5862
func (b *Buffer) BufferType() storage.CDNBufferType {
5963
return b.bufferType
6064
}
65+
66+
func (b *Buffer) ResyncWithDatabase(ctx context.Context, db gorp.SqlExecutor, t sdk.CDNItemType, dryRun bool) {
67+
root := fmt.Sprintf("%s/%s", b.config.Path, string(t))
68+
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
69+
if info == nil {
70+
return nil
71+
}
72+
if info.IsDir() {
73+
log.Warn(ctx, "local-buffer: found directory inside %s: %s", string(t), path)
74+
return nil
75+
}
76+
_, fileName := filepath.Split(path)
77+
has, err := storage.HashItemUnitByApiRefHash(db, fileName, b.ID())
78+
if err != nil {
79+
log.Error(ctx, "local-buffer: unable to check if unit item exist for api ref hash %s: %v", fileName, err)
80+
return nil
81+
}
82+
if has {
83+
return nil
84+
}
85+
if !dryRun {
86+
if err := os.Remove(path); err != nil {
87+
log.Error(ctx, "local-buffer: unable to remove file %s: %v", path, err)
88+
return nil
89+
}
90+
log.Info(ctx, "local-buffer: file %s has been deleted", fileName)
91+
} else {
92+
log.Info(ctx, "local-buffer: file %s should be deleted", fileName)
93+
}
94+
return nil
95+
}); err != nil {
96+
log.Error(ctx, "local-buffer: error during walk operation: %v", err)
97+
}
98+
99+
}

‎engine/cdn/storage/local/local.go

+4
Original file line numberDiff line numberDiff line change
@@ -193,3 +193,7 @@ func (s *AbstractLocal) Remove(ctx context.Context, i sdk.CDNItemUnit) error {
193193
}
194194
return nil
195195
}
196+
197+
func (s *Local) ResyncWithDatabase(ctx context.Context, _ gorp.SqlExecutor, _ sdk.CDNItemType, _ bool) {
198+
log.Error(ctx, "Resynchronization with database not implemented for local storage unit")
199+
}

‎engine/cdn/storage/nfs/nfs.go

+44
Original file line numberDiff line numberDiff line change
@@ -332,3 +332,47 @@ func (n *Buffer) ls(v *gonfs.Target, path string) ([]*gonfs.EntryPlus, error) {
332332
}
333333
return dirs, nil
334334
}
335+
336+
func (n *Buffer) ResyncWithDatabase(ctx context.Context, db gorp.SqlExecutor, t sdk.CDNItemType, dryRun bool) {
337+
dial, target, err := n.Connect()
338+
if err != nil {
339+
log.Error(ctx, "nfs-buffer: unable to connect to NFS: %v", err)
340+
return
341+
}
342+
defer dial.Close() // nolint
343+
defer target.Close() //
344+
345+
entries, err := n.ls(target, string(t))
346+
if err != nil {
347+
log.Error(ctx, "nfs-buffer: unable to list directory %s", string(t))
348+
return
349+
}
350+
for _, e := range entries {
351+
if e.IsDir() {
352+
log.Warn(ctx, "nfs-buffer: found directory inside %s: %s", string(t), e)
353+
continue
354+
}
355+
if e.FileName == "" {
356+
log.Warn(ctx, "nfs-buffer: missing file name")
357+
continue
358+
}
359+
has, err := storage.HashItemUnitByApiRefHash(db, e.FileName, n.ID())
360+
if err != nil {
361+
log.Error(ctx, "nfs-buffer: unable to check if unit item exist for api ref hash %s: %v", e.FileName, err)
362+
continue
363+
}
364+
if has {
365+
continue
366+
}
367+
if !dryRun {
368+
if err := target.Remove(string(t) + "/" + e.FileName); err != nil {
369+
log.Error(ctx, "nfs-buffer: unable to remove file %s: %v", string(t)+"/"+e.FileName, err)
370+
continue
371+
}
372+
log.Info(ctx, "nfs-buffer: file %s has been deleted", e.FileName)
373+
} else {
374+
log.Info(ctx, "nfs-buffer: file %s should be deleted", e.FileName)
375+
}
376+
}
377+
return
378+
}

‎engine/cdn/storage/redis/redis.go

+5
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"strconv"
88

99
"github.com/go-gorp/gorp"
10+
"github.com/rockbears/log"
1011

1112
"github.com/ovh/cds/engine/cache"
1213
"github.com/ovh/cds/engine/cdn/redis"
@@ -142,3 +143,7 @@ func (s *Redis) Status(_ context.Context) []sdk.MonitoringStatusLine {
142143
func (s *Redis) Remove(_ context.Context, i sdk.CDNItemUnit) error {
143144
return sdk.WithStack(s.store.Delete(cache.Key(keyBuffer, i.ItemID)))
144145
}
146+
147+
func (s *Redis) ResyncWithDatabase(ctx context.Context, _ gorp.SqlExecutor, _ sdk.CDNItemType, _ bool) {
148+
log.Error(ctx, "Resynchronization with database not implemented for redis buffer unit")
149+
}

‎engine/cdn/storage/s3/s3.go

+4
Original file line numberDiff line numberDiff line change
@@ -203,3 +203,7 @@ func (s *S3) Remove(ctx context.Context, i sdk.CDNItemUnit) error {
203203
})
204204
return sdk.WithStack(err)
205205
}
206+
207+
func (s *S3) ResyncWithDatabase(ctx context.Context, _ gorp.SqlExecutor, _ sdk.CDNItemType, _ bool) {
208+
log.Error(ctx, "Resynchronization with database not implemented for s3 storage unit")
209+
}

‎engine/cdn/storage/swift/swift.go

+4
Original file line numberDiff line numberDiff line change
@@ -139,3 +139,7 @@ func (s *Swift) Remove(ctx context.Context, i sdk.CDNItemUnit) error {
139139
}
140140
return nil
141141
}
142+
143+
func (s *Swift) ResyncWithDatabase(ctx context.Context, _ gorp.SqlExecutor, _ sdk.CDNItemType, _ bool) {
144+
log.Error(ctx, "Resynchronization with database not implemented for swift storage unit")
145+
}

‎engine/cdn/storage/types.go

+1
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ type Unit interface {
9898
Read(i sdk.CDNItemUnit, r io.Reader, w io.Writer) error
9999
NewReader(ctx context.Context, i sdk.CDNItemUnit) (io.ReadCloser, error)
100100
GetDriverName() string
101+
ResyncWithDatabase(ctx context.Context, db gorp.SqlExecutor, t sdk.CDNItemType, dryRun bool)
101102
}
102103

103104
type BufferUnit interface {

‎engine/cdn/storage/webdav/webdav.go

+4
Original file line numberDiff line numberDiff line change
@@ -120,3 +120,7 @@ func (s *Webdav) Remove(ctx context.Context, i sdk.CDNItemUnit) error {
120120
}
121121
return sdk.WithStack(s.client.Remove(f))
122122
}
123+
124+
func (s *Webdav) ResyncWithDatabase(ctx context.Context, _ gorp.SqlExecutor, _ sdk.CDNItemType, _ bool) {
125+
log.Error(ctx, "Resynchronization with database not implemented for webdav storage unit")
126+
}

‎engine/cdn/unit_handler.go

+32
Original file line numberDiff line numberDiff line change
@@ -108,3 +108,35 @@ func (s *Service) markItemUnitAsDeleteHandler() service.Handler {
108108
return nil
109109
}
110110
}
111+
112+
func (s *Service) postAdminResyncBackendWithDatabaseHandler() service.Handler {
113+
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
114+
vars := mux.Vars(r)
115+
unitID := vars["id"]
116+
itemType := vars["type"]
117+
118+
it := sdk.CDNItemType(itemType)
119+
if err := it.Validate(); err != nil {
120+
return err
121+
}
122+
123+
dryRunString := r.FormValue("dryRun")
124+
dryRun := dryRunString != "false"
125+
126+
for _, u := range s.Units.Buffers {
127+
if u.ID() == unitID {
128+
s.GoRoutines.Exec(context.Background(), "ResyncWithDB-"+unitID, func(ctx context.Context) {
129+
u.ResyncWithDatabase(ctx, s.mustDBWithCtx(ctx), it, dryRun)
130+
})
131+
}
132+
}
133+
for _, u := range s.Units.Storages {
134+
if u.ID() == unitID {
135+
s.GoRoutines.Exec(context.Background(), "ResyncWithDB-"+unitID, func(ctx context.Context) {
136+
u.ResyncWithDatabase(ctx, s.mustDBWithCtx(ctx), it, dryRun)
137+
})
138+
}
139+
}
140+
return nil
141+
}
142+
}

‎engine/cdn/unit_handler_test.go

+144
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,21 @@ package cdn
22

33
import (
44
"context"
5+
"fmt"
56
"github.com/ovh/cds/engine/cdn/item"
67
"github.com/ovh/cds/engine/cdn/storage"
78
cdntest "github.com/ovh/cds/engine/cdn/test"
9+
"github.com/ovh/cds/engine/test"
810
"github.com/ovh/cds/sdk"
11+
"github.com/ovh/cds/sdk/cdn"
12+
"github.com/ovh/symmecrypt/ciphers/aesgcm"
13+
"github.com/ovh/symmecrypt/convergent"
14+
"github.com/ovh/symmecrypt/keyloader"
915
"github.com/rockbears/log"
1016
"github.com/stretchr/testify/require"
17+
"io/ioutil"
1118
"net/http/httptest"
19+
"os"
1220
"testing"
1321
"time"
1422
)
@@ -94,3 +102,139 @@ func TestMarkItemUnitAsDeleteHandler(t *testing.T) {
94102
require.Equal(t, 204, recDel.Code)
95103

96104
}
105+
106+
func TestPostAdminResyncBackendWithDatabaseHandler(t *testing.T) {
107+
s, db := newTestService(t)
108+
109+
cfg := test.LoadTestingConf(t, sdk.TypeCDN)
110+
111+
cdntest.ClearItem(t, context.TODO(), s.Mapper, db)
112+
cdntest.ClearItem(t, context.TODO(), s.Mapper, db)
113+
cdntest.ClearSyncRedisSet(t, s.Cache, "local_storage")
114+
115+
// Start CDN
116+
ctx, cancel := context.WithCancel(context.Background())
117+
t.Cleanup(cancel)
118+
tmpDir, err := ioutil.TempDir("", t.Name()+"-cdn-1-*")
119+
require.NoError(t, err)
120+
121+
tmpDir2, err := ioutil.TempDir("", t.Name()+"-cdn-2-*")
122+
require.NoError(t, err)
123+
124+
cdnUnits, err := storage.Init(ctx, s.Mapper, s.Cache, db.DbMap, sdk.NewGoRoutines(ctx), storage.Configuration{
125+
SyncSeconds: 1,
126+
SyncNbElements: 1000,
127+
PurgeNbElements: 1000,
128+
PurgeSeconds: 30,
129+
HashLocatorSalt: "thisismysalt",
130+
Buffers: map[string]storage.BufferConfiguration{
131+
"refis_buffer": {
132+
Redis: &storage.RedisBufferConfiguration{
133+
Host: cfg["redisHost"],
134+
Password: cfg["redisPassword"],
135+
},
136+
BufferType: storage.CDNBufferTypeLog,
137+
},
138+
"local_buffer": {
139+
Local: &storage.LocalBufferConfiguration{
140+
Path: tmpDir,
141+
Encryption: []*keyloader.KeyConfig{
142+
{
143+
Key: "iamakey.iamakey.iamakey.iamakey.",
144+
Cipher: aesgcm.CipherName,
145+
Identifier: "local-bukker-id",
146+
},
147+
},
148+
},
149+
BufferType: storage.CDNBufferTypeFile,
150+
},
151+
},
152+
Storages: map[string]storage.StorageConfiguration{
153+
"local_storage": {
154+
SyncParallel: 10,
155+
SyncBandwidth: int64(1024 * 1024),
156+
Local: &storage.LocalStorageConfiguration{
157+
Path: tmpDir2,
158+
Encryption: []convergent.ConvergentEncryptionConfig{
159+
{
160+
Cipher: aesgcm.CipherName,
161+
LocatorSalt: "secret_locator_salt",
162+
SecretValue: "secret_value",
163+
},
164+
},
165+
},
166+
},
167+
},
168+
})
169+
require.NoError(t, err)
170+
s.Units = cdnUnits
171+
cdnUnits.Start(ctx, sdk.NewGoRoutines(ctx))
172+
173+
// Create an Item
174+
it, err := s.loadOrCreateItem(context.TODO(), sdk.CDNTypeItemRunResult, cdn.Signature{
175+
RunID: 1,
176+
JobID: 1,
177+
WorkflowID: 1,
178+
NodeRunID: 1,
179+
Worker: &cdn.SignatureWorker{
180+
WorkerID: "1",
181+
FileName: sdk.RandomString(10),
182+
RunResultType: string(sdk.WorkflowRunResultTypeArtifact),
183+
FilePerm: 0777,
184+
StepOrder: 0,
185+
WorkerName: sdk.RandomString(10),
186+
},
187+
})
188+
require.NoError(t, err)
189+
_, err = s.loadOrCreateItemUnitBuffer(context.TODO(), it.ID, sdk.CDNTypeItemRunResult)
190+
require.NoError(t, err)
191+
192+
require.NoError(t, os.Mkdir(fmt.Sprintf("%s/%s", tmpDir, string(sdk.CDNTypeItemRunResult)), 0755))
193+
194+
file1Path := fmt.Sprintf("%s/%s/%s", tmpDir, string(sdk.CDNTypeItemRunResult), it.APIRefHash)
195+
t.Logf("Creating file %s", file1Path)
196+
content1 := []byte("I'm the real one")
197+
f1, err := os.Create(file1Path)
198+
require.NoError(t, err)
199+
defer f1.Close()
200+
_, err = f1.Write(content1)
201+
require.NoError(t, err)
202+
203+
file2Path := fmt.Sprintf("%s/%s/%s", tmpDir, string(sdk.CDNTypeItemRunResult), "wronghash")
204+
t.Logf("Creating file %s", file2Path)
205+
content2 := []byte("I'm not the real one")
206+
f2, err := os.Create(file2Path)
207+
require.NoError(t, err)
208+
defer f2.Close()
209+
_, err = f2.Write(content2)
210+
require.NoError(t, err)
211+
212+
vars := make(map[string]string)
213+
vars["id"] = s.Units.GetBuffer(sdk.CDNTypeItemRunResult).ID()
214+
vars["type"] = string(sdk.CDNTypeItemRunResult)
215+
uri := s.Router.GetRoute("POST", s.postAdminResyncBackendWithDatabaseHandler, vars) + "?dryRun=false"
216+
require.NotEmpty(t, uri)
217+
req := newRequest(t, "POST", uri, nil)
218+
rec := httptest.NewRecorder()
219+
s.Router.Mux.ServeHTTP(rec, req)
220+
require.Equal(t, 204, rec.Code)
221+
222+
cpt := 0
223+
for {
224+
_, err := os.Stat(file2Path)
225+
if os.IsNotExist(err) {
226+
break
227+
}
228+
if cpt >= 20 {
229+
t.FailNow()
230+
}
231+
cpt++
232+
time.Sleep(250 * time.Millisecond)
233+
}
234+
_, err = os.Stat(file2Path)
235+
require.True(t, os.IsNotExist(err))
236+
237+
_, err = os.Stat(file1Path)
238+
require.NoError(t, err)
239+
240+
}

0 commit comments

Comments
 (0)