@@ -690,90 +690,96 @@ func (repo *Repo) index0(memo string, checkChunks bool, context map[string]inter
690690 init = true
691691 }
692692
693- var workerErrs []error
694- workerErrLock := sync.Mutex {}
695693 var upserts , removes , latestFiles []* entity.File
696- if ! init {
697- start = time .Now ()
698- count := atomic.Int32 {}
699- total := len (files )
700- eventbus .Publish (eventbus .EvtIndexBeforeGetLatestFiles , context , total )
701- lock := & sync.Mutex {}
702- waitGroup := & sync.WaitGroup {}
703- p , _ := ants .NewPoolWithFunc (4 , func (arg interface {}) {
704- defer waitGroup .Done ()
705-
706- count .Add (1 )
707- eventbus .Publish (eventbus .EvtIndexGetLatestFile , context , int (count .Load ()), total )
708-
709- fileID := arg .(string )
710- file , getErr := repo .store .GetFile (fileID )
711- if nil != getErr {
712- logging .LogErrorf ("get file [%s] failed: %s" , fileID , getErr )
713- workerErrLock .Lock ()
714- workerErrs = append (workerErrs , ErrRepoFatal )
715- workerErrLock .Unlock ()
716- return
717- }
718-
719- lock .Lock ()
720- latestFiles = append (latestFiles , file )
721- lock .Unlock ()
694+ fullLatest := repo .getFullLatest (latest )
695+ if nil != fullLatest {
696+ latestFiles = fullLatest .Files
697+ } else {
698+ var workerErrs []error
699+ workerErrLock := sync.Mutex {}
700+ if ! init {
701+ start = time .Now ()
702+ count := atomic.Int32 {}
703+ total := len (files )
704+ eventbus .Publish (eventbus .EvtIndexBeforeGetLatestFiles , context , total )
705+ lock := & sync.Mutex {}
706+ waitGroup := & sync.WaitGroup {}
707+ p , _ := ants .NewPoolWithFunc (4 , func (arg interface {}) {
708+ defer waitGroup .Done ()
709+
710+ count .Add (1 )
711+ eventbus .Publish (eventbus .EvtIndexGetLatestFile , context , int (count .Load ()), total )
712+
713+ fileID := arg .(string )
714+ file , getErr := repo .store .GetFile (fileID )
715+ if nil != getErr {
716+ logging .LogErrorf ("get file [%s] failed: %s" , fileID , getErr )
717+ workerErrLock .Lock ()
718+ workerErrs = append (workerErrs , ErrRepoFatal )
719+ workerErrLock .Unlock ()
720+ return
721+ }
722722
723- if checkChunks { // 仅在非移动端校验，因为移动端私有数据空间不会存在外部操作导致分块损坏的情况 https://github.com/siyuan-note/siyuan/issues/13216
724- // Check local data chunk integrity before data synchronization https://github.com/siyuan-note/siyuan/issues/8853
725- for _ , chunk := range file .Chunks {
726- info , statErr := repo .store .Stat (chunk )
727- if nil == statErr {
728- continue
729- }
723+ lock .Lock ()
724+ latestFiles = append (latestFiles , file )
725+ lock .Unlock ()
730726
731- if nil != info {
732- logging . LogWarnf ( "stat file [%s, %s, %s, %d] chunk [%s, perm=%04o] failed: %s" ,
733- file . ID , file . Path , time . UnixMilli ( file . Updated ). Format ( "2006-01-02 15:04:05" ), file .Size , chunk , info . Mode (). Perm (), statErr )
734- } else {
735- logging . LogWarnf ( "stat file [%s, %s, %s, %d] chunk [%s] failed: %s" ,
736- file . ID , file . Path , time . UnixMilli ( file . Updated ). Format ( "2006-01-02 15:04:05" ), file . Size , chunk , statErr )
737- }
727+ if checkChunks { // 仅在非移动端校验，因为移动端私有数据空间不会存在外部操作导致分块损坏的情况 https://github.com/siyuan-note/siyuan/issues/13216
728+ // Check local data chunk integrity before data synchronization https://github.com/siyuan-note/siyuan/issues/8853
729+ for _ , chunk := range file .Chunks {
730+ info , statErr := repo . store . Stat ( chunk )
731+ if nil == statErr {
732+ continue
733+ }
738734
739- if errors .Is (statErr , os .ErrPermission ) {
740- // 如果是权限问题，则尝试修改权限，不认为是分块文件损坏
741- // Improve checking local data chunk integrity before data sync https://github.com/siyuan-note/siyuan/issues/9688
742- if chmodErr := os .Chmod (chunk , 0644 ); nil != chmodErr {
743- logging .LogWarnf ("chmod file [%s] failed: %s" , chunk , chmodErr )
735+ if nil != info {
736+ logging .LogWarnf ("stat file [%s, %s, %s, %d] chunk [%s, perm=%04o] failed: %s" ,
737+ file .ID , file .Path , time .UnixMilli (file .Updated ).Format ("2006-01-02 15:04:05" ), file .Size , chunk , info .Mode ().Perm (), statErr )
744738 } else {
745- logging .LogInfof ("chmod file [%s] to [0644]" , chunk )
739+ logging .LogWarnf ("stat file [%s, %s, %s, %d] chunk [%s] failed: %s" ,
740+ file .ID , file .Path , time .UnixMilli (file .Updated ).Format ("2006-01-02 15:04:05" ), file .Size , chunk , statErr )
746741 }
747- continue
748- }
749742
750- if errors .Is (statErr , os .ErrNotExist ) {
751- workerErrLock .Lock ()
752- workerErrs = append (workerErrs , ErrRepoFatal )
753- workerErrLock .Unlock ()
754- return
743+ if errors .Is (statErr , os .ErrPermission ) {
744+ // 如果是权限问题，则尝试修改权限，不认为是分块文件损坏
745+ // Improve checking local data chunk integrity before data sync https://github.com/siyuan-note/siyuan/issues/9688
746+ if chmodErr := os .Chmod (chunk , 0644 ); nil != chmodErr {
747+ logging .LogWarnf ("chmod file [%s] failed: %s" , chunk , chmodErr )
748+ } else {
749+ logging .LogInfof ("chmod file [%s] to [0644]" , chunk )
750+ }
751+ continue
752+ }
753+
754+ if errors .Is (statErr , os .ErrNotExist ) {
755+ workerErrLock .Lock ()
756+ workerErrs = append (workerErrs , ErrRepoFatal )
757+ workerErrLock .Unlock ()
758+ return
759+ }
755760 }
756761 }
762+ })
763+
764+ for _ , f := range latest .Files {
765+ waitGroup .Add (1 )
766+ err = p .Invoke (f )
767+ if nil != err {
768+ logging .LogErrorf ("invoke failed: %s" , err )
769+ return
770+ }
757771 }
758- })
759-
760- for _ , f := range latest .Files {
761- waitGroup .Add (1 )
762- err = p .Invoke (f )
763- if nil != err {
764- logging .LogErrorf ("invoke failed: %s" , err )
772+ waitGroup .Wait ()
773+ p .Release ()
774+ logging .LogInfof ("get latest files [files=%d] cost [%s]" , len (latestFiles ), time .Since (start ))
775+ if 0 < len (workerErrs ) {
776+ err = workerErrs [0 ]
777+ logging .LogErrorf ("get latest files failed: %s" , err )
765778 return
766779 }
767780 }
768- waitGroup .Wait ()
769- p .Release ()
770- logging .LogInfof ("get latest files [files=%d] cost [%s]" , len (latestFiles ), time .Since (start ))
771- if 0 < len (workerErrs ) {
772- err = workerErrs [0 ]
773- logging .LogErrorf ("get latest files failed: %s" , err )
774- return
775- }
776781 }
782+
777783 upserts , removes = repo .diffUpsertRemove (files , latestFiles , false )
778784 if 1 > len (upserts ) && 1 > len (removes ) {
779785 ret = latest
@@ -795,7 +801,8 @@ func (repo *Repo) index0(memo string, checkChunks bool, context map[string]inter
795801
796802 count := atomic.Int32 {}
797803 total := len (upserts )
798- workerErrs = nil
804+ var workerErrs []error
805+ workerErrLock := sync.Mutex {}
799806 eventbus .Publish (eventbus .EvtIndexUpsertFiles , context , total )
800807 waitGroup := & sync.WaitGroup {}
801808 p , _ := ants .NewPoolWithFunc (4 , func (arg interface {}) {
0 commit comments