From 0ec518973c93805c5f87b8fd232c4a517958eaba Mon Sep 17 00:00:00 2001 From: zyxkad Date: Sun, 18 Aug 2024 08:18:36 -0600 Subject: [PATCH] fix report API, and a few translations --- cluster/cluster.go | 6 ++++++ cluster/requests.go | 14 +++++++++++--- cluster/socket.go | 4 ++-- cluster/storage.go | 27 +++++++-------------------- config.go | 5 ++--- lang/en/us.go | 10 +++++----- lang/zh/cn.go | 10 +++++----- main.go | 3 +++ runner.go | 26 ++++++++------------------ utils/error.go | 2 +- utils/http.go | 18 ++++++++++-------- 11 files changed, 60 insertions(+), 65 deletions(-) diff --git a/cluster/cluster.go b/cluster/cluster.go index e2347935..fa10f638 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -42,6 +42,8 @@ var ( reFileHashMismatchError = regexp.MustCompile(` hash mismatch, expected ([0-9a-f]+), got ([0-9a-f]+)`) ) +const DefaultBMCLAPIServer = "https://openbmclapi.bangbang93.com" + type Cluster struct { name string opts config.ClusterOptions @@ -77,7 +79,11 @@ func NewCluster( for i, name := range opts.Storages { storages[i] = storageManager.GetIndex(name) } + if opts.Server == "" { + opts.Server = DefaultBMCLAPIServer + } cr = &Cluster{ + name: name, opts: opts, gcfg: gcfg, diff --git a/cluster/requests.go b/cluster/requests.go index 83c6403f..0899c10e 100644 --- a/cluster/requests.go +++ b/cluster/requests.go @@ -26,6 +26,7 @@ import ( "crypto/hmac" "encoding/hex" "encoding/json" + "errors" "fmt" "net/http" "net/url" @@ -261,12 +262,18 @@ func (cr *Cluster) RequestCert(ctx context.Context) (ckp *CertKeyPair, err error } func (cr *Cluster) ReportDownload(ctx context.Context, response *http.Response, err error) error { + if errors.Is(err, context.Canceled) { + return nil + } + type ReportPayload struct { - Urls []string `json:"urls"` - Error utils.EmbedJSON[struct{ Message string }] `json:"error"` + Urls []string `json:"urls"` + Error utils.EmbedJSON[struct { + Message string `json:"message"` + }] `json:"error"` } var payload ReportPayload - redirects := utils.GetRedirects(response.Request) + redirects := utils.GetRedirects(response) payload.Urls = make([]string, len(redirects)) for i, u := range redirects { payload.Urls[i] = u.String() @@ -280,6 +287,7 @@ func (cr *Cluster) ReportDownload(ctx context.Context, response *http.Response, if err != nil { return err } + req.Header.Set("Content-Type", "application/json") resp, err := cr.client.Do(req) if err != nil { return err diff --git a/cluster/socket.go b/cluster/socket.go index db238397..ccce2bd5 100644 --- a/cluster/socket.go +++ b/cluster/socket.go @@ -68,7 +68,7 @@ func (cr *Cluster) Connect(ctx context.Context) error { }) } engio.OnConnect(func(s *engine.Socket) { - log.Info("Engine.IO %s connected for cluster %s", s.ID(), cr.ID()) + log.Infof("Engine.IO %s connected for cluster %s", s.ID(), cr.ID()) }) engio.OnDisconnect(cr.onDisconnected) engio.OnDialError(func(s *engine.Socket, err *engine.DialErrorContext) { @@ -102,7 +102,7 @@ func (cr *Cluster) Connect(ctx context.Context) error { log.Infof("[remote]: %v", data[0]) } }) - log.Info("Connecting to socket.io namespace") + log.Infof("Cluster %s is connecting to socket.io namespace", cr.Name()) if err := cr.socket.Connect(""); err != nil { return fmt.Errorf("Namespace connect error: %w", err) } diff --git a/cluster/storage.go b/cluster/storage.go index 7a234b81..2750b5d1 100644 --- a/cluster/storage.go +++ b/cluster/storage.go @@ -174,23 +174,6 @@ func storageIdSortFunc(a, b storage.Storage) int { return 1 } -// func SyncFiles(ctx context.Context, manager *storage.Manager, files map[string]*StorageFileInfo, heavyCheck bool) bool { -// log.TrInfof("info.sync.prepare", len(files)) - -// slices.SortFunc(files, func(a, b *StorageFileInfo) int { return a.Size - b.Size }) -// if cr.syncFiles(ctx, files, heavyCheck) != nil { -// return false -// } - -// cr.filesetMux.Lock() -// for _, f := range files { -// cr.fileset[f.Hash] = f.Size -// } -// cr.filesetMux.Unlock() - -// return true -// } - var emptyStr string func checkFile( @@ -393,12 +376,15 @@ func (c *HTTPClient) SyncFiles( return err } - totalFiles := len(files) + totalFiles := len(missingMap) var stats syncStats stats.pg = pg stats.slots = limited.NewBufSlots(slots) stats.totalFiles = totalFiles + for _, f := range missingMap { + stats.totalSize += f.Size + } var barUnit decor.SizeB1024 stats.lastInc.Store(time.Now().UnixNano()) @@ -620,7 +606,8 @@ func (c *HTTPClient) fetchFile(ctx context.Context, stats *syncStats, f *Storage return utils.ProxyPBReader(r, bar, stats.totalBar, &stats.lastInc) }); err != nil { reqInd = (reqInd + 1) % len(reqs) - if rerr, ok := err.(*utils.RedirectError); ok { + var rerr *utils.RedirectError + if errors.As(err, &rerr) { go func() { if err := rp.Cluster.ReportDownload(context.WithoutCancel(ctx), rerr.GetResponse(), rerr.Unwrap()); err != nil { log.Warnf("Report API error: %v", err) @@ -642,7 +629,7 @@ func (c *HTTPClient) fetchFile(ctx context.Context, stats *syncStats, f *Storage bar.SetRefill(bar.Current()) c := tried.Add(1) - if c > maxRetryCount { + if c > maxRetryCount || errors.Is(err, context.Canceled) { log.TrErrorf("error.sync.download.failed", f.Hash, err) stats.failCount.Add(1) return diff --git a/config.go b/config.go index 29b5cd49..02dd529c 100644 --- a/config.go +++ b/config.go @@ -33,14 +33,13 @@ import ( "gopkg.in/yaml.v3" "github.com/LiterMC/go-openbmclapi/api" + "github.com/LiterMC/go-openbmclapi/cluster" "github.com/LiterMC/go-openbmclapi/config" "github.com/LiterMC/go-openbmclapi/log" "github.com/LiterMC/go-openbmclapi/storage" "github.com/LiterMC/go-openbmclapi/utils" ) -const DefaultBMCLAPIServer = "https://openbmclapi.bangbang93.com" - func migrateConfig(data []byte, cfg *config.Config) { var oldConfig map[string]any if err := yaml.Unmarshal(data, &oldConfig); err != nil { @@ -97,7 +96,7 @@ func readAndRewriteConfig() (cfg *config.Config, err error) { Id: "${CLUSTER_ID}", Secret: "${CLUSTER_SECRET}", PublicHosts: []string{}, - Server: DefaultBMCLAPIServer, + Server: cluster.DefaultBMCLAPIServer, SkipSignatureCheck: false, }, } diff --git a/lang/en/us.go b/lang/en/us.go index 7f0df686..95f453ee 100644 --- a/lang/en/us.go +++ b/lang/en/us.go @@ -10,12 +10,12 @@ var areaUS = map[string]string{ "warn.exit.detected.windows.open.browser": "Detected that you are in windows environment, we are helping you to open the browser", "warn.cluster.detected.hash.mismatch": "Detected hash mismatch error, removing bad file %s", - "info.filelist.fetching": "Fetching file list", + "info.filelist.fetching": "Fetching file list for %s", "error.filelist.fetch.failed": "Cannot fetch cluster file list: %v", "error.address.listen.failed": "Cannot listen address %s: %v", - "info.cert.requesting": "Requesting certificates, please wait ...", + "info.cert.requesting": "Requesting certificates for %s, please wait ...", "info.cert.requested": "Requested certificate for %s", "error.cert.not.set": "No certificates was set in the config", "error.cert.parse.failed": "Cannot parse certificate key pair[%d]: %v", @@ -27,7 +27,7 @@ var areaUS = map[string]string{ "info.wait.first.sync": "Waiting for the first sync ...", "info.cluster.enable.sending": "Sending enable packet", "info.cluster.enabled": "Cluster enabled", - "error.cluster.enable.failed": "Cannot enable cluster: %v", + "error.cluster.enable.failed": "Cannot enable cluster %s: %v", "error.cluster.disconnected": "Cluster disconnected from remote. exit.", "info.cluster.reconnect.keepalive": "Reconnecting due to keepalive failed", "info.cluster.reconnecting": "Reconnecting ...", @@ -49,8 +49,8 @@ var areaUS = map[string]string{ "warn.cluster.disabled": "Cluster disabled", "warn.httpserver.closing": "Closing HTTP server ...", - "info.check.start": "Start checking files for %s, heavy = %v", - "info.check.done": "File check finished for %s, missing %d files", + "info.check.start": "Start checking files, heavy = %v", + "info.check.done": "File check finished, missing %d files", "error.check.failed": "Failed to check %s: %v", "hint.check.checking": "> Checking ", "warn.check.modified.size": "Found modified file: size of %q is %d, expect %d", diff --git a/lang/zh/cn.go b/lang/zh/cn.go index 35c359e0..2b62e4d2 100644 --- a/lang/zh/cn.go +++ b/lang/zh/cn.go @@ -10,12 +10,12 @@ var areaCN = map[string]string{ "warn.exit.detected.windows.open.browser": "检测到您是新手 Windows 用户. 我们正在帮助您打开浏览器 ...", "warn.cluster.detected.hash.mismatch": "检测到文件哈希值不匹配, 正在删除 %s", - "info.filelist.fetching": "获取文件列表中", + "info.filelist.fetching": "为 %s 获取文件列表中", "error.filelist.fetch.failed": "文件列表获取失败: %v", "error.address.listen.failed": "无法监听地址 %s: %v", - "info.cert.requesting": "请求证书中, 请稍候 ...", + "info.cert.requesting": "正在为 %s 请求证书, 请稍候 ...", "info.cert.requested": "证书请求完毕, 域名为 %s", "error.cert.not.set": "配置文件内没有提供证书", "error.cert.parse.failed": "无法解析证书密钥对[%d]: %v", @@ -27,7 +27,7 @@ var areaCN = map[string]string{ "info.wait.first.sync": "正在等待第一次同步 ...", "info.cluster.enable.sending": "正在发送启用数据包", "info.cluster.enabled": "节点已启用", - "error.cluster.enable.failed": "无法启用节点: %v", + "error.cluster.enable.failed": "无法启用节点 %s: %v", "error.cluster.disconnected": "节点从主控断开. exit.", "info.cluster.reconnect.keepalive": "保活失败, 重连中 ...", "info.cluster.reconnecting": "重连中 ...", @@ -49,8 +49,8 @@ var areaCN = map[string]string{ "warn.cluster.disabled": "节点已禁用", "warn.httpserver.closing": "正在关闭 HTTP 服务器 ...", - "info.check.start": "开始在 %s 检测文件. 强检查 = %v", - "info.check.done": "文件在 %s 检查完毕, 缺失 %d 个文件", + "info.check.start": "开始检测文件. 强检查 = %v", + "info.check.done": "文件检查完毕, 缺失 %d 个文件", "error.check.failed": "无法检查 %s: %v", "hint.check.checking": "> 检查中 ", "warn.check.modified.size": "找到修改过的文件: %q 的大小为 %d, 预期 %d", diff --git a/main.go b/main.go index 14ce0549..0a869dd3 100644 --- a/main.go +++ b/main.go @@ -167,6 +167,9 @@ func main() { log.TrInfof("info.wait.first.sync") r.InitSynchronizer(ctx) + if ctx.Err() != nil { + return + } r.EnableClusterAll(ctx) }(ctx) diff --git a/runner.go b/runner.go index d2d67bd9..4f89fa5e 100644 --- a/runner.go +++ b/runner.go @@ -158,6 +158,7 @@ func NewRunner(cfg *config.Config) *Runner { log.Errorf("Stat load failed: %v", err) } r.apiRateLimiter = limited.NewAPIRateMiddleWare(api.RealAddrCtxKey, "go-openbmclapi.cluster.logged.user" /* api/v0.loggedUserKey */) + r.certificates = make(map[string]*tls.Certificate) return r } @@ -227,25 +228,11 @@ func (r *Runner) InitClusters(ctx context.Context) { for name, opts := range r.Config.Clusters { cr := cluster.NewCluster(name, opts, gcfg, r.storageManager, r.statManager, r.client) if err := cr.Init(ctx); err != nil { - log.TrErrorf("error.init.failed", err) - } else { - r.clusters[name] = cr + log.TrErrorf("error.init.failed", cr.Name(), err) + continue } + r.clusters[name] = cr } - - // r.cluster = NewCluster(ctx, - // ClusterServerURL, - // baseDir, - // config.PublicHost, r.getPublicPort(), - // config.ClusterId, config.ClusterSecret, - // config.Byoc, dialer, - // config.Storages, - // cache, - // ) - // if err := r.cluster.Init(ctx); err != nil { - // log.TrErrorf("error.init.failed"), err) - // os.Exit(1) - // } } func (r *Runner) ListenSignals(ctx context.Context, cancel context.CancelFunc) int { @@ -363,7 +350,6 @@ func (r *Runner) SetupLogger(ctx context.Context) error { } func (r *Runner) StopServer(ctx context.Context) { - r.tunnelCancel() shutCtx, cancelShut := context.WithTimeout(context.Background(), time.Second*15) defer cancelShut() log.TrWarnf("warn.server.closing") @@ -373,6 +359,7 @@ func (r *Runner) StopServer(ctx context.Context) { defer cancelShut() var wg sync.WaitGroup for _, cr := range r.clusters { + wg.Add(1) go func() { defer wg.Done() cr.Disable(shutCtx) @@ -381,6 +368,9 @@ func (r *Runner) StopServer(ctx context.Context) { wg.Wait() log.TrWarnf("warn.httpserver.closing") r.server.Shutdown(shutCtx) + if r.tunnelCancel != nil { + r.tunnelCancel() + } r.listener.Close() r.listener = nil }() diff --git a/utils/error.go b/utils/error.go index 31d62f76..627450bb 100644 --- a/utils/error.go +++ b/utils/error.go @@ -38,7 +38,7 @@ func NewHTTPStatusErrorFromResponse(res *http.Response) (e *HTTPStatusError) { e.URL = res.Request.URL.String() } if res.Body != nil { - var buf [512]byte + var buf [1024]byte n, _ := res.Body.Read(buf[:]) msg := (string)(buf[:n]) for _, b := range msg { diff --git a/utils/http.go b/utils/http.go index cd5b3863..c4169da8 100644 --- a/utils/http.go +++ b/utils/http.go @@ -22,6 +22,7 @@ package utils import ( "bufio" "bytes" + "context" "crypto/tls" "errors" "io" @@ -506,15 +507,16 @@ func (c *connHeadReader) Read(buf []byte) (n int, err error) { return c.Conn.Read(buf) } -func GetRedirects(req *http.Request) []*url.URL { +func GetRedirects(resp *http.Response) []*url.URL { redirects := make([]*url.URL, 0, 5) - for req != nil { - redirects = append(redirects, req.URL) - resp := req.Response - if resp == nil { - break - } + if u, _ := resp.Location(); u != nil { + redirects = append(redirects, u) + } + var req *http.Request + for resp != nil { req = resp.Request + redirects = append(redirects, req.URL) + resp = req.Response } if len(redirects) == 0 { return nil @@ -531,7 +533,7 @@ type RedirectError struct { func ErrorFromRedirect(err error, resp *http.Response) *RedirectError { return &RedirectError{ - Redirects: GetRedirects(resp.Request), + Redirects: GetRedirects(resp), Response: resp, Err: err, }