From 75b1182178b6882fc6ce9d1ab85b29521d00f8a9 Mon Sep 17 00:00:00 2001 From: CorrectRoadH Date: Wed, 22 May 2024 08:34:53 +0000 Subject: [PATCH 1/9] fix: fix latest tag image always can update --- pkg/docker/digest.go | 50 ++++++++++++++++++++++++++++++++++ service/appstore_management.go | 28 +++---------------- 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/pkg/docker/digest.go b/pkg/docker/digest.go index b912bba1..60a01787 100644 --- a/pkg/docker/digest.go +++ b/pkg/docker/digest.go @@ -20,6 +20,8 @@ import ( "github.com/docker/distribution/manifest/manifestlist" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" v1 "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -194,3 +196,51 @@ func addDefaultHeaders(header *http.Header, token string) { // header.Add("Accept", schema1.MediaTypeManifest) header.Add("Accept", v1.MediaTypeImageIndex) } + +func IsLatestImageUpdateAvailiable(imageName string) (bool, error) { + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return false, err + } + + localDigest, err := getLocalImageDigest(cli, imageName) + if err != nil { + return false, err + } + + remoteDigest, err := getRemoteImageDigest(cli, imageName) + if err != nil { + return false, err + } + + return localDigest != remoteDigest, nil +} + +// 获取本地镜像的Digest +func getLocalImageDigest(cli *client.Client, imageName string) (string, error) { + images, err := cli.ImageList(context.Background(), types.ImageListOptions{}) + if err != nil { + return "", err + } + + for _, image := range images { + for _, tag := range image.RepoTags { + if tag == imageName { + return image.ID, nil + } + } + } + + return "", fmt.Errorf("未找到本地镜像：%s", imageName) +} + +// 获取远程镜像的Digest +func getRemoteImageDigest(cli *client.Client, imageName string) (string, error) { + refStr := "docker.io/library/" + imageName + inspect, _, err := cli.ImageInspectWithRaw(context.Background(), refStr) + if err != nil { + return "", err + } + + return inspect.ID, nil +} diff --git a/service/appstore_management.go b/service/appstore_management.go index 5c70755d..2335915c 100644 --- a/service/appstore_management.go +++ b/service/appstore_management.go @@ -15,18 +15,16 @@ import ( "github.com/IceWhaleTech/CasaOS-Common/utils/file" "github.com/IceWhaleTech/CasaOS-Common/utils/logger" "github.com/bluele/gcache" - "github.com/docker/docker/client" "github.com/samber/lo" "go.uber.org/zap" ) type AppStoreManagement struct { + isAppUpgradable gcache.Cache + defaultAppStore AppStore + isAppUpgrading sync.Map onAppStoreRegister []func(string) error onAppStoreUnregister []func(string) error - - isAppUpgradable gcache.Cache - isAppUpgrading sync.Map - defaultAppStore AppStore } func (a *AppStoreManagement) AppStoreList() []codegen.AppStoreMetadata { @@ -482,28 +480,10 @@ func (a *AppStoreManagement) IsUpdateAvailableWith(composeApp *ComposeApp, store return false, err } if currentTag == "latest" { - ctx := context.Background() - cli, clientErr := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if clientErr != nil { - logger.Error("failed to create docker client", zap.Error(clientErr)) - return false, clientErr - } - defer cli.Close() image, _ := docker.ExtractImageAndTag(mainService.Image) - imageInfo, _, clientErr := cli.ImageInspectWithRaw(ctx, image) - if clientErr != nil { - logger.Error("failed to inspect image", zap.Error(clientErr)) - return false, clientErr - } - match, clientErr := docker.CompareDigest(mainService.Image, imageInfo.RepoDigests) - if clientErr != nil { - logger.Error("failed to compare digest", zap.Error(clientErr)) - return false, clientErr - } - // match means no update available - return !match, nil + return docker.IsLatestImageUpdateAvailiable(image) } storeTag, err := storeComposeApp.MainTag() return currentTag != storeTag, err From 5d9619348483679346363b9bd72882ab3ca0832c Mon Sep 17 00:00:00 2001 From: CorrectRoadH Date: Thu, 23 May 2024 03:37:43 +0000 Subject: [PATCH 2/9] refactor logic code --- common/constants.go | 2 ++ pkg/docker/digest.go | 14 +++++++++----- route/v2/compose_app.go | 2 +- service/appstore_management.go | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/common/constants.go b/common/constants.go index 15ef4c10..d3ced044 100644 --- a/common/constants.go +++ b/common/constants.go @@ -30,3 +30,5 @@ const ( CategoryListFileName = "category-list.json" RecommendListFileName = "recommend-list.json" ) + +var NeedCheckDigestTags = []string{"latest"} diff --git a/pkg/docker/digest.go b/pkg/docker/digest.go index 60a01787..67369ded 100644 --- a/pkg/docker/digest.go +++ b/pkg/docker/digest.go @@ -16,6 +16,7 @@ import ( "strings" "time" + "github.com/IceWhaleTech/CasaOS-AppManagement/common" "github.com/docker/distribution/manifest" "github.com/docker/distribution/manifest/manifestlist" "github.com/docker/distribution/manifest/schema1" @@ -225,8 +226,11 @@ func getLocalImageDigest(cli *client.Client, imageName string) (string, error) { for _, image := range images { for _, tag := range image.RepoTags { - if tag == imageName { - return image.ID, nil + // tag like "image:latest" + for _, needCheckTag := range common.NeedCheckDigestTags { + if tag == fmt.Sprintf("%s:%s", imageName, needCheckTag) { + return image.ID, nil + } } } } @@ -236,11 +240,11 @@ func getLocalImageDigest(cli *client.Client, imageName string) (string, error) { // 获取远程镜像的Digest func getRemoteImageDigest(cli *client.Client, imageName string) (string, error) { - refStr := "docker.io/library/" + imageName - inspect, _, err := cli.ImageInspectWithRaw(context.Background(), refStr) + // 获取远程镜像的Digest + imageInspect, _, err := cli.ImageInspectWithRaw(context.Background(), imageName) if err != nil { return "", err } - return inspect.ID, nil + return imageInspect.ID, nil } diff --git a/route/v2/compose_app.go b/route/v2/compose_app.go index ae71bf4b..a0cea6b4 100644 --- a/route/v2/compose_app.go +++ b/route/v2/compose_app.go @@ -108,7 +108,7 @@ func (a *AppManagement) IsNewComposeUncontrolled(newComposeApp *service.ComposeA } // TODO refactor this. because if user not update. the status will be uncontrolled. - if newTag == "latest" { + if lo.Contains(common.NeedCheckDigestTags, newTag) { return false, nil } diff --git a/service/appstore_management.go b/service/appstore_management.go index 2335915c..7cd17160 100644 --- a/service/appstore_management.go +++ b/service/appstore_management.go @@ -479,7 +479,7 @@ func (a *AppStoreManagement) IsUpdateAvailableWith(composeApp *ComposeApp, store logger.Error("failed to get main service", zap.Error(err)) return false, err } - if currentTag == "latest" { + if lo.Contains(common.NeedCheckDigestTags, currentTag) { image, _ := docker.ExtractImageAndTag(mainService.Image) From c0d9addae8071a638ca444a83ff162caf4eafa51 Mon Sep 17 00:00:00 2001 From: CorrectRoadH Date: Thu, 23 May 2024 03:44:46 +0000 Subject: [PATCH 3/9] rename --- common/constants.go | 1 + pkg/docker/digest.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/common/constants.go b/common/constants.go index d3ced044..fdbd1b6b 100644 --- a/common/constants.go +++ b/common/constants.go @@ -31,4 +31,5 @@ const ( RecommendListFileName = "recommend-list.json" ) +// the tags can add more. like "latest", "stable", "edge", "beta", "alpha" var NeedCheckDigestTags = []string{"latest"} diff --git a/pkg/docker/digest.go b/pkg/docker/digest.go index 67369ded..bb2660e3 100644 --- a/pkg/docker/digest.go +++ b/pkg/docker/digest.go @@ -198,7 +198,7 @@ func addDefaultHeaders(header *http.Header, token string) { header.Add("Accept", v1.MediaTypeImageIndex) } -func IsLatestImageUpdateAvailiable(imageName string) (bool, error) { +func IsImageDigestChanged(imageName string) (bool, error) { cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { return false, err From 616ab641cb59e6d82eb879ee4718dedf35f9dafb Mon Sep 17 00:00:00 2001 From: CorrectRoadH Date: Thu, 23 May 2024 03:49:07 +0000 Subject: [PATCH 4/9] update func name --- service/appstore_management.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/appstore_management.go b/service/appstore_management.go index 7cd17160..d51c34c2 100644 --- a/service/appstore_management.go +++ b/service/appstore_management.go @@ -483,7 +483,7 @@ func (a *AppStoreManagement) IsUpdateAvailableWith(composeApp *ComposeApp, store image, _ := docker.ExtractImageAndTag(mainService.Image) - return docker.IsLatestImageUpdateAvailiable(image) + return docker.IsImageDigestChanged(image) } storeTag, err := storeComposeApp.MainTag() return currentTag != storeTag, err From 060ab023730a929f773d4beb5ca133089e636f8d Mon Sep 17 00:00:00 2001 From: CorrectRoadH Date: Mon, 27 May 2024 06:01:51 +0000 Subject: [PATCH 5/9] add more comment --- route/v2/compose_app.go | 19 ++++++++++++++----- service/appstore_management.go | 18 +++++++++--------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/route/v2/compose_app.go b/route/v2/compose_app.go index a0cea6b4..75dd67d6 100644 --- a/route/v2/compose_app.go +++ b/route/v2/compose_app.go @@ -84,18 +84,27 @@ func (a *AppManagement) MyComposeApp(ctx echo.Context, id codegen.ComposeAppID) logger.Error("failed to get compose app status", zap.Error(err), zap.String("composeAppID", id)) } + // disable the because performance issue + // check update is hard and cost a lot of time. specially when the tag is latest + // such as Stable Diffusion. the check by ZimaOS GPU Application by @LinkLeong + // Alought @LinkLeong Didn't need the field. + // We should add a new API to get app info without update info + // and restore the following code + // check if updateAvailable - updateAvailable := service.MyService.AppStoreManagement().IsUpdateAvailable(composeApp) + // updateAvailable := service.MyService.AppStoreManagement().IsUpdateAvailable(composeApp) message := fmt.Sprintf("!! JSON format is for debugging purpose only - use `Accept: %s` HTTP header to get YAML instead !!", common.MIMEApplicationYAML) return ctx.JSON(http.StatusOK, codegen.ComposeAppOK{ // extension properties aren't marshalled - https://github.com/golang/go/issues/6213 Message: &message, Data: &codegen.ComposeAppWithStoreInfo{ - StoreInfo: storeInfo, - Compose: (*types.Project)(composeApp), - Status: &status, - UpdateAvailable: &updateAvailable, + StoreInfo: storeInfo, + Compose: (*types.Project)(composeApp), + Status: &status, + + // see above comment + UpdateAvailable: nil, }, }) } diff --git a/service/appstore_management.go b/service/appstore_management.go index d51c34c2..c670dd88 100644 --- a/service/appstore_management.go +++ b/service/appstore_management.go @@ -418,15 +418,15 @@ func (a *AppStoreManagement) WorkDir() (string, error) { func (a *AppStoreManagement) IsUpdateAvailable(composeApp *ComposeApp) bool { storeID := composeApp.Name - if value, err := a.isAppUpgradable.Get(storeID); err == nil { - switch value := value.(type) { - case bool: - return value - default: - logger.Error("invalid type in cache", zap.String("storeID", storeID), zap.Any("value", value)) - return false - } - } + // if value, err := a.isAppUpgradable.Get(storeID); err == nil { + // switch value := value.(type) { + // case bool: + // return value + // default: + // logger.Error("invalid type in cache", zap.String("storeID", storeID), zap.Any("value", value)) + // return false + // } + // } isUpdate, err := a.isUpdateAvailable(composeApp) if err != nil { From 09f6367a99cb990b3a08ce491bab18119b26f3cf Mon Sep 17 00:00:00 2001 From: CorrectRoadH Date: Mon, 27 May 2024 06:02:42 +0000 Subject: [PATCH 6/9] revert code --- service/appstore_management.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/service/appstore_management.go b/service/appstore_management.go index c670dd88..d51c34c2 100644 --- a/service/appstore_management.go +++ b/service/appstore_management.go @@ -418,15 +418,15 @@ func (a *AppStoreManagement) WorkDir() (string, error) { func (a *AppStoreManagement) IsUpdateAvailable(composeApp *ComposeApp) bool { storeID := composeApp.Name - // if value, err := a.isAppUpgradable.Get(storeID); err == nil { - // switch value := value.(type) { - // case bool: - // return value - // default: - // logger.Error("invalid type in cache", zap.String("storeID", storeID), zap.Any("value", value)) - // return false - // } - // } + if value, err := a.isAppUpgradable.Get(storeID); err == nil { + switch value := value.(type) { + case bool: + return value + default: + logger.Error("invalid type in cache", zap.String("storeID", storeID), zap.Any("value", value)) + return false + } + } isUpdate, err := a.isUpdateAvailable(composeApp) if err != nil { From 679013375c39cb698741e88bb5fc63bc4c6cf8f7 Mon Sep 17 00:00:00 2001 From: CorrectRoadH Date: Tue, 28 May 2024 03:21:27 +0000 Subject: [PATCH 7/9] wip --- route/v2/appstore.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/route/v2/appstore.go b/route/v2/appstore.go index 0b0068c7..ec5b0f63 100644 --- a/route/v2/appstore.go +++ b/route/v2/appstore.go @@ -406,7 +406,8 @@ func (a *AppManagement) UpgradableAppList(ctx echo.Context) error { if service.MyService.AppStoreManagement().IsUpdateAvailable(composeApp) { upgradableAppList = append(upgradableAppList, codegen.UpgradableAppInfo{ - Title: string(title), + Title: string(title), + // TODO: the tag can be latest Version: tag, StoreAppID: lo.ToPtr(id), Status: status, From 6038614c6741d1e12dad8b3de26deb8e16b731ab Mon Sep 17 00:00:00 2001 From: CorrectRoadH Date: Tue, 28 May 2024 06:44:04 +0000 Subject: [PATCH 8/9] wip --- route/v2/appstore.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/route/v2/appstore.go b/route/v2/appstore.go index ec5b0f63..118bb0ed 100644 --- a/route/v2/appstore.go +++ b/route/v2/appstore.go @@ -338,7 +338,7 @@ func FilterCatalogByCategory(catalog map[string]*service.ComposeApp, category st return false } - return strings.ToLower(storeInfo.Category) == strings.ToLower(category) + return strings.EqualFold(storeInfo.Category, category) }) } @@ -352,13 +352,13 @@ func FilterCatalogByAuthorType(catalog map[string]*service.ComposeApp, authorTyp return map[string]*service.ComposeApp{} } - return lo.PickBy(catalog, func(storeAppID string, composeApp *service.ComposeApp) bool { + return lo.PickBy(catalog, func(_ string, composeApp *service.ComposeApp) bool { return composeApp.AuthorType() == authorType }) } func FilterCatalogByAppStoreID(catalog map[string]*service.ComposeApp, appStoreIDs []string) map[string]*service.ComposeApp { - return lo.PickBy(catalog, func(storeAppID string, composeApp *service.ComposeApp) bool { + return lo.PickBy(catalog, func(storeAppID string, _ *service.ComposeApp) bool { return lo.Contains(appStoreIDs, storeAppID) }) } @@ -404,11 +404,22 @@ func (a *AppManagement) UpgradableAppList(ctx echo.Context) error { status = codegen.Updating } + // not change the main tag + mainTag, err := composeApp.MainTag() + if err != nil { + logger.Error("failed to get main tag", zap.Error(err), zap.String("name", composeApp.Name)) + continue + } + + targetTag := tag + if lo.Contains(common.NeedCheckDigestTags, mainTag) { + targetTag = mainTag + } + if service.MyService.AppStoreManagement().IsUpdateAvailable(composeApp) { upgradableAppList = append(upgradableAppList, codegen.UpgradableAppInfo{ - Title: string(title), - // TODO: the tag can be latest - Version: tag, + Title: string(title), + Version: targetTag, StoreAppID: lo.ToPtr(id), Status: status, Icon: storeInfo.Icon, From 56611726ba95737093b30e804b57cfa6cbc965a8 Mon Sep 17 00:00:00 2001 From: CorrectRoadH Date: Tue, 28 May 2024 08:13:16 +0000 Subject: [PATCH 9/9] remove latest checkout --- pkg/docker/digest.go | 54 ---------------------------------- service/appstore_management.go | 8 +++-- 2 files changed, 5 insertions(+), 57 deletions(-) diff --git a/pkg/docker/digest.go b/pkg/docker/digest.go index bb2660e3..b912bba1 100644 --- a/pkg/docker/digest.go +++ b/pkg/docker/digest.go @@ -16,13 +16,10 @@ import ( "strings" "time" - "github.com/IceWhaleTech/CasaOS-AppManagement/common" "github.com/docker/distribution/manifest" "github.com/docker/distribution/manifest/manifestlist" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" - "github.com/docker/docker/api/types" - "github.com/docker/docker/client" v1 "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -197,54 +194,3 @@ func addDefaultHeaders(header *http.Header, token string) { // header.Add("Accept", schema1.MediaTypeManifest) header.Add("Accept", v1.MediaTypeImageIndex) } - -func IsImageDigestChanged(imageName string) (bool, error) { - cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if err != nil { - return false, err - } - - localDigest, err := getLocalImageDigest(cli, imageName) - if err != nil { - return false, err - } - - remoteDigest, err := getRemoteImageDigest(cli, imageName) - if err != nil { - return false, err - } - - return localDigest != remoteDigest, nil -} - -// 获取本地镜像的Digest -func getLocalImageDigest(cli *client.Client, imageName string) (string, error) { - images, err := cli.ImageList(context.Background(), types.ImageListOptions{}) - if err != nil { - return "", err - } - - for _, image := range images { - for _, tag := range image.RepoTags { - // tag like "image:latest" - for _, needCheckTag := range common.NeedCheckDigestTags { - if tag == fmt.Sprintf("%s:%s", imageName, needCheckTag) { - return image.ID, nil - } - } - } - } - - return "", fmt.Errorf("未找到本地镜像：%s", imageName) -} - -// 获取远程镜像的Digest -func getRemoteImageDigest(cli *client.Client, imageName string) (string, error) { - // 获取远程镜像的Digest - imageInspect, _, err := cli.ImageInspectWithRaw(context.Background(), imageName) - if err != nil { - return "", err - } - - return imageInspect.ID, nil -} diff --git a/service/appstore_management.go b/service/appstore_management.go index d51c34c2..02b2691f 100644 --- a/service/appstore_management.go +++ b/service/appstore_management.go @@ -481,9 +481,11 @@ func (a *AppStoreManagement) IsUpdateAvailableWith(composeApp *ComposeApp, store } if lo.Contains(common.NeedCheckDigestTags, currentTag) { - image, _ := docker.ExtractImageAndTag(mainService.Image) - - return docker.IsImageDigestChanged(image) + _, _ = docker.ExtractImageAndTag(mainService.Image) + // TODO: refactor this. + // need to check the digest of the image + // to see if the image is updated + return false, nil } storeTag, err := storeComposeApp.MainTag() return currentTag != storeTag, err