diff --git a/agent/app/service/app.go b/agent/app/service/app.go index d87541ab22d2..65b9f76530b5 100644 --- a/agent/app/service/app.go +++ b/agent/app/service/app.go @@ -925,247 +925,7 @@ func (a AppService) SyncAppListFromRemote(taskID string) (err error) { if err != nil { return err } - syncTask.AddSubTask(task.GetTaskName(i18n.GetMsgByKey("App"), task.TaskSync, task.TaskScopeAppStore), func(t *task.Task) (err error) { - updateRes, err := a.GetAppUpdate() - if err != nil { - return err - } - if !updateRes.CanUpdate { - if updateRes.IsSyncing { - t.Log(i18n.GetMsgByKey("AppStoreIsSyncing")) - return nil - } - t.Log(i18n.GetMsgByKey("AppStoreIsUpToDate")) - return nil - } - list := &dto.AppList{} - if updateRes.AppList == nil { - list, err = getAppList() - if err != nil { - return err - } - } else { - list = updateRes.AppList - } - settingService := NewISettingService() - _ = settingService.Update("AppStoreSyncStatus", constant.StatusSyncing) - - setting, err := settingService.GetSettingInfo() - if err != nil { - return err - } - var ( - appTags []*model.AppTag - oldAppIds []uint - ) - if err = SyncTags(list.Extra); err != nil { - return err - } - deleteCustomApp() - oldApps, err := appRepo.GetBy(appRepo.WithNotLocal()) - if err != nil { - return err - } - for _, old := range oldApps { - oldAppIds = append(oldAppIds, old.ID) - } - - baseRemoteUrl := fmt.Sprintf("%s/%s/1panel", global.CONF.RemoteURL.AppRepo, global.CONF.Base.Mode) - - appsMap := getApps(oldApps, list.Apps, setting.SystemVersion, t) - - t.LogStart(i18n.GetMsgByKey("SyncAppDetail")) - for _, l := range list.Apps { - app, ok := appsMap[l.AppProperty.Key] - if !ok { - continue - } - iconStr := "" - _, iconRes, err := req_helper.HandleRequest(l.Icon, http.MethodGet, constant.TimeOut20s) - if err == nil { - if !strings.Contains(string(iconRes), "") { - iconStr = base64.StdEncoding.EncodeToString(iconRes) - } - } - app.Icon = iconStr - app.TagsKey = l.AppProperty.Tags - if l.AppProperty.Recommend > 0 { - app.Recommend = l.AppProperty.Recommend - } else { - app.Recommend = 9999 - } - app.ReadMe = l.ReadMe - app.LastModified = l.LastModified - versions := l.Versions - detailsMap := getAppDetails(app.Details, versions) - for _, v := range versions { - version := v.Name - detail := detailsMap[version] - versionUrl := fmt.Sprintf("%s/%s/%s", baseRemoteUrl, app.Key, version) - paramByte, _ := json.Marshal(v.AppForm) - var appForm dto.AppForm - _ = json.Unmarshal(paramByte, &appForm) - if appForm.SupportVersion > 0 && common.CompareVersion(strconv.FormatFloat(appForm.SupportVersion, 'f', -1, 64), setting.SystemVersion) { - delete(detailsMap, version) - continue - } - if _, ok := InitTypes[app.Type]; ok { - dockerComposeUrl := fmt.Sprintf("%s/%s", versionUrl, "docker-compose.yml") - _, composeRes, err := req_helper.HandleRequest(dockerComposeUrl, http.MethodGet, constant.TimeOut20s) - if err == nil { - detail.DockerCompose = string(composeRes) - } - } else { - detail.DockerCompose = "" - } - - detail.Params = string(paramByte) - detail.DownloadUrl = fmt.Sprintf("%s/%s", versionUrl, app.Key+"-"+version+".tar.gz") - detail.DownloadCallBackUrl = v.DownloadCallBackUrl - detail.Update = true - detail.LastModified = v.LastModified - detailsMap[version] = detail - } - var newDetails []model.AppDetail - for _, detail := range detailsMap { - newDetails = append(newDetails, detail) - } - app.Details = newDetails - appsMap[l.AppProperty.Key] = app - } - t.LogSuccess(i18n.GetMsgByKey("SyncAppDetail")) - - tags, _ := tagRepo.All() - var ( - addAppArray []model.App - updateAppArray []model.App - deleteAppArray []model.App - deleteIds []uint - tagMap = make(map[string]uint, len(tags)) - ) - - for _, v := range appsMap { - if v.ID == 0 { - addAppArray = append(addAppArray, v) - } else { - if v.Status == constant.AppTakeDown { - installs, _ := appInstallRepo.ListBy(context.Background(), appInstallRepo.WithAppId(v.ID)) - if len(installs) > 0 { - updateAppArray = append(updateAppArray, v) - continue - } - deleteAppArray = append(deleteAppArray, v) - deleteIds = append(deleteIds, v.ID) - } else { - updateAppArray = append(updateAppArray, v) - } - } - } - - tx, ctx := getTxAndContext() - defer func() { - if err != nil { - tx.Rollback() - return - } - }() - if len(addAppArray) > 0 { - if err = appRepo.BatchCreate(ctx, addAppArray); err != nil { - return - } - } - if len(deleteAppArray) > 0 { - if err = appRepo.BatchDelete(ctx, deleteAppArray); err != nil { - return - } - if err = appDetailRepo.DeleteByAppIds(ctx, deleteIds); err != nil { - return - } - } - for _, tag := range tags { - tagMap[tag.Key] = tag.ID - } - for _, update := range updateAppArray { - if err = appRepo.Save(ctx, &update); err != nil { - return - } - } - apps := append(addAppArray, updateAppArray...) - - var ( - addDetails []model.AppDetail - updateDetails []model.AppDetail - deleteDetails []model.AppDetail - ) - for _, app := range apps { - for _, tag := range app.TagsKey { - tagId, ok := tagMap[tag] - if ok { - exist, _ := appTagRepo.GetFirst(ctx, appTagRepo.WithByTagID(tagId), appTagRepo.WithByAppID(app.ID)) - if exist == nil { - appTags = append(appTags, &model.AppTag{ - AppId: app.ID, - TagId: tagId, - }) - } - } - } - for _, d := range app.Details { - d.AppId = app.ID - if d.ID == 0 { - addDetails = append(addDetails, d) - } else { - if d.Status == constant.AppTakeDown { - runtime, _ := runtimeRepo.GetFirst(ctx, runtimeRepo.WithDetailId(d.ID)) - if runtime != nil { - updateDetails = append(updateDetails, d) - continue - } - installs, _ := appInstallRepo.ListBy(ctx, appInstallRepo.WithDetailIdsIn([]uint{d.ID})) - if len(installs) > 0 { - updateDetails = append(updateDetails, d) - continue - } - deleteDetails = append(deleteDetails, d) - } else { - updateDetails = append(updateDetails, d) - } - } - } - } - if len(addDetails) > 0 { - if err = appDetailRepo.BatchCreate(ctx, addDetails); err != nil { - return - } - } - if len(deleteDetails) > 0 { - if err = appDetailRepo.BatchDelete(ctx, deleteDetails); err != nil { - return - } - } - for _, u := range updateDetails { - if err = appDetailRepo.Update(ctx, u); err != nil { - return - } - } - - if len(oldAppIds) > 0 { - if err = appTagRepo.DeleteByAppIds(ctx, deleteIds); err != nil { - return - } - } - - if len(appTags) > 0 { - if err = appTagRepo.BatchCreate(ctx, appTags); err != nil { - return - } - } - tx.Commit() - - _ = settingService.Update("AppStoreSyncStatus", constant.StatusSyncSuccess) - _ = settingService.Update("AppStoreLastModified", strconv.Itoa(list.LastModified)) - return nil - }, nil) + syncTask.AddSubTask(task.GetTaskName(i18n.GetMsgByKey("App"), task.TaskSync, task.TaskScopeAppStore), a.syncAppStoreTask, nil) go func() { if err := syncTask.Execute(); err != nil { diff --git a/agent/app/service/app_sync_task.go b/agent/app/service/app_sync_task.go new file mode 100644 index 000000000000..7e584c9589aa --- /dev/null +++ b/agent/app/service/app_sync_task.go @@ -0,0 +1,347 @@ +package service + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/1Panel-dev/1Panel/agent/app/dto" + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/app/task" + "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/i18n" + "github.com/1Panel-dev/1Panel/agent/utils/common" + "github.com/1Panel-dev/1Panel/agent/utils/req_helper" + "github.com/1Panel-dev/1Panel/agent/utils/xpack" +) + +type appSyncContext struct { + task *task.Task + httpClient http.Client + baseRemoteUrl string + systemVersion string + appsMap map[string]model.App + settingService ISettingService + list *dto.AppList + oldAppIds []uint + appTags []*model.AppTag +} + +func (a AppService) syncAppStoreTask(t *task.Task) (err error) { + updateRes, err := a.GetAppUpdate() + if err != nil { + return err + } + if !updateRes.CanUpdate { + if updateRes.IsSyncing { + t.Log(i18n.GetMsgByKey("AppStoreIsSyncing")) + return nil + } + global.LOG.Infof("[AppStore] Appstore is up to date") + t.Log(i18n.GetMsgByKey("AppStoreIsUpToDate")) + return nil + } + + list := &dto.AppList{} + if updateRes.AppList == nil { + list, err = getAppList() + if err != nil { + return err + } + } else { + list = updateRes.AppList + } + + settingService := NewISettingService() + _ = settingService.Update("AppStoreSyncStatus", constant.StatusSyncing) + + setting, err := settingService.GetSettingInfo() + if err != nil { + return err + } + + ctx := &appSyncContext{ + task: t, + httpClient: http.Client{Timeout: time.Duration(constant.TimeOut20s) * time.Second, Transport: xpack.LoadRequestTransport()}, + baseRemoteUrl: fmt.Sprintf("%s/%s/1panel", global.CONF.RemoteURL.AppRepo, global.CONF.Base.Mode), + systemVersion: setting.SystemVersion, + settingService: settingService, + list: list, + appTags: make([]*model.AppTag, 0), + } + + if err = SyncTags(list.Extra); err != nil { + return err + } + deleteCustomApp() + + oldApps, err := appRepo.GetBy(appRepo.WithNotLocal()) + if err != nil { + return err + } + ctx.oldAppIds = make([]uint, 0, len(oldApps)) + for _, old := range oldApps { + ctx.oldAppIds = append(ctx.oldAppIds, old.ID) + } + + ctx.appsMap = getApps(oldApps, list.Apps, setting.SystemVersion, t) + + if err = ctx.syncAppIconsAndDetails(); err != nil { + return err + } + + if err = ctx.classifyAndPersistApps(); err != nil { + return err + } + + _ = settingService.Update("AppStoreSyncStatus", constant.StatusSyncSuccess) + _ = settingService.Update("AppStoreLastModified", strconv.Itoa(list.LastModified)) + global.LOG.Infof("[AppStore] Appstore sync completed") + return nil +} + +func (c *appSyncContext) syncAppIconsAndDetails() error { + c.task.LogStart(i18n.GetMsgByKey("SyncAppDetail")) + global.LOG.Infof("[AppStore] sync app detail start, total: %d", len(c.list.Apps)) + + downloadIconNum := 0 + total := len(c.list.Apps) + + for _, l := range c.list.Apps { + downloadIconNum++ + if downloadIconNum%10 == 0 { + c.task.LogWithProgress(i18n.GetMsgByKey("SyncAppDetail"), downloadIconNum, total) + } + + app, ok := c.appsMap[l.AppProperty.Key] + if !ok { + continue + } + + iconStr := c.downloadAppIcon(l.Icon) + if iconStr == "" { + global.LOG.Infof("[AppStore] save failed url=%s", l.Icon) + } + app.Icon = iconStr + + app.TagsKey = l.AppProperty.Tags + if l.AppProperty.Recommend > 0 { + app.Recommend = l.AppProperty.Recommend + } else { + app.Recommend = 9999 + } + app.ReadMe = l.ReadMe + app.LastModified = l.LastModified + + versions := l.Versions + detailsMap := getAppDetails(app.Details, versions) + for _, v := range versions { + version := v.Name + detail := detailsMap[version] + versionUrl := fmt.Sprintf("%s/%s/%s", c.baseRemoteUrl, app.Key, version) + + paramByte, _ := json.Marshal(v.AppForm) + var appForm dto.AppForm + _ = json.Unmarshal(paramByte, &appForm) + + if appForm.SupportVersion > 0 && common.CompareVersion(strconv.FormatFloat(appForm.SupportVersion, 'f', -1, 64), c.systemVersion) { + delete(detailsMap, version) + continue + } + + if _, ok := InitTypes[app.Type]; ok { + dockerComposeUrl := fmt.Sprintf("%s/%s", versionUrl, "docker-compose.yml") + _, composeRes, err := req_helper.HandleRequestWithClient(&c.httpClient, dockerComposeUrl, http.MethodGet, constant.TimeOut20s) + if err == nil { + detail.DockerCompose = string(composeRes) + } + } else { + detail.DockerCompose = "" + } + + detail.Params = string(paramByte) + detail.DownloadUrl = fmt.Sprintf("%s/%s", versionUrl, app.Key+"-"+version+".tar.gz") + detail.DownloadCallBackUrl = v.DownloadCallBackUrl + detail.Update = true + detail.LastModified = v.LastModified + detailsMap[version] = detail + } + + var newDetails []model.AppDetail + for _, detail := range detailsMap { + newDetails = append(newDetails, detail) + } + app.Details = newDetails + c.appsMap[l.AppProperty.Key] = app + } + + global.LOG.Infof("[AppStore] download icon success: %d, total: %d", + downloadIconNum, total) + + c.task.LogSuccess(i18n.GetMsgByKey("SyncAppDetail")) + return nil +} + +func (c *appSyncContext) downloadAppIcon(iconUrl string) string { + iconStr := "" + + code, iconRes, err := req_helper.HandleRequestWithClient(&c.httpClient, iconUrl, http.MethodGet, constant.TimeOut20s) + if err == nil { + if code == http.StatusOK { + if len(iconRes) > 0 { + if iconRes[0] != '<' { + iconStr = base64.StdEncoding.EncodeToString(iconRes) + } + } + } else { + global.LOG.Infof("[AppStore] download failed status=%d", code) + } + } + return iconStr +} + +func (c *appSyncContext) classifyAndPersistApps() (err error) { + tags, _ := tagRepo.All() + var ( + addAppArray []model.App + updateAppArray []model.App + deleteAppArray []model.App + deleteIds []uint + tagMap = make(map[string]uint, len(tags)) + ) + + for _, v := range c.appsMap { + if v.ID == 0 { + addAppArray = append(addAppArray, v) + } else { + if v.Status == constant.AppTakeDown { + installs, _ := appInstallRepo.ListBy(context.Background(), appInstallRepo.WithAppId(v.ID)) + if len(installs) > 0 { + updateAppArray = append(updateAppArray, v) + continue + } + deleteAppArray = append(deleteAppArray, v) + deleteIds = append(deleteIds, v.ID) + } else { + updateAppArray = append(updateAppArray, v) + } + } + } + + tx, ctx := getTxAndContext() + defer func() { + if err != nil { + tx.Rollback() + return + } + }() + + if len(addAppArray) > 0 { + if err = appRepo.BatchCreate(ctx, addAppArray); err != nil { + return + } + } + + if len(deleteAppArray) > 0 { + if err = appRepo.BatchDelete(ctx, deleteAppArray); err != nil { + return + } + if err = appDetailRepo.DeleteByAppIds(ctx, deleteIds); err != nil { + return + } + } + + for _, tag := range tags { + tagMap[tag.Key] = tag.ID + } + + for _, update := range updateAppArray { + if err = appRepo.Save(ctx, &update); err != nil { + return + } + } + + apps := append(addAppArray, updateAppArray...) + + var ( + addDetails []model.AppDetail + updateDetails []model.AppDetail + deleteDetails []model.AppDetail + ) + + for _, app := range apps { + for _, tag := range app.TagsKey { + tagId, ok := tagMap[tag] + if ok { + exist, _ := appTagRepo.GetFirst(ctx, appTagRepo.WithByTagID(tagId), appTagRepo.WithByAppID(app.ID)) + if exist == nil { + c.appTags = append(c.appTags, &model.AppTag{ + AppId: app.ID, + TagId: tagId, + }) + } + } + } + + for _, d := range app.Details { + d.AppId = app.ID + if d.ID == 0 { + addDetails = append(addDetails, d) + } else { + if d.Status == constant.AppTakeDown { + runtime, _ := runtimeRepo.GetFirst(ctx, runtimeRepo.WithDetailId(d.ID)) + if runtime != nil { + updateDetails = append(updateDetails, d) + continue + } + installs, _ := appInstallRepo.ListBy(ctx, appInstallRepo.WithDetailIdsIn([]uint{d.ID})) + if len(installs) > 0 { + updateDetails = append(updateDetails, d) + continue + } + deleteDetails = append(deleteDetails, d) + } else { + updateDetails = append(updateDetails, d) + } + } + } + } + + if len(addDetails) > 0 { + if err = appDetailRepo.BatchCreate(ctx, addDetails); err != nil { + return + } + } + + if len(deleteDetails) > 0 { + if err = appDetailRepo.BatchDelete(ctx, deleteDetails); err != nil { + return + } + } + + for _, u := range updateDetails { + if err = appDetailRepo.Update(ctx, u); err != nil { + return + } + } + + if len(c.oldAppIds) > 0 { + if err = appTagRepo.DeleteByAppIds(ctx, deleteIds); err != nil { + return + } + } + + if len(c.appTags) > 0 { + if err = appTagRepo.BatchCreate(ctx, c.appTags); err != nil { + return + } + } + + tx.Commit() + return nil +} diff --git a/agent/app/task/task.go b/agent/app/task/task.go index 50ab4e97dad8..4222acd38563 100644 --- a/agent/app/task/task.go +++ b/agent/app/task/task.go @@ -7,6 +7,7 @@ import ( "os" "path" "strconv" + "strings" "time" "github.com/1Panel-dev/1Panel/agent/buserr" @@ -353,7 +354,20 @@ func (t *Task) LogSuccessWithOps(operate, msg string) { } func (t *Task) LogFailedWithOps(operate, msg string, err error) { - t.Logger.Printf("%s%s%s : %s ", i18n.GetMsgByKey(operate), msg, i18n.GetMsgByKey("Failed"), err.Error()) + t.Logger.Printf("%s%s : %s ", msg, i18n.GetMsgByKey("Failed"), err.Error()) +} + +func (t *Task) LogWithProgress(msg string, current int, total int) { + const barWidth = 10 + filled := int(float64(current) / float64(total) * 100 / 10) + if filled > barWidth { + filled = barWidth + } + if filled < 0 { + filled = 0 + } + bar := strings.Repeat("=", filled) + strings.Repeat("-", barWidth-filled) + t.Logger.Printf("%s [%s] %.2f%%", msg, bar, float64(current)/float64(total)*100) } type SimpleFormatter struct{} diff --git a/agent/utils/req_helper/request.go b/agent/utils/req_helper/request.go index 5b29cdb2be46..60f660651643 100644 --- a/agent/utils/req_helper/request.go +++ b/agent/utils/req_helper/request.go @@ -4,13 +4,14 @@ import ( "context" "crypto/tls" "errors" - "github.com/1Panel-dev/1Panel/agent/utils/xpack" "io" "net" "net/http" "strings" "time" + "github.com/1Panel-dev/1Panel/agent/utils/xpack" + "github.com/1Panel-dev/1Panel/agent/buserr" "github.com/1Panel-dev/1Panel/agent/global" ) @@ -46,6 +47,12 @@ func HandleGet(url string) (*http.Response, error) { } func HandleRequest(url, method string, timeout int) (int, []byte, error) { + transport := xpack.LoadRequestTransport() + client := http.Client{Timeout: time.Duration(timeout) * time.Second, Transport: transport} + return HandleRequestWithClient(&client, url, method, timeout) +} + +func HandleRequestWithClient(client *http.Client, url, method string, timeout int) (int, []byte, error) { defer func() { if r := recover(); r != nil { global.LOG.Errorf("handle request failed, error message: %v", r) @@ -53,8 +60,6 @@ func HandleRequest(url, method string, timeout int) (int, []byte, error) { } }() - transport := xpack.LoadRequestTransport() - client := http.Client{Timeout: time.Duration(timeout) * time.Second, Transport: transport} ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) defer cancel() request, err := http.NewRequestWithContext(ctx, method, url, nil)