From 1a8b7603899f30f052ad123b7607b45a6d4c8772 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Thu, 4 Jan 2018 23:40:25 +0100 Subject: Improved and fixed local SDK tarball installation. Signed-off-by: Sebastien Douheret --- lib/xdsserver/apiv1-sdks.go | 2 +- lib/xdsserver/folders.go | 2 +- lib/xdsserver/sdk.go | 81 ++++++++++++++++++++----- lib/xdsserver/sdks.go | 143 ++++++++++++++++++++++++++++++++++++-------- 4 files changed, 185 insertions(+), 43 deletions(-) (limited to 'lib/xdsserver') diff --git a/lib/xdsserver/apiv1-sdks.go b/lib/xdsserver/apiv1-sdks.go index ba32dc9..e69b82d 100644 --- a/lib/xdsserver/apiv1-sdks.go +++ b/lib/xdsserver/apiv1-sdks.go @@ -74,7 +74,7 @@ func (s *APIService) installSdk(c *gin.Context) { return } - sdk, err := s.sdks.Install(id, args.Filename, args.Force, args.Timeout, sess) + sdk, err := s.sdks.Install(id, args.Filename, args.Force, args.Timeout, args.InstallArgs, sess) if err != nil { common.APIError(c, err.Error()) return diff --git a/lib/xdsserver/folders.go b/lib/xdsserver/folders.go index b0c198a..93d2898 100644 --- a/lib/xdsserver/folders.go +++ b/lib/xdsserver/folders.go @@ -177,7 +177,7 @@ func (f *Folders) ResolveID(id string) (string, error) { } else if len(match) == 0 { return id, fmt.Errorf("Unknown id") } - return id, fmt.Errorf("Multiple IDs found with provided prefix: " + id) + return id, fmt.Errorf("Multiple IDs found %v", match) } // Get returns the folder config or nil if not existing diff --git a/lib/xdsserver/sdk.go b/lib/xdsserver/sdk.go index cafd182..e5d4d7c 100644 --- a/lib/xdsserver/sdk.go +++ b/lib/xdsserver/sdk.go @@ -36,19 +36,21 @@ import ( // Definition of scripts used to managed SDKs const ( - scriptAdd = "add" - scriptGetConfig = "get-config" - scriptList = "list" - scriptRemove = "remove" - scriptUpdate = "update" + scriptAdd = "add" + scriptDbDump = "db-dump" + scriptDbUpdate = "db-update" + scriptGetFamConfig = "get-family-config" + scriptGetSdkInfo = "get-sdk-info" + scriptRemove = "remove" ) var scriptsAll = []string{ scriptAdd, - scriptGetConfig, - scriptList, + scriptDbDump, + scriptDbUpdate, + scriptGetFamConfig, + scriptGetSdkInfo, scriptRemove, - scriptUpdate, } var sdkCmdID = 0 @@ -65,26 +67,55 @@ type CrossSDK struct { bufStderr string } -// ListCrossSDK List all available and installed SDK (call "list" script) +// ListCrossSDK List all available and installed SDK (call "db-dump" script) func ListCrossSDK(scriptDir string, log *logrus.Logger) ([]xsapiv1.SDK, error) { sdksList := []xsapiv1.SDK{} // Retrieve SDKs list and info - cmd := exec.Command(path.Join(scriptDir, scriptList)) + cmd := exec.Command(path.Join(scriptDir, scriptDbDump)) stdout, err := cmd.CombinedOutput() if err != nil { return sdksList, fmt.Errorf("Cannot get sdks list: %v", err) } if err = json.Unmarshal(stdout, &sdksList); err != nil { - log.Errorf("SDK list script output:\n%v\n", string(stdout)) + log.Errorf("SDK %s script output:\n%v\n", scriptDbDump, string(stdout)) return sdksList, fmt.Errorf("Cannot decode sdk list %v", err) } return sdksList, nil } -// NewCrossSDK creates a new instance of Syncthing +// GetSDKInfo Used get-sdk-info script to extract SDK get info from a SDK file/tarball +func GetSDKInfo(scriptDir, url, filename, md5sum string, log *logrus.Logger) (xsapiv1.SDK, error) { + sdk := xsapiv1.SDK{} + + args := []string{} + if url != "" { + args = append(args, "--url", url) + } else if filename != "" { + args = append(args, "--file", filename) + if md5sum != "" { + args = append(args, "--md5", md5sum) + } + } else { + return sdk, fmt.Errorf("url of filename must be set") + } + + cmd := exec.Command(path.Join(scriptDir, scriptGetSdkInfo), args...) + stdout, err := cmd.CombinedOutput() + if err != nil { + return sdk, fmt.Errorf("%v %v", string(stdout), err) + } + + if err = json.Unmarshal(stdout, &sdk); err != nil { + log.Errorf("SDK %s script output:\n%v\n", scriptGetSdkInfo, string(stdout)) + return sdk, fmt.Errorf("Cannot decode sdk info %v", err) + } + return sdk, nil +} + +// NewCrossSDK creates a new instance of CrossSDK func NewCrossSDK(ctx *Context, sdk xsapiv1.SDK, scriptDir string) (*CrossSDK, error) { s := CrossSDK{ Context: ctx, @@ -93,9 +124,9 @@ func NewCrossSDK(ctx *Context, sdk xsapiv1.SDK, scriptDir string) (*CrossSDK, er } // Execute get-config script to retrieve SDK configuration - getConfFile := path.Join(scriptDir, scriptGetConfig) + getConfFile := path.Join(scriptDir, scriptGetFamConfig) if !common.Exists(getConfFile) { - return &s, fmt.Errorf("'%s' script file not found in %s", scriptGetConfig, scriptDir) + return &s, fmt.Errorf("'%s' script file not found in %s", scriptGetFamConfig, scriptDir) } cmd := exec.Command(getConfFile) @@ -170,7 +201,7 @@ func NewCrossSDK(ctx *Context, sdk xsapiv1.SDK, scriptDir string) (*CrossSDK, er } // Install a SDK (non blocking command, IOW run in background) -func (s *CrossSDK) Install(file string, force bool, timeout int, sess *ClientSession) error { +func (s *CrossSDK) Install(file string, force bool, timeout int, args []string, sess *ClientSession) error { if s.sdk.Status == xsapiv1.SdkStatusInstalled { return fmt.Errorf("already installed") @@ -190,6 +221,11 @@ func (s *CrossSDK) Install(file string, force bool, timeout int, sess *ClientSes cmdArgs = append(cmdArgs, "--force") } + // Append additional args (passthrough arguments) + if len(args) > 0 { + cmdArgs = append(cmdArgs, args...) + } + // Unique command id sdkCmdID++ cmdID := "sdk-install-" + strconv.Itoa(sdkCmdID) @@ -308,6 +344,21 @@ func (s *CrossSDK) Install(file string, force bool, timeout int, sess *ClientSes if code == 0 && exitError == nil { s.sdk.LastError = "" s.sdk.Status = xsapiv1.SdkStatusInstalled + + // FIXME: better update it using monitoring install dir (inotify) + // (see sdks.go / monitorSDKInstallation ) + // Update SetupFile when n + if s.sdk.SetupFile == "" { + sdkDef, err := GetSDKInfo(s.sdk.FamilyConf.ScriptsDir, s.sdk.URL, "", "", s.Log) + if err != nil || sdkDef.SetupFile == "" { + code = 1 + s.sdk.LastError = "Installation failed (cannot init SetupFile path)" + s.sdk.Status = xsapiv1.SdkStatusNotInstalled + } else { + s.sdk.SetupFile = sdkDef.SetupFile + } + } + } else { s.sdk.LastError = "Installation failed (code " + strconv.Itoa(code) + ")" diff --git a/lib/xdsserver/sdks.go b/lib/xdsserver/sdks.go index 9c10747..b7f7e2e 100644 --- a/lib/xdsserver/sdks.go +++ b/lib/xdsserver/sdks.go @@ -26,13 +26,13 @@ import ( common "github.com/iotbzh/xds-common/golib" "github.com/iotbzh/xds-server/lib/xsapiv1" - uuid "github.com/satori/go.uuid" ) // SDKs List of installed SDK type SDKs struct { *Context - Sdks map[string]*CrossSDK + Sdks map[string]*CrossSDK + SdksFamilies map[string]*xsapiv1.SDKFamilyConfig mutex sync.Mutex stop chan struct{} // signals intentional stop @@ -41,9 +41,10 @@ type SDKs struct { // NewSDKs creates a new instance of SDKs func NewSDKs(ctx *Context) (*SDKs, error) { s := SDKs{ - Context: ctx, - Sdks: make(map[string]*CrossSDK), - stop: make(chan struct{}), + Context: ctx, + Sdks: make(map[string]*CrossSDK), + SdksFamilies: make(map[string]*xsapiv1.SDKFamilyConfig), + stop: make(chan struct{}), } scriptsDir := ctx.Config.FileConf.SdkScriptsDir @@ -67,7 +68,6 @@ func NewSDKs(ctx *Context) (*SDKs, error) { // Foreach directories in scripts/sdk nbInstalled := 0 - monSdksPath := make(map[string]*xsapiv1.SDKFamilyConfig) for _, d := range dirs { if !common.IsDir(d) { continue @@ -81,42 +81,82 @@ func NewSDKs(ctx *Context) (*SDKs, error) { s.LogSillyf("'%s' SDKs list: %v", d, sdksList) for _, sdk := range sdksList { - cSdk, err := NewCrossSDK(ctx, sdk, d) + cSdk, err := s._createNewCrossSDK(sdk, d, false, false) if err != nil { s.Log.Debugf("Error while processing SDK sdk=%v\n err=%s", sdk, err.Error()) continue } - if _, exist := s.Sdks[cSdk.sdk.ID]; exist { - s.Log.Warningf("Duplicate SDK ID : %v", cSdk.sdk.ID) - cSdk.sdk.ID += "_DUPLICATE_" + uuid.NewV1().String() - } - s.Sdks[cSdk.sdk.ID] = cSdk + if cSdk.sdk.Status == xsapiv1.SdkStatusInstalled { nbInstalled++ } - monSdksPath[cSdk.sdk.FamilyConf.RootDir] = &cSdk.sdk.FamilyConf + s.SdksFamilies[cSdk.sdk.FamilyConf.FamilyName] = &cSdk.sdk.FamilyConf } } ctx.Log.Debugf("Cross SDKs: %d defined, %d installed", len(s.Sdks), nbInstalled) // Start monitor thread to detect new SDKs - if len(monSdksPath) == 0 { + sdksDirs := []string{} + for _, sf := range s.SdksFamilies { + sdksDirs = append(sdksDirs, sf.RootDir) + } + + if len(s.SdksFamilies) == 0 { s.Log.Warningf("No cross SDKs definition found") + /* TODO: used it or cleanup + } else { + go s.monitorSDKInstallation(sdksDirs) + */ } return &s, nil } +// _createNewCrossSDK Private function to create a new Cross SDK +func (s *SDKs) _createNewCrossSDK(sdk xsapiv1.SDK, scriptDir string, installing bool, force bool) (*CrossSDK, error) { + + cSdk, err := NewCrossSDK(s.Context, sdk, scriptDir) + if err != nil { + return cSdk, err + } + + // Allow to overwrite not installed SDK or when force is set + if _, exist := s.Sdks[cSdk.sdk.ID]; exist { + if !force && cSdk.sdk.Path != "" && common.Exists(cSdk.sdk.Path) { + return cSdk, fmt.Errorf("SDK ID %s already installed in %s", cSdk.sdk.ID, cSdk.sdk.Path) + } + if !force && cSdk.sdk.Status != xsapiv1.SdkStatusNotInstalled { + return cSdk, fmt.Errorf("Duplicate SDK ID %s (use force to overwrite)", cSdk.sdk.ID) + } + } + + // Sanity check + errMsg := "Invalid SDK definition " + if installing && cSdk.sdk.Path == "" { + return cSdk, fmt.Errorf(errMsg + "(path not set)") + } + if installing && cSdk.sdk.URL == "" { + return cSdk, fmt.Errorf(errMsg + "(url not set)") + } + + // Add to list + s.Sdks[cSdk.sdk.ID] = cSdk + + return cSdk, err +} + // Stop SDKs management func (s *SDKs) Stop() { close(s.stop) } // monitorSDKInstallation -/* TODO: cleanup -func (s *SDKs) monitorSDKInstallation(monSDKs map[string]*xsapiv1.SDKFamilyConfig) { +/* TODO: used it or cleanup +import "github.com/zillode/notify" + +func (s *SDKs) monitorSDKInstallation(watchingDirs []string) { // Set up a watchpoint listening for inotify-specific events c := make(chan notify.EventInfo, 1) @@ -131,7 +171,7 @@ func (s *SDKs) monitorSDKInstallation(monSDKs map[string]*xsapiv1.SDKFamilyConfi } // Add directory watchers - for dir := range monSDKs { + for _, dir := range watchingDirs { if err := addWatcher(dir); err != nil { s.Log.Errorln(err.Error()) } @@ -162,6 +202,14 @@ func (s *SDKs) monitorSDKInstallation(monSDKs map[string]*xsapiv1.SDKFamilyConfi switch ei.Event() { case notify.Create: + sdkDef, err := GetSDKInfo(scriptDir, sdk.URL, "", "", s.Log) + if err != nil { + s.Log.Warningf("Cannot get sdk info: %v", err) + continue + } + sdk.Path = sdkDef.Path + sdk.Path = sdkDef.SetupFile + // Emit Folder state change event if err := s.events.Emit(xsapiv1.EVTSDKInstall, sdk, ""); err != nil { s.Log.Warningf("Cannot notify SDK install: %v", err) @@ -196,7 +244,7 @@ func (s *SDKs) ResolveID(id string) (string, error) { } else if len(match) == 0 { return id, fmt.Errorf("Unknown sdk id") } - return id, fmt.Errorf("Multiple sdk IDs found with provided prefix: " + id) + return id, fmt.Errorf("Multiple sdk IDs found: %v", match) } // Get returns an SDK from id @@ -260,30 +308,73 @@ func (s *SDKs) GetEnvCmd(id string, defaultID string) []string { } // Install Used to install a new SDK -func (s *SDKs) Install(id, filepath string, force bool, timeout int, sess *ClientSession) (*xsapiv1.SDK, error) { - var cSdk *CrossSDK +func (s *SDKs) Install(id, filepath string, force bool, timeout int, args []string, sess *ClientSession) (*xsapiv1.SDK, error) { + + var sdk *xsapiv1.SDK + var err error + scriptDir := "" + sdkFilename := "" + if id != "" && filepath != "" { return nil, fmt.Errorf("invalid parameter, both id and filepath are set") } + + s.mutex.Lock() + defer s.mutex.Unlock() + if id != "" { - var exist bool - cSdk, exist = s.Sdks[id] + curSdk, exist := s.Sdks[id] if !exist { return nil, fmt.Errorf("unknown id") } + + sdk = &curSdk.sdk + scriptDir = sdk.FamilyConf.ScriptsDir + + // Update path when not set + if sdk.Path == "" { + sdkDef, err := GetSDKInfo(scriptDir, sdk.URL, "", "", s.Log) + if err != nil || sdkDef.Path == "" { + return nil, fmt.Errorf("cannot retrieve sdk path %v", err) + } + sdk.Path = sdkDef.Path + } + } else if filepath != "" { - // TODO check that file is accessible + // FIXME support any location and also sharing either by pathmap or Syncthing + baseDir := "${HOME}/xds-workspace/sdks" + sdkFilename, _ = common.ResolveEnvVar(path.Join(baseDir, path.Base(filepath))) + if !common.Exists(sdkFilename) { + return nil, fmt.Errorf("SDK file not accessible, must be in %s", baseDir) + } + + for _, sf := range s.SdksFamilies { + sdkDef, err := GetSDKInfo(sf.ScriptsDir, "", sdkFilename, "", s.Log) + if err == nil { + // OK, sdk found + sdk = &sdkDef + scriptDir = sf.ScriptsDir + break + } + + s.Log.Debugf("GetSDKInfo error: family=%s, sdkFilename=%s, err=%v", sf.FamilyName, path.Base(sdkFilename), err) + } + if sdk == nil { + return nil, fmt.Errorf("Cannot identify SDK family for %s", path.Base(filepath)) + } } else { return nil, fmt.Errorf("invalid parameter, id or filepath must be set") } - s.mutex.Lock() - defer s.mutex.Unlock() + cSdk, err := s._createNewCrossSDK(*sdk, scriptDir, true, force) + if err != nil { + return nil, err + } // Launch script to install // (note that add event will be generated by monitoring thread) - if err := cSdk.Install(filepath, force, timeout, sess); err != nil { + if err := cSdk.Install(sdkFilename, force, timeout, args, sess); err != nil { return &cSdk.sdk, err } -- cgit 1.2.3-korg