Improved and fixed local SDK tarball installation.

Signed-off-by: Sebastien Douheret <sebastien.douheret@iot.bzh>

5 files changed, 190 insertions, 47 deletions

diff --git a/lib/xdsserver/apiv1-sdks.go b/lib/xdsserver/apiv1-sdks.go
index ba32dc9..e69b82d 100644
--- a/lib/xdsserver/apiv1-sdks.go
+++ b/lib/xdsserver/apiv1-sdks.go
@@ -74,7 +74,7 @@ func (s *APIService) installSdk(c *gin.Context) {
 		return
 	}
 
-	sdk, err := s.sdks.Install(id, args.Filename, args.Force, args.Timeout, sess)
+	sdk, err := s.sdks.Install(id, args.Filename, args.Force, args.Timeout, args.InstallArgs, sess)
 	if err != nil {
 		common.APIError(c, err.Error())
 		return
diff --git a/lib/xdsserver/folders.go b/lib/xdsserver/folders.go
index b0c198a..93d2898 100644
--- a/lib/xdsserver/folders.go
+++ b/lib/xdsserver/folders.go
@@ -177,7 +177,7 @@ func (f *Folders) ResolveID(id string) (string, error) {
 	} else if len(match) == 0 {
 		return id, fmt.Errorf("Unknown id")
 	}
-	return id, fmt.Errorf("Multiple IDs found with provided prefix: " + id)
+	return id, fmt.Errorf("Multiple IDs found %v", match)
 }
 
 // Get returns the folder config or nil if not existing
diff --git a/lib/xdsserver/sdk.go b/lib/xdsserver/sdk.go
index cafd182..e5d4d7c 100644
--- a/lib/xdsserver/sdk.go
+++ b/lib/xdsserver/sdk.go
@@ -36,19 +36,21 @@ import (
 
 // Definition of scripts used to managed SDKs
 const (
-	scriptAdd       = "add"
-	scriptGetConfig = "get-config"
-	scriptList      = "list"
-	scriptRemove    = "remove"
-	scriptUpdate    = "update"
+	scriptAdd          = "add"
+	scriptDbDump       = "db-dump"
+	scriptDbUpdate     = "db-update"
+	scriptGetFamConfig = "get-family-config"
+	scriptGetSdkInfo   = "get-sdk-info"
+	scriptRemove       = "remove"
 )
 
 var scriptsAll = []string{
 	scriptAdd,
-	scriptGetConfig,
-	scriptList,
+	scriptDbDump,
+	scriptDbUpdate,
+	scriptGetFamConfig,
+	scriptGetSdkInfo,
 	scriptRemove,
-	scriptUpdate,
 }
 
 var sdkCmdID = 0
@@ -65,26 +67,55 @@ type CrossSDK struct {
 	bufStderr string
 }
 
-// ListCrossSDK List all available and installed SDK  (call "list" script)
+// ListCrossSDK List all available and installed SDK  (call "db-dump" script)
 func ListCrossSDK(scriptDir string, log *logrus.Logger) ([]xsapiv1.SDK, error) {
 	sdksList := []xsapiv1.SDK{}
 
 	// Retrieve SDKs list and info
-	cmd := exec.Command(path.Join(scriptDir, scriptList))
+	cmd := exec.Command(path.Join(scriptDir, scriptDbDump))
 	stdout, err := cmd.CombinedOutput()
 	if err != nil {
 		return sdksList, fmt.Errorf("Cannot get sdks list: %v", err)
 	}
 
 	if err = json.Unmarshal(stdout, &sdksList); err != nil {
-		log.Errorf("SDK list script output:\n%v\n", string(stdout))
+		log.Errorf("SDK %s script output:\n%v\n", scriptDbDump, string(stdout))
 		return sdksList, fmt.Errorf("Cannot decode sdk list %v", err)
 	}
 
 	return sdksList, nil
 }
 
-// NewCrossSDK creates a new instance of Syncthing
+// GetSDKInfo Used get-sdk-info script to extract SDK get info from a SDK file/tarball
+func GetSDKInfo(scriptDir, url, filename, md5sum string, log *logrus.Logger) (xsapiv1.SDK, error) {
+	sdk := xsapiv1.SDK{}
+
+	args := []string{}
+	if url != "" {
+		args = append(args, "--url", url)
+	} else if filename != "" {
+		args = append(args, "--file", filename)
+		if md5sum != "" {
+			args = append(args, "--md5", md5sum)
+		}
+	} else {
+		return sdk, fmt.Errorf("url of filename must be set")
+	}
+
+	cmd := exec.Command(path.Join(scriptDir, scriptGetSdkInfo), args...)
+	stdout, err := cmd.CombinedOutput()
+	if err != nil {
+		return sdk, fmt.Errorf("%v %v", string(stdout), err)
+	}
+
+	if err = json.Unmarshal(stdout, &sdk); err != nil {
+		log.Errorf("SDK %s script output:\n%v\n", scriptGetSdkInfo, string(stdout))
+		return sdk, fmt.Errorf("Cannot decode sdk info %v", err)
+	}
+	return sdk, nil
+}
+
+// NewCrossSDK creates a new instance of CrossSDK
 func NewCrossSDK(ctx *Context, sdk xsapiv1.SDK, scriptDir string) (*CrossSDK, error) {
 	s := CrossSDK{
 		Context: ctx,
@@ -93,9 +124,9 @@ func NewCrossSDK(ctx *Context, sdk xsapiv1.SDK, scriptDir string) (*CrossSDK, er
 	}
 
 	// Execute get-config script to retrieve SDK configuration
-	getConfFile := path.Join(scriptDir, scriptGetConfig)
+	getConfFile := path.Join(scriptDir, scriptGetFamConfig)
 	if !common.Exists(getConfFile) {
-		return &s, fmt.Errorf("'%s' script file not found in %s", scriptGetConfig, scriptDir)
+		return &s, fmt.Errorf("'%s' script file not found in %s", scriptGetFamConfig, scriptDir)
 	}
 
 	cmd := exec.Command(getConfFile)
@@ -170,7 +201,7 @@ func NewCrossSDK(ctx *Context, sdk xsapiv1.SDK, scriptDir string) (*CrossSDK, er
 }
 
 // Install a SDK (non blocking command, IOW run in background)
-func (s *CrossSDK) Install(file string, force bool, timeout int, sess *ClientSession) error {
+func (s *CrossSDK) Install(file string, force bool, timeout int, args []string, sess *ClientSession) error {
 
 	if s.sdk.Status == xsapiv1.SdkStatusInstalled {
 		return fmt.Errorf("already installed")
@@ -190,6 +221,11 @@ func (s *CrossSDK) Install(file string, force bool, timeout int, sess *ClientSes
 		cmdArgs = append(cmdArgs, "--force")
 	}
 
+	// Append additional args (passthrough arguments)
+	if len(args) > 0 {
+		cmdArgs = append(cmdArgs, args...)
+	}
+
 	// Unique command id
 	sdkCmdID++
 	cmdID := "sdk-install-" + strconv.Itoa(sdkCmdID)
@@ -308,6 +344,21 @@ func (s *CrossSDK) Install(file string, force bool, timeout int, sess *ClientSes
 		if code == 0 && exitError == nil {
 			s.sdk.LastError = ""
 			s.sdk.Status = xsapiv1.SdkStatusInstalled
+
+			// FIXME: better update it using monitoring install dir (inotify)
+			// (see sdks.go / monitorSDKInstallation )
+			// Update SetupFile when n
+			if s.sdk.SetupFile == "" {
+				sdkDef, err := GetSDKInfo(s.sdk.FamilyConf.ScriptsDir, s.sdk.URL, "", "", s.Log)
+				if err != nil || sdkDef.SetupFile == "" {
+					code = 1
+					s.sdk.LastError = "Installation failed (cannot init SetupFile path)"
+					s.sdk.Status = xsapiv1.SdkStatusNotInstalled
+				} else {
+					s.sdk.SetupFile = sdkDef.SetupFile
+				}
+			}
+
 		} else {
 			s.sdk.LastError = "Installation failed (code " + strconv.Itoa(code) +
 				")"
diff --git a/lib/xdsserver/sdks.go b/lib/xdsserver/sdks.go
index 9c10747..b7f7e2e 100644
--- a/lib/xdsserver/sdks.go
+++ b/lib/xdsserver/sdks.go
@@ -26,13 +26,13 @@ import (
 
 	common "github.com/iotbzh/xds-common/golib"
 	"github.com/iotbzh/xds-server/lib/xsapiv1"
-	uuid "github.com/satori/go.uuid"
 )
 
 // SDKs List of installed SDK
 type SDKs struct {
 	*Context
-	Sdks map[string]*CrossSDK
+	Sdks         map[string]*CrossSDK
+	SdksFamilies map[string]*xsapiv1.SDKFamilyConfig
 
 	mutex sync.Mutex
 	stop  chan struct{} // signals intentional stop
@@ -41,9 +41,10 @@ type SDKs struct {
 // NewSDKs creates a new instance of SDKs
 func NewSDKs(ctx *Context) (*SDKs, error) {
 	s := SDKs{
-		Context: ctx,
-		Sdks:    make(map[string]*CrossSDK),
-		stop:    make(chan struct{}),
+		Context:      ctx,
+		Sdks:         make(map[string]*CrossSDK),
+		SdksFamilies: make(map[string]*xsapiv1.SDKFamilyConfig),
+		stop:         make(chan struct{}),
 	}
 
 	scriptsDir := ctx.Config.FileConf.SdkScriptsDir
@@ -67,7 +68,6 @@ func NewSDKs(ctx *Context) (*SDKs, error) {
 
 	// Foreach directories in scripts/sdk
 	nbInstalled := 0
-	monSdksPath := make(map[string]*xsapiv1.SDKFamilyConfig)
 	for _, d := range dirs {
 		if !common.IsDir(d) {
 			continue
@@ -81,42 +81,82 @@ func NewSDKs(ctx *Context) (*SDKs, error) {
 		s.LogSillyf("'%s' SDKs list: %v", d, sdksList)
 
 		for _, sdk := range sdksList {
-			cSdk, err := NewCrossSDK(ctx, sdk, d)
+			cSdk, err := s._createNewCrossSDK(sdk, d, false, false)
 			if err != nil {
 				s.Log.Debugf("Error while processing SDK sdk=%v\n err=%s", sdk, err.Error())
 				continue
 			}
-			if _, exist := s.Sdks[cSdk.sdk.ID]; exist {
-				s.Log.Warningf("Duplicate SDK ID : %v", cSdk.sdk.ID)
-				cSdk.sdk.ID += "_DUPLICATE_" + uuid.NewV1().String()
-			}
-			s.Sdks[cSdk.sdk.ID] = cSdk
+
 			if cSdk.sdk.Status == xsapiv1.SdkStatusInstalled {
 				nbInstalled++
 			}
 
-			monSdksPath[cSdk.sdk.FamilyConf.RootDir] = &cSdk.sdk.FamilyConf
+			s.SdksFamilies[cSdk.sdk.FamilyConf.FamilyName] = &cSdk.sdk.FamilyConf
 		}
 	}
 
 	ctx.Log.Debugf("Cross SDKs: %d defined, %d installed", len(s.Sdks), nbInstalled)
 
 	// Start monitor thread to detect new SDKs
-	if len(monSdksPath) == 0 {
+	sdksDirs := []string{}
+	for _, sf := range s.SdksFamilies {
+		sdksDirs = append(sdksDirs, sf.RootDir)
+	}
+
+	if len(s.SdksFamilies) == 0 {
 		s.Log.Warningf("No cross SDKs definition found")
+		/* TODO: used it or cleanup
+		} else {
+			go s.monitorSDKInstallation(sdksDirs)
+		*/
 	}
 
 	return &s, nil
 }
 
+// _createNewCrossSDK Private function to create a new Cross SDK
+func (s *SDKs) _createNewCrossSDK(sdk xsapiv1.SDK, scriptDir string, installing bool, force bool) (*CrossSDK, error) {
+
+	cSdk, err := NewCrossSDK(s.Context, sdk, scriptDir)
+	if err != nil {
+		return cSdk, err
+	}
+
+	// Allow to overwrite not installed SDK or when force is set
+	if _, exist := s.Sdks[cSdk.sdk.ID]; exist {
+		if !force && cSdk.sdk.Path != "" && common.Exists(cSdk.sdk.Path) {
+			return cSdk, fmt.Errorf("SDK ID %s already installed in %s", cSdk.sdk.ID, cSdk.sdk.Path)
+		}
+		if !force && cSdk.sdk.Status != xsapiv1.SdkStatusNotInstalled {
+			return cSdk, fmt.Errorf("Duplicate SDK ID %s (use force to overwrite)", cSdk.sdk.ID)
+		}
+	}
+
+	// Sanity check
+	errMsg := "Invalid SDK definition "
+	if installing && cSdk.sdk.Path == "" {
+		return cSdk, fmt.Errorf(errMsg + "(path not set)")
+	}
+	if installing && cSdk.sdk.URL == "" {
+		return cSdk, fmt.Errorf(errMsg + "(url not set)")
+	}
+
+	// Add to list
+	s.Sdks[cSdk.sdk.ID] = cSdk
+
+	return cSdk, err
+}
+
 // Stop SDKs management
 func (s *SDKs) Stop() {
 	close(s.stop)
 }
 
 // monitorSDKInstallation
-/* TODO: cleanup
-func (s *SDKs) monitorSDKInstallation(monSDKs map[string]*xsapiv1.SDKFamilyConfig) {
+/* TODO: used it or cleanup
+import 	"github.com/zillode/notify"
+
+func (s *SDKs) monitorSDKInstallation(watchingDirs []string) {
 
 	// Set up a watchpoint listening for inotify-specific events
 	c := make(chan notify.EventInfo, 1)
@@ -131,7 +171,7 @@ func (s *SDKs) monitorSDKInstallation(monSDKs map[string]*xsapiv1.SDKFamilyConfi
 	}
 
 	// Add directory watchers
-	for dir := range monSDKs {
+	for _, dir := range watchingDirs {
 		if err := addWatcher(dir); err != nil {
 			s.Log.Errorln(err.Error())
 		}
@@ -162,6 +202,14 @@ func (s *SDKs) monitorSDKInstallation(monSDKs map[string]*xsapiv1.SDKFamilyConfi
 
 			switch ei.Event() {
 			case notify.Create:
+				sdkDef, err := GetSDKInfo(scriptDir, sdk.URL, "", "", s.Log)
+				if err != nil {
+					s.Log.Warningf("Cannot get sdk info: %v", err)
+					continue
+				}
+				sdk.Path = sdkDef.Path
+				sdk.Path = sdkDef.SetupFile
+
 				// Emit Folder state change event
 				if err := s.events.Emit(xsapiv1.EVTSDKInstall, sdk, ""); err != nil {
 					s.Log.Warningf("Cannot notify SDK install: %v", err)
@@ -196,7 +244,7 @@ func (s *SDKs) ResolveID(id string) (string, error) {
 	} else if len(match) == 0 {
 		return id, fmt.Errorf("Unknown sdk id")
 	}
-	return id, fmt.Errorf("Multiple sdk IDs found with provided prefix: " + id)
+	return id, fmt.Errorf("Multiple sdk IDs found: %v", match)
 }
 
 // Get returns an SDK from id
@@ -260,30 +308,73 @@ func (s *SDKs) GetEnvCmd(id string, defaultID string) []string {
 }
 
 // Install Used to install a new SDK
-func (s *SDKs) Install(id, filepath string, force bool, timeout int, sess *ClientSession) (*xsapiv1.SDK, error) {
-	var cSdk *CrossSDK
+func (s *SDKs) Install(id, filepath string, force bool, timeout int, args []string, sess *ClientSession) (*xsapiv1.SDK, error) {
+
+	var sdk *xsapiv1.SDK
+	var err error
+	scriptDir := ""
+	sdkFilename := ""
+
 	if id != "" && filepath != "" {
 		return nil, fmt.Errorf("invalid parameter, both id and filepath are set")
 	}
+
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
 	if id != "" {
-		var exist bool
-		cSdk, exist = s.Sdks[id]
+		curSdk, exist := s.Sdks[id]
 		if !exist {
 			return nil, fmt.Errorf("unknown id")
 		}
+
+		sdk = &curSdk.sdk
+		scriptDir = sdk.FamilyConf.ScriptsDir
+
+		// Update path when not set
+		if sdk.Path == "" {
+			sdkDef, err := GetSDKInfo(scriptDir, sdk.URL, "", "", s.Log)
+			if err != nil || sdkDef.Path == "" {
+				return nil, fmt.Errorf("cannot retrieve sdk path %v", err)
+			}
+			sdk.Path = sdkDef.Path
+		}
+
 	} else if filepath != "" {
-		// TODO check that file is accessible
+		// FIXME support any location and also sharing either by pathmap or Syncthing
+		baseDir := "${HOME}/xds-workspace/sdks"
+		sdkFilename, _ = common.ResolveEnvVar(path.Join(baseDir, path.Base(filepath)))
+		if !common.Exists(sdkFilename) {
+			return nil, fmt.Errorf("SDK file not accessible, must be in %s", baseDir)
+		}
+
+		for _, sf := range s.SdksFamilies {
+			sdkDef, err := GetSDKInfo(sf.ScriptsDir, "", sdkFilename, "", s.Log)
+			if err == nil {
+				// OK, sdk found
+				sdk = &sdkDef
+				scriptDir = sf.ScriptsDir
+				break
+			}
+
+			s.Log.Debugf("GetSDKInfo error: family=%s, sdkFilename=%s, err=%v", sf.FamilyName, path.Base(sdkFilename), err)
+		}
+		if sdk == nil {
+			return nil, fmt.Errorf("Cannot identify SDK family for %s", path.Base(filepath))
+		}
 
 	} else {
 		return nil, fmt.Errorf("invalid parameter, id or filepath must be set")
 	}
 
-	s.mutex.Lock()
-	defer s.mutex.Unlock()
+	cSdk, err := s._createNewCrossSDK(*sdk, scriptDir, true, force)
+	if err != nil {
+		return nil, err
+	}
 
 	// Launch script to install
 	// (note that add event will be generated by monitoring thread)
-	if err := cSdk.Install(filepath, force, timeout, sess); err != nil {
+	if err := cSdk.Install(sdkFilename, force, timeout, args, sess); err != nil {
 		return &cSdk.sdk, err
 	}
 
diff --git a/lib/xsapiv1/sdks.go b/lib/xsapiv1/sdks.go
index 3a79c99..44b0940 100644
--- a/lib/xsapiv1/sdks.go
+++ b/lib/xsapiv1/sdks.go
@@ -58,10 +58,11 @@ type SDKFamilyConfig struct {
 
 // SDKInstallArgs JSON parameters of POST /sdks or /sdks/abortinstall commands
 type SDKInstallArgs struct {
-	ID       string `json:"id" binding:"required"` // install by ID (must be part of GET /sdks result)
-	Filename string `json:"filename"`              // install by using a file
-	Force    bool   `json:"force"`                 // force SDK install when already existing
-	Timeout  int    `json:"timeout"`               // 1800 == default 30 minutes
+	ID          string   `json:"id"`          // install by ID (must be part of GET /sdks result)
+	Filename    string   `json:"filename"`    // install by using a file
+	Force       bool     `json:"force"`       // force SDK install when already existing
+	Timeout     int      `json:"timeout"`     // 1800 == default 30 minutes
+	InstallArgs []string `json:"installArgs"` // args directly passed to add/install script
 }
 
 // SDKManagementMsg Message send during SDK installation or when installation is complete