From 1efdb28f1bf9246004a7b145e8d91d89be785772 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Tue, 4 Jul 2017 12:03:05 +0200 Subject: Report an error when sdkid not found. --- lib/apiv1/exec.go | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/apiv1/exec.go b/lib/apiv1/exec.go index 654ff64..6c70a98 100644 --- a/lib/apiv1/exec.go +++ b/lib/apiv1/exec.go @@ -177,6 +177,12 @@ func (s *APIService) execCmd(c *gin.Context) { if envCmd := s.sdks.GetEnvCmd(args.SdkID, prj.DefaultSdk); len(envCmd) > 0 { cmd = append(cmd, envCmd...) cmd = append(cmd, "&&") + } else { + // It's an error if no envcmd found while a sdkid has been provided + if args.SdkID != "" { + common.APIError(c, "Unknown sdkid") + return + } } cmd = append(cmd, "cd", prj.GetFullPath(args.RPath), "&&", args.Cmd) -- cgit From bf2487c4e4c925f437f9e72f09f6ef5099a0c3cb Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Sun, 9 Jul 2017 15:43:18 +0200 Subject: Add stdin support to /exec Signed-off-by: Sebastien Douheret --- lib/apiv1/apiv1.go | 1 + lib/apiv1/exec.go | 61 +++++++++++++++++++++++++++++++++++++++++++++++------- lib/apiv1/make.go | 16 +++++++------- 3 files changed, 63 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/apiv1/apiv1.go b/lib/apiv1/apiv1.go index 7fa69e9..cde2526 100644 --- a/lib/apiv1/apiv1.go +++ b/lib/apiv1/apiv1.go @@ -52,6 +52,7 @@ func New(r *gin.Engine, sess *session.Sessions, cfg *xdsconfig.Config, mfolder * s.apiRouter.POST("/exec", s.execCmd) s.apiRouter.POST("/exec/:id", s.execCmd) + s.apiRouter.POST("/signal", s.execSignalCmd) return s } diff --git a/lib/apiv1/exec.go b/lib/apiv1/exec.go index 6c70a98..ce0241a 100644 --- a/lib/apiv1/exec.go +++ b/lib/apiv1/exec.go @@ -6,6 +6,8 @@ import ( "strings" "time" + "fmt" + "github.com/gin-gonic/gin" common "github.com/iotbzh/xds-common/golib" ) @@ -38,6 +40,12 @@ type ExecExitMsg struct { Error error `json:"error"` } +// ExecSignalArgs JSON parameters of /exec/signal command +type ExecSignalArgs struct { + CmdID string `json:"cmdID" binding:"required"` // command id + Signal string `json:"signal" binding:"required"` // signal number +} + // ExecOutEvent Event send in WS when characters are received const ExecOutEvent = "exec:output" @@ -85,14 +93,26 @@ func (s *APIService) execCmd(c *gin.Context) { } execTmo := args.CmdTimeout - if execTmo == 0 { + if execTmo == -1 { + // -1 : no timeout + execTmo = 365 * 24 * 60 * 60 // 1 year == no timeout + } else if execTmo == 0 { + // 0 : default timeout // TODO get default timeout from config.json file execTmo = 24 * 60 * 60 // 1 day } + // Define callback for input + /* SEB TODO + var iCB common.OnInputCB + iCB = func() { + + } + */ + // Define callback for output var oCB common.EmitOutputCB - oCB = func(sid string, id int, stdout, stderr string, data *map[string]interface{}) { + oCB = func(sid string, id string, stdout, stderr string, data *map[string]interface{}) { // IO socket can be nil when disconnected so := s.sessions.IOSocketGet(sid) if so == nil { @@ -110,9 +130,11 @@ func (s *APIService) execCmd(c *gin.Context) { s.log.Debugf("%s emitted - WS sid %s - id:%d - prjID:%s", ExecOutEvent, sid, id, prjID) + fmt.Printf("SEB SEND out <%v>, err <%v>\n", stdout, stderr) + // FIXME replace by .BroadcastTo a room err := (*so).Emit(ExecOutEvent, ExecOutMsg{ - CmdID: strconv.Itoa(id), + CmdID: id, Timestamp: time.Now().String(), Stdout: stdout, Stderr: stderr, @@ -123,7 +145,7 @@ func (s *APIService) execCmd(c *gin.Context) { } // Define callback for output - eCB := func(sid string, id int, code int, err error, data *map[string]interface{}) { + eCB := func(sid string, id string, code int, err error, data *map[string]interface{}) { s.log.Debugf("Command [Cmd ID %d] exited: code %d, error: %v", id, code, err) // IO socket can be nil when disconnected @@ -159,7 +181,7 @@ func (s *APIService) execCmd(c *gin.Context) { // FIXME replace by .BroadcastTo a room e := (*so).Emit(ExecExitEvent, ExecExitMsg{ - CmdID: strconv.Itoa(id), + CmdID: id, Timestamp: time.Now().String(), Code: code, Error: err, @@ -169,7 +191,7 @@ func (s *APIService) execCmd(c *gin.Context) { } } - cmdID := execCommandID + cmdID := strconv.Itoa(execCommandID) execCommandID++ cmd := []string{} @@ -190,10 +212,13 @@ func (s *APIService) execCmd(c *gin.Context) { cmd = append(cmd, args.Args...) } + // SEB Workaround for stderr issue (order not respected with stdout) + cmd = append(cmd, " 2>&1") + // Append client project dir to environment args.Env = append(args.Env, "CLIENT_PROJECT_DIR="+prj.RelativePath) - s.log.Debugf("Execute [Cmd ID %d]: %v", cmdID, cmd) + s.log.Debugf("Execute [Cmd ID %s]: %v", cmdID, cmd) data := make(map[string]interface{}) data["ID"] = prj.ID @@ -212,3 +237,25 @@ func (s *APIService) execCmd(c *gin.Context) { "cmdID": cmdID, }) } + +// ExecCmd executes remotely a command +func (s *APIService) execSignalCmd(c *gin.Context) { + var args ExecSignalArgs + + if c.BindJSON(&args) != nil { + common.APIError(c, "Invalid arguments") + return + } + + s.log.Debugf("Signal %s for command ID %s", args.Signal, args.CmdID) + err := common.ExecSignal(args.CmdID, args.Signal) + if err != nil { + common.APIError(c, err.Error()) + return + } + + c.JSON(http.StatusOK, + gin.H{ + "status": "OK", + }) +} diff --git a/lib/apiv1/make.go b/lib/apiv1/make.go index 5cd98c6..6ae840b 100644 --- a/lib/apiv1/make.go +++ b/lib/apiv1/make.go @@ -92,11 +92,11 @@ func (s *APIService) buildMake(c *gin.Context) { // Define callback for output var oCB common.EmitOutputCB - oCB = func(sid string, id int, stdout, stderr string, data *map[string]interface{}) { + oCB = func(sid string, cmdID string, stdout, stderr string, data *map[string]interface{}) { // IO socket can be nil when disconnected so := s.sessions.IOSocketGet(sid) if so == nil { - s.log.Infof("%s not emitted: WS closed - sid: %s - msg id:%d", MakeOutEvent, sid, id) + s.log.Infof("%s not emitted: WS closed - sid: %s - msg id:%s", MakeOutEvent, sid, cmdID) return } @@ -112,7 +112,7 @@ func (s *APIService) buildMake(c *gin.Context) { // FIXME replace by .BroadcastTo a room err := (*so).Emit(MakeOutEvent, MakeOutMsg{ - CmdID: strconv.Itoa(id), + CmdID: cmdID, Timestamp: time.Now().String(), Stdout: stdout, Stderr: stderr, @@ -123,13 +123,13 @@ func (s *APIService) buildMake(c *gin.Context) { } // Define callback for output - eCB := func(sid string, id int, code int, err error, data *map[string]interface{}) { - s.log.Debugf("Command [Cmd ID %d] exited: code %d, error: %v", id, code, err) + eCB := func(sid string, cmdID string, code int, err error, data *map[string]interface{}) { + s.log.Debugf("Command [Cmd ID %s] exited: code %d, error: %v", cmdID, code, err) // IO socket can be nil when disconnected so := s.sessions.IOSocketGet(sid) if so == nil { - s.log.Infof("%s not emitted - WS closed (id:%d", MakeExitEvent, id) + s.log.Infof("%s not emitted - WS closed (id:%s", MakeExitEvent, cmdID) return } @@ -159,7 +159,7 @@ func (s *APIService) buildMake(c *gin.Context) { // FIXME replace by .BroadcastTo a room e := (*so).Emit(MakeExitEvent, MakeExitMsg{ - CmdID: strconv.Itoa(id), + CmdID: id, Timestamp: time.Now().String(), Code: code, Error: err, @@ -169,7 +169,7 @@ func (s *APIService) buildMake(c *gin.Context) { } } - cmdID := makeCommandID + cmdID := strconv.Itoa(makeCommandID) makeCommandID++ cmd := []string{} -- cgit From ab1170e65d6d03dd1eb2542b5fc47694d7785e70 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Mon, 7 Aug 2017 17:22:15 +0200 Subject: Improved /exec to support gdb execution. /exec now supports stdin and stdout/stderr tunneling over an websocket (socketio). This also supports redirection of inferior process output (stdout only) in particular case of gdb command (set gdb --tty option). --- lib/apiv1/exec.go | 312 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 216 insertions(+), 96 deletions(-) (limited to 'lib') diff --git a/lib/apiv1/exec.go b/lib/apiv1/exec.go index ce0241a..eb93af8 100644 --- a/lib/apiv1/exec.go +++ b/lib/apiv1/exec.go @@ -1,61 +1,88 @@ package apiv1 import ( + "fmt" "net/http" + "os" + "regexp" "strconv" "strings" "time" - "fmt" - "github.com/gin-gonic/gin" common "github.com/iotbzh/xds-common/golib" + "github.com/iotbzh/xds-common/golib/eows" + "github.com/kr/pty" ) -// ExecArgs JSON parameters of /exec command -type ExecArgs struct { - ID string `json:"id" binding:"required"` - SdkID string `json:"sdkid"` // sdk ID to use for setting env - Cmd string `json:"cmd" binding:"required"` - Args []string `json:"args"` - Env []string `json:"env"` - RPath string `json:"rpath"` // relative path into project - ExitImmediate bool `json:"exitImmediate"` // when true, exit event sent immediately when command exited (IOW, don't wait file synchronization) - CmdTimeout int `json:"timeout"` // command completion timeout in Second -} +type ( + // ExecArgs JSON parameters of /exec command + ExecArgs struct { + ID string `json:"id" binding:"required"` + SdkID string `json:"sdkid"` // sdk ID to use for setting env + Cmd string `json:"cmd" binding:"required"` + Args []string `json:"args"` + Env []string `json:"env"` + RPath string `json:"rpath"` // relative path into project + TTY bool `json:"tty"` // Use a tty, specific to gdb --tty option + TTYGdbserverFix bool `json:"ttyGdbserverFix"` // Set to true to activate gdbserver workaround about inferior output + ExitImmediate bool `json:"exitImmediate"` // when true, exit event sent immediately when command exited (IOW, don't wait file synchronization) + CmdTimeout int `json:"timeout"` // command completion timeout in Second + } -// ExecOutMsg Message send on each output (stdout+stderr) of executed command -type ExecOutMsg struct { - CmdID string `json:"cmdID"` - Timestamp string `json:"timestamp"` - Stdout string `json:"stdout"` - Stderr string `json:"stderr"` -} + // ExecInMsg Message used to received input characters (stdin) + ExecInMsg struct { + CmdID string `json:"cmdID"` + Timestamp string `json:"timestamp"` + Stdin string `json:"stdin"` + } -// ExecExitMsg Message send when executed command exited -type ExecExitMsg struct { - CmdID string `json:"cmdID"` - Timestamp string `json:"timestamp"` - Code int `json:"code"` - Error error `json:"error"` -} + // ExecOutMsg Message used to send output characters (stdout+stderr) + ExecOutMsg struct { + CmdID string `json:"cmdID"` + Timestamp string `json:"timestamp"` + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + } -// ExecSignalArgs JSON parameters of /exec/signal command -type ExecSignalArgs struct { - CmdID string `json:"cmdID" binding:"required"` // command id - Signal string `json:"signal" binding:"required"` // signal number -} + // ExecExitMsg Message sent when executed command exited + ExecExitMsg struct { + CmdID string `json:"cmdID"` + Timestamp string `json:"timestamp"` + Code int `json:"code"` + Error error `json:"error"` + } -// ExecOutEvent Event send in WS when characters are received -const ExecOutEvent = "exec:output" + // ExecSignalArgs JSON parameters of /exec/signal command + ExecSignalArgs struct { + CmdID string `json:"cmdID" binding:"required"` // command id + Signal string `json:"signal" binding:"required"` // signal number + } +) + +const ( + // ExecInEvent Event send in WS when characters are sent (stdin) + ExecInEvent = "exec:input" + + // ExecOutEvent Event send in WS when characters are received (stdout or stderr) + ExecOutEvent = "exec:output" -// ExecExitEvent Event send in WS when program exited -const ExecExitEvent = "exec:exit" + // ExecExitEvent Event send in WS when program exited + ExecExitEvent = "exec:exit" + + // ExecInferiorInEvent Event send in WS when characters are sent to an inferior (used by gdb inferior/tty) + ExecInferiorInEvent = "exec:inferior-input" + + // ExecInferiorOutEvent Event send in WS when characters are received by an inferior + ExecInferiorOutEvent = "exec:inferior-output" +) var execCommandID = 1 // ExecCmd executes remotely a command func (s *APIService) execCmd(c *gin.Context) { + var gdbPty, gdbTty *os.File + var err error var args ExecArgs if c.BindJSON(&args) != nil { common.APIError(c, "Invalid arguments") @@ -92,49 +119,112 @@ func (s *APIService) execCmd(c *gin.Context) { return } - execTmo := args.CmdTimeout - if execTmo == -1 { - // -1 : no timeout - execTmo = 365 * 24 * 60 * 60 // 1 year == no timeout - } else if execTmo == 0 { + // Build command line + cmd := []string{} + // Setup env var regarding Sdk ID (used for example to setup cross toolchain) + if envCmd := s.sdks.GetEnvCmd(args.SdkID, prj.DefaultSdk); len(envCmd) > 0 { + cmd = append(cmd, envCmd...) + cmd = append(cmd, "&&") + } else { + // It's an error if no envcmd found while a sdkid has been provided + if args.SdkID != "" { + common.APIError(c, "Unknown sdkid") + return + } + } + + // FIXME - SEB: exec prevents to use syntax: + // xds-exec -l debug -c xds-config.env -- "cd build && cmake .." + cmd = append(cmd, "cd", prj.GetFullPath(args.RPath)) + cmd = append(cmd, "&&", "exec", args.Cmd) + + // Process command arguments + cmdArgs := make([]string, len(args.Args)+1) + copy(cmdArgs, args.Args) + + // Allocate pts if tty if used + if args.TTY { + gdbPty, gdbTty, err = pty.Open() + if err != nil { + common.APIError(c, err.Error()) + return + } + + s.log.Debugf("Client command tty: %v %v\n", gdbTty.Name(), gdbTty.Name()) + cmdArgs = append(cmdArgs, "--tty="+gdbTty.Name()) + } + + // Unique ID for each commands + cmdID := strconv.Itoa(execCommandID) + execCommandID++ + + // Create new execution over WS context + execWS := eows.New(strings.Join(cmd, " "), cmdArgs, sop, sess.ID, cmdID) + execWS.Log = s.log + + // Append client project dir to environment + execWS.Env = append(args.Env, "CLIENT_PROJECT_DIR="+prj.RelativePath) + + // Set command execution timeout + if args.CmdTimeout == 0 { // 0 : default timeout // TODO get default timeout from config.json file - execTmo = 24 * 60 * 60 // 1 day + execWS.CmdExecTimeout = 24 * 60 * 60 // 1 day + } else { + execWS.CmdExecTimeout = args.CmdTimeout } - // Define callback for input - /* SEB TODO - var iCB common.OnInputCB - iCB = func() { + // Define callback for input (stdin) + execWS.InputEvent = ExecInEvent + execWS.InputCB = func(e *eows.ExecOverWS, stdin string) (string, error) { + s.log.Debugf("STDIN <<%v>>", strings.Replace(stdin, "\n", "\\n", -1)) + + // Handle Ctrl-D + if len(stdin) == 1 && stdin == "\x04" { + // Close stdin + errMsg := fmt.Errorf("close stdin: %v", stdin) + return "", errMsg + } + + // Set correct path + data := e.UserData + rootPath := (*data)["RootPath"].(string) + relaPath := (*data)["RelativePath"].(string) + stdin = strings.Replace(stdin, relaPath, rootPath+"/"+relaPath, -1) + return stdin, nil } - */ - // Define callback for output - var oCB common.EmitOutputCB - oCB = func(sid string, id string, stdout, stderr string, data *map[string]interface{}) { + // Define callback for output (stdout+stderr) + execWS.OutputCB = func(e *eows.ExecOverWS, stdout, stderr string) { // IO socket can be nil when disconnected - so := s.sessions.IOSocketGet(sid) + so := s.sessions.IOSocketGet(e.Sid) if so == nil { - s.log.Infof("%s not emitted: WS closed - sid: %s - msg id:%d", ExecOutEvent, sid, id) + s.log.Infof("%s not emitted: WS closed (sid:%s, msgid:%s)", ExecOutEvent, e.Sid, e.CmdID) return } // Retrieve project ID and RootPath + data := e.UserData prjID := (*data)["ID"].(string) prjRootPath := (*data)["RootPath"].(string) + gdbServerTTY := (*data)["gdbServerTTY"].(string) // Cleanup any references to internal rootpath in stdout & stderr stdout = strings.Replace(stdout, prjRootPath, "", -1) stderr = strings.Replace(stderr, prjRootPath, "", -1) - s.log.Debugf("%s emitted - WS sid %s - id:%d - prjID:%s", ExecOutEvent, sid, id, prjID) - - fmt.Printf("SEB SEND out <%v>, err <%v>\n", stdout, stderr) + s.log.Debugf("%s emitted - WS sid[4:] %s - id:%s - prjID:%s", ExecOutEvent, e.Sid[4:], e.CmdID, prjID) + if stdout != "" { + s.log.Debugf("STDOUT <<%v>>", strings.Replace(stdout, "\n", "\\n", -1)) + } + if stderr != "" { + s.log.Debugf("STDERR <<%v>>", strings.Replace(stderr, "\n", "\\n", -1)) + } // FIXME replace by .BroadcastTo a room err := (*so).Emit(ExecOutEvent, ExecOutMsg{ - CmdID: id, + CmdID: e.CmdID, Timestamp: time.Now().String(), Stdout: stdout, Stderr: stderr, @@ -142,20 +232,53 @@ func (s *APIService) execCmd(c *gin.Context) { if err != nil { s.log.Errorf("WS Emit : %v", err) } + + // XXX - Workaround due to gdbserver bug that doesn't redirect + // inferior output (https://bugs.eclipse.org/bugs/show_bug.cgi?id=437532#c13) + if gdbServerTTY == "workaround" && len(stdout) > 1 && stdout[0] == '&' { + + // Extract and cleanup string like &"bla bla\n" + re := regexp.MustCompile("&\"(.*)\"") + rer := re.FindAllStringSubmatch(stdout, -1) + out := "" + if rer != nil && len(rer) > 0 { + for _, o := range rer { + if len(o) >= 1 { + out = strings.Replace(o[1], "\\n", "\n", -1) + out = strings.Replace(out, "\\r", "\r", -1) + out = strings.Replace(out, "\\t", "\t", -1) + + s.log.Debugf("STDOUT INFERIOR: <<%v>>", out) + err := (*so).Emit(ExecInferiorOutEvent, ExecOutMsg{ + CmdID: e.CmdID, + Timestamp: time.Now().String(), + Stdout: out, + Stderr: "", + }) + if err != nil { + s.log.Errorf("WS Emit : %v", err) + } + } + } + } else { + s.log.Errorf("INFERIOR out parsing error: stdout=<%v>", stdout) + } + } } // Define callback for output - eCB := func(sid string, id string, code int, err error, data *map[string]interface{}) { - s.log.Debugf("Command [Cmd ID %d] exited: code %d, error: %v", id, code, err) + execWS.ExitCB = func(e *eows.ExecOverWS, code int, err error) { + s.log.Debugf("Command [Cmd ID %s] exited: code %d, error: %v", e.CmdID, code, err) // IO socket can be nil when disconnected - so := s.sessions.IOSocketGet(sid) + so := s.sessions.IOSocketGet(e.Sid) if so == nil { - s.log.Infof("%s not emitted - WS closed (id:%d", ExecExitEvent, id) + s.log.Infof("%s not emitted - WS closed (id:%s)", ExecExitEvent, e.CmdID) return } // Retrieve project ID and RootPath + data := e.UserData prjID := (*data)["ID"].(string) exitImm := (*data)["ExitImmediate"].(bool) @@ -179,53 +302,43 @@ func (s *APIService) execCmd(c *gin.Context) { } } + // Close client tty + if gdbPty != nil { + gdbPty.Close() + } + if gdbTty != nil { + gdbTty.Close() + } + // FIXME replace by .BroadcastTo a room - e := (*so).Emit(ExecExitEvent, ExecExitMsg{ - CmdID: id, + errSoEmit := (*so).Emit(ExecExitEvent, ExecExitMsg{ + CmdID: e.CmdID, Timestamp: time.Now().String(), Code: code, Error: err, }) - if e != nil { - s.log.Errorf("WS Emit : %v", e) + if errSoEmit != nil { + s.log.Errorf("WS Emit : %v", errSoEmit) } } - cmdID := strconv.Itoa(execCommandID) - execCommandID++ - cmd := []string{} - - // Setup env var regarding Sdk ID (used for example to setup cross toolchain) - if envCmd := s.sdks.GetEnvCmd(args.SdkID, prj.DefaultSdk); len(envCmd) > 0 { - cmd = append(cmd, envCmd...) - cmd = append(cmd, "&&") - } else { - // It's an error if no envcmd found while a sdkid has been provided - if args.SdkID != "" { - common.APIError(c, "Unknown sdkid") - return - } - } - - cmd = append(cmd, "cd", prj.GetFullPath(args.RPath), "&&", args.Cmd) - if len(args.Args) > 0 { - cmd = append(cmd, args.Args...) - } - - // SEB Workaround for stderr issue (order not respected with stdout) - cmd = append(cmd, " 2>&1") - - // Append client project dir to environment - args.Env = append(args.Env, "CLIENT_PROJECT_DIR="+prj.RelativePath) - - s.log.Debugf("Execute [Cmd ID %s]: %v", cmdID, cmd) - + // User data (used within callbacks) data := make(map[string]interface{}) data["ID"] = prj.ID data["RootPath"] = prj.RootPath + data["RelativePath"] = prj.RelativePath data["ExitImmediate"] = args.ExitImmediate + if args.TTY && args.TTYGdbserverFix { + data["gdbServerTTY"] = "workaround" + } else { + data["gdbServerTTY"] = "" + } + execWS.UserData = &data - err := common.ExecPipeWs(cmd, args.Env, sop, sess.ID, cmdID, execTmo, s.log, oCB, eCB, &data) + // Start command execution + s.log.Debugf("Execute [Cmd ID %s]: %v %v", execWS.CmdID, execWS.Cmd, execWS.Args) + + err = execWS.Start() if err != nil { common.APIError(c, err.Error()) return @@ -234,7 +347,7 @@ func (s *APIService) execCmd(c *gin.Context) { c.JSON(http.StatusOK, gin.H{ "status": "OK", - "cmdID": cmdID, + "cmdID": execWS.CmdID, }) } @@ -248,7 +361,14 @@ func (s *APIService) execSignalCmd(c *gin.Context) { } s.log.Debugf("Signal %s for command ID %s", args.Signal, args.CmdID) - err := common.ExecSignal(args.CmdID, args.Signal) + + e := eows.GetEows(args.CmdID) + if e == nil { + common.APIError(c, "unknown cmdID") + return + } + + err := e.Signal(args.Signal) if err != nil { common.APIError(c, err.Error()) return -- cgit From 0262f5bef6ff67e77b844a04733c57740fba9f00 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Tue, 8 Aug 2017 16:59:56 +0200 Subject: Added -logfile option. --- lib/webserver/server.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/webserver/server.go b/lib/webserver/server.go index 8fd7e44..7649cce 100644 --- a/lib/webserver/server.go +++ b/lib/webserver/server.go @@ -2,6 +2,7 @@ package webserver import ( "fmt" + "log" "net/http" "os" @@ -36,20 +37,21 @@ const indexFilename = "index.html" const cookieMaxAge = "3600" // New creates an instance of Server -func New(cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs, log *logrus.Logger) *Server { +func New(cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs, logr *logrus.Logger) *Server { // Setup logging for gin router - if log.Level == logrus.DebugLevel { + if logr.Level == logrus.DebugLevel { gin.SetMode(gin.DebugMode) } else { gin.SetMode(gin.ReleaseMode) } - // TODO - // - try to bind gin DefaultWriter & DefaultErrorWriter to logrus logger - // - try to fix pb about isTerminal=false when out is in VSC Debug Console - //gin.DefaultWriter = ?? - //gin.DefaultErrorWriter = ?? + // Redirect gin logs into logrus logger + gin.DefaultWriter = logr.Out + gin.DefaultErrorWriter = logr.Out + log.SetOutput(logr.Out) + + // FIXME - fix pb about isTerminal=false when out is in VSC Debug Console // Creates gin router r := gin.New() @@ -63,7 +65,7 @@ func New(cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs, log sessions: nil, mfolder: mfolder, sdks: sdks, - log: log, + log: logr, stop: make(chan struct{}), } -- cgit From dd6f08b10b1597f44e3dc25509ac9a45336b0914 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Thu, 10 Aug 2017 12:19:34 +0200 Subject: Add folder interface and support native pathmap folder type. Signed-off-by: Sebastien Douheret --- lib/apiv1/agent.go | 3 + lib/apiv1/apiv1.go | 6 +- lib/apiv1/config.go | 7 +- lib/apiv1/exec.go | 22 +-- lib/apiv1/folders.go | 36 ++--- lib/apiv1/make.go | 12 +- lib/crosssdk/sdks.go | 3 + lib/folder/folder-interface.go | 59 ++++++++ lib/folder/folder-pathmap.go | 88 +++++++++++ lib/model/folder.go | 110 -------------- lib/model/folders.go | 333 +++++++++++++++++++++++++++++++++++++++++ lib/syncthing/folder-st.go | 97 ++++++++++++ lib/syncthing/st.go | 50 +------ lib/syncthing/stfolder.go | 123 +++++++++++++-- lib/webserver/server.go | 8 +- lib/xdsconfig/config.go | 21 ++- lib/xdsconfig/fileconfig.go | 40 +++-- lib/xdsconfig/folderconfig.go | 85 ----------- lib/xdsconfig/foldersconfig.go | 47 ------ 19 files changed, 785 insertions(+), 365 deletions(-) create mode 100644 lib/folder/folder-interface.go create mode 100644 lib/folder/folder-pathmap.go delete mode 100644 lib/model/folder.go create mode 100644 lib/model/folders.go create mode 100644 lib/syncthing/folder-st.go delete mode 100644 lib/xdsconfig/folderconfig.go delete mode 100644 lib/xdsconfig/foldersconfig.go (limited to 'lib') diff --git a/lib/apiv1/agent.go b/lib/apiv1/agent.go index 651f246..925f12b 100644 --- a/lib/apiv1/agent.go +++ b/lib/apiv1/agent.go @@ -11,6 +11,7 @@ import ( common "github.com/iotbzh/xds-common/golib" ) +// XDSAgentTarball . type XDSAgentTarball struct { OS string `json:"os"` Arch string `json:"arch"` @@ -18,6 +19,8 @@ type XDSAgentTarball struct { RawVersion string `json:"raw-version"` FileURL string `json:"fileUrl"` } + +// XDSAgentInfo . type XDSAgentInfo struct { Tarballs []XDSAgentTarball `json:"tarballs"` } diff --git a/lib/apiv1/apiv1.go b/lib/apiv1/apiv1.go index cde2526..f32e53b 100644 --- a/lib/apiv1/apiv1.go +++ b/lib/apiv1/apiv1.go @@ -16,19 +16,19 @@ type APIService struct { apiRouter *gin.RouterGroup sessions *session.Sessions cfg *xdsconfig.Config - mfolder *model.Folder + mfolders *model.Folders sdks *crosssdk.SDKs log *logrus.Logger } // New creates a new instance of API service -func New(r *gin.Engine, sess *session.Sessions, cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs) *APIService { +func New(r *gin.Engine, sess *session.Sessions, cfg *xdsconfig.Config, mfolders *model.Folders, sdks *crosssdk.SDKs) *APIService { s := &APIService{ router: r, sessions: sess, apiRouter: r.Group("/api/v1"), cfg: cfg, - mfolder: mfolder, + mfolders: mfolders, sdks: sdks, log: cfg.Log, } diff --git a/lib/apiv1/config.go b/lib/apiv1/config.go index 662ec8e..4b53217 100644 --- a/lib/apiv1/config.go +++ b/lib/apiv1/config.go @@ -36,10 +36,5 @@ func (s *APIService) setConfig(c *gin.Context) { s.log.Debugln("SET config: ", cfgArg) - if err := s.mfolder.UpdateAll(cfgArg); err != nil { - common.APIError(c, err.Error()) - return - } - - c.JSON(http.StatusOK, s.cfg) + common.APIError(c, "Not Supported") } diff --git a/lib/apiv1/exec.go b/lib/apiv1/exec.go index eb93af8..4a591be 100644 --- a/lib/apiv1/exec.go +++ b/lib/apiv1/exec.go @@ -113,11 +113,13 @@ func (s *APIService) execCmd(c *gin.Context) { return } - prj := s.mfolder.GetFolderFromID(id) - if prj == nil { + f := s.mfolders.Get(id) + if f == nil { common.APIError(c, "Unknown id") return } + folder := *f + prj := folder.GetConfig() // Build command line cmd := []string{} @@ -135,7 +137,7 @@ func (s *APIService) execCmd(c *gin.Context) { // FIXME - SEB: exec prevents to use syntax: // xds-exec -l debug -c xds-config.env -- "cd build && cmake .." - cmd = append(cmd, "cd", prj.GetFullPath(args.RPath)) + cmd = append(cmd, "cd", folder.GetFullPath(args.RPath)) cmd = append(cmd, "&&", "exec", args.Cmd) // Process command arguments @@ -163,7 +165,7 @@ func (s *APIService) execCmd(c *gin.Context) { execWS.Log = s.log // Append client project dir to environment - execWS.Env = append(args.Env, "CLIENT_PROJECT_DIR="+prj.RelativePath) + execWS.Env = append(args.Env, "CLIENT_PROJECT_DIR="+prj.ClientPath) // Set command execution timeout if args.CmdTimeout == 0 { @@ -189,8 +191,8 @@ func (s *APIService) execCmd(c *gin.Context) { // Set correct path data := e.UserData rootPath := (*data)["RootPath"].(string) - relaPath := (*data)["RelativePath"].(string) - stdin = strings.Replace(stdin, relaPath, rootPath+"/"+relaPath, -1) + clientPath := (*data)["ClientPath"].(string) + stdin = strings.Replace(stdin, clientPath, rootPath+"/"+clientPath, -1) return stdin, nil } @@ -283,7 +285,7 @@ func (s *APIService) execCmd(c *gin.Context) { exitImm := (*data)["ExitImmediate"].(bool) // XXX - workaround to be sure that Syncthing detected all changes - if err := s.mfolder.ForceSync(prjID); err != nil { + if err := s.mfolders.ForceSync(prjID); err != nil { s.log.Errorf("Error while syncing folder %s: %v", prjID, err) } if !exitImm { @@ -291,8 +293,8 @@ func (s *APIService) execCmd(c *gin.Context) { // FIXME pass as argument tmo := 60 for t := tmo; t > 0; t-- { - s.log.Debugf("Wait file insync for %s (%d/%d)", prjID, t, tmo) - if sync, err := s.mfolder.IsFolderInSync(prjID); sync || err != nil { + s.log.Debugf("Wait file in-sync for %s (%d/%d)", prjID, t, tmo) + if sync, err := s.mfolders.IsFolderInSync(prjID); sync || err != nil { if err != nil { s.log.Errorf("ERROR IsFolderInSync (%s): %v", prjID, err) } @@ -326,7 +328,7 @@ func (s *APIService) execCmd(c *gin.Context) { data := make(map[string]interface{}) data["ID"] = prj.ID data["RootPath"] = prj.RootPath - data["RelativePath"] = prj.RelativePath + data["ClientPath"] = prj.ClientPath data["ExitImmediate"] = args.ExitImmediate if args.TTY && args.TTYGdbserverFix { data["gdbServerTTY"] = "workaround" diff --git a/lib/apiv1/folders.go b/lib/apiv1/folders.go index 44bda24..f957c6d 100644 --- a/lib/apiv1/folders.go +++ b/lib/apiv1/folders.go @@ -2,49 +2,39 @@ package apiv1 import ( "net/http" - "strconv" "github.com/gin-gonic/gin" common "github.com/iotbzh/xds-common/golib" - "github.com/iotbzh/xds-server/lib/xdsconfig" + "github.com/iotbzh/xds-server/lib/folder" ) // getFolders returns all folders configuration func (s *APIService) getFolders(c *gin.Context) { - confMut.Lock() - defer confMut.Unlock() - - c.JSON(http.StatusOK, s.cfg.Folders) + c.JSON(http.StatusOK, s.mfolders.GetConfigArr()) } // getFolder returns a specific folder configuration func (s *APIService) getFolder(c *gin.Context) { - id, err := strconv.Atoi(c.Param("id")) - if err != nil || id < 0 || id > len(s.cfg.Folders) { + f := s.mfolders.Get(c.Param("id")) + if f == nil { common.APIError(c, "Invalid id") return } - confMut.Lock() - defer confMut.Unlock() - - c.JSON(http.StatusOK, s.cfg.Folders[id]) + c.JSON(http.StatusOK, (*f).GetConfig()) } // addFolder adds a new folder to server config func (s *APIService) addFolder(c *gin.Context) { - var cfgArg xdsconfig.FolderConfig + var cfgArg folder.FolderConfig if c.BindJSON(&cfgArg) != nil { common.APIError(c, "Invalid arguments") return } - confMut.Lock() - defer confMut.Unlock() - s.log.Debugln("Add folder config: ", cfgArg) - newFld, err := s.mfolder.UpdateFolder(cfgArg) + newFld, err := s.mfolders.Add(cfgArg) if err != nil { common.APIError(c, err.Error()) return @@ -56,19 +46,11 @@ func (s *APIService) addFolder(c *gin.Context) { // delFolder deletes folder from server config func (s *APIService) delFolder(c *gin.Context) { id := c.Param("id") - if id == "" { - common.APIError(c, "Invalid id") - return - } - - confMut.Lock() - defer confMut.Unlock() s.log.Debugln("Delete folder id ", id) - var delEntry xdsconfig.FolderConfig - var err error - if delEntry, err = s.mfolder.DeleteFolder(id); err != nil { + delEntry, err := s.mfolders.Delete(id) + if err != nil { common.APIError(c, err.Error()) return } diff --git a/lib/apiv1/make.go b/lib/apiv1/make.go index 6ae840b..cf76476 100644 --- a/lib/apiv1/make.go +++ b/lib/apiv1/make.go @@ -76,11 +76,13 @@ func (s *APIService) buildMake(c *gin.Context) { return } - prj := s.mfolder.GetFolderFromID(id) - if prj == nil { + pf := s.mfolders.Get(id) + if pf == nil { common.APIError(c, "Unknown id") return } + folder := *pf + prj := folder.GetConfig() execTmo := args.CmdTimeout if execTmo == 0 { @@ -138,7 +140,7 @@ func (s *APIService) buildMake(c *gin.Context) { exitImm := (*data)["ExitImmediate"].(bool) // XXX - workaround to be sure that Syncthing detected all changes - if err := s.mfolder.ForceSync(prjID); err != nil { + if err := s.mfolders.ForceSync(prjID); err != nil { s.log.Errorf("Error while syncing folder %s: %v", prjID, err) } if !exitImm { @@ -147,7 +149,7 @@ func (s *APIService) buildMake(c *gin.Context) { tmo := 60 for t := tmo; t > 0; t-- { s.log.Debugf("Wait file insync for %s (%d/%d)", prjID, t, tmo) - if sync, err := s.mfolder.IsFolderInSync(prjID); sync || err != nil { + if sync, err := s.mfolders.IsFolderInSync(prjID); sync || err != nil { if err != nil { s.log.Errorf("ERROR IsFolderInSync (%s): %v", prjID, err) } @@ -179,7 +181,7 @@ func (s *APIService) buildMake(c *gin.Context) { cmd = append(cmd, "&&") } - cmd = append(cmd, "cd", prj.GetFullPath(args.RPath), "&&", "make") + cmd = append(cmd, "cd", folder.GetFullPath(args.RPath), "&&", "make") if len(args.Args) > 0 { cmd = append(cmd, args.Args...) } diff --git a/lib/crosssdk/sdks.go b/lib/crosssdk/sdks.go index 35a9998..0da0d1b 100644 --- a/lib/crosssdk/sdks.go +++ b/lib/crosssdk/sdks.go @@ -36,6 +36,9 @@ func Init(cfg *xdsconfig.Config, log *logrus.Logger) (*SDKs, error) { defer s.mutex.Unlock() for _, d := range dirs { + if !common.IsDir(d) { + continue + } sdk, err := NewCrossSDK(d) if err != nil { log.Debugf("Error while processing SDK dir=%s, err=%s", d, err.Error()) diff --git a/lib/folder/folder-interface.go b/lib/folder/folder-interface.go new file mode 100644 index 0000000..b76b3f3 --- /dev/null +++ b/lib/folder/folder-interface.go @@ -0,0 +1,59 @@ +package folder + +// FolderType definition +type FolderType int + +const ( + TypePathMap = 1 + TypeCloudSync = 2 + TypeCifsSmb = 3 +) + +// Folder Status definition +const ( + StatusErrorConfig = "ErrorConfig" + StatusDisable = "Disable" + StatusEnable = "Enable" +) + +// IFOLDER Folder interface +type IFOLDER interface { + Add(cfg FolderConfig) (*FolderConfig, error) // Add a new folder + GetConfig() FolderConfig // Get folder public configuration + GetFullPath(dir string) string // Get folder full path + Remove() error // Remove a folder + Sync() error // Force folder files synchronization + IsInSync() (bool, error) // Check if folder files are in-sync +} + +// FolderConfig is the config for one folder +type FolderConfig struct { + ID string `json:"id"` + Label string `json:"label"` + ClientPath string `json:"path"` + Type FolderType `json:"type"` + Status string `json:"status"` + DefaultSdk string `json:"defaultSdk"` + + // Not exported fields from REST API point of view + RootPath string `json:"-"` + + // FIXME: better to define an equivalent to union data and then implement + // UnmarshalJSON/MarshalJSON to decode/encode according to Type value + // Data interface{} `json:"data"` + + // Specific data depending on which Type is used + DataPathMap PathMapConfig `json:"dataPathMap,omitempty"` + DataCloudSync CloudSyncConfig `json:"dataCloudSync,omitempty"` +} + +// PathMapConfig Path mapping specific data +type PathMapConfig struct { + ServerPath string `json:"serverPath"` +} + +// CloudSyncConfig CloudSync (AKA Syncthing) specific data +type CloudSyncConfig struct { + SyncThingID string `json:"syncThingID"` + BuilderSThgID string `json:"builderSThgID"` +} diff --git a/lib/folder/folder-pathmap.go b/lib/folder/folder-pathmap.go new file mode 100644 index 0000000..8711df2 --- /dev/null +++ b/lib/folder/folder-pathmap.go @@ -0,0 +1,88 @@ +package folder + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + common "github.com/iotbzh/xds-common/golib" +) + +// IFOLDER interface implementation for native/path mapping folders + +// PathMap . +type PathMap struct { + config FolderConfig +} + +// NewFolderPathMap Create a new instance of PathMap +func NewFolderPathMap() *PathMap { + f := PathMap{} + return &f +} + +// Add a new folder +func (f *PathMap) Add(cfg FolderConfig) (*FolderConfig, error) { + if cfg.DataPathMap.ServerPath == "" { + return nil, fmt.Errorf("ServerPath must be set") + } + + // Sanity check + dir := cfg.DataPathMap.ServerPath + if !common.Exists(dir) { + // try to create if not existing + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("Cannot create ServerPath directory: %s", dir) + } + } + if !common.Exists(dir) { + return nil, fmt.Errorf("ServerPath directory is not accessible: %s", dir) + } + file, err := ioutil.TempFile(dir, "xds_pathmap_check") + if err != nil { + return nil, fmt.Errorf("ServerPath sanity check error: %s", err.Error()) + } + defer os.Remove(file.Name()) + + msg := "sanity check PathMap Add folder" + n, err := file.Write([]byte(msg)) + if err != nil || n != len(msg) { + return nil, fmt.Errorf("ServerPath sanity check error: %s", err.Error()) + } + + f.config = cfg + f.config.RootPath = cfg.DataPathMap.ServerPath + f.config.Status = StatusEnable + + return &f.config, nil +} + +// GetConfig Get public part of folder config +func (f *PathMap) GetConfig() FolderConfig { + return f.config +} + +// GetFullPath returns the full path +func (f *PathMap) GetFullPath(dir string) string { + if &dir == nil { + return f.config.DataPathMap.ServerPath + } + return filepath.Join(f.config.DataPathMap.ServerPath, dir) +} + +// Remove a folder +func (f *PathMap) Remove() error { + // nothing to do + return nil +} + +// Sync Force folder files synchronization +func (f *PathMap) Sync() error { + return nil +} + +// IsInSync Check if folder files are in-sync +func (f *PathMap) IsInSync() (bool, error) { + return true, nil +} diff --git a/lib/model/folder.go b/lib/model/folder.go deleted file mode 100644 index 56a46b1..0000000 --- a/lib/model/folder.go +++ /dev/null @@ -1,110 +0,0 @@ -package model - -import ( - "fmt" - - common "github.com/iotbzh/xds-common/golib" - "github.com/iotbzh/xds-server/lib/syncthing" - "github.com/iotbzh/xds-server/lib/xdsconfig" -) - -// Folder Represent a an XDS folder -type Folder struct { - Conf *xdsconfig.Config - SThg *st.SyncThing -} - -// NewFolder Create a new instance of Model Folder -func NewFolder(cfg *xdsconfig.Config, st *st.SyncThing) *Folder { - return &Folder{ - Conf: cfg, - SThg: st, - } -} - -// GetFolderFromID retrieves the Folder config from id -func (c *Folder) GetFolderFromID(id string) *xdsconfig.FolderConfig { - if idx := c.Conf.Folders.GetIdx(id); idx != -1 { - return &c.Conf.Folders[idx] - } - return nil -} - -// UpdateAll updates all the current configuration -func (c *Folder) UpdateAll(newCfg xdsconfig.Config) error { - return fmt.Errorf("Not Supported") - /* - if err := VerifyConfig(newCfg); err != nil { - return err - } - - // TODO: c.Builder = c.Builder.Update(newCfg.Builder) - c.Folders = c.Folders.Update(newCfg.Folders) - - // FIXME To be tested & improved error handling - for _, f := range c.Folders { - if err := c.SThg.FolderChange(st.FolderChangeArg{ - ID: f.ID, - Label: f.Label, - RelativePath: f.RelativePath, - SyncThingID: f.SyncThingID, - ShareRootDir: c.FileConf.ShareRootDir, - }); err != nil { - return err - } - } - - return nil - */ -} - -// UpdateFolder updates a specific folder into the current configuration -func (c *Folder) UpdateFolder(newFolder xdsconfig.FolderConfig) (xdsconfig.FolderConfig, error) { - // rootPath should not be empty - if newFolder.RootPath == "" { - newFolder.RootPath = c.Conf.FileConf.ShareRootDir - } - - // Sanity check of folder settings - if err := newFolder.Verify(); err != nil { - return xdsconfig.FolderConfig{}, err - } - - // Normalize path (needed for Windows path including bashlashes) - newFolder.RelativePath = common.PathNormalize(newFolder.RelativePath) - - // Update config folder - c.Conf.Folders = c.Conf.Folders.Update(xdsconfig.FoldersConfig{newFolder}) - - // Update Syncthing folder - err := c.SThg.FolderChange(newFolder) - - newFolder.BuilderSThgID = c.Conf.Builder.SyncThingID // FIXME - should be removed after local ST config rework - newFolder.Status = xdsconfig.FolderStatusEnable - - return newFolder, err -} - -// DeleteFolder deletes a specific folder -func (c *Folder) DeleteFolder(id string) (xdsconfig.FolderConfig, error) { - var fld xdsconfig.FolderConfig - var err error - - if err = c.SThg.FolderDelete(id); err != nil { - return fld, err - } - - c.Conf.Folders, fld, err = c.Conf.Folders.Delete(id) - - return fld, err -} - -// ForceSync Force the synchronization of a folder -func (c *Folder) ForceSync(id string) error { - return c.SThg.FolderScan(id, "") -} - -// IsFolderInSync Returns true when folder is in sync -func (c *Folder) IsFolderInSync(id string) (bool, error) { - return c.SThg.IsFolderInSync(id) -} diff --git a/lib/model/folders.go b/lib/model/folders.go new file mode 100644 index 0000000..3c2457c --- /dev/null +++ b/lib/model/folders.go @@ -0,0 +1,333 @@ +package model + +import ( + "encoding/xml" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/Sirupsen/logrus" + common "github.com/iotbzh/xds-common/golib" + "github.com/iotbzh/xds-server/lib/folder" + "github.com/iotbzh/xds-server/lib/syncthing" + "github.com/iotbzh/xds-server/lib/xdsconfig" + uuid "github.com/satori/go.uuid" + "github.com/syncthing/syncthing/lib/sync" +) + +// Folders Represent a an XDS folders +type Folders struct { + fileOnDisk string + Conf *xdsconfig.Config + Log *logrus.Logger + SThg *st.SyncThing + folders map[string]*folder.IFOLDER +} + +// Mutex to make add/delete atomic +var fcMutex = sync.NewMutex() +var ffMutex = sync.NewMutex() + +// FoldersNew Create a new instance of Model Folders +func FoldersNew(cfg *xdsconfig.Config, st *st.SyncThing) *Folders { + file, _ := xdsconfig.FoldersConfigFilenameGet() + return &Folders{ + fileOnDisk: file, + Conf: cfg, + Log: cfg.Log, + SThg: st, + folders: make(map[string]*folder.IFOLDER), + } +} + +// LoadConfig Load folders configuration from disk +func (f *Folders) LoadConfig() error { + var flds []folder.FolderConfig + var stFlds []folder.FolderConfig + + // load from disk + if f.Conf.Options.NoFolderConfig { + f.Log.Infof("Don't read folder config file (-no-folderconfig option is set)") + } else if f.fileOnDisk != "" { + f.Log.Infof("Use folder config file: %s", f.fileOnDisk) + err := foldersConfigRead(f.fileOnDisk, &flds) + if err != nil { + if strings.HasPrefix(err.Error(), "No folder config") { + f.Log.Warnf(err.Error()) + } else { + return err + } + } + } else { + f.Log.Warnf("Folders config filename not set") + } + + // Retrieve initial Syncthing config (just append don't overwrite existing ones) + if f.SThg != nil { + f.Log.Infof("Retrieve syncthing folder config") + if err := f.SThg.FolderLoadFromStConfig(&stFlds); err != nil { + // Don't exit on such error, just log it + f.Log.Errorf(err.Error()) + } + } + + // Merge syncthing folders into XDS folders + for _, stf := range stFlds { + found := false + for i, xf := range flds { + if xf.ID == stf.ID { + found = true + // sanity check + if xf.Type != folder.TypeCloudSync { + flds[i].Status = folder.StatusErrorConfig + } + break + } + } + // add it + if !found { + flds = append(flds, stf) + } + } + + // Detect ghost project + // (IOW existing in xds file config and not in syncthing database) + for i, xf := range flds { + // only for syncthing project + if xf.Type != folder.TypeCloudSync { + continue + } + found := false + for _, stf := range stFlds { + if stf.ID == xf.ID { + found = true + break + } + } + if !found { + flds[i].Status = folder.StatusErrorConfig + } + } + + // Update folders + f.Log.Infof("Loading initial folders config: %d folders found", len(flds)) + for _, fc := range flds { + if _, err := f.createUpdate(fc, false); err != nil { + return err + } + } + + return nil +} + +// SaveConfig Save folders configuration to disk +func (f *Folders) SaveConfig() error { + if f.fileOnDisk == "" { + return fmt.Errorf("Folders config filename not set") + } + + // FIXME: buffered save or avoid to write on disk each time + return foldersConfigWrite(f.fileOnDisk, f.getConfigArrUnsafe()) +} + +// Get returns the folder config or nil if not existing +func (f *Folders) Get(id string) *folder.IFOLDER { + if id == "" { + return nil + } + fc, exist := f.folders[id] + if !exist { + return nil + } + return fc +} + +// GetConfigArr returns the config of all folders as an array +func (f *Folders) GetConfigArr() []folder.FolderConfig { + fcMutex.Lock() + defer fcMutex.Unlock() + + return f.getConfigArrUnsafe() +} + +// getConfigArrUnsafe Same as GetConfigArr without mutex protection +func (f *Folders) getConfigArrUnsafe() []folder.FolderConfig { + var conf []folder.FolderConfig + + for _, v := range f.folders { + conf = append(conf, (*v).GetConfig()) + } + return conf +} + +// Add adds a new folder +func (f *Folders) Add(newF folder.FolderConfig) (*folder.FolderConfig, error) { + return f.createUpdate(newF, true) +} + +// CreateUpdate creates or update a folder +func (f *Folders) createUpdate(newF folder.FolderConfig, create bool) (*folder.FolderConfig, error) { + + fcMutex.Lock() + defer fcMutex.Unlock() + + // Sanity check + if _, exist := f.folders[newF.ID]; create && exist { + return nil, fmt.Errorf("ID already exists") + } + if newF.ClientPath == "" { + return nil, fmt.Errorf("ClientPath must be set") + } + + // Allocate a new UUID + if create { + newF.ID = uuid.NewV1().String() + } + if !create && newF.ID == "" { + return nil, fmt.Errorf("Cannot update folder with null ID") + } + + // Set default value if needed + if newF.Status == "" { + newF.Status = folder.StatusDisable + } + + if newF.Label == "" { + newF.Label = filepath.Base(newF.ClientPath) + "_" + newF.ID[0:8] + } + + var fld folder.IFOLDER + switch newF.Type { + // SYNCTHING + case folder.TypeCloudSync: + if f.SThg == nil { + return nil, fmt.Errorf("ClownSync type not supported (syncthing not initialized)") + } + fld = f.SThg.NewFolderST(f.Conf) + // PATH MAP + case folder.TypePathMap: + fld = folder.NewFolderPathMap() + default: + return nil, fmt.Errorf("Unsupported folder type") + } + + // Normalize path (needed for Windows path including bashlashes) + newF.ClientPath = common.PathNormalize(newF.ClientPath) + + // Add new folder + newFolder, err := fld.Add(newF) + if err != nil { + newF.Status = folder.StatusErrorConfig + log.Printf("ERROR Adding folder: %v\n", err) + return newFolder, err + } + + // Register folder object + f.folders[newF.ID] = &fld + + // Save config on disk + err = f.SaveConfig() + + return newFolder, err +} + +// Delete deletes a specific folder +func (f *Folders) Delete(id string) (folder.FolderConfig, error) { + var err error + + fcMutex.Lock() + defer fcMutex.Unlock() + + fld := folder.FolderConfig{} + fc, exist := f.folders[id] + if !exist { + return fld, fmt.Errorf("unknown id") + } + + fld = (*fc).GetConfig() + + if err = (*fc).Remove(); err != nil { + return fld, err + } + + delete(f.folders, id) + + // Save config on disk + err = f.SaveConfig() + + return fld, err +} + +// ForceSync Force the synchronization of a folder +func (f *Folders) ForceSync(id string) error { + fc := f.Get(id) + if fc == nil { + return fmt.Errorf("Unknown id") + } + return (*fc).Sync() +} + +// IsFolderInSync Returns true when folder is in sync +func (f *Folders) IsFolderInSync(id string) (bool, error) { + fc := f.Get(id) + if fc == nil { + return false, fmt.Errorf("Unknown id") + } + return (*fc).IsInSync() +} + +//*** Private functions *** + +// Use XML format and not json to be able to save/load all fields including +// ones that are masked in json (IOW defined with `json:"-"`) +type xmlFolders struct { + XMLName xml.Name `xml:"folders"` + Version string `xml:"version,attr"` + Folders []folder.FolderConfig `xml:"folders"` +} + +// foldersConfigRead reads folders config from disk +func foldersConfigRead(file string, folders *[]folder.FolderConfig) error { + if !common.Exists(file) { + return fmt.Errorf("No folder config file found (%s)", file) + } + + ffMutex.Lock() + defer ffMutex.Unlock() + + fd, err := os.Open(file) + defer fd.Close() + if err != nil { + return err + } + + data := xmlFolders{} + err = xml.NewDecoder(fd).Decode(&data) + if err == nil { + *folders = data.Folders + } + return err +} + +// foldersConfigWrite writes folders config on disk +func foldersConfigWrite(file string, folders []folder.FolderConfig) error { + ffMutex.Lock() + defer ffMutex.Unlock() + + fd, err := os.OpenFile(file, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + defer fd.Close() + if err != nil { + return err + } + + data := &xmlFolders{ + Version: "1", + Folders: folders, + } + + enc := xml.NewEncoder(fd) + enc.Indent("", " ") + return enc.Encode(data) +} diff --git a/lib/syncthing/folder-st.go b/lib/syncthing/folder-st.go new file mode 100644 index 0000000..ffcd284 --- /dev/null +++ b/lib/syncthing/folder-st.go @@ -0,0 +1,97 @@ +package st + +import ( + "fmt" + "path/filepath" + + "github.com/iotbzh/xds-server/lib/folder" + "github.com/iotbzh/xds-server/lib/xdsconfig" + "github.com/syncthing/syncthing/lib/config" +) + +// IFOLDER interface implementation for syncthing + +// STFolder . +type STFolder struct { + globalConfig *xdsconfig.Config + st *SyncThing + fConfig folder.FolderConfig + stfConfig config.FolderConfiguration +} + +// NewFolderST Create a new instance of STFolder +func (s *SyncThing) NewFolderST(gc *xdsconfig.Config) *STFolder { + return &STFolder{ + globalConfig: gc, + st: s, + } +} + +// Add a new folder +func (f *STFolder) Add(cfg folder.FolderConfig) (*folder.FolderConfig, error) { + + // Sanity check + if cfg.DataCloudSync.SyncThingID == "" { + return nil, fmt.Errorf("device id not set (SyncThingID field)") + } + + // rootPath should not be empty + if cfg.RootPath == "" { + cfg.RootPath = f.globalConfig.FileConf.ShareRootDir + } + + f.fConfig = cfg + + f.fConfig.DataCloudSync.BuilderSThgID = f.st.MyID // FIXME - should be removed after local ST config rework + + // Update Syncthing folder + // (expect if status is ErrorConfig) + // TODO: add cache to avoid multiple requests on startup + if f.fConfig.Status != folder.StatusErrorConfig { + id, err := f.st.FolderChange(f.fConfig) + if err != nil { + return nil, err + } + + f.stfConfig, err = f.st.FolderConfigGet(id) + if err != nil { + f.fConfig.Status = folder.StatusErrorConfig + return nil, err + } + + f.fConfig.Status = folder.StatusEnable + } + + return &f.fConfig, nil +} + +// GetConfig Get public part of folder config +func (f *STFolder) GetConfig() folder.FolderConfig { + return f.fConfig +} + +// GetFullPath returns the full path +func (f *STFolder) GetFullPath(dir string) string { + if &dir == nil { + dir = "" + } + if filepath.IsAbs(dir) { + return filepath.Join(f.fConfig.RootPath, dir) + } + return filepath.Join(f.fConfig.RootPath, f.fConfig.ClientPath, dir) +} + +// Remove a folder +func (f *STFolder) Remove() error { + return f.st.FolderDelete(f.stfConfig.ID) +} + +// Sync Force folder files synchronization +func (f *STFolder) Sync() error { + return f.st.FolderScan(f.stfConfig.ID, "") +} + +// IsInSync Check if folder files are in-sync +func (f *STFolder) IsInSync() (bool, error) { + return f.st.IsFolderInSync(f.stfConfig.ID) +} diff --git a/lib/syncthing/st.go b/lib/syncthing/st.go index 3380cda..9bdb48f 100644 --- a/lib/syncthing/st.go +++ b/lib/syncthing/st.go @@ -32,6 +32,7 @@ type SyncThing struct { Home string STCmd *exec.Cmd STICmd *exec.Cmd + MyID string // Private fields binDir string @@ -211,13 +212,13 @@ func (s *SyncThing) Start() (*exec.Cmd, error) { env := []string{ "STNODEFAULTFOLDER=1", "STNOUPGRADE=1", - "STNORESTART=1", + "STNORESTART=1", // FIXME SEB remove ? } s.STCmd, err = s.startProc("syncthing", args, env, &s.exitSTChan) // Use autogenerated apikey if not set by config.json - if s.APIKey == "" { + if err == nil && s.APIKey == "" { if fd, err := os.Open(filepath.Join(s.Home, "config.xml")); err == nil { defer fd.Close() if b, err := ioutil.ReadAll(fd); err == nil { @@ -314,7 +315,9 @@ func (s *SyncThing) Connect() error { s.client.SetLogger(s.log) - return nil + s.MyID, err = s.IDGet() + + return err } // IDGet returns the Syncthing ID of Syncthing instance running locally @@ -360,44 +363,3 @@ func (s *SyncThing) IsConfigInSync() (bool, error) { } return d.ConfigInSync, nil } - -// FolderStatus Returns all information about the current -func (s *SyncThing) FolderStatus(folderID string) (*FolderStatus, error) { - var data []byte - var res FolderStatus - if folderID == "" { - return nil, fmt.Errorf("folderID not set") - } - if err := s.client.HTTPGet("db/status?folder="+folderID, &data); err != nil { - return nil, err - } - if err := json.Unmarshal(data, &res); err != nil { - return nil, err - } - return &res, nil -} - -// IsFolderInSync Returns true when folder is in sync -func (s *SyncThing) IsFolderInSync(folderID string) (bool, error) { - // FIXME better to detected FolderCompletion event (/rest/events) - // See https://docs.syncthing.net/dev/events.html - sts, err := s.FolderStatus(folderID) - if err != nil { - return false, err - } - return sts.NeedBytes == 0, nil -} - -// FolderScan Request immediate folder scan. -// Scan all folders if folderID param is empty -func (s *SyncThing) FolderScan(folderID string, subpath string) error { - url := "db/scan" - if folderID != "" { - url += "?folder=" + folderID - - if subpath != "" { - url += "&sub=" + subpath - } - } - return s.client.HTTPPost(url, "") -} diff --git a/lib/syncthing/stfolder.go b/lib/syncthing/stfolder.go index 661e19d..bbdcc43 100644 --- a/lib/syncthing/stfolder.go +++ b/lib/syncthing/stfolder.go @@ -1,34 +1,77 @@ package st import ( + "encoding/json" + "fmt" "path/filepath" "strings" - "github.com/iotbzh/xds-server/lib/xdsconfig" + "github.com/iotbzh/xds-server/lib/folder" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/protocol" ) +// FolderLoadFromStConfig Load/Retrieve folder config from syncthing database +func (s *SyncThing) FolderLoadFromStConfig(f *[]folder.FolderConfig) error { + + defaultSdk := "" // cannot know which was the default sdk + + stCfg, err := s.ConfigGet() + if err != nil { + return err + } + if len(stCfg.Devices) < 1 { + return fmt.Errorf("Cannot load syncthing config: no device defined") + } + devID := stCfg.Devices[0].DeviceID.String() + if devID == s.MyID { + if len(stCfg.Devices) < 2 { + return fmt.Errorf("Cannot load syncthing config: no valid device found") + } + devID = stCfg.Devices[1].DeviceID.String() + } + + for _, stFld := range stCfg.Folders { + cliPath := strings.TrimPrefix(stFld.RawPath, s.conf.FileConf.ShareRootDir) + if cliPath == "" { + cliPath = stFld.RawPath + } + *f = append(*f, folder.FolderConfig{ + ID: stFld.ID, + Label: stFld.Label, + ClientPath: strings.TrimRight(cliPath, "/"), + Type: folder.TypeCloudSync, + Status: folder.StatusDisable, + DefaultSdk: defaultSdk, + RootPath: s.conf.FileConf.ShareRootDir, + DataCloudSync: folder.CloudSyncConfig{SyncThingID: devID}, + }) + } + + return nil +} + // FolderChange is called when configuration has changed -func (s *SyncThing) FolderChange(f xdsconfig.FolderConfig) error { +func (s *SyncThing) FolderChange(f folder.FolderConfig) (string, error) { // Get current config stCfg, err := s.ConfigGet() if err != nil { s.log.Errorln(err) - return err + return "", err } + stClientID := f.DataCloudSync.SyncThingID // Add new Device if needed var devID protocol.DeviceID - if err := devID.UnmarshalText([]byte(f.SyncThingID)); err != nil { - s.log.Errorf("not a valid device id (err %v)\n", err) - return err + if err := devID.UnmarshalText([]byte(stClientID)); err != nil { + s.log.Errorf("not a valid device id (err %v)", err) + return "", err } newDevice := config.DeviceConfiguration{ DeviceID: devID, - Name: f.SyncThingID, + Name: stClientID, Addresses: []string{"dynamic"}, } @@ -49,13 +92,13 @@ func (s *SyncThing) FolderChange(f xdsconfig.FolderConfig) error { label = strings.Split(id, "/")[0] } if id = f.ID; id == "" { - id = f.SyncThingID[0:15] + "_" + label + id = stClientID[0:15] + "_" + label } folder := config.FolderConfiguration{ ID: id, Label: label, - RawPath: filepath.Join(s.conf.FileConf.ShareRootDir, f.RelativePath), + RawPath: filepath.Join(s.conf.FileConf.ShareRootDir, f.ClientPath), } if s.conf.FileConf.SThgConf.RescanIntervalS > 0 { @@ -85,7 +128,7 @@ func (s *SyncThing) FolderChange(f xdsconfig.FolderConfig) error { s.log.Errorln(err) } - return nil + return id, nil } // FolderDelete is called to delete a folder config @@ -110,3 +153,63 @@ func (s *SyncThing) FolderDelete(id string) error { return nil } + +// FolderConfigGet Returns the configuration of a specific folder +func (s *SyncThing) FolderConfigGet(folderID string) (config.FolderConfiguration, error) { + fc := config.FolderConfiguration{} + if folderID == "" { + return fc, fmt.Errorf("folderID not set") + } + cfg, err := s.ConfigGet() + if err != nil { + return fc, err + } + for _, f := range cfg.Folders { + if f.ID == folderID { + fc = f + return fc, nil + } + } + return fc, fmt.Errorf("id not found") +} + +// FolderStatus Returns all information about the current +func (s *SyncThing) FolderStatus(folderID string) (*FolderStatus, error) { + var data []byte + var res FolderStatus + if folderID == "" { + return nil, fmt.Errorf("folderID not set") + } + if err := s.client.HTTPGet("db/status?folder="+folderID, &data); err != nil { + return nil, err + } + if err := json.Unmarshal(data, &res); err != nil { + return nil, err + } + return &res, nil +} + +// IsFolderInSync Returns true when folder is in sync +func (s *SyncThing) IsFolderInSync(folderID string) (bool, error) { + // FIXME better to detected FolderCompletion event (/rest/events) + // See https://docs.syncthing.net/dev/events.html + sts, err := s.FolderStatus(folderID) + if err != nil { + return false, err + } + return sts.NeedBytes == 0, nil +} + +// FolderScan Request immediate folder scan. +// Scan all folders if folderID param is empty +func (s *SyncThing) FolderScan(folderID string, subpath string) error { + url := "db/scan" + if folderID != "" { + url += "?folder=" + folderID + + if subpath != "" { + url += "&sub=" + subpath + } + } + return s.client.HTTPPost(url, "") +} diff --git a/lib/webserver/server.go b/lib/webserver/server.go index 7649cce..5183208 100644 --- a/lib/webserver/server.go +++ b/lib/webserver/server.go @@ -27,7 +27,7 @@ type Server struct { webApp *gin.RouterGroup cfg *xdsconfig.Config sessions *session.Sessions - mfolder *model.Folder + mfolders *model.Folders sdks *crosssdk.SDKs log *logrus.Logger stop chan struct{} // signals intentional stop @@ -37,7 +37,7 @@ const indexFilename = "index.html" const cookieMaxAge = "3600" // New creates an instance of Server -func New(cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs, logr *logrus.Logger) *Server { +func New(cfg *xdsconfig.Config, mfolders *model.Folders, sdks *crosssdk.SDKs, logr *logrus.Logger) *Server { // Setup logging for gin router if logr.Level == logrus.DebugLevel { @@ -63,7 +63,7 @@ func New(cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs, logr webApp: nil, cfg: cfg, sessions: nil, - mfolder: mfolder, + mfolders: mfolders, sdks: sdks, log: logr, stop: make(chan struct{}), @@ -86,7 +86,7 @@ func (s *Server) Serve() error { s.sessions = session.NewClientSessions(s.router, s.log, cookieMaxAge) // Create REST API - s.api = apiv1.New(s.router, s.sessions, s.cfg, s.mfolder, s.sdks) + s.api = apiv1.New(s.router, s.sessions, s.cfg, s.mfolders, s.sdks) // Websocket routes s.sIOServer, err = socketio.NewServer(nil) diff --git a/lib/xdsconfig/config.go b/lib/xdsconfig/config.go index f2d0710..a3e5a7e 100644 --- a/lib/xdsconfig/config.go +++ b/lib/xdsconfig/config.go @@ -2,7 +2,6 @@ package xdsconfig import ( "fmt" - "os" "github.com/Sirupsen/logrus" @@ -16,13 +15,21 @@ type Config struct { APIVersion string `json:"apiVersion"` VersionGitTag string `json:"gitTag"` Builder BuilderConfig `json:"builder"` - Folders FoldersConfig `json:"folders"` // Private (un-exported fields in REST GET /config route) + Options Options `json:"-"` FileConf FileConfig `json:"-"` Log *logrus.Logger `json:"-"` } +// Options set at the command line +type Options struct { + ConfigFile string + LogLevel string + LogFile string + NoFolderConfig bool +} + // Config default values const ( DefaultAPIVersion = "1" @@ -41,7 +48,13 @@ func Init(cliCtx *cli.Context, log *logrus.Logger) (*Config, error) { APIVersion: DefaultAPIVersion, VersionGitTag: cliCtx.App.Metadata["git-tag"].(string), Builder: BuilderConfig{}, - Folders: FoldersConfig{}, + + Options: Options{ + ConfigFile: cliCtx.GlobalString("config"), + LogLevel: cliCtx.GlobalString("log"), + LogFile: cliCtx.GlobalString("logfile"), + NoFolderConfig: cliCtx.GlobalBool("no-folderconfig"), + }, FileConf: FileConfig{ WebAppDir: "webapp/dist", ShareRootDir: DefaultShareDir, @@ -52,7 +65,7 @@ func Init(cliCtx *cli.Context, log *logrus.Logger) (*Config, error) { } // config file settings overwrite default config - err = updateConfigFromFile(&c, cliCtx.GlobalString("config")) + err = readGlobalConfig(&c, c.Options.ConfigFile) if err != nil { return nil, err } diff --git a/lib/xdsconfig/fileconfig.go b/lib/xdsconfig/fileconfig.go index 90c1aad..2dbf884 100644 --- a/lib/xdsconfig/fileconfig.go +++ b/lib/xdsconfig/fileconfig.go @@ -11,6 +11,16 @@ import ( common "github.com/iotbzh/xds-common/golib" ) +const ( + // ConfigDir Directory in user HOME directory where xds config will be saved + ConfigDir = ".xds" + // GlobalConfigFilename Global config filename + GlobalConfigFilename = "config.json" + // FoldersConfigFilename Folders config filename + FoldersConfigFilename = "server-config_folders.xml" +) + +// SyncThingConf definition type SyncThingConf struct { BinDir string `json:"binDir"` Home string `json:"home"` @@ -19,6 +29,7 @@ type SyncThingConf struct { RescanIntervalS int `json:"rescanIntervalS"` } +// FileConfig is the JSON structure of xds-server config file (config.json) type FileConfig struct { WebAppDir string `json:"webAppDir"` ShareRootDir string `json:"shareRootDir"` @@ -28,21 +39,21 @@ type FileConfig struct { LogsDir string `json:"logsDir"` } -// getConfigFromFile reads configuration from a config file. +// readGlobalConfig reads configuration from a config file. // Order to determine which config file is used: // 1/ from command line option: "--config myConfig.json" // 2/ $HOME/.xds/config.json file // 3/ /config.json file // 4/ /config.json file - -func updateConfigFromFile(c *Config, confFile string) error { +func readGlobalConfig(c *Config, confFile string) error { searchIn := make([]string, 0, 3) if confFile != "" { searchIn = append(searchIn, confFile) } if usr, err := user.Current(); err == nil { - searchIn = append(searchIn, path.Join(usr.HomeDir, ".xds", "config.json")) + searchIn = append(searchIn, path.Join(usr.HomeDir, ConfigDir, + GlobalConfigFilename)) } cwd, err := os.Getwd() if err == nil { @@ -70,7 +81,6 @@ func updateConfigFromFile(c *Config, confFile string) error { // TODO move on viper package to support comments in JSON and also // bind with flags (command line options) // see https://github.com/spf13/viper#working-with-flags - fd, _ := os.Open(*cFile) defer fd.Close() fCfg := FileConfig{} @@ -79,14 +89,15 @@ func updateConfigFromFile(c *Config, confFile string) error { } // Support environment variables (IOW ${MY_ENV_VAR} syntax) in config.json - for _, field := range []*string{ + vars := []*string{ &fCfg.WebAppDir, &fCfg.ShareRootDir, &fCfg.SdkRootDir, - &fCfg.LogsDir, - &fCfg.SThgConf.Home, - &fCfg.SThgConf.BinDir} { - + &fCfg.LogsDir} + if fCfg.SThgConf != nil { + vars = append(vars, &fCfg.SThgConf.Home, &fCfg.SThgConf.BinDir) + } + for _, field := range vars { var err error if *field, err = common.ResolveEnvVar(*field); err != nil { return err @@ -123,3 +134,12 @@ func updateConfigFromFile(c *Config, confFile string) error { c.FileConf = fCfg return nil } + +// FoldersConfigFilenameGet +func FoldersConfigFilenameGet() (string, error) { + usr, err := user.Current() + if err != nil { + return "", err + } + return path.Join(usr.HomeDir, ConfigDir, FoldersConfigFilename), nil +} diff --git a/lib/xdsconfig/folderconfig.go b/lib/xdsconfig/folderconfig.go deleted file mode 100644 index bb2b56f..0000000 --- a/lib/xdsconfig/folderconfig.go +++ /dev/null @@ -1,85 +0,0 @@ -package xdsconfig - -import ( - "fmt" - "log" - "path/filepath" -) - -// FolderType constances -const ( - FolderTypeDocker = 0 - FolderTypeWindowsSubsystem = 1 - FolderTypeCloudSync = 2 - - FolderStatusErrorConfig = "ErrorConfig" - FolderStatusDisable = "Disable" - FolderStatusEnable = "Enable" -) - -// FolderType is the type of sharing folder -type FolderType int - -// FolderConfig is the config for one folder -type FolderConfig struct { - ID string `json:"id" binding:"required"` - Label string `json:"label"` - RelativePath string `json:"path"` - Type FolderType `json:"type"` - SyncThingID string `json:"syncThingID"` - BuilderSThgID string `json:"builderSThgID"` - Status string `json:"status"` - DefaultSdk string `json:"defaultSdk"` - - // Not exported fields - RootPath string `json:"-"` -} - -// NewFolderConfig creates a new folder object -func NewFolderConfig(id, label, rootDir, path string, defaultSdk string) FolderConfig { - return FolderConfig{ - ID: id, - Label: label, - RelativePath: path, - Type: FolderTypeCloudSync, - SyncThingID: "", - Status: FolderStatusDisable, - RootPath: rootDir, - DefaultSdk: defaultSdk, - } -} - -// GetFullPath returns the full path -func (c *FolderConfig) GetFullPath(dir string) string { - if &dir == nil { - dir = "" - } - if filepath.IsAbs(dir) { - return filepath.Join(c.RootPath, dir) - } - return filepath.Join(c.RootPath, c.RelativePath, dir) -} - -// Verify is called to verify that a configuration is valid -func (c *FolderConfig) Verify() error { - var err error - - if c.Type != FolderTypeCloudSync { - err = fmt.Errorf("Unsupported folder type") - } - - if c.SyncThingID == "" { - err = fmt.Errorf("device id not set (SyncThingID field)") - } - - if c.RootPath == "" { - err = fmt.Errorf("RootPath must not be empty") - } - - if err != nil { - c.Status = FolderStatusErrorConfig - log.Printf("ERROR Verify: %v\n", err) - } - - return err -} diff --git a/lib/xdsconfig/foldersconfig.go b/lib/xdsconfig/foldersconfig.go deleted file mode 100644 index 4ad16df..0000000 --- a/lib/xdsconfig/foldersconfig.go +++ /dev/null @@ -1,47 +0,0 @@ -package xdsconfig - -import ( - "fmt" -) - -// FoldersConfig contains all the folder configurations -type FoldersConfig []FolderConfig - -// GetIdx returns the index of the folder matching id in FoldersConfig array -func (c FoldersConfig) GetIdx(id string) int { - for i := range c { - if id == c[i].ID { - return i - } - } - return -1 -} - -// Update is used to fully update or add a new FolderConfig -func (c FoldersConfig) Update(newCfg FoldersConfig) FoldersConfig { - for i := range newCfg { - found := false - for j := range c { - if newCfg[i].ID == c[j].ID { - c[j] = newCfg[i] - found = true - break - } - } - if !found { - c = append(c, newCfg[i]) - } - } - return c -} - -// Delete is used to delete a folder matching id in FoldersConfig array -func (c FoldersConfig) Delete(id string) (FoldersConfig, FolderConfig, error) { - if idx := c.GetIdx(id); idx != -1 { - f := c[idx] - c = append(c[:idx], c[idx+1:]...) - return c, f, nil - } - - return c, FolderConfig{}, fmt.Errorf("invalid id") -} -- cgit From 4feef5296bf3aea331fdde4cd7b94ee2322a907e Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Wed, 16 Aug 2017 14:24:49 +0200 Subject: Moved project creation in a modal windows Signed-off-by: Sebastien Douheret --- lib/folder/folder-pathmap.go | 20 +++++++++++++++----- lib/model/folders.go | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/folder/folder-pathmap.go b/lib/folder/folder-pathmap.go index 8711df2..2ad8a93 100644 --- a/lib/folder/folder-pathmap.go +++ b/lib/folder/folder-pathmap.go @@ -7,18 +7,22 @@ import ( "path/filepath" common "github.com/iotbzh/xds-common/golib" + "github.com/iotbzh/xds-server/lib/xdsconfig" ) // IFOLDER interface implementation for native/path mapping folders // PathMap . type PathMap struct { - config FolderConfig + globalConfig *xdsconfig.Config + config FolderConfig } // NewFolderPathMap Create a new instance of PathMap -func NewFolderPathMap() *PathMap { - f := PathMap{} +func NewFolderPathMap(gc *xdsconfig.Config) *PathMap { + f := PathMap{ + globalConfig: gc, + } return &f } @@ -28,8 +32,13 @@ func (f *PathMap) Add(cfg FolderConfig) (*FolderConfig, error) { return nil, fmt.Errorf("ServerPath must be set") } - // Sanity check + // Use shareRootDir if ServerPath is a relative path dir := cfg.DataPathMap.ServerPath + if !filepath.IsAbs(dir) { + dir = filepath.Join(f.globalConfig.FileConf.ShareRootDir, dir) + } + + // Sanity check if !common.Exists(dir) { // try to create if not existing if err := os.MkdirAll(dir, 0755); err != nil { @@ -52,7 +61,8 @@ func (f *PathMap) Add(cfg FolderConfig) (*FolderConfig, error) { } f.config = cfg - f.config.RootPath = cfg.DataPathMap.ServerPath + f.config.RootPath = dir + f.config.DataPathMap.ServerPath = dir f.config.Status = StatusEnable return &f.config, nil diff --git a/lib/model/folders.go b/lib/model/folders.go index 3c2457c..02c3254 100644 --- a/lib/model/folders.go +++ b/lib/model/folders.go @@ -208,7 +208,7 @@ func (f *Folders) createUpdate(newF folder.FolderConfig, create bool) (*folder.F fld = f.SThg.NewFolderST(f.Conf) // PATH MAP case folder.TypePathMap: - fld = folder.NewFolderPathMap() + fld = folder.NewFolderPathMap(f.Conf) default: return nil, fmt.Errorf("Unsupported folder type") } -- cgit From 8f44cc7217ce48f3f94c8ea3f037cdf011c4493b Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Fri, 18 Aug 2017 01:04:02 +0200 Subject: Add folder synchronization status. Also add ability to force re-synchronization. --- lib/apiv1/apiv1.go | 5 + lib/apiv1/events.go | 147 +++++++++++++++++++++++++ lib/apiv1/folders.go | 16 ++- lib/folder/folder-interface.go | 21 +++- lib/folder/folder-pathmap.go | 17 +++ lib/model/folders.go | 105 +++++++++++++----- lib/syncthing/folder-st.go | 83 +++++++++++++- lib/syncthing/st.go | 10 ++ lib/syncthing/stEvent.go | 242 +++++++++++++++++++++++++++++++++++++++++ lib/syncthing/stfolder.go | 4 +- 10 files changed, 610 insertions(+), 40 deletions(-) create mode 100644 lib/apiv1/events.go create mode 100644 lib/syncthing/stEvent.go (limited to 'lib') diff --git a/lib/apiv1/apiv1.go b/lib/apiv1/apiv1.go index f32e53b..262f513 100644 --- a/lib/apiv1/apiv1.go +++ b/lib/apiv1/apiv1.go @@ -42,6 +42,7 @@ func New(r *gin.Engine, sess *session.Sessions, cfg *xdsconfig.Config, mfolders s.apiRouter.GET("/folders", s.getFolders) s.apiRouter.GET("/folder/:id", s.getFolder) s.apiRouter.POST("/folder", s.addFolder) + s.apiRouter.POST("/folder/sync/:id", s.syncFolder) s.apiRouter.DELETE("/folder/:id", s.delFolder) s.apiRouter.GET("/sdks", s.getSdks) @@ -54,5 +55,9 @@ func New(r *gin.Engine, sess *session.Sessions, cfg *xdsconfig.Config, mfolders s.apiRouter.POST("/exec/:id", s.execCmd) s.apiRouter.POST("/signal", s.execSignalCmd) + s.apiRouter.GET("/events", s.eventsList) + s.apiRouter.POST("/events/register", s.eventsRegister) + s.apiRouter.POST("/events/unregister", s.eventsUnRegister) + return s } diff --git a/lib/apiv1/events.go b/lib/apiv1/events.go new file mode 100644 index 0000000..da8298c --- /dev/null +++ b/lib/apiv1/events.go @@ -0,0 +1,147 @@ +package apiv1 + +import ( + "net/http" + "time" + + "github.com/iotbzh/xds-server/lib/folder" + + "github.com/gin-gonic/gin" + common "github.com/iotbzh/xds-common/golib" +) + +// EventArgs is the parameters (json format) of /events/register command +type EventRegisterArgs struct { + Name string `json:"name"` + ProjectID string `json:"filterProjectID"` +} + +type EventUnRegisterArgs struct { + Name string `json:"name"` + ID int `json:"id"` +} + +// EventMsg Message send +type EventMsg struct { + Time string `json:"time"` + Type string `json:"type"` + Folder folder.FolderConfig `json:"folder"` +} + +// EventEvent Event send in WS when an internal event (eg. Syncthing event is received) +const EventEventAll = "event:all" +const EventEventType = "event:" // following by event type + +// eventsList Registering for events that will be send over a WS +func (s *APIService) eventsList(c *gin.Context) { + +} + +// eventsRegister Registering for events that will be send over a WS +func (s *APIService) eventsRegister(c *gin.Context) { + var args EventRegisterArgs + + if c.BindJSON(&args) != nil { + common.APIError(c, "Invalid arguments") + return + } + + sess := s.sessions.Get(c) + if sess == nil { + common.APIError(c, "Unknown sessions") + return + } + + evType := "FolderStateChanged" + if args.Name != evType { + common.APIError(c, "Unsupported event name") + return + } + + /* XXX - to be removed if no plan to support "generic" event + var cbFunc st.EventsCB + cbFunc = func(ev st.Event, data *st.EventsCBData) { + + evid, _ := strconv.Atoi((*data)["id"].(string)) + ssid := (*data)["sid"].(string) + so := s.sessions.IOSocketGet(ssid) + if so == nil { + s.log.Infof("Event %s not emitted - sid: %s", ev.Type, ssid) + + // Consider that client disconnected, so unregister this event + s.mfolders.SThg.Events.UnRegister(ev.Type, evid) + return + } + + msg := EventMsg{ + Time: ev.Time, + Type: ev.Type, + Data: ev.Data, + } + + if err := (*so).Emit(EventEventAll, msg); err != nil { + s.log.Errorf("WS Emit Event : %v", err) + } + + if err := (*so).Emit(EventEventType+ev.Type, msg); err != nil { + s.log.Errorf("WS Emit Event : %v", err) + } + } + + data := make(st.EventsCBData) + data["sid"] = sess.ID + + id, err := s.mfolders.SThg.Events.Register(args.Name, cbFunc, args.ProjectID, &data) + */ + + var cbFunc folder.EventCB + cbFunc = func(cfg *folder.FolderConfig, data *folder.EventCBData) { + ssid := (*data)["sid"].(string) + so := s.sessions.IOSocketGet(ssid) + if so == nil { + //s.log.Infof("Event %s not emitted - sid: %s", ev.Type, ssid) + + // Consider that client disconnected, so unregister this event + // SEB FIXMEs.mfolders.RegisterEventChange(ev.Type) + return + } + + msg := EventMsg{ + Time: time.Now().String(), + Type: evType, + Folder: *cfg, + } + + if err := (*so).Emit(EventEventType+evType, msg); err != nil { + s.log.Errorf("WS Emit Folder StateChanged event : %v", err) + } + } + data := make(folder.EventCBData) + data["sid"] = sess.ID + + err := s.mfolders.RegisterEventChange(args.ProjectID, &cbFunc, &data) + if err != nil { + common.APIError(c, err.Error()) + return + } + + c.JSON(http.StatusOK, gin.H{"status": "OK"}) +} + +// eventsRegister Registering for events that will be send over a WS +func (s *APIService) eventsUnRegister(c *gin.Context) { + var args EventUnRegisterArgs + + if c.BindJSON(&args) != nil || args.Name == "" || args.ID < 0 { + common.APIError(c, "Invalid arguments") + return + } + /* TODO + if err := s.mfolders.SThg.Events.UnRegister(args.Name, args.ID); err != nil { + common.APIError(c, err.Error()) + return + } + c.JSON(http.StatusOK, gin.H{"status": "OK"}) + */ + common.APIError(c, "Not implemented yet") +} diff --git a/lib/apiv1/folders.go b/lib/apiv1/folders.go index f957c6d..cf56c3f 100644 --- a/lib/apiv1/folders.go +++ b/lib/apiv1/folders.go @@ -43,6 +43,21 @@ func (s *APIService) addFolder(c *gin.Context) { c.JSON(http.StatusOK, newFld) } +// syncFolder force synchronization of folder files +func (s *APIService) syncFolder(c *gin.Context) { + id := c.Param("id") + + s.log.Debugln("Sync folder id: ", id) + + err := s.mfolders.ForceSync(id) + if err != nil { + common.APIError(c, err.Error()) + return + } + + c.JSON(http.StatusOK, "") +} + // delFolder deletes folder from server config func (s *APIService) delFolder(c *gin.Context) { id := c.Param("id") @@ -55,5 +70,4 @@ func (s *APIService) delFolder(c *gin.Context) { return } c.JSON(http.StatusOK, delEntry) - } diff --git a/lib/folder/folder-interface.go b/lib/folder/folder-interface.go index b76b3f3..c04cbd7 100644 --- a/lib/folder/folder-interface.go +++ b/lib/folder/folder-interface.go @@ -14,16 +14,24 @@ const ( StatusErrorConfig = "ErrorConfig" StatusDisable = "Disable" StatusEnable = "Enable" + StatusPause = "Pause" + StatusSyncing = "Syncing" ) +type EventCBData map[string]interface{} +type EventCB func(cfg *FolderConfig, data *EventCBData) + // IFOLDER Folder interface type IFOLDER interface { - Add(cfg FolderConfig) (*FolderConfig, error) // Add a new folder - GetConfig() FolderConfig // Get folder public configuration - GetFullPath(dir string) string // Get folder full path - Remove() error // Remove a folder - Sync() error // Force folder files synchronization - IsInSync() (bool, error) // Check if folder files are in-sync + NewUID(suffix string) string // Get a new folder UUID + Add(cfg FolderConfig) (*FolderConfig, error) // Add a new folder + GetConfig() FolderConfig // Get folder public configuration + GetFullPath(dir string) string // Get folder full path + Remove() error // Remove a folder + RegisterEventChange(cb *EventCB, data *EventCBData) error // Request events registration (sent through WS) + UnRegisterEventChange() error // Un-register events + Sync() error // Force folder files synchronization + IsInSync() (bool, error) // Check if folder files are in-sync } // FolderConfig is the config for one folder @@ -33,6 +41,7 @@ type FolderConfig struct { ClientPath string `json:"path"` Type FolderType `json:"type"` Status string `json:"status"` + IsInSync bool `json:"isInSync"` DefaultSdk string `json:"defaultSdk"` // Not exported fields from REST API point of view diff --git a/lib/folder/folder-pathmap.go b/lib/folder/folder-pathmap.go index 2ad8a93..f73f271 100644 --- a/lib/folder/folder-pathmap.go +++ b/lib/folder/folder-pathmap.go @@ -8,6 +8,7 @@ import ( common "github.com/iotbzh/xds-common/golib" "github.com/iotbzh/xds-server/lib/xdsconfig" + uuid "github.com/satori/go.uuid" ) // IFOLDER interface implementation for native/path mapping folders @@ -26,6 +27,11 @@ func NewFolderPathMap(gc *xdsconfig.Config) *PathMap { return &f } +// NewUID Get a UUID +func (f *PathMap) NewUID(suffix string) string { + return uuid.NewV1().String() + "_" + suffix +} + // Add a new folder func (f *PathMap) Add(cfg FolderConfig) (*FolderConfig, error) { if cfg.DataPathMap.ServerPath == "" { @@ -63,6 +69,7 @@ func (f *PathMap) Add(cfg FolderConfig) (*FolderConfig, error) { f.config = cfg f.config.RootPath = dir f.config.DataPathMap.ServerPath = dir + f.config.IsInSync = true f.config.Status = StatusEnable return &f.config, nil @@ -87,6 +94,16 @@ func (f *PathMap) Remove() error { return nil } +// RegisterEventChange requests registration for folder change event +func (f *PathMap) RegisterEventChange(cb *EventCB, data *EventCBData) error { + return nil +} + +// UnRegisterEventChange remove registered callback +func (f *PathMap) UnRegisterEventChange() error { + return nil +} + // Sync Force folder files synchronization func (f *PathMap) Sync() error { return nil diff --git a/lib/model/folders.go b/lib/model/folders.go index 02c3254..ed0078e 100644 --- a/lib/model/folders.go +++ b/lib/model/folders.go @@ -7,13 +7,13 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/Sirupsen/logrus" common "github.com/iotbzh/xds-common/golib" "github.com/iotbzh/xds-server/lib/folder" "github.com/iotbzh/xds-server/lib/syncthing" "github.com/iotbzh/xds-server/lib/xdsconfig" - uuid "github.com/satori/go.uuid" "github.com/syncthing/syncthing/lib/sync" ) @@ -24,6 +24,12 @@ type Folders struct { Log *logrus.Logger SThg *st.SyncThing folders map[string]*folder.IFOLDER + registerCB []RegisteredCB +} + +type RegisteredCB struct { + cb *folder.EventCB + data *folder.EventCBData } // Mutex to make add/delete atomic @@ -39,6 +45,7 @@ func FoldersNew(cfg *xdsconfig.Config, st *st.SyncThing) *Folders { Log: cfg.Log, SThg: st, folders: make(map[string]*folder.IFOLDER), + registerCB: []RegisteredCB{}, } } @@ -114,12 +121,15 @@ func (f *Folders) LoadConfig() error { // Update folders f.Log.Infof("Loading initial folders config: %d folders found", len(flds)) for _, fc := range flds { - if _, err := f.createUpdate(fc, false); err != nil { + if _, err := f.createUpdate(fc, false, true); err != nil { return err } } - return nil + // Save config on disk + err := f.SaveConfig() + + return err } // SaveConfig Save folders configuration to disk @@ -164,11 +174,11 @@ func (f *Folders) getConfigArrUnsafe() []folder.FolderConfig { // Add adds a new folder func (f *Folders) Add(newF folder.FolderConfig) (*folder.FolderConfig, error) { - return f.createUpdate(newF, true) + return f.createUpdate(newF, true, false) } // CreateUpdate creates or update a folder -func (f *Folders) createUpdate(newF folder.FolderConfig, create bool) (*folder.FolderConfig, error) { +func (f *Folders) createUpdate(newF folder.FolderConfig, create bool, initial bool) (*folder.FolderConfig, error) { fcMutex.Lock() defer fcMutex.Unlock() @@ -181,23 +191,7 @@ func (f *Folders) createUpdate(newF folder.FolderConfig, create bool) (*folder.F return nil, fmt.Errorf("ClientPath must be set") } - // Allocate a new UUID - if create { - newF.ID = uuid.NewV1().String() - } - if !create && newF.ID == "" { - return nil, fmt.Errorf("Cannot update folder with null ID") - } - - // Set default value if needed - if newF.Status == "" { - newF.Status = folder.StatusDisable - } - - if newF.Label == "" { - newF.Label = filepath.Base(newF.ClientPath) + "_" + newF.ID[0:8] - } - + // Create a new folder object var fld folder.IFOLDER switch newF.Type { // SYNCTHING @@ -213,6 +207,26 @@ func (f *Folders) createUpdate(newF folder.FolderConfig, create bool) (*folder.F return nil, fmt.Errorf("Unsupported folder type") } + // Set default value if needed + if newF.Status == "" { + newF.Status = folder.StatusDisable + } + if newF.Label == "" { + newF.Label = filepath.Base(newF.ClientPath) + "_" + newF.ID[0:8] + } + + // Allocate a new UUID + if create { + i := len(newF.Label) + if i > 20 { + i = 20 + } + newF.ID = fld.NewUID(newF.Label[:i]) + } + if !create && newF.ID == "" { + return nil, fmt.Errorf("Cannot update folder with null ID") + } + // Normalize path (needed for Windows path including bashlashes) newF.ClientPath = common.PathNormalize(newF.ClientPath) @@ -224,13 +238,31 @@ func (f *Folders) createUpdate(newF folder.FolderConfig, create bool) (*folder.F return newFolder, err } - // Register folder object + // Add to folders list f.folders[newF.ID] = &fld // Save config on disk - err = f.SaveConfig() + if !initial { + if err := f.SaveConfig(); err != nil { + return newFolder, err + } + } + + // Register event change callback + for _, rcb := range f.registerCB { + if err := fld.RegisterEventChange(rcb.cb, rcb.data); err != nil { + return newFolder, err + } + } + + // Force sync after creation + // (need to defer to be sure that WS events will arrive after HTTP creation reply) + go func() { + time.Sleep(time.Millisecond * 500) + fld.Sync() + }() - return newFolder, err + return newFolder, nil } // Delete deletes a specific folder @@ -260,6 +292,29 @@ func (f *Folders) Delete(id string) (folder.FolderConfig, error) { return fld, err } +// RegisterEventChange requests registration for folder event change +func (f *Folders) RegisterEventChange(id string, cb *folder.EventCB, data *folder.EventCBData) error { + + flds := make(map[string]*folder.IFOLDER) + if id != "" { + // Register to a specific folder + flds[id] = f.Get(id) + } else { + // Register to all folders + flds = f.folders + f.registerCB = append(f.registerCB, RegisteredCB{cb: cb, data: data}) + } + + for _, fld := range flds { + err := (*fld).RegisterEventChange(cb, data) + if err != nil { + return err + } + } + + return nil +} + // ForceSync Force the synchronization of a folder func (f *Folders) ForceSync(id string) error { fc := f.Get(id) diff --git a/lib/syncthing/folder-st.go b/lib/syncthing/folder-st.go index ffcd284..da27062 100644 --- a/lib/syncthing/folder-st.go +++ b/lib/syncthing/folder-st.go @@ -6,6 +6,7 @@ import ( "github.com/iotbzh/xds-server/lib/folder" "github.com/iotbzh/xds-server/lib/xdsconfig" + uuid "github.com/satori/go.uuid" "github.com/syncthing/syncthing/lib/config" ) @@ -13,10 +14,13 @@ import ( // STFolder . type STFolder struct { - globalConfig *xdsconfig.Config - st *SyncThing - fConfig folder.FolderConfig - stfConfig config.FolderConfiguration + globalConfig *xdsconfig.Config + st *SyncThing + fConfig folder.FolderConfig + stfConfig config.FolderConfiguration + eventIDs []int + eventChangeCB *folder.EventCB + eventChangeCBData *folder.EventCBData } // NewFolderST Create a new instance of STFolder @@ -27,6 +31,15 @@ func (s *SyncThing) NewFolderST(gc *xdsconfig.Config) *STFolder { } } +// NewUID Get a UUID +func (f *STFolder) NewUID(suffix string) string { + i := len(f.st.MyID) + if i > 15 { + i = 15 + } + return uuid.NewV1().String()[:14] + f.st.MyID[:i] + "_" + suffix +} + // Add a new folder func (f *STFolder) Add(cfg folder.FolderConfig) (*folder.FolderConfig, error) { @@ -59,6 +72,16 @@ func (f *STFolder) Add(cfg folder.FolderConfig) (*folder.FolderConfig, error) { return nil, err } + // Register to events to update folder status + for _, evName := range []string{EventStateChanged, EventFolderPaused} { + evID, err := f.st.Events.Register(evName, f.cbEventState, id, nil) + if err != nil { + return nil, err + } + f.eventIDs = append(f.eventIDs, evID) + } + + f.fConfig.IsInSync = false // will be updated later by events f.fConfig.Status = folder.StatusEnable } @@ -86,6 +109,20 @@ func (f *STFolder) Remove() error { return f.st.FolderDelete(f.stfConfig.ID) } +// RegisterEventChange requests registration for folder event change +func (f *STFolder) RegisterEventChange(cb *folder.EventCB, data *folder.EventCBData) error { + f.eventChangeCB = cb + f.eventChangeCBData = data + return nil +} + +// UnRegisterEventChange remove registered callback +func (f *STFolder) UnRegisterEventChange() error { + f.eventChangeCB = nil + f.eventChangeCBData = nil + return nil +} + // Sync Force folder files synchronization func (f *STFolder) Sync() error { return f.st.FolderScan(f.stfConfig.ID, "") @@ -93,5 +130,41 @@ func (f *STFolder) Sync() error { // IsInSync Check if folder files are in-sync func (f *STFolder) IsInSync() (bool, error) { - return f.st.IsFolderInSync(f.stfConfig.ID) + sts, err := f.st.IsFolderInSync(f.stfConfig.ID) + if err != nil { + return false, err + } + f.fConfig.IsInSync = sts + return sts, nil +} + +// callback use to update IsInSync status +func (f *STFolder) cbEventState(ev Event, data *EventsCBData) { + prevSync := f.fConfig.IsInSync + prevStatus := f.fConfig.Status + + switch ev.Type { + + case EventStateChanged: + to := ev.Data["to"] + switch to { + case "scanning", "syncing": + f.fConfig.Status = folder.StatusSyncing + case "idle": + f.fConfig.Status = folder.StatusEnable + } + f.fConfig.IsInSync = (to == "idle") + + case EventFolderPaused: + if f.fConfig.Status == folder.StatusEnable { + f.fConfig.Status = folder.StatusPause + } + f.fConfig.IsInSync = false + } + + if f.eventChangeCB != nil && + (prevSync != f.fConfig.IsInSync || prevStatus != f.fConfig.Status) { + cpConf := f.fConfig + (*f.eventChangeCB)(&cpConf, f.eventChangeCBData) + } } diff --git a/lib/syncthing/st.go b/lib/syncthing/st.go index 9bdb48f..10210a4 100644 --- a/lib/syncthing/st.go +++ b/lib/syncthing/st.go @@ -42,6 +42,7 @@ type SyncThing struct { conf *xdsconfig.Config client *common.HTTPClient log *logrus.Logger + Events *Events } // ExitChan Channel used for process exit @@ -126,6 +127,9 @@ func NewSyncThing(conf *xdsconfig.Config, log *logrus.Logger) *SyncThing { conf: conf, } + // Create Events monitoring + s.Events = s.NewEventListener() + return &s } @@ -316,6 +320,12 @@ func (s *SyncThing) Connect() error { s.client.SetLogger(s.log) s.MyID, err = s.IDGet() + if err != nil { + return fmt.Errorf("ERROR: cannot retrieve ID") + } + + // Start events monitoring + err = s.Events.Start() return err } diff --git a/lib/syncthing/stEvent.go b/lib/syncthing/stEvent.go new file mode 100644 index 0000000..bf2a809 --- /dev/null +++ b/lib/syncthing/stEvent.go @@ -0,0 +1,242 @@ +package st + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/Sirupsen/logrus" +) + +// Events . +type Events struct { + MonitorTime time.Duration + Debug bool + + stop chan bool + st *SyncThing + log *logrus.Logger + cbArr map[string][]cbMap +} + +type Event struct { + Type string `json:"type"` + Time time.Time `json:"time"` + Data map[string]string `json:"data"` +} + +type EventsCBData map[string]interface{} +type EventsCB func(ev Event, cbData *EventsCBData) + +const ( + EventFolderCompletion string = "FolderCompletion" + EventFolderSummary string = "FolderSummary" + EventFolderPaused string = "FolderPaused" + EventFolderResumed string = "FolderResumed" + EventFolderErrors string = "FolderErrors" + EventStateChanged string = "StateChanged" +) + +var EventsAll string = EventFolderCompletion + "|" + + EventFolderSummary + "|" + + EventFolderPaused + "|" + + EventFolderResumed + "|" + + EventFolderErrors + "|" + + EventStateChanged + +type STEvent struct { + // Per-subscription sequential event ID. Named "id" for backwards compatibility with the REST API + SubscriptionID int `json:"id"` + // Global ID of the event across all subscriptions + GlobalID int `json:"globalID"` + Time time.Time `json:"time"` + Type string `json:"type"` + Data map[string]interface{} `json:"data"` +} + +type cbMap struct { + id int + cb EventsCB + filterID string + data *EventsCBData +} + +// NewEventListener Create a new instance of Event listener +func (s *SyncThing) NewEventListener() *Events { + _, dbg := os.LookupEnv("XDS_DEBUG_STEVENTS") // set to add more debug log + return &Events{ + MonitorTime: 100, // in Milliseconds + Debug: dbg, + stop: make(chan bool, 1), + st: s, + log: s.log, + cbArr: make(map[string][]cbMap), + } +} + +// Start starts event monitoring loop +func (e *Events) Start() error { + go e.monitorLoop() + return nil +} + +// Stop stops event monitoring loop +func (e *Events) Stop() { + e.stop <- true +} + +// Register Add a listener on an event +func (e *Events) Register(evName string, cb EventsCB, filterID string, data *EventsCBData) (int, error) { + if evName == "" || !strings.Contains(EventsAll, evName) { + return -1, fmt.Errorf("Unknown event name") + } + if data == nil { + data = &EventsCBData{} + } + + cbList := []cbMap{} + if _, ok := e.cbArr[evName]; ok { + cbList = e.cbArr[evName] + } + + id := len(cbList) + (*data)["id"] = strconv.Itoa(id) + + e.cbArr[evName] = append(cbList, cbMap{id: id, cb: cb, filterID: filterID, data: data}) + + return id, nil +} + +// UnRegister Remove a listener event +func (e *Events) UnRegister(evName string, id int) error { + cbKey, ok := e.cbArr[evName] + if !ok { + return fmt.Errorf("No event registered to such name") + } + + // FIXME - NOT TESTED + if id >= len(cbKey) { + return fmt.Errorf("Invalid id") + } else if id == len(cbKey) { + e.cbArr[evName] = cbKey[:id-1] + } else { + e.cbArr[evName] = cbKey[id : id+1] + } + + return nil +} + +// GetEvents returns the Syncthing events +func (e *Events) getEvents(since int) ([]STEvent, error) { + var data []byte + ev := []STEvent{} + url := "events" + if since != -1 { + url += "?since=" + strconv.Itoa(since) + } + if err := e.st.client.HTTPGet(url, &data); err != nil { + return ev, err + } + err := json.Unmarshal(data, &ev) + return ev, err +} + +// Loop to monitor Syncthing events +func (e *Events) monitorLoop() { + e.log.Infof("Event monitoring running...") + since := 0 + for { + select { + case <-e.stop: + e.log.Infof("Event monitoring exited") + return + + case <-time.After(e.MonitorTime * time.Millisecond): + stEvArr, err := e.getEvents(since) + if err != nil { + e.log.Errorf("Syncthing Get Events: %v", err) + continue + } + // Process events + for _, stEv := range stEvArr { + since = stEv.SubscriptionID + if e.Debug { + e.log.Warnf("ST EVENT: %d %s\n %v", stEv.GlobalID, stEv.Type, stEv) + } + + cbKey, ok := e.cbArr[stEv.Type] + if !ok { + continue + } + + evData := Event{ + Type: stEv.Type, + Time: stEv.Time, + } + + // Decode Events + // FIXME: re-define data struct for each events + // instead of map of string and use JSON marshing/unmarshing + fID := "" + evData.Data = make(map[string]string) + switch stEv.Type { + + case EventFolderCompletion: + fID = convString(stEv.Data["folder"]) + evData.Data["completion"] = convFloat64(stEv.Data["completion"]) + + case EventFolderSummary: + fID = convString(stEv.Data["folder"]) + evData.Data["needBytes"] = convInt64(stEv.Data["needBytes"]) + evData.Data["state"] = convString(stEv.Data["state"]) + + case EventFolderPaused, EventFolderResumed: + fID = convString(stEv.Data["id"]) + evData.Data["label"] = convString(stEv.Data["label"]) + + case EventFolderErrors: + fID = convString(stEv.Data["folder"]) + // TODO decode array evData.Data["errors"] = convString(stEv.Data["errors"]) + + case EventStateChanged: + fID = convString(stEv.Data["folder"]) + evData.Data["from"] = convString(stEv.Data["from"]) + evData.Data["to"] = convString(stEv.Data["to"]) + + default: + e.log.Warnf("Unsupported event type") + } + + if fID != "" { + evData.Data["id"] = fID + } + + // Call all registered callbacks + for _, c := range cbKey { + if e.Debug { + e.log.Warnf("EVENT CB fID=%s, filterID=%s", fID, c.filterID) + } + // Call when filterID is not set or when it matches + if c.filterID == "" || (fID != "" && fID == c.filterID) { + c.cb(evData, c.data) + } + } + } + } + } +} + +func convString(d interface{}) string { + return d.(string) +} + +func convFloat64(d interface{}) string { + return strconv.FormatFloat(d.(float64), 'f', -1, 64) +} + +func convInt64(d interface{}) string { + return strconv.FormatInt(d.(int64), 10) +} diff --git a/lib/syncthing/stfolder.go b/lib/syncthing/stfolder.go index bbdcc43..70ac70a 100644 --- a/lib/syncthing/stfolder.go +++ b/lib/syncthing/stfolder.go @@ -191,13 +191,11 @@ func (s *SyncThing) FolderStatus(folderID string) (*FolderStatus, error) { // IsFolderInSync Returns true when folder is in sync func (s *SyncThing) IsFolderInSync(folderID string) (bool, error) { - // FIXME better to detected FolderCompletion event (/rest/events) - // See https://docs.syncthing.net/dev/events.html sts, err := s.FolderStatus(folderID) if err != nil { return false, err } - return sts.NeedBytes == 0, nil + return sts.NeedBytes == 0 && sts.State == "idle", nil } // FolderScan Request immediate folder scan. -- cgit From 2fed63ed0087df0c79f7f30f7f397611381bfccd Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Thu, 24 Aug 2017 17:09:57 +0200 Subject: Fixed /exec command (don't prefix command with exec binary). adding 'exec' prevents to use "script" syntax: xds-exec -c xds-config.env -- "cd build && cmake .." But 'exec' binary is mandatory to allow to pass correctly signals to subprocess such as gdb. As workaround, exec is set for now on client side (eg. in xds-gdb). --- lib/apiv1/exec.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/apiv1/exec.go b/lib/apiv1/exec.go index 4a591be..6300dba 100644 --- a/lib/apiv1/exec.go +++ b/lib/apiv1/exec.go @@ -135,10 +135,13 @@ func (s *APIService) execCmd(c *gin.Context) { } } - // FIXME - SEB: exec prevents to use syntax: - // xds-exec -l debug -c xds-config.env -- "cd build && cmake .." cmd = append(cmd, "cd", folder.GetFullPath(args.RPath)) - cmd = append(cmd, "&&", "exec", args.Cmd) + // FIXME - add 'exec' prevents to use syntax: + // xds-exec -l debug -c xds-config.env -- "cd build && cmake .." + // but exec is mandatory to allow to pass correctly signals + // As workaround, exec is set for now on client side (eg. in xds-gdb) + //cmd = append(cmd, "&&", "exec", args.Cmd) + cmd = append(cmd, "&&", args.Cmd) // Process command arguments cmdArgs := make([]string, len(args.Args)+1) -- cgit From 347bd1674bbf67ccb6209951a4bf8f2971715532 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Thu, 24 Aug 2017 21:23:40 +0200 Subject: Redirect HTTP and Gin server logs into a file (xds-server-verbose.log). Signed-off-by: Sebastien Douheret --- lib/syncthing/st.go | 5 ++++- lib/webserver/server.go | 8 ++++---- lib/xdsconfig/config.go | 8 +++++--- 3 files changed, 13 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/syncthing/st.go b/lib/syncthing/st.go index 10210a4..b622970 100644 --- a/lib/syncthing/st.go +++ b/lib/syncthing/st.go @@ -317,7 +317,10 @@ func (s *SyncThing) Connect() error { return fmt.Errorf("ERROR: cannot connect to Syncthing (null client)") } - s.client.SetLogger(s.log) + // Redirect HTTP log into a file + s.client.SetLogLevel(s.conf.Log.Level.String()) + s.client.LoggerPrefix = "SYNCTHING: " + s.client.LoggerOut = s.conf.LogVerboseOut s.MyID, err = s.IDGet() if err != nil { diff --git a/lib/webserver/server.go b/lib/webserver/server.go index 5183208..8639b66 100644 --- a/lib/webserver/server.go +++ b/lib/webserver/server.go @@ -46,10 +46,10 @@ func New(cfg *xdsconfig.Config, mfolders *model.Folders, sdks *crosssdk.SDKs, lo gin.SetMode(gin.ReleaseMode) } - // Redirect gin logs into logrus logger - gin.DefaultWriter = logr.Out - gin.DefaultErrorWriter = logr.Out - log.SetOutput(logr.Out) + // Redirect gin logs into another logger (LogVerboseOut may be stderr or a file) + gin.DefaultWriter = cfg.LogVerboseOut + gin.DefaultErrorWriter = cfg.LogVerboseOut + log.SetOutput(cfg.LogVerboseOut) // FIXME - fix pb about isTerminal=false when out is in VSC Debug Console diff --git a/lib/xdsconfig/config.go b/lib/xdsconfig/config.go index a3e5a7e..82ca97f 100644 --- a/lib/xdsconfig/config.go +++ b/lib/xdsconfig/config.go @@ -2,6 +2,7 @@ package xdsconfig import ( "fmt" + "io" "os" "github.com/Sirupsen/logrus" @@ -17,9 +18,10 @@ type Config struct { Builder BuilderConfig `json:"builder"` // Private (un-exported fields in REST GET /config route) - Options Options `json:"-"` - FileConf FileConfig `json:"-"` - Log *logrus.Logger `json:"-"` + Options Options `json:"-"` + FileConf FileConfig `json:"-"` + Log *logrus.Logger `json:"-"` + LogVerboseOut io.Writer `json:"-"` } // Options set at the command line -- cgit From 3f0a0e6f9cf9e5963d89c6d1be515d8f43ee08bb Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Thu, 24 Aug 2017 21:45:32 +0200 Subject: Take care of ST connection lost in ST event monitor. --- lib/syncthing/st.go | 16 ++++++++++------ lib/syncthing/stEvent.go | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/syncthing/st.go b/lib/syncthing/st.go index b622970..5086994 100644 --- a/lib/syncthing/st.go +++ b/lib/syncthing/st.go @@ -27,12 +27,13 @@ import ( // SyncThing . type SyncThing struct { - BaseURL string - APIKey string - Home string - STCmd *exec.Cmd - STICmd *exec.Cmd - MyID string + BaseURL string + APIKey string + Home string + STCmd *exec.Cmd + STICmd *exec.Cmd + MyID string + Connected bool // Private fields binDir string @@ -301,6 +302,7 @@ func (s *SyncThing) StopInotify() { // Connect Establish HTTP connection with Syncthing func (s *SyncThing) Connect() error { var err error + s.Connected = false s.client, err = common.HTTPNewClient(s.BaseURL, common.HTTPClientConfig{ URLPrefix: "/rest", @@ -327,6 +329,8 @@ func (s *SyncThing) Connect() error { return fmt.Errorf("ERROR: cannot retrieve ID") } + s.Connected = true + // Start events monitoring err = s.Events.Start() diff --git a/lib/syncthing/stEvent.go b/lib/syncthing/stEvent.go index bf2a809..9ca8b78 100644 --- a/lib/syncthing/stEvent.go +++ b/lib/syncthing/stEvent.go @@ -148,6 +148,8 @@ func (e *Events) getEvents(since int) ([]STEvent, error) { func (e *Events) monitorLoop() { e.log.Infof("Event monitoring running...") since := 0 + cntErrConn := 0 + cntErrRetry := 1 for { select { case <-e.stop: @@ -155,11 +157,32 @@ func (e *Events) monitorLoop() { return case <-time.After(e.MonitorTime * time.Millisecond): + + if !e.st.Connected { + cntErrConn++ + time.Sleep(time.Second) + if cntErrConn > cntErrRetry { + e.log.Error("ST Event monitor: ST connection down") + cntErrConn = 0 + cntErrRetry *= 2 + if _, err := e.getEvents(since); err == nil { + e.st.Connected = true + cntErrRetry = 1 + // XXX - should we reset since value ? + goto readEvent + } + } + continue + } + + readEvent: stEvArr, err := e.getEvents(since) if err != nil { e.log.Errorf("Syncthing Get Events: %v", err) + e.st.Connected = false continue } + // Process events for _, stEv := range stEvArr { since = stEv.SubscriptionID -- cgit