From 1efdb28f1bf9246004a7b145e8d91d89be785772 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Tue, 4 Jul 2017 12:03:05 +0200 Subject: Report an error when sdkid not found. --- lib/apiv1/exec.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/apiv1/exec.go b/lib/apiv1/exec.go index 654ff64..6c70a98 100644 --- a/lib/apiv1/exec.go +++ b/lib/apiv1/exec.go @@ -177,6 +177,12 @@ func (s *APIService) execCmd(c *gin.Context) { if envCmd := s.sdks.GetEnvCmd(args.SdkID, prj.DefaultSdk); len(envCmd) > 0 { cmd = append(cmd, envCmd...) cmd = append(cmd, "&&") + } else { + // It's an error if no envcmd found while a sdkid has been provided + if args.SdkID != "" { + common.APIError(c, "Unknown sdkid") + return + } } cmd = append(cmd, "cd", prj.GetFullPath(args.RPath), "&&", args.Cmd) -- cgit 1.2.3-korg From 5c45a5d016f7738ac66f9dedcff6d4712aab2a3d Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Fri, 7 Jul 2017 15:12:19 +0200 Subject: Add httpPort value to config example. Signed-off-by: Sebastien Douheret --- config.json.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.json.in b/config.json.in index 751bb29..1668e05 100644 --- a/config.json.in +++ b/config.json.in @@ -1,5 +1,6 @@ { "webAppDir": "webapp/dist", + "httpPort": "8000", "shareRootDir": "${ROOT_DIR}/tmp/builder_dev_host/share", "logsDir": "/tmp/xds-server/logs", "sdkRootDir": "/xdt/sdk", @@ -8,4 +9,4 @@ "home": "${ROOT_DIR}/tmp/builder_dev_host/syncthing-config", "gui-address": "http://localhost:8384" } -} \ No newline at end of file +} -- cgit 1.2.3-korg From 090c194da6d5da2c0c68faddf3879fed2997d2f8 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Sun, 9 Jul 2017 15:43:45 +0200 Subject: Set exit code Signed-off-by: Sebastien Douheret --- scripts/xds-server-start.sh | 2 ++ scripts/xds-server-stop.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/scripts/xds-server-start.sh b/scripts/xds-server-start.sh index dc108fe..7985759 100755 --- a/scripts/xds-server-start.sh +++ b/scripts/xds-server-start.sh @@ -73,3 +73,5 @@ if [ "$1" != "-dryrun" ]; then pid_xds=$(jobs -p) echo "pid=${pid_xds}" fi + +exit 0 diff --git a/scripts/xds-server-stop.sh b/scripts/xds-server-stop.sh index 8a6bf5e..674ed25 100755 --- a/scripts/xds-server-stop.sh +++ b/scripts/xds-server-stop.sh @@ -16,3 +16,4 @@ if [ "$nbProc" != "0" ]; then pkill -KILL syncthing-inotify fi +exit 0 -- cgit 1.2.3-korg From e9b18cd409f82928e1c4de3029ee1cc2d3816552 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Mon, 24 Jul 2017 11:18:59 +0200 Subject: Support SDK directory with spaces Signed-off-by: Sebastien Douheret --- scripts/xds-utils/install-agl-sdks.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/xds-utils/install-agl-sdks.sh b/scripts/xds-utils/install-agl-sdks.sh index 9fbacbb..f230569 100755 --- a/scripts/xds-utils/install-agl-sdks.sh +++ b/scripts/xds-utils/install-agl-sdks.sh @@ -75,7 +75,7 @@ if [ "$FILE" = "" ]; then exit 1 fi SDK_FILE=${XDT_SDK}/${FILE} -elif [ ! -f $FILE ]; then +elif [ ! -f "$FILE" ]; then echo "SDK file not found: $FILE" exit 1 else @@ -108,14 +108,14 @@ cleanExit () } # Get SDK installer -if [ ! -f ${SDK_FILE} ]; then +if [ ! -f "${SDK_FILE}" ]; then do_cleanup=true - wget "$SDK_BASEURL/$FILE" -O ${SDK_FILE} || exit 1 + wget "$SDK_BASEURL/$FILE" -O "${SDK_FILE}" || exit 1 fi # Retreive default install dir to extract version -offset=$(grep -na -m1 "^MARKER:$" ${SDK_FILE} | cut -d':' -f1) -eval $(head -n $offset ${SDK_FILE} | grep ^DEFAULT_INSTALL_DIR= ) +offset=$(grep -na -m1 "^MARKER:$" "${SDK_FILE}" | cut -d':' -f1) +eval $(head -n $offset "${SDK_FILE}" | grep ^DEFAULT_INSTALL_DIR= ) VERSION=$(basename $DEFAULT_INSTALL_DIR) [ "$PROFILE" = "" ] && { echo "PROFILE is not set"; exit 1; } -- cgit 1.2.3-korg From bf2487c4e4c925f437f9e72f09f6ef5099a0c3cb Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Sun, 9 Jul 2017 15:43:18 +0200 Subject: Add stdin support to /exec Signed-off-by: Sebastien Douheret --- lib/apiv1/apiv1.go | 1 + lib/apiv1/exec.go | 61 +++++++++++++++++++++++++++++++++++++++++++++++------- lib/apiv1/make.go | 16 +++++++------- 3 files changed, 63 insertions(+), 15 deletions(-) diff --git a/lib/apiv1/apiv1.go b/lib/apiv1/apiv1.go index 7fa69e9..cde2526 100644 --- a/lib/apiv1/apiv1.go +++ b/lib/apiv1/apiv1.go @@ -52,6 +52,7 @@ func New(r *gin.Engine, sess *session.Sessions, cfg *xdsconfig.Config, mfolder * s.apiRouter.POST("/exec", s.execCmd) s.apiRouter.POST("/exec/:id", s.execCmd) + s.apiRouter.POST("/signal", s.execSignalCmd) return s } diff --git a/lib/apiv1/exec.go b/lib/apiv1/exec.go index 6c70a98..ce0241a 100644 --- a/lib/apiv1/exec.go +++ b/lib/apiv1/exec.go @@ -6,6 +6,8 @@ import ( "strings" "time" + "fmt" + "github.com/gin-gonic/gin" common "github.com/iotbzh/xds-common/golib" ) @@ -38,6 +40,12 @@ type ExecExitMsg struct { Error error `json:"error"` } +// ExecSignalArgs JSON parameters of /exec/signal command +type ExecSignalArgs struct { + CmdID string `json:"cmdID" binding:"required"` // command id + Signal string `json:"signal" binding:"required"` // signal number +} + // ExecOutEvent Event send in WS when characters are received const ExecOutEvent = "exec:output" @@ -85,14 +93,26 @@ func (s *APIService) execCmd(c *gin.Context) { } execTmo := args.CmdTimeout - if execTmo == 0 { + if execTmo == -1 { + // -1 : no timeout + execTmo = 365 * 24 * 60 * 60 // 1 year == no timeout + } else if execTmo == 0 { + // 0 : default timeout // TODO get default timeout from config.json file execTmo = 24 * 60 * 60 // 1 day } + // Define callback for input + /* SEB TODO + var iCB common.OnInputCB + iCB = func() { + + } + */ + // Define callback for output var oCB common.EmitOutputCB - oCB = func(sid string, id int, stdout, stderr string, data *map[string]interface{}) { + oCB = func(sid string, id string, stdout, stderr string, data *map[string]interface{}) { // IO socket can be nil when disconnected so := s.sessions.IOSocketGet(sid) if so == nil { @@ -110,9 +130,11 @@ func (s *APIService) execCmd(c *gin.Context) { s.log.Debugf("%s emitted - WS sid %s - id:%d - prjID:%s", ExecOutEvent, sid, id, prjID) + fmt.Printf("SEB SEND out <%v>, err <%v>\n", stdout, stderr) + // FIXME replace by .BroadcastTo a room err := (*so).Emit(ExecOutEvent, ExecOutMsg{ - CmdID: strconv.Itoa(id), + CmdID: id, Timestamp: time.Now().String(), Stdout: stdout, Stderr: stderr, @@ -123,7 +145,7 @@ func (s *APIService) execCmd(c *gin.Context) { } // Define callback for output - eCB := func(sid string, id int, code int, err error, data *map[string]interface{}) { + eCB := func(sid string, id string, code int, err error, data *map[string]interface{}) { s.log.Debugf("Command [Cmd ID %d] exited: code %d, error: %v", id, code, err) // IO socket can be nil when disconnected @@ -159,7 +181,7 @@ func (s *APIService) execCmd(c *gin.Context) { // FIXME replace by .BroadcastTo a room e := (*so).Emit(ExecExitEvent, ExecExitMsg{ - CmdID: strconv.Itoa(id), + CmdID: id, Timestamp: time.Now().String(), Code: code, Error: err, @@ -169,7 +191,7 @@ func (s *APIService) execCmd(c *gin.Context) { } } - cmdID := execCommandID + cmdID := strconv.Itoa(execCommandID) execCommandID++ cmd := []string{} @@ -190,10 +212,13 @@ func (s *APIService) execCmd(c *gin.Context) { cmd = append(cmd, args.Args...) } + // SEB Workaround for stderr issue (order not respected with stdout) + cmd = append(cmd, " 2>&1") + // Append client project dir to environment args.Env = append(args.Env, "CLIENT_PROJECT_DIR="+prj.RelativePath) - s.log.Debugf("Execute [Cmd ID %d]: %v", cmdID, cmd) + s.log.Debugf("Execute [Cmd ID %s]: %v", cmdID, cmd) data := make(map[string]interface{}) data["ID"] = prj.ID @@ -212,3 +237,25 @@ func (s *APIService) execCmd(c *gin.Context) { "cmdID": cmdID, }) } + +// ExecCmd executes remotely a command +func (s *APIService) execSignalCmd(c *gin.Context) { + var args ExecSignalArgs + + if c.BindJSON(&args) != nil { + common.APIError(c, "Invalid arguments") + return + } + + s.log.Debugf("Signal %s for command ID %s", args.Signal, args.CmdID) + err := common.ExecSignal(args.CmdID, args.Signal) + if err != nil { + common.APIError(c, err.Error()) + return + } + + c.JSON(http.StatusOK, + gin.H{ + "status": "OK", + }) +} diff --git a/lib/apiv1/make.go b/lib/apiv1/make.go index 5cd98c6..6ae840b 100644 --- a/lib/apiv1/make.go +++ b/lib/apiv1/make.go @@ -92,11 +92,11 @@ func (s *APIService) buildMake(c *gin.Context) { // Define callback for output var oCB common.EmitOutputCB - oCB = func(sid string, id int, stdout, stderr string, data *map[string]interface{}) { + oCB = func(sid string, cmdID string, stdout, stderr string, data *map[string]interface{}) { // IO socket can be nil when disconnected so := s.sessions.IOSocketGet(sid) if so == nil { - s.log.Infof("%s not emitted: WS closed - sid: %s - msg id:%d", MakeOutEvent, sid, id) + s.log.Infof("%s not emitted: WS closed - sid: %s - msg id:%s", MakeOutEvent, sid, cmdID) return } @@ -112,7 +112,7 @@ func (s *APIService) buildMake(c *gin.Context) { // FIXME replace by .BroadcastTo a room err := (*so).Emit(MakeOutEvent, MakeOutMsg{ - CmdID: strconv.Itoa(id), + CmdID: cmdID, Timestamp: time.Now().String(), Stdout: stdout, Stderr: stderr, @@ -123,13 +123,13 @@ func (s *APIService) buildMake(c *gin.Context) { } // Define callback for output - eCB := func(sid string, id int, code int, err error, data *map[string]interface{}) { - s.log.Debugf("Command [Cmd ID %d] exited: code %d, error: %v", id, code, err) + eCB := func(sid string, cmdID string, code int, err error, data *map[string]interface{}) { + s.log.Debugf("Command [Cmd ID %s] exited: code %d, error: %v", cmdID, code, err) // IO socket can be nil when disconnected so := s.sessions.IOSocketGet(sid) if so == nil { - s.log.Infof("%s not emitted - WS closed (id:%d", MakeExitEvent, id) + s.log.Infof("%s not emitted - WS closed (id:%s", MakeExitEvent, cmdID) return } @@ -159,7 +159,7 @@ func (s *APIService) buildMake(c *gin.Context) { // FIXME replace by .BroadcastTo a room e := (*so).Emit(MakeExitEvent, MakeExitMsg{ - CmdID: strconv.Itoa(id), + CmdID: id, Timestamp: time.Now().String(), Code: code, Error: err, @@ -169,7 +169,7 @@ func (s *APIService) buildMake(c *gin.Context) { } } - cmdID := makeCommandID + cmdID := strconv.Itoa(makeCommandID) makeCommandID++ cmd := []string{} -- cgit 1.2.3-korg From ab1170e65d6d03dd1eb2542b5fc47694d7785e70 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Mon, 7 Aug 2017 17:22:15 +0200 Subject: Improved /exec to support gdb execution. /exec now supports stdin and stdout/stderr tunneling over an websocket (socketio). This also supports redirection of inferior process output (stdout only) in particular case of gdb command (set gdb --tty option). --- .vscode/settings.json | 6 +- Makefile | 4 + glide.yaml | 3 + lib/apiv1/exec.go | 312 ++++++++++++++++++++++++++++++++++---------------- 4 files changed, 226 insertions(+), 99 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index bb7040e..429cbbe 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,7 +12,6 @@ "webapp/dist": true, "webapp/node_modules": true }, - // Words to add to dictionary for a workspace. "cSpell.words": [ "apiv", @@ -40,6 +39,7 @@ "pkill", "sdkid", "CLOUDSYNC", - "xdsagent" + "xdsagent", + "eows" ] -} \ No newline at end of file +} diff --git a/Makefile b/Makefile index d088c5d..236a415 100644 --- a/Makefile +++ b/Makefile @@ -157,6 +157,10 @@ package-all: vendor: tools/glide glide.yaml $(LOCAL_TOOLSDIR)/glide install --strip-vendor +vendor/debug: vendor + (cd vendor/github.com/iotbzh && \ + rm -rf xds-common && ln -s ../../../../xds-common ) + .PHONY: tools/glide tools/glide: @test -f $(LOCAL_TOOLSDIR)/glide || { \ diff --git a/glide.yaml b/glide.yaml index aecb56c..8b1e84c 100644 --- a/glide.yaml +++ b/glide.yaml @@ -24,3 +24,6 @@ import: - package: github.com/iotbzh/xds-common subpackages: - golib/common + - golib/eows +- package: github.com/kr/pty + version: ^1.0.0 diff --git a/lib/apiv1/exec.go b/lib/apiv1/exec.go index ce0241a..eb93af8 100644 --- a/lib/apiv1/exec.go +++ b/lib/apiv1/exec.go @@ -1,61 +1,88 @@ package apiv1 import ( + "fmt" "net/http" + "os" + "regexp" "strconv" "strings" "time" - "fmt" - "github.com/gin-gonic/gin" common "github.com/iotbzh/xds-common/golib" + "github.com/iotbzh/xds-common/golib/eows" + "github.com/kr/pty" ) -// ExecArgs JSON parameters of /exec command -type ExecArgs struct { - ID string `json:"id" binding:"required"` - SdkID string `json:"sdkid"` // sdk ID to use for setting env - Cmd string `json:"cmd" binding:"required"` - Args []string `json:"args"` - Env []string `json:"env"` - RPath string `json:"rpath"` // relative path into project - ExitImmediate bool `json:"exitImmediate"` // when true, exit event sent immediately when command exited (IOW, don't wait file synchronization) - CmdTimeout int `json:"timeout"` // command completion timeout in Second -} +type ( + // ExecArgs JSON parameters of /exec command + ExecArgs struct { + ID string `json:"id" binding:"required"` + SdkID string `json:"sdkid"` // sdk ID to use for setting env + Cmd string `json:"cmd" binding:"required"` + Args []string `json:"args"` + Env []string `json:"env"` + RPath string `json:"rpath"` // relative path into project + TTY bool `json:"tty"` // Use a tty, specific to gdb --tty option + TTYGdbserverFix bool `json:"ttyGdbserverFix"` // Set to true to activate gdbserver workaround about inferior output + ExitImmediate bool `json:"exitImmediate"` // when true, exit event sent immediately when command exited (IOW, don't wait file synchronization) + CmdTimeout int `json:"timeout"` // command completion timeout in Second + } -// ExecOutMsg Message send on each output (stdout+stderr) of executed command -type ExecOutMsg struct { - CmdID string `json:"cmdID"` - Timestamp string `json:"timestamp"` - Stdout string `json:"stdout"` - Stderr string `json:"stderr"` -} + // ExecInMsg Message used to received input characters (stdin) + ExecInMsg struct { + CmdID string `json:"cmdID"` + Timestamp string `json:"timestamp"` + Stdin string `json:"stdin"` + } -// ExecExitMsg Message send when executed command exited -type ExecExitMsg struct { - CmdID string `json:"cmdID"` - Timestamp string `json:"timestamp"` - Code int `json:"code"` - Error error `json:"error"` -} + // ExecOutMsg Message used to send output characters (stdout+stderr) + ExecOutMsg struct { + CmdID string `json:"cmdID"` + Timestamp string `json:"timestamp"` + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + } -// ExecSignalArgs JSON parameters of /exec/signal command -type ExecSignalArgs struct { - CmdID string `json:"cmdID" binding:"required"` // command id - Signal string `json:"signal" binding:"required"` // signal number -} + // ExecExitMsg Message sent when executed command exited + ExecExitMsg struct { + CmdID string `json:"cmdID"` + Timestamp string `json:"timestamp"` + Code int `json:"code"` + Error error `json:"error"` + } -// ExecOutEvent Event send in WS when characters are received -const ExecOutEvent = "exec:output" + // ExecSignalArgs JSON parameters of /exec/signal command + ExecSignalArgs struct { + CmdID string `json:"cmdID" binding:"required"` // command id + Signal string `json:"signal" binding:"required"` // signal number + } +) + +const ( + // ExecInEvent Event send in WS when characters are sent (stdin) + ExecInEvent = "exec:input" + + // ExecOutEvent Event send in WS when characters are received (stdout or stderr) + ExecOutEvent = "exec:output" -// ExecExitEvent Event send in WS when program exited -const ExecExitEvent = "exec:exit" + // ExecExitEvent Event send in WS when program exited + ExecExitEvent = "exec:exit" + + // ExecInferiorInEvent Event send in WS when characters are sent to an inferior (used by gdb inferior/tty) + ExecInferiorInEvent = "exec:inferior-input" + + // ExecInferiorOutEvent Event send in WS when characters are received by an inferior + ExecInferiorOutEvent = "exec:inferior-output" +) var execCommandID = 1 // ExecCmd executes remotely a command func (s *APIService) execCmd(c *gin.Context) { + var gdbPty, gdbTty *os.File + var err error var args ExecArgs if c.BindJSON(&args) != nil { common.APIError(c, "Invalid arguments") @@ -92,49 +119,112 @@ func (s *APIService) execCmd(c *gin.Context) { return } - execTmo := args.CmdTimeout - if execTmo == -1 { - // -1 : no timeout - execTmo = 365 * 24 * 60 * 60 // 1 year == no timeout - } else if execTmo == 0 { + // Build command line + cmd := []string{} + // Setup env var regarding Sdk ID (used for example to setup cross toolchain) + if envCmd := s.sdks.GetEnvCmd(args.SdkID, prj.DefaultSdk); len(envCmd) > 0 { + cmd = append(cmd, envCmd...) + cmd = append(cmd, "&&") + } else { + // It's an error if no envcmd found while a sdkid has been provided + if args.SdkID != "" { + common.APIError(c, "Unknown sdkid") + return + } + } + + // FIXME - SEB: exec prevents to use syntax: + // xds-exec -l debug -c xds-config.env -- "cd build && cmake .." + cmd = append(cmd, "cd", prj.GetFullPath(args.RPath)) + cmd = append(cmd, "&&", "exec", args.Cmd) + + // Process command arguments + cmdArgs := make([]string, len(args.Args)+1) + copy(cmdArgs, args.Args) + + // Allocate pts if tty if used + if args.TTY { + gdbPty, gdbTty, err = pty.Open() + if err != nil { + common.APIError(c, err.Error()) + return + } + + s.log.Debugf("Client command tty: %v %v\n", gdbTty.Name(), gdbTty.Name()) + cmdArgs = append(cmdArgs, "--tty="+gdbTty.Name()) + } + + // Unique ID for each commands + cmdID := strconv.Itoa(execCommandID) + execCommandID++ + + // Create new execution over WS context + execWS := eows.New(strings.Join(cmd, " "), cmdArgs, sop, sess.ID, cmdID) + execWS.Log = s.log + + // Append client project dir to environment + execWS.Env = append(args.Env, "CLIENT_PROJECT_DIR="+prj.RelativePath) + + // Set command execution timeout + if args.CmdTimeout == 0 { // 0 : default timeout // TODO get default timeout from config.json file - execTmo = 24 * 60 * 60 // 1 day + execWS.CmdExecTimeout = 24 * 60 * 60 // 1 day + } else { + execWS.CmdExecTimeout = args.CmdTimeout } - // Define callback for input - /* SEB TODO - var iCB common.OnInputCB - iCB = func() { + // Define callback for input (stdin) + execWS.InputEvent = ExecInEvent + execWS.InputCB = func(e *eows.ExecOverWS, stdin string) (string, error) { + s.log.Debugf("STDIN <<%v>>", strings.Replace(stdin, "\n", "\\n", -1)) + + // Handle Ctrl-D + if len(stdin) == 1 && stdin == "\x04" { + // Close stdin + errMsg := fmt.Errorf("close stdin: %v", stdin) + return "", errMsg + } + + // Set correct path + data := e.UserData + rootPath := (*data)["RootPath"].(string) + relaPath := (*data)["RelativePath"].(string) + stdin = strings.Replace(stdin, relaPath, rootPath+"/"+relaPath, -1) + return stdin, nil } - */ - // Define callback for output - var oCB common.EmitOutputCB - oCB = func(sid string, id string, stdout, stderr string, data *map[string]interface{}) { + // Define callback for output (stdout+stderr) + execWS.OutputCB = func(e *eows.ExecOverWS, stdout, stderr string) { // IO socket can be nil when disconnected - so := s.sessions.IOSocketGet(sid) + so := s.sessions.IOSocketGet(e.Sid) if so == nil { - s.log.Infof("%s not emitted: WS closed - sid: %s - msg id:%d", ExecOutEvent, sid, id) + s.log.Infof("%s not emitted: WS closed (sid:%s, msgid:%s)", ExecOutEvent, e.Sid, e.CmdID) return } // Retrieve project ID and RootPath + data := e.UserData prjID := (*data)["ID"].(string) prjRootPath := (*data)["RootPath"].(string) + gdbServerTTY := (*data)["gdbServerTTY"].(string) // Cleanup any references to internal rootpath in stdout & stderr stdout = strings.Replace(stdout, prjRootPath, "", -1) stderr = strings.Replace(stderr, prjRootPath, "", -1) - s.log.Debugf("%s emitted - WS sid %s - id:%d - prjID:%s", ExecOutEvent, sid, id, prjID) - - fmt.Printf("SEB SEND out <%v>, err <%v>\n", stdout, stderr) + s.log.Debugf("%s emitted - WS sid[4:] %s - id:%s - prjID:%s", ExecOutEvent, e.Sid[4:], e.CmdID, prjID) + if stdout != "" { + s.log.Debugf("STDOUT <<%v>>", strings.Replace(stdout, "\n", "\\n", -1)) + } + if stderr != "" { + s.log.Debugf("STDERR <<%v>>", strings.Replace(stderr, "\n", "\\n", -1)) + } // FIXME replace by .BroadcastTo a room err := (*so).Emit(ExecOutEvent, ExecOutMsg{ - CmdID: id, + CmdID: e.CmdID, Timestamp: time.Now().String(), Stdout: stdout, Stderr: stderr, @@ -142,20 +232,53 @@ func (s *APIService) execCmd(c *gin.Context) { if err != nil { s.log.Errorf("WS Emit : %v", err) } + + // XXX - Workaround due to gdbserver bug that doesn't redirect + // inferior output (https://bugs.eclipse.org/bugs/show_bug.cgi?id=437532#c13) + if gdbServerTTY == "workaround" && len(stdout) > 1 && stdout[0] == '&' { + + // Extract and cleanup string like &"bla bla\n" + re := regexp.MustCompile("&\"(.*)\"") + rer := re.FindAllStringSubmatch(stdout, -1) + out := "" + if rer != nil && len(rer) > 0 { + for _, o := range rer { + if len(o) >= 1 { + out = strings.Replace(o[1], "\\n", "\n", -1) + out = strings.Replace(out, "\\r", "\r", -1) + out = strings.Replace(out, "\\t", "\t", -1) + + s.log.Debugf("STDOUT INFERIOR: <<%v>>", out) + err := (*so).Emit(ExecInferiorOutEvent, ExecOutMsg{ + CmdID: e.CmdID, + Timestamp: time.Now().String(), + Stdout: out, + Stderr: "", + }) + if err != nil { + s.log.Errorf("WS Emit : %v", err) + } + } + } + } else { + s.log.Errorf("INFERIOR out parsing error: stdout=<%v>", stdout) + } + } } // Define callback for output - eCB := func(sid string, id string, code int, err error, data *map[string]interface{}) { - s.log.Debugf("Command [Cmd ID %d] exited: code %d, error: %v", id, code, err) + execWS.ExitCB = func(e *eows.ExecOverWS, code int, err error) { + s.log.Debugf("Command [Cmd ID %s] exited: code %d, error: %v", e.CmdID, code, err) // IO socket can be nil when disconnected - so := s.sessions.IOSocketGet(sid) + so := s.sessions.IOSocketGet(e.Sid) if so == nil { - s.log.Infof("%s not emitted - WS closed (id:%d", ExecExitEvent, id) + s.log.Infof("%s not emitted - WS closed (id:%s)", ExecExitEvent, e.CmdID) return } // Retrieve project ID and RootPath + data := e.UserData prjID := (*data)["ID"].(string) exitImm := (*data)["ExitImmediate"].(bool) @@ -179,53 +302,43 @@ func (s *APIService) execCmd(c *gin.Context) { } } + // Close client tty + if gdbPty != nil { + gdbPty.Close() + } + if gdbTty != nil { + gdbTty.Close() + } + // FIXME replace by .BroadcastTo a room - e := (*so).Emit(ExecExitEvent, ExecExitMsg{ - CmdID: id, + errSoEmit := (*so).Emit(ExecExitEvent, ExecExitMsg{ + CmdID: e.CmdID, Timestamp: time.Now().String(), Code: code, Error: err, }) - if e != nil { - s.log.Errorf("WS Emit : %v", e) + if errSoEmit != nil { + s.log.Errorf("WS Emit : %v", errSoEmit) } } - cmdID := strconv.Itoa(execCommandID) - execCommandID++ - cmd := []string{} - - // Setup env var regarding Sdk ID (used for example to setup cross toolchain) - if envCmd := s.sdks.GetEnvCmd(args.SdkID, prj.DefaultSdk); len(envCmd) > 0 { - cmd = append(cmd, envCmd...) - cmd = append(cmd, "&&") - } else { - // It's an error if no envcmd found while a sdkid has been provided - if args.SdkID != "" { - common.APIError(c, "Unknown sdkid") - return - } - } - - cmd = append(cmd, "cd", prj.GetFullPath(args.RPath), "&&", args.Cmd) - if len(args.Args) > 0 { - cmd = append(cmd, args.Args...) - } - - // SEB Workaround for stderr issue (order not respected with stdout) - cmd = append(cmd, " 2>&1") - - // Append client project dir to environment - args.Env = append(args.Env, "CLIENT_PROJECT_DIR="+prj.RelativePath) - - s.log.Debugf("Execute [Cmd ID %s]: %v", cmdID, cmd) - + // User data (used within callbacks) data := make(map[string]interface{}) data["ID"] = prj.ID data["RootPath"] = prj.RootPath + data["RelativePath"] = prj.RelativePath data["ExitImmediate"] = args.ExitImmediate + if args.TTY && args.TTYGdbserverFix { + data["gdbServerTTY"] = "workaround" + } else { + data["gdbServerTTY"] = "" + } + execWS.UserData = &data - err := common.ExecPipeWs(cmd, args.Env, sop, sess.ID, cmdID, execTmo, s.log, oCB, eCB, &data) + // Start command execution + s.log.Debugf("Execute [Cmd ID %s]: %v %v", execWS.CmdID, execWS.Cmd, execWS.Args) + + err = execWS.Start() if err != nil { common.APIError(c, err.Error()) return @@ -234,7 +347,7 @@ func (s *APIService) execCmd(c *gin.Context) { c.JSON(http.StatusOK, gin.H{ "status": "OK", - "cmdID": cmdID, + "cmdID": execWS.CmdID, }) } @@ -248,7 +361,14 @@ func (s *APIService) execSignalCmd(c *gin.Context) { } s.log.Debugf("Signal %s for command ID %s", args.Signal, args.CmdID) - err := common.ExecSignal(args.CmdID, args.Signal) + + e := eows.GetEows(args.CmdID) + if e == nil { + common.APIError(c, "unknown cmdID") + return + } + + err := e.Signal(args.Signal) if err != nil { common.APIError(c, err.Error()) return -- cgit 1.2.3-korg From 356af1eaeae9bb7d38b84d87134b1afe7a496e85 Mon Sep 17 00:00:00 2001 From: Romain Forlot Date: Tue, 4 Jul 2017 10:54:24 +0200 Subject: Needs super user rights using install-agl-sdks.sh Change-Id: I18b767e7e1cba943f7ab293e4a4acc7de1fa1027 Signed-off-by: Romain Forlot --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 938360c..95bd565 100644 --- a/README.md +++ b/README.md @@ -124,10 +124,10 @@ Use provided `install-agl-sdks` script, for example to install SDK for ARM64 and ```bash # Install ARM64 SDK (automatic download) -/usr/local/bin/xds-utils/install-agl-sdks.sh --arch aarch64 +sudo /usr/local/bin/xds-utils/install-agl-sdks.sh --arch aarch64 # Install Intel corei7-64 SDK (using an SDK tarball that has been built or downloaded manually) -/usr/local/bin/xds-utils/install-agl-sdks.sh --arch corei7-64 --file /tmp/poky-agl-glibc-x86_64-agl-demo-platform-crosssdk-corei7-64-toolchain- +sudo /usr/local/bin/xds-utils/install-agl-sdks.sh --arch corei7-64 --file /tmp/poky-agl-glibc-x86_64-agl-demo-platform-crosssdk-corei7-64-toolchain- 3.99.1+snapshot.sh ``` -- cgit 1.2.3-korg From ed89fa27ad268921f2598eae4aa5df975f75553d Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Mon, 7 Aug 2017 19:42:43 +0200 Subject: Update vsc launcher. --- .vscode/launch.json | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 8bdde69..3637b39 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,7 +1,7 @@ { "version": "0.2.0", "configurations": [{ - "name": "XDS-Server local", + "name": "XDS-Server", "type": "go", "request": "launch", "mode": "debug", @@ -16,6 +16,22 @@ "args": ["-log", "debug", "-c", "config.json.in"], "showLog": false }, +{ + "name": "XDS-Server local dev", + "type": "go", + "request": "launch", + "mode": "debug", + "remotePath": "", + "port": 2345, + "host": "127.0.0.1", + "program": "${workspaceRoot}", + "env": { + "GOPATH": "${workspaceRoot}/../../../..:${env:GOPATH}", + "ROOT_DIR": "${workspaceRoot}/../../../.." + }, + "args": ["-log", "debug", "-c", "__config_local_dev.json"], + "showLog": false + }, { "name": "XDS-Server IN DOCKER", "type": "go", @@ -34,4 +50,4 @@ } ] -} \ No newline at end of file +} -- cgit 1.2.3-korg From ae723466f745347c931a8a8bd9ff6dbf5237ece3 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Tue, 8 Aug 2017 16:51:23 +0200 Subject: Define all packages version --- glide.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/glide.yaml b/glide.yaml index 8b1e84c..54cfc78 100644 --- a/glide.yaml +++ b/glide.yaml @@ -7,6 +7,7 @@ import: - package: github.com/gin-gonic/gin version: ^1.1.4 - package: github.com/gin-contrib/static + version: master - package: github.com/syncthing/syncthing version: =0.14.28 subpackages: @@ -18,10 +19,13 @@ import: - package: github.com/Sirupsen/logrus version: ^0.11.5 - package: github.com/googollee/go-socket.io + version: 5447e71f36d3947 - package: github.com/zhouhui8915/go-socket.io-client + version: master - package: github.com/satori/go.uuid version: ^1.1.0 - package: github.com/iotbzh/xds-common + version: master subpackages: - golib/common - golib/eows -- cgit 1.2.3-korg From 0262f5bef6ff67e77b844a04733c57740fba9f00 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Tue, 8 Aug 2017 16:59:56 +0200 Subject: Added -logfile option. --- lib/webserver/server.go | 18 ++++++++++-------- main.go | 28 ++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/lib/webserver/server.go b/lib/webserver/server.go index 8fd7e44..7649cce 100644 --- a/lib/webserver/server.go +++ b/lib/webserver/server.go @@ -2,6 +2,7 @@ package webserver import ( "fmt" + "log" "net/http" "os" @@ -36,20 +37,21 @@ const indexFilename = "index.html" const cookieMaxAge = "3600" // New creates an instance of Server -func New(cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs, log *logrus.Logger) *Server { +func New(cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs, logr *logrus.Logger) *Server { // Setup logging for gin router - if log.Level == logrus.DebugLevel { + if logr.Level == logrus.DebugLevel { gin.SetMode(gin.DebugMode) } else { gin.SetMode(gin.ReleaseMode) } - // TODO - // - try to bind gin DefaultWriter & DefaultErrorWriter to logrus logger - // - try to fix pb about isTerminal=false when out is in VSC Debug Console - //gin.DefaultWriter = ?? - //gin.DefaultErrorWriter = ?? + // Redirect gin logs into logrus logger + gin.DefaultWriter = logr.Out + gin.DefaultErrorWriter = logr.Out + log.SetOutput(logr.Out) + + // FIXME - fix pb about isTerminal=false when out is in VSC Debug Console // Creates gin router r := gin.New() @@ -63,7 +65,7 @@ func New(cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs, log sessions: nil, mfolder: mfolder, sdks: sdks, - log: log, + log: logr, stop: make(chan struct{}), } diff --git a/main.go b/main.go index fd1480e..060a927 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "os/signal" + "path/filepath" "strings" "syscall" "time" @@ -115,8 +116,25 @@ func xdsApp(cliCtx *cli.Context) error { } ctx.Config = cfg - // TODO allow to redirect stdout/sterr into logs file - //logFilename := filepath.Join(ctx.Config.FileConf.LogsDir + "xds-server.log") + // Logs redirected into a file when logsDir is set + logfilename := cliCtx.GlobalString("logfile") + if ctx.Config.FileConf.LogsDir != "" && logfilename != "stdout" { + if logfilename == "" { + logfilename = "xds-server.log" + } + // is it an absolute path ? + logFile := logfilename + if logfilename[0] == '.' || logfilename[0] != '/' { + logFile = filepath.Join(ctx.Config.FileConf.LogsDir, logfilename) + } + fmt.Printf("Logging file: %s\n", logFile) + fdL, err := os.OpenFile(logFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err != nil { + msgErr := fmt.Sprintf("Cannot create log file %s", logFile) + return cli.NewExitError(msgErr, int(syscall.EPERM)) + } + ctx.Log.Out = fdL + } // FIXME - add a builder interface and support other builder type (eg. native) builderType := "syncthing" @@ -247,6 +265,12 @@ func main() { Usage: "logging level (supported levels: panic, fatal, error, warn, info, debug)\n\t", EnvVar: "LOG_LEVEL", }, + cli.StringFlag{ + Name: "logfile", + Value: "stdout", + Usage: "filename where logs will be redirected (default stdout)\n\t", + EnvVar: "LOG_FILENAME", + }, } // only one action: Web Server -- cgit 1.2.3-korg From dd6f08b10b1597f44e3dc25509ac9a45336b0914 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Thu, 10 Aug 2017 12:19:34 +0200 Subject: Add folder interface and support native pathmap folder type. Signed-off-by: Sebastien Douheret --- .vscode/settings.json | 17 +- Makefile | 2 +- lib/apiv1/agent.go | 3 + lib/apiv1/apiv1.go | 6 +- lib/apiv1/config.go | 7 +- lib/apiv1/exec.go | 22 +- lib/apiv1/folders.go | 36 +-- lib/apiv1/make.go | 12 +- lib/crosssdk/sdks.go | 3 + lib/folder/folder-interface.go | 59 ++++ lib/folder/folder-pathmap.go | 88 ++++++ lib/model/folder.go | 110 -------- lib/model/folders.go | 333 +++++++++++++++++++++++ lib/syncthing/folder-st.go | 97 +++++++ lib/syncthing/st.go | 50 +--- lib/syncthing/stfolder.go | 123 ++++++++- lib/webserver/server.go | 8 +- lib/xdsconfig/config.go | 21 +- lib/xdsconfig/fileconfig.go | 40 ++- lib/xdsconfig/folderconfig.go | 85 ------ lib/xdsconfig/foldersconfig.go | 47 ---- main.go | 83 ++---- webapp/src/app/config/config.component.html | 17 +- webapp/src/app/config/config.component.ts | 35 ++- webapp/src/app/devel/deploy/deploy.component.ts | 6 +- webapp/src/app/projects/projectCard.component.ts | 33 ++- webapp/src/app/services/config.service.ts | 80 +++--- webapp/src/app/services/syncthing.service.ts | 4 +- webapp/src/app/services/xdsserver.service.ts | 44 +-- 29 files changed, 961 insertions(+), 510 deletions(-) create mode 100644 lib/folder/folder-interface.go create mode 100644 lib/folder/folder-pathmap.go delete mode 100644 lib/model/folder.go create mode 100644 lib/model/folders.go create mode 100644 lib/syncthing/folder-st.go delete mode 100644 lib/xdsconfig/folderconfig.go delete mode 100644 lib/xdsconfig/foldersconfig.go diff --git a/.vscode/settings.json b/.vscode/settings.json index 429cbbe..60fab57 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,6 +12,16 @@ "webapp/dist": true, "webapp/node_modules": true }, + + // Specify paths/files to ignore. (Supports Globs) + "cSpell.ignorePaths": [ + "**/node_modules/**", + "**/vscode-extension/**", + "**/.git/**", + "**/vendor/**", + ".vscode", + "typings" + ], // Words to add to dictionary for a workspace. "cSpell.words": [ "apiv", @@ -40,6 +50,11 @@ "sdkid", "CLOUDSYNC", "xdsagent", - "eows" + "gdbserver", + "golib", + "eows", + "mfolders", + "IFOLDER", + "flds" ] } diff --git a/Makefile b/Makefile index 236a415..d839539 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ all: tools/syncthing build build: vendor xds webapp xds: scripts tools/syncthing/copytobin - @echo "### Build XDS server (version $(VERSION), subversion $(SUB_VERSION))"; + @echo "### Build XDS server (version $(VERSION), subversion $(SUB_VERSION), $(BUILD_MODE))"; @cd $(ROOT_SRCDIR); $(BUILD_ENV_FLAGS) go build $(VERBOSE_$(V)) -i -o $(LOCAL_BINDIR)/xds-server$(EXT) -ldflags "$(GORELEASE) -X main.AppVersion=$(VERSION) -X main.AppSubVersion=$(SUB_VERSION)" . test: tools/glide diff --git a/lib/apiv1/agent.go b/lib/apiv1/agent.go index 651f246..925f12b 100644 --- a/lib/apiv1/agent.go +++ b/lib/apiv1/agent.go @@ -11,6 +11,7 @@ import ( common "github.com/iotbzh/xds-common/golib" ) +// XDSAgentTarball . type XDSAgentTarball struct { OS string `json:"os"` Arch string `json:"arch"` @@ -18,6 +19,8 @@ type XDSAgentTarball struct { RawVersion string `json:"raw-version"` FileURL string `json:"fileUrl"` } + +// XDSAgentInfo . type XDSAgentInfo struct { Tarballs []XDSAgentTarball `json:"tarballs"` } diff --git a/lib/apiv1/apiv1.go b/lib/apiv1/apiv1.go index cde2526..f32e53b 100644 --- a/lib/apiv1/apiv1.go +++ b/lib/apiv1/apiv1.go @@ -16,19 +16,19 @@ type APIService struct { apiRouter *gin.RouterGroup sessions *session.Sessions cfg *xdsconfig.Config - mfolder *model.Folder + mfolders *model.Folders sdks *crosssdk.SDKs log *logrus.Logger } // New creates a new instance of API service -func New(r *gin.Engine, sess *session.Sessions, cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs) *APIService { +func New(r *gin.Engine, sess *session.Sessions, cfg *xdsconfig.Config, mfolders *model.Folders, sdks *crosssdk.SDKs) *APIService { s := &APIService{ router: r, sessions: sess, apiRouter: r.Group("/api/v1"), cfg: cfg, - mfolder: mfolder, + mfolders: mfolders, sdks: sdks, log: cfg.Log, } diff --git a/lib/apiv1/config.go b/lib/apiv1/config.go index 662ec8e..4b53217 100644 --- a/lib/apiv1/config.go +++ b/lib/apiv1/config.go @@ -36,10 +36,5 @@ func (s *APIService) setConfig(c *gin.Context) { s.log.Debugln("SET config: ", cfgArg) - if err := s.mfolder.UpdateAll(cfgArg); err != nil { - common.APIError(c, err.Error()) - return - } - - c.JSON(http.StatusOK, s.cfg) + common.APIError(c, "Not Supported") } diff --git a/lib/apiv1/exec.go b/lib/apiv1/exec.go index eb93af8..4a591be 100644 --- a/lib/apiv1/exec.go +++ b/lib/apiv1/exec.go @@ -113,11 +113,13 @@ func (s *APIService) execCmd(c *gin.Context) { return } - prj := s.mfolder.GetFolderFromID(id) - if prj == nil { + f := s.mfolders.Get(id) + if f == nil { common.APIError(c, "Unknown id") return } + folder := *f + prj := folder.GetConfig() // Build command line cmd := []string{} @@ -135,7 +137,7 @@ func (s *APIService) execCmd(c *gin.Context) { // FIXME - SEB: exec prevents to use syntax: // xds-exec -l debug -c xds-config.env -- "cd build && cmake .." - cmd = append(cmd, "cd", prj.GetFullPath(args.RPath)) + cmd = append(cmd, "cd", folder.GetFullPath(args.RPath)) cmd = append(cmd, "&&", "exec", args.Cmd) // Process command arguments @@ -163,7 +165,7 @@ func (s *APIService) execCmd(c *gin.Context) { execWS.Log = s.log // Append client project dir to environment - execWS.Env = append(args.Env, "CLIENT_PROJECT_DIR="+prj.RelativePath) + execWS.Env = append(args.Env, "CLIENT_PROJECT_DIR="+prj.ClientPath) // Set command execution timeout if args.CmdTimeout == 0 { @@ -189,8 +191,8 @@ func (s *APIService) execCmd(c *gin.Context) { // Set correct path data := e.UserData rootPath := (*data)["RootPath"].(string) - relaPath := (*data)["RelativePath"].(string) - stdin = strings.Replace(stdin, relaPath, rootPath+"/"+relaPath, -1) + clientPath := (*data)["ClientPath"].(string) + stdin = strings.Replace(stdin, clientPath, rootPath+"/"+clientPath, -1) return stdin, nil } @@ -283,7 +285,7 @@ func (s *APIService) execCmd(c *gin.Context) { exitImm := (*data)["ExitImmediate"].(bool) // XXX - workaround to be sure that Syncthing detected all changes - if err := s.mfolder.ForceSync(prjID); err != nil { + if err := s.mfolders.ForceSync(prjID); err != nil { s.log.Errorf("Error while syncing folder %s: %v", prjID, err) } if !exitImm { @@ -291,8 +293,8 @@ func (s *APIService) execCmd(c *gin.Context) { // FIXME pass as argument tmo := 60 for t := tmo; t > 0; t-- { - s.log.Debugf("Wait file insync for %s (%d/%d)", prjID, t, tmo) - if sync, err := s.mfolder.IsFolderInSync(prjID); sync || err != nil { + s.log.Debugf("Wait file in-sync for %s (%d/%d)", prjID, t, tmo) + if sync, err := s.mfolders.IsFolderInSync(prjID); sync || err != nil { if err != nil { s.log.Errorf("ERROR IsFolderInSync (%s): %v", prjID, err) } @@ -326,7 +328,7 @@ func (s *APIService) execCmd(c *gin.Context) { data := make(map[string]interface{}) data["ID"] = prj.ID data["RootPath"] = prj.RootPath - data["RelativePath"] = prj.RelativePath + data["ClientPath"] = prj.ClientPath data["ExitImmediate"] = args.ExitImmediate if args.TTY && args.TTYGdbserverFix { data["gdbServerTTY"] = "workaround" diff --git a/lib/apiv1/folders.go b/lib/apiv1/folders.go index 44bda24..f957c6d 100644 --- a/lib/apiv1/folders.go +++ b/lib/apiv1/folders.go @@ -2,49 +2,39 @@ package apiv1 import ( "net/http" - "strconv" "github.com/gin-gonic/gin" common "github.com/iotbzh/xds-common/golib" - "github.com/iotbzh/xds-server/lib/xdsconfig" + "github.com/iotbzh/xds-server/lib/folder" ) // getFolders returns all folders configuration func (s *APIService) getFolders(c *gin.Context) { - confMut.Lock() - defer confMut.Unlock() - - c.JSON(http.StatusOK, s.cfg.Folders) + c.JSON(http.StatusOK, s.mfolders.GetConfigArr()) } // getFolder returns a specific folder configuration func (s *APIService) getFolder(c *gin.Context) { - id, err := strconv.Atoi(c.Param("id")) - if err != nil || id < 0 || id > len(s.cfg.Folders) { + f := s.mfolders.Get(c.Param("id")) + if f == nil { common.APIError(c, "Invalid id") return } - confMut.Lock() - defer confMut.Unlock() - - c.JSON(http.StatusOK, s.cfg.Folders[id]) + c.JSON(http.StatusOK, (*f).GetConfig()) } // addFolder adds a new folder to server config func (s *APIService) addFolder(c *gin.Context) { - var cfgArg xdsconfig.FolderConfig + var cfgArg folder.FolderConfig if c.BindJSON(&cfgArg) != nil { common.APIError(c, "Invalid arguments") return } - confMut.Lock() - defer confMut.Unlock() - s.log.Debugln("Add folder config: ", cfgArg) - newFld, err := s.mfolder.UpdateFolder(cfgArg) + newFld, err := s.mfolders.Add(cfgArg) if err != nil { common.APIError(c, err.Error()) return @@ -56,19 +46,11 @@ func (s *APIService) addFolder(c *gin.Context) { // delFolder deletes folder from server config func (s *APIService) delFolder(c *gin.Context) { id := c.Param("id") - if id == "" { - common.APIError(c, "Invalid id") - return - } - - confMut.Lock() - defer confMut.Unlock() s.log.Debugln("Delete folder id ", id) - var delEntry xdsconfig.FolderConfig - var err error - if delEntry, err = s.mfolder.DeleteFolder(id); err != nil { + delEntry, err := s.mfolders.Delete(id) + if err != nil { common.APIError(c, err.Error()) return } diff --git a/lib/apiv1/make.go b/lib/apiv1/make.go index 6ae840b..cf76476 100644 --- a/lib/apiv1/make.go +++ b/lib/apiv1/make.go @@ -76,11 +76,13 @@ func (s *APIService) buildMake(c *gin.Context) { return } - prj := s.mfolder.GetFolderFromID(id) - if prj == nil { + pf := s.mfolders.Get(id) + if pf == nil { common.APIError(c, "Unknown id") return } + folder := *pf + prj := folder.GetConfig() execTmo := args.CmdTimeout if execTmo == 0 { @@ -138,7 +140,7 @@ func (s *APIService) buildMake(c *gin.Context) { exitImm := (*data)["ExitImmediate"].(bool) // XXX - workaround to be sure that Syncthing detected all changes - if err := s.mfolder.ForceSync(prjID); err != nil { + if err := s.mfolders.ForceSync(prjID); err != nil { s.log.Errorf("Error while syncing folder %s: %v", prjID, err) } if !exitImm { @@ -147,7 +149,7 @@ func (s *APIService) buildMake(c *gin.Context) { tmo := 60 for t := tmo; t > 0; t-- { s.log.Debugf("Wait file insync for %s (%d/%d)", prjID, t, tmo) - if sync, err := s.mfolder.IsFolderInSync(prjID); sync || err != nil { + if sync, err := s.mfolders.IsFolderInSync(prjID); sync || err != nil { if err != nil { s.log.Errorf("ERROR IsFolderInSync (%s): %v", prjID, err) } @@ -179,7 +181,7 @@ func (s *APIService) buildMake(c *gin.Context) { cmd = append(cmd, "&&") } - cmd = append(cmd, "cd", prj.GetFullPath(args.RPath), "&&", "make") + cmd = append(cmd, "cd", folder.GetFullPath(args.RPath), "&&", "make") if len(args.Args) > 0 { cmd = append(cmd, args.Args...) } diff --git a/lib/crosssdk/sdks.go b/lib/crosssdk/sdks.go index 35a9998..0da0d1b 100644 --- a/lib/crosssdk/sdks.go +++ b/lib/crosssdk/sdks.go @@ -36,6 +36,9 @@ func Init(cfg *xdsconfig.Config, log *logrus.Logger) (*SDKs, error) { defer s.mutex.Unlock() for _, d := range dirs { + if !common.IsDir(d) { + continue + } sdk, err := NewCrossSDK(d) if err != nil { log.Debugf("Error while processing SDK dir=%s, err=%s", d, err.Error()) diff --git a/lib/folder/folder-interface.go b/lib/folder/folder-interface.go new file mode 100644 index 0000000..b76b3f3 --- /dev/null +++ b/lib/folder/folder-interface.go @@ -0,0 +1,59 @@ +package folder + +// FolderType definition +type FolderType int + +const ( + TypePathMap = 1 + TypeCloudSync = 2 + TypeCifsSmb = 3 +) + +// Folder Status definition +const ( + StatusErrorConfig = "ErrorConfig" + StatusDisable = "Disable" + StatusEnable = "Enable" +) + +// IFOLDER Folder interface +type IFOLDER interface { + Add(cfg FolderConfig) (*FolderConfig, error) // Add a new folder + GetConfig() FolderConfig // Get folder public configuration + GetFullPath(dir string) string // Get folder full path + Remove() error // Remove a folder + Sync() error // Force folder files synchronization + IsInSync() (bool, error) // Check if folder files are in-sync +} + +// FolderConfig is the config for one folder +type FolderConfig struct { + ID string `json:"id"` + Label string `json:"label"` + ClientPath string `json:"path"` + Type FolderType `json:"type"` + Status string `json:"status"` + DefaultSdk string `json:"defaultSdk"` + + // Not exported fields from REST API point of view + RootPath string `json:"-"` + + // FIXME: better to define an equivalent to union data and then implement + // UnmarshalJSON/MarshalJSON to decode/encode according to Type value + // Data interface{} `json:"data"` + + // Specific data depending on which Type is used + DataPathMap PathMapConfig `json:"dataPathMap,omitempty"` + DataCloudSync CloudSyncConfig `json:"dataCloudSync,omitempty"` +} + +// PathMapConfig Path mapping specific data +type PathMapConfig struct { + ServerPath string `json:"serverPath"` +} + +// CloudSyncConfig CloudSync (AKA Syncthing) specific data +type CloudSyncConfig struct { + SyncThingID string `json:"syncThingID"` + BuilderSThgID string `json:"builderSThgID"` +} diff --git a/lib/folder/folder-pathmap.go b/lib/folder/folder-pathmap.go new file mode 100644 index 0000000..8711df2 --- /dev/null +++ b/lib/folder/folder-pathmap.go @@ -0,0 +1,88 @@ +package folder + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + common "github.com/iotbzh/xds-common/golib" +) + +// IFOLDER interface implementation for native/path mapping folders + +// PathMap . +type PathMap struct { + config FolderConfig +} + +// NewFolderPathMap Create a new instance of PathMap +func NewFolderPathMap() *PathMap { + f := PathMap{} + return &f +} + +// Add a new folder +func (f *PathMap) Add(cfg FolderConfig) (*FolderConfig, error) { + if cfg.DataPathMap.ServerPath == "" { + return nil, fmt.Errorf("ServerPath must be set") + } + + // Sanity check + dir := cfg.DataPathMap.ServerPath + if !common.Exists(dir) { + // try to create if not existing + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("Cannot create ServerPath directory: %s", dir) + } + } + if !common.Exists(dir) { + return nil, fmt.Errorf("ServerPath directory is not accessible: %s", dir) + } + file, err := ioutil.TempFile(dir, "xds_pathmap_check") + if err != nil { + return nil, fmt.Errorf("ServerPath sanity check error: %s", err.Error()) + } + defer os.Remove(file.Name()) + + msg := "sanity check PathMap Add folder" + n, err := file.Write([]byte(msg)) + if err != nil || n != len(msg) { + return nil, fmt.Errorf("ServerPath sanity check error: %s", err.Error()) + } + + f.config = cfg + f.config.RootPath = cfg.DataPathMap.ServerPath + f.config.Status = StatusEnable + + return &f.config, nil +} + +// GetConfig Get public part of folder config +func (f *PathMap) GetConfig() FolderConfig { + return f.config +} + +// GetFullPath returns the full path +func (f *PathMap) GetFullPath(dir string) string { + if &dir == nil { + return f.config.DataPathMap.ServerPath + } + return filepath.Join(f.config.DataPathMap.ServerPath, dir) +} + +// Remove a folder +func (f *PathMap) Remove() error { + // nothing to do + return nil +} + +// Sync Force folder files synchronization +func (f *PathMap) Sync() error { + return nil +} + +// IsInSync Check if folder files are in-sync +func (f *PathMap) IsInSync() (bool, error) { + return true, nil +} diff --git a/lib/model/folder.go b/lib/model/folder.go deleted file mode 100644 index 56a46b1..0000000 --- a/lib/model/folder.go +++ /dev/null @@ -1,110 +0,0 @@ -package model - -import ( - "fmt" - - common "github.com/iotbzh/xds-common/golib" - "github.com/iotbzh/xds-server/lib/syncthing" - "github.com/iotbzh/xds-server/lib/xdsconfig" -) - -// Folder Represent a an XDS folder -type Folder struct { - Conf *xdsconfig.Config - SThg *st.SyncThing -} - -// NewFolder Create a new instance of Model Folder -func NewFolder(cfg *xdsconfig.Config, st *st.SyncThing) *Folder { - return &Folder{ - Conf: cfg, - SThg: st, - } -} - -// GetFolderFromID retrieves the Folder config from id -func (c *Folder) GetFolderFromID(id string) *xdsconfig.FolderConfig { - if idx := c.Conf.Folders.GetIdx(id); idx != -1 { - return &c.Conf.Folders[idx] - } - return nil -} - -// UpdateAll updates all the current configuration -func (c *Folder) UpdateAll(newCfg xdsconfig.Config) error { - return fmt.Errorf("Not Supported") - /* - if err := VerifyConfig(newCfg); err != nil { - return err - } - - // TODO: c.Builder = c.Builder.Update(newCfg.Builder) - c.Folders = c.Folders.Update(newCfg.Folders) - - // FIXME To be tested & improved error handling - for _, f := range c.Folders { - if err := c.SThg.FolderChange(st.FolderChangeArg{ - ID: f.ID, - Label: f.Label, - RelativePath: f.RelativePath, - SyncThingID: f.SyncThingID, - ShareRootDir: c.FileConf.ShareRootDir, - }); err != nil { - return err - } - } - - return nil - */ -} - -// UpdateFolder updates a specific folder into the current configuration -func (c *Folder) UpdateFolder(newFolder xdsconfig.FolderConfig) (xdsconfig.FolderConfig, error) { - // rootPath should not be empty - if newFolder.RootPath == "" { - newFolder.RootPath = c.Conf.FileConf.ShareRootDir - } - - // Sanity check of folder settings - if err := newFolder.Verify(); err != nil { - return xdsconfig.FolderConfig{}, err - } - - // Normalize path (needed for Windows path including bashlashes) - newFolder.RelativePath = common.PathNormalize(newFolder.RelativePath) - - // Update config folder - c.Conf.Folders = c.Conf.Folders.Update(xdsconfig.FoldersConfig{newFolder}) - - // Update Syncthing folder - err := c.SThg.FolderChange(newFolder) - - newFolder.BuilderSThgID = c.Conf.Builder.SyncThingID // FIXME - should be removed after local ST config rework - newFolder.Status = xdsconfig.FolderStatusEnable - - return newFolder, err -} - -// DeleteFolder deletes a specific folder -func (c *Folder) DeleteFolder(id string) (xdsconfig.FolderConfig, error) { - var fld xdsconfig.FolderConfig - var err error - - if err = c.SThg.FolderDelete(id); err != nil { - return fld, err - } - - c.Conf.Folders, fld, err = c.Conf.Folders.Delete(id) - - return fld, err -} - -// ForceSync Force the synchronization of a folder -func (c *Folder) ForceSync(id string) error { - return c.SThg.FolderScan(id, "") -} - -// IsFolderInSync Returns true when folder is in sync -func (c *Folder) IsFolderInSync(id string) (bool, error) { - return c.SThg.IsFolderInSync(id) -} diff --git a/lib/model/folders.go b/lib/model/folders.go new file mode 100644 index 0000000..3c2457c --- /dev/null +++ b/lib/model/folders.go @@ -0,0 +1,333 @@ +package model + +import ( + "encoding/xml" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/Sirupsen/logrus" + common "github.com/iotbzh/xds-common/golib" + "github.com/iotbzh/xds-server/lib/folder" + "github.com/iotbzh/xds-server/lib/syncthing" + "github.com/iotbzh/xds-server/lib/xdsconfig" + uuid "github.com/satori/go.uuid" + "github.com/syncthing/syncthing/lib/sync" +) + +// Folders Represent a an XDS folders +type Folders struct { + fileOnDisk string + Conf *xdsconfig.Config + Log *logrus.Logger + SThg *st.SyncThing + folders map[string]*folder.IFOLDER +} + +// Mutex to make add/delete atomic +var fcMutex = sync.NewMutex() +var ffMutex = sync.NewMutex() + +// FoldersNew Create a new instance of Model Folders +func FoldersNew(cfg *xdsconfig.Config, st *st.SyncThing) *Folders { + file, _ := xdsconfig.FoldersConfigFilenameGet() + return &Folders{ + fileOnDisk: file, + Conf: cfg, + Log: cfg.Log, + SThg: st, + folders: make(map[string]*folder.IFOLDER), + } +} + +// LoadConfig Load folders configuration from disk +func (f *Folders) LoadConfig() error { + var flds []folder.FolderConfig + var stFlds []folder.FolderConfig + + // load from disk + if f.Conf.Options.NoFolderConfig { + f.Log.Infof("Don't read folder config file (-no-folderconfig option is set)") + } else if f.fileOnDisk != "" { + f.Log.Infof("Use folder config file: %s", f.fileOnDisk) + err := foldersConfigRead(f.fileOnDisk, &flds) + if err != nil { + if strings.HasPrefix(err.Error(), "No folder config") { + f.Log.Warnf(err.Error()) + } else { + return err + } + } + } else { + f.Log.Warnf("Folders config filename not set") + } + + // Retrieve initial Syncthing config (just append don't overwrite existing ones) + if f.SThg != nil { + f.Log.Infof("Retrieve syncthing folder config") + if err := f.SThg.FolderLoadFromStConfig(&stFlds); err != nil { + // Don't exit on such error, just log it + f.Log.Errorf(err.Error()) + } + } + + // Merge syncthing folders into XDS folders + for _, stf := range stFlds { + found := false + for i, xf := range flds { + if xf.ID == stf.ID { + found = true + // sanity check + if xf.Type != folder.TypeCloudSync { + flds[i].Status = folder.StatusErrorConfig + } + break + } + } + // add it + if !found { + flds = append(flds, stf) + } + } + + // Detect ghost project + // (IOW existing in xds file config and not in syncthing database) + for i, xf := range flds { + // only for syncthing project + if xf.Type != folder.TypeCloudSync { + continue + } + found := false + for _, stf := range stFlds { + if stf.ID == xf.ID { + found = true + break + } + } + if !found { + flds[i].Status = folder.StatusErrorConfig + } + } + + // Update folders + f.Log.Infof("Loading initial folders config: %d folders found", len(flds)) + for _, fc := range flds { + if _, err := f.createUpdate(fc, false); err != nil { + return err + } + } + + return nil +} + +// SaveConfig Save folders configuration to disk +func (f *Folders) SaveConfig() error { + if f.fileOnDisk == "" { + return fmt.Errorf("Folders config filename not set") + } + + // FIXME: buffered save or avoid to write on disk each time + return foldersConfigWrite(f.fileOnDisk, f.getConfigArrUnsafe()) +} + +// Get returns the folder config or nil if not existing +func (f *Folders) Get(id string) *folder.IFOLDER { + if id == "" { + return nil + } + fc, exist := f.folders[id] + if !exist { + return nil + } + return fc +} + +// GetConfigArr returns the config of all folders as an array +func (f *Folders) GetConfigArr() []folder.FolderConfig { + fcMutex.Lock() + defer fcMutex.Unlock() + + return f.getConfigArrUnsafe() +} + +// getConfigArrUnsafe Same as GetConfigArr without mutex protection +func (f *Folders) getConfigArrUnsafe() []folder.FolderConfig { + var conf []folder.FolderConfig + + for _, v := range f.folders { + conf = append(conf, (*v).GetConfig()) + } + return conf +} + +// Add adds a new folder +func (f *Folders) Add(newF folder.FolderConfig) (*folder.FolderConfig, error) { + return f.createUpdate(newF, true) +} + +// CreateUpdate creates or update a folder +func (f *Folders) createUpdate(newF folder.FolderConfig, create bool) (*folder.FolderConfig, error) { + + fcMutex.Lock() + defer fcMutex.Unlock() + + // Sanity check + if _, exist := f.folders[newF.ID]; create && exist { + return nil, fmt.Errorf("ID already exists") + } + if newF.ClientPath == "" { + return nil, fmt.Errorf("ClientPath must be set") + } + + // Allocate a new UUID + if create { + newF.ID = uuid.NewV1().String() + } + if !create && newF.ID == "" { + return nil, fmt.Errorf("Cannot update folder with null ID") + } + + // Set default value if needed + if newF.Status == "" { + newF.Status = folder.StatusDisable + } + + if newF.Label == "" { + newF.Label = filepath.Base(newF.ClientPath) + "_" + newF.ID[0:8] + } + + var fld folder.IFOLDER + switch newF.Type { + // SYNCTHING + case folder.TypeCloudSync: + if f.SThg == nil { + return nil, fmt.Errorf("ClownSync type not supported (syncthing not initialized)") + } + fld = f.SThg.NewFolderST(f.Conf) + // PATH MAP + case folder.TypePathMap: + fld = folder.NewFolderPathMap() + default: + return nil, fmt.Errorf("Unsupported folder type") + } + + // Normalize path (needed for Windows path including bashlashes) + newF.ClientPath = common.PathNormalize(newF.ClientPath) + + // Add new folder + newFolder, err := fld.Add(newF) + if err != nil { + newF.Status = folder.StatusErrorConfig + log.Printf("ERROR Adding folder: %v\n", err) + return newFolder, err + } + + // Register folder object + f.folders[newF.ID] = &fld + + // Save config on disk + err = f.SaveConfig() + + return newFolder, err +} + +// Delete deletes a specific folder +func (f *Folders) Delete(id string) (folder.FolderConfig, error) { + var err error + + fcMutex.Lock() + defer fcMutex.Unlock() + + fld := folder.FolderConfig{} + fc, exist := f.folders[id] + if !exist { + return fld, fmt.Errorf("unknown id") + } + + fld = (*fc).GetConfig() + + if err = (*fc).Remove(); err != nil { + return fld, err + } + + delete(f.folders, id) + + // Save config on disk + err = f.SaveConfig() + + return fld, err +} + +// ForceSync Force the synchronization of a folder +func (f *Folders) ForceSync(id string) error { + fc := f.Get(id) + if fc == nil { + return fmt.Errorf("Unknown id") + } + return (*fc).Sync() +} + +// IsFolderInSync Returns true when folder is in sync +func (f *Folders) IsFolderInSync(id string) (bool, error) { + fc := f.Get(id) + if fc == nil { + return false, fmt.Errorf("Unknown id") + } + return (*fc).IsInSync() +} + +//*** Private functions *** + +// Use XML format and not json to be able to save/load all fields including +// ones that are masked in json (IOW defined with `json:"-"`) +type xmlFolders struct { + XMLName xml.Name `xml:"folders"` + Version string `xml:"version,attr"` + Folders []folder.FolderConfig `xml:"folders"` +} + +// foldersConfigRead reads folders config from disk +func foldersConfigRead(file string, folders *[]folder.FolderConfig) error { + if !common.Exists(file) { + return fmt.Errorf("No folder config file found (%s)", file) + } + + ffMutex.Lock() + defer ffMutex.Unlock() + + fd, err := os.Open(file) + defer fd.Close() + if err != nil { + return err + } + + data := xmlFolders{} + err = xml.NewDecoder(fd).Decode(&data) + if err == nil { + *folders = data.Folders + } + return err +} + +// foldersConfigWrite writes folders config on disk +func foldersConfigWrite(file string, folders []folder.FolderConfig) error { + ffMutex.Lock() + defer ffMutex.Unlock() + + fd, err := os.OpenFile(file, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + defer fd.Close() + if err != nil { + return err + } + + data := &xmlFolders{ + Version: "1", + Folders: folders, + } + + enc := xml.NewEncoder(fd) + enc.Indent("", " ") + return enc.Encode(data) +} diff --git a/lib/syncthing/folder-st.go b/lib/syncthing/folder-st.go new file mode 100644 index 0000000..ffcd284 --- /dev/null +++ b/lib/syncthing/folder-st.go @@ -0,0 +1,97 @@ +package st + +import ( + "fmt" + "path/filepath" + + "github.com/iotbzh/xds-server/lib/folder" + "github.com/iotbzh/xds-server/lib/xdsconfig" + "github.com/syncthing/syncthing/lib/config" +) + +// IFOLDER interface implementation for syncthing + +// STFolder . +type STFolder struct { + globalConfig *xdsconfig.Config + st *SyncThing + fConfig folder.FolderConfig + stfConfig config.FolderConfiguration +} + +// NewFolderST Create a new instance of STFolder +func (s *SyncThing) NewFolderST(gc *xdsconfig.Config) *STFolder { + return &STFolder{ + globalConfig: gc, + st: s, + } +} + +// Add a new folder +func (f *STFolder) Add(cfg folder.FolderConfig) (*folder.FolderConfig, error) { + + // Sanity check + if cfg.DataCloudSync.SyncThingID == "" { + return nil, fmt.Errorf("device id not set (SyncThingID field)") + } + + // rootPath should not be empty + if cfg.RootPath == "" { + cfg.RootPath = f.globalConfig.FileConf.ShareRootDir + } + + f.fConfig = cfg + + f.fConfig.DataCloudSync.BuilderSThgID = f.st.MyID // FIXME - should be removed after local ST config rework + + // Update Syncthing folder + // (expect if status is ErrorConfig) + // TODO: add cache to avoid multiple requests on startup + if f.fConfig.Status != folder.StatusErrorConfig { + id, err := f.st.FolderChange(f.fConfig) + if err != nil { + return nil, err + } + + f.stfConfig, err = f.st.FolderConfigGet(id) + if err != nil { + f.fConfig.Status = folder.StatusErrorConfig + return nil, err + } + + f.fConfig.Status = folder.StatusEnable + } + + return &f.fConfig, nil +} + +// GetConfig Get public part of folder config +func (f *STFolder) GetConfig() folder.FolderConfig { + return f.fConfig +} + +// GetFullPath returns the full path +func (f *STFolder) GetFullPath(dir string) string { + if &dir == nil { + dir = "" + } + if filepath.IsAbs(dir) { + return filepath.Join(f.fConfig.RootPath, dir) + } + return filepath.Join(f.fConfig.RootPath, f.fConfig.ClientPath, dir) +} + +// Remove a folder +func (f *STFolder) Remove() error { + return f.st.FolderDelete(f.stfConfig.ID) +} + +// Sync Force folder files synchronization +func (f *STFolder) Sync() error { + return f.st.FolderScan(f.stfConfig.ID, "") +} + +// IsInSync Check if folder files are in-sync +func (f *STFolder) IsInSync() (bool, error) { + return f.st.IsFolderInSync(f.stfConfig.ID) +} diff --git a/lib/syncthing/st.go b/lib/syncthing/st.go index 3380cda..9bdb48f 100644 --- a/lib/syncthing/st.go +++ b/lib/syncthing/st.go @@ -32,6 +32,7 @@ type SyncThing struct { Home string STCmd *exec.Cmd STICmd *exec.Cmd + MyID string // Private fields binDir string @@ -211,13 +212,13 @@ func (s *SyncThing) Start() (*exec.Cmd, error) { env := []string{ "STNODEFAULTFOLDER=1", "STNOUPGRADE=1", - "STNORESTART=1", + "STNORESTART=1", // FIXME SEB remove ? } s.STCmd, err = s.startProc("syncthing", args, env, &s.exitSTChan) // Use autogenerated apikey if not set by config.json - if s.APIKey == "" { + if err == nil && s.APIKey == "" { if fd, err := os.Open(filepath.Join(s.Home, "config.xml")); err == nil { defer fd.Close() if b, err := ioutil.ReadAll(fd); err == nil { @@ -314,7 +315,9 @@ func (s *SyncThing) Connect() error { s.client.SetLogger(s.log) - return nil + s.MyID, err = s.IDGet() + + return err } // IDGet returns the Syncthing ID of Syncthing instance running locally @@ -360,44 +363,3 @@ func (s *SyncThing) IsConfigInSync() (bool, error) { } return d.ConfigInSync, nil } - -// FolderStatus Returns all information about the current -func (s *SyncThing) FolderStatus(folderID string) (*FolderStatus, error) { - var data []byte - var res FolderStatus - if folderID == "" { - return nil, fmt.Errorf("folderID not set") - } - if err := s.client.HTTPGet("db/status?folder="+folderID, &data); err != nil { - return nil, err - } - if err := json.Unmarshal(data, &res); err != nil { - return nil, err - } - return &res, nil -} - -// IsFolderInSync Returns true when folder is in sync -func (s *SyncThing) IsFolderInSync(folderID string) (bool, error) { - // FIXME better to detected FolderCompletion event (/rest/events) - // See https://docs.syncthing.net/dev/events.html - sts, err := s.FolderStatus(folderID) - if err != nil { - return false, err - } - return sts.NeedBytes == 0, nil -} - -// FolderScan Request immediate folder scan. -// Scan all folders if folderID param is empty -func (s *SyncThing) FolderScan(folderID string, subpath string) error { - url := "db/scan" - if folderID != "" { - url += "?folder=" + folderID - - if subpath != "" { - url += "&sub=" + subpath - } - } - return s.client.HTTPPost(url, "") -} diff --git a/lib/syncthing/stfolder.go b/lib/syncthing/stfolder.go index 661e19d..bbdcc43 100644 --- a/lib/syncthing/stfolder.go +++ b/lib/syncthing/stfolder.go @@ -1,34 +1,77 @@ package st import ( + "encoding/json" + "fmt" "path/filepath" "strings" - "github.com/iotbzh/xds-server/lib/xdsconfig" + "github.com/iotbzh/xds-server/lib/folder" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/protocol" ) +// FolderLoadFromStConfig Load/Retrieve folder config from syncthing database +func (s *SyncThing) FolderLoadFromStConfig(f *[]folder.FolderConfig) error { + + defaultSdk := "" // cannot know which was the default sdk + + stCfg, err := s.ConfigGet() + if err != nil { + return err + } + if len(stCfg.Devices) < 1 { + return fmt.Errorf("Cannot load syncthing config: no device defined") + } + devID := stCfg.Devices[0].DeviceID.String() + if devID == s.MyID { + if len(stCfg.Devices) < 2 { + return fmt.Errorf("Cannot load syncthing config: no valid device found") + } + devID = stCfg.Devices[1].DeviceID.String() + } + + for _, stFld := range stCfg.Folders { + cliPath := strings.TrimPrefix(stFld.RawPath, s.conf.FileConf.ShareRootDir) + if cliPath == "" { + cliPath = stFld.RawPath + } + *f = append(*f, folder.FolderConfig{ + ID: stFld.ID, + Label: stFld.Label, + ClientPath: strings.TrimRight(cliPath, "/"), + Type: folder.TypeCloudSync, + Status: folder.StatusDisable, + DefaultSdk: defaultSdk, + RootPath: s.conf.FileConf.ShareRootDir, + DataCloudSync: folder.CloudSyncConfig{SyncThingID: devID}, + }) + } + + return nil +} + // FolderChange is called when configuration has changed -func (s *SyncThing) FolderChange(f xdsconfig.FolderConfig) error { +func (s *SyncThing) FolderChange(f folder.FolderConfig) (string, error) { // Get current config stCfg, err := s.ConfigGet() if err != nil { s.log.Errorln(err) - return err + return "", err } + stClientID := f.DataCloudSync.SyncThingID // Add new Device if needed var devID protocol.DeviceID - if err := devID.UnmarshalText([]byte(f.SyncThingID)); err != nil { - s.log.Errorf("not a valid device id (err %v)\n", err) - return err + if err := devID.UnmarshalText([]byte(stClientID)); err != nil { + s.log.Errorf("not a valid device id (err %v)", err) + return "", err } newDevice := config.DeviceConfiguration{ DeviceID: devID, - Name: f.SyncThingID, + Name: stClientID, Addresses: []string{"dynamic"}, } @@ -49,13 +92,13 @@ func (s *SyncThing) FolderChange(f xdsconfig.FolderConfig) error { label = strings.Split(id, "/")[0] } if id = f.ID; id == "" { - id = f.SyncThingID[0:15] + "_" + label + id = stClientID[0:15] + "_" + label } folder := config.FolderConfiguration{ ID: id, Label: label, - RawPath: filepath.Join(s.conf.FileConf.ShareRootDir, f.RelativePath), + RawPath: filepath.Join(s.conf.FileConf.ShareRootDir, f.ClientPath), } if s.conf.FileConf.SThgConf.RescanIntervalS > 0 { @@ -85,7 +128,7 @@ func (s *SyncThing) FolderChange(f xdsconfig.FolderConfig) error { s.log.Errorln(err) } - return nil + return id, nil } // FolderDelete is called to delete a folder config @@ -110,3 +153,63 @@ func (s *SyncThing) FolderDelete(id string) error { return nil } + +// FolderConfigGet Returns the configuration of a specific folder +func (s *SyncThing) FolderConfigGet(folderID string) (config.FolderConfiguration, error) { + fc := config.FolderConfiguration{} + if folderID == "" { + return fc, fmt.Errorf("folderID not set") + } + cfg, err := s.ConfigGet() + if err != nil { + return fc, err + } + for _, f := range cfg.Folders { + if f.ID == folderID { + fc = f + return fc, nil + } + } + return fc, fmt.Errorf("id not found") +} + +// FolderStatus Returns all information about the current +func (s *SyncThing) FolderStatus(folderID string) (*FolderStatus, error) { + var data []byte + var res FolderStatus + if folderID == "" { + return nil, fmt.Errorf("folderID not set") + } + if err := s.client.HTTPGet("db/status?folder="+folderID, &data); err != nil { + return nil, err + } + if err := json.Unmarshal(data, &res); err != nil { + return nil, err + } + return &res, nil +} + +// IsFolderInSync Returns true when folder is in sync +func (s *SyncThing) IsFolderInSync(folderID string) (bool, error) { + // FIXME better to detected FolderCompletion event (/rest/events) + // See https://docs.syncthing.net/dev/events.html + sts, err := s.FolderStatus(folderID) + if err != nil { + return false, err + } + return sts.NeedBytes == 0, nil +} + +// FolderScan Request immediate folder scan. +// Scan all folders if folderID param is empty +func (s *SyncThing) FolderScan(folderID string, subpath string) error { + url := "db/scan" + if folderID != "" { + url += "?folder=" + folderID + + if subpath != "" { + url += "&sub=" + subpath + } + } + return s.client.HTTPPost(url, "") +} diff --git a/lib/webserver/server.go b/lib/webserver/server.go index 7649cce..5183208 100644 --- a/lib/webserver/server.go +++ b/lib/webserver/server.go @@ -27,7 +27,7 @@ type Server struct { webApp *gin.RouterGroup cfg *xdsconfig.Config sessions *session.Sessions - mfolder *model.Folder + mfolders *model.Folders sdks *crosssdk.SDKs log *logrus.Logger stop chan struct{} // signals intentional stop @@ -37,7 +37,7 @@ const indexFilename = "index.html" const cookieMaxAge = "3600" // New creates an instance of Server -func New(cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs, logr *logrus.Logger) *Server { +func New(cfg *xdsconfig.Config, mfolders *model.Folders, sdks *crosssdk.SDKs, logr *logrus.Logger) *Server { // Setup logging for gin router if logr.Level == logrus.DebugLevel { @@ -63,7 +63,7 @@ func New(cfg *xdsconfig.Config, mfolder *model.Folder, sdks *crosssdk.SDKs, logr webApp: nil, cfg: cfg, sessions: nil, - mfolder: mfolder, + mfolders: mfolders, sdks: sdks, log: logr, stop: make(chan struct{}), @@ -86,7 +86,7 @@ func (s *Server) Serve() error { s.sessions = session.NewClientSessions(s.router, s.log, cookieMaxAge) // Create REST API - s.api = apiv1.New(s.router, s.sessions, s.cfg, s.mfolder, s.sdks) + s.api = apiv1.New(s.router, s.sessions, s.cfg, s.mfolders, s.sdks) // Websocket routes s.sIOServer, err = socketio.NewServer(nil) diff --git a/lib/xdsconfig/config.go b/lib/xdsconfig/config.go index f2d0710..a3e5a7e 100644 --- a/lib/xdsconfig/config.go +++ b/lib/xdsconfig/config.go @@ -2,7 +2,6 @@ package xdsconfig import ( "fmt" - "os" "github.com/Sirupsen/logrus" @@ -16,13 +15,21 @@ type Config struct { APIVersion string `json:"apiVersion"` VersionGitTag string `json:"gitTag"` Builder BuilderConfig `json:"builder"` - Folders FoldersConfig `json:"folders"` // Private (un-exported fields in REST GET /config route) + Options Options `json:"-"` FileConf FileConfig `json:"-"` Log *logrus.Logger `json:"-"` } +// Options set at the command line +type Options struct { + ConfigFile string + LogLevel string + LogFile string + NoFolderConfig bool +} + // Config default values const ( DefaultAPIVersion = "1" @@ -41,7 +48,13 @@ func Init(cliCtx *cli.Context, log *logrus.Logger) (*Config, error) { APIVersion: DefaultAPIVersion, VersionGitTag: cliCtx.App.Metadata["git-tag"].(string), Builder: BuilderConfig{}, - Folders: FoldersConfig{}, + + Options: Options{ + ConfigFile: cliCtx.GlobalString("config"), + LogLevel: cliCtx.GlobalString("log"), + LogFile: cliCtx.GlobalString("logfile"), + NoFolderConfig: cliCtx.GlobalBool("no-folderconfig"), + }, FileConf: FileConfig{ WebAppDir: "webapp/dist", ShareRootDir: DefaultShareDir, @@ -52,7 +65,7 @@ func Init(cliCtx *cli.Context, log *logrus.Logger) (*Config, error) { } // config file settings overwrite default config - err = updateConfigFromFile(&c, cliCtx.GlobalString("config")) + err = readGlobalConfig(&c, c.Options.ConfigFile) if err != nil { return nil, err } diff --git a/lib/xdsconfig/fileconfig.go b/lib/xdsconfig/fileconfig.go index 90c1aad..2dbf884 100644 --- a/lib/xdsconfig/fileconfig.go +++ b/lib/xdsconfig/fileconfig.go @@ -11,6 +11,16 @@ import ( common "github.com/iotbzh/xds-common/golib" ) +const ( + // ConfigDir Directory in user HOME directory where xds config will be saved + ConfigDir = ".xds" + // GlobalConfigFilename Global config filename + GlobalConfigFilename = "config.json" + // FoldersConfigFilename Folders config filename + FoldersConfigFilename = "server-config_folders.xml" +) + +// SyncThingConf definition type SyncThingConf struct { BinDir string `json:"binDir"` Home string `json:"home"` @@ -19,6 +29,7 @@ type SyncThingConf struct { RescanIntervalS int `json:"rescanIntervalS"` } +// FileConfig is the JSON structure of xds-server config file (config.json) type FileConfig struct { WebAppDir string `json:"webAppDir"` ShareRootDir string `json:"shareRootDir"` @@ -28,21 +39,21 @@ type FileConfig struct { LogsDir string `json:"logsDir"` } -// getConfigFromFile reads configuration from a config file. +// readGlobalConfig reads configuration from a config file. // Order to determine which config file is used: // 1/ from command line option: "--config myConfig.json" // 2/ $HOME/.xds/config.json file // 3/ /config.json file // 4/ /config.json file - -func updateConfigFromFile(c *Config, confFile string) error { +func readGlobalConfig(c *Config, confFile string) error { searchIn := make([]string, 0, 3) if confFile != "" { searchIn = append(searchIn, confFile) } if usr, err := user.Current(); err == nil { - searchIn = append(searchIn, path.Join(usr.HomeDir, ".xds", "config.json")) + searchIn = append(searchIn, path.Join(usr.HomeDir, ConfigDir, + GlobalConfigFilename)) } cwd, err := os.Getwd() if err == nil { @@ -70,7 +81,6 @@ func updateConfigFromFile(c *Config, confFile string) error { // TODO move on viper package to support comments in JSON and also // bind with flags (command line options) // see https://github.com/spf13/viper#working-with-flags - fd, _ := os.Open(*cFile) defer fd.Close() fCfg := FileConfig{} @@ -79,14 +89,15 @@ func updateConfigFromFile(c *Config, confFile string) error { } // Support environment variables (IOW ${MY_ENV_VAR} syntax) in config.json - for _, field := range []*string{ + vars := []*string{ &fCfg.WebAppDir, &fCfg.ShareRootDir, &fCfg.SdkRootDir, - &fCfg.LogsDir, - &fCfg.SThgConf.Home, - &fCfg.SThgConf.BinDir} { - + &fCfg.LogsDir} + if fCfg.SThgConf != nil { + vars = append(vars, &fCfg.SThgConf.Home, &fCfg.SThgConf.BinDir) + } + for _, field := range vars { var err error if *field, err = common.ResolveEnvVar(*field); err != nil { return err @@ -123,3 +134,12 @@ func updateConfigFromFile(c *Config, confFile string) error { c.FileConf = fCfg return nil } + +// FoldersConfigFilenameGet +func FoldersConfigFilenameGet() (string, error) { + usr, err := user.Current() + if err != nil { + return "", err + } + return path.Join(usr.HomeDir, ConfigDir, FoldersConfigFilename), nil +} diff --git a/lib/xdsconfig/folderconfig.go b/lib/xdsconfig/folderconfig.go deleted file mode 100644 index bb2b56f..0000000 --- a/lib/xdsconfig/folderconfig.go +++ /dev/null @@ -1,85 +0,0 @@ -package xdsconfig - -import ( - "fmt" - "log" - "path/filepath" -) - -// FolderType constances -const ( - FolderTypeDocker = 0 - FolderTypeWindowsSubsystem = 1 - FolderTypeCloudSync = 2 - - FolderStatusErrorConfig = "ErrorConfig" - FolderStatusDisable = "Disable" - FolderStatusEnable = "Enable" -) - -// FolderType is the type of sharing folder -type FolderType int - -// FolderConfig is the config for one folder -type FolderConfig struct { - ID string `json:"id" binding:"required"` - Label string `json:"label"` - RelativePath string `json:"path"` - Type FolderType `json:"type"` - SyncThingID string `json:"syncThingID"` - BuilderSThgID string `json:"builderSThgID"` - Status string `json:"status"` - DefaultSdk string `json:"defaultSdk"` - - // Not exported fields - RootPath string `json:"-"` -} - -// NewFolderConfig creates a new folder object -func NewFolderConfig(id, label, rootDir, path string, defaultSdk string) FolderConfig { - return FolderConfig{ - ID: id, - Label: label, - RelativePath: path, - Type: FolderTypeCloudSync, - SyncThingID: "", - Status: FolderStatusDisable, - RootPath: rootDir, - DefaultSdk: defaultSdk, - } -} - -// GetFullPath returns the full path -func (c *FolderConfig) GetFullPath(dir string) string { - if &dir == nil { - dir = "" - } - if filepath.IsAbs(dir) { - return filepath.Join(c.RootPath, dir) - } - return filepath.Join(c.RootPath, c.RelativePath, dir) -} - -// Verify is called to verify that a configuration is valid -func (c *FolderConfig) Verify() error { - var err error - - if c.Type != FolderTypeCloudSync { - err = fmt.Errorf("Unsupported folder type") - } - - if c.SyncThingID == "" { - err = fmt.Errorf("device id not set (SyncThingID field)") - } - - if c.RootPath == "" { - err = fmt.Errorf("RootPath must not be empty") - } - - if err != nil { - c.Status = FolderStatusErrorConfig - log.Printf("ERROR Verify: %v\n", err) - } - - return err -} diff --git a/lib/xdsconfig/foldersconfig.go b/lib/xdsconfig/foldersconfig.go deleted file mode 100644 index 4ad16df..0000000 --- a/lib/xdsconfig/foldersconfig.go +++ /dev/null @@ -1,47 +0,0 @@ -package xdsconfig - -import ( - "fmt" -) - -// FoldersConfig contains all the folder configurations -type FoldersConfig []FolderConfig - -// GetIdx returns the index of the folder matching id in FoldersConfig array -func (c FoldersConfig) GetIdx(id string) int { - for i := range c { - if id == c[i].ID { - return i - } - } - return -1 -} - -// Update is used to fully update or add a new FolderConfig -func (c FoldersConfig) Update(newCfg FoldersConfig) FoldersConfig { - for i := range newCfg { - found := false - for j := range c { - if newCfg[i].ID == c[j].ID { - c[j] = newCfg[i] - found = true - break - } - } - if !found { - c = append(c, newCfg[i]) - } - } - return c -} - -// Delete is used to delete a folder matching id in FoldersConfig array -func (c FoldersConfig) Delete(id string) (FoldersConfig, FolderConfig, error) { - if idx := c.GetIdx(id); idx != -1 { - f := c[idx] - c = append(c[:idx], c[idx+1:]...) - return c, f, nil - } - - return c, FolderConfig{}, fmt.Errorf("invalid id") -} diff --git a/main.go b/main.go index 060a927..65ab7a0 100644 --- a/main.go +++ b/main.go @@ -8,7 +8,6 @@ import ( "os/exec" "os/signal" "path/filepath" - "strings" "syscall" "time" @@ -48,7 +47,7 @@ type Context struct { SThg *st.SyncThing SThgCmd *exec.Cmd SThgInotCmd *exec.Cmd - MFolder *model.Folder + MFolders *model.Folders SDKs *crosssdk.SDKs WWWServer *webserver.Server Exit chan os.Signal @@ -99,7 +98,7 @@ func handlerSigTerm(ctx *Context) { ctx.Log.Infof("Stoping Web server...") ctx.WWWServer.Stop() } - os.Exit(1) + os.Exit(0) } // XDS Server application main routine @@ -112,7 +111,7 @@ func xdsApp(cliCtx *cli.Context) error { // Load config cfg, err := xdsconfig.Init(ctx.Cli, ctx.Log) if err != nil { - return cli.NewExitError(err, 2) + return cli.NewExitError(err, -2) } ctx.Config = cfg @@ -136,26 +135,24 @@ func xdsApp(cliCtx *cli.Context) error { ctx.Log.Out = fdL } - // FIXME - add a builder interface and support other builder type (eg. native) - builderType := "syncthing" - - switch builderType { - case "syncthing": - - // Start local instance of Syncthing and Syncthing-notify + // Create syncthing instance when section "syncthing" is present in config.json + if ctx.Config.FileConf.SThgConf != nil { ctx.SThg = st.NewSyncThing(ctx.Config, ctx.Log) + } + // Start local instance of Syncthing and Syncthing-notify + if ctx.SThg != nil { ctx.Log.Infof("Starting Syncthing...") ctx.SThgCmd, err = ctx.SThg.Start() if err != nil { - return cli.NewExitError(err, 2) + return cli.NewExitError(err, -4) } fmt.Printf("Syncthing started (PID %d)\n", ctx.SThgCmd.Process.Pid) ctx.Log.Infof("Starting Syncthing-inotify...") ctx.SThgInotCmd, err = ctx.SThg.StartInotify() if err != nil { - return cli.NewExitError(err, 2) + return cli.NewExitError(err, -4) } fmt.Printf("Syncthing-inotify started (PID %d)\n", ctx.SThgInotCmd.Process.Pid) @@ -174,64 +171,37 @@ func xdsApp(cliCtx *cli.Context) error { retry-- } if err != nil || retry == 0 { - return cli.NewExitError(err, 2) - } - - // Retrieve Syncthing config - id, err := ctx.SThg.IDGet() - if err != nil { - return cli.NewExitError(err, 2) - } - - if ctx.Config.Builder, err = xdsconfig.NewBuilderConfig(id); err != nil { - return cli.NewExitError(err, 2) - } - - // Retrieve initial Syncthing config - - // FIXME: cannot retrieve default SDK, need to save on disk or somewhere - // else all config to be able to restore it. - defaultSdk := "" - stCfg, err := ctx.SThg.ConfigGet() - if err != nil { - return cli.NewExitError(err, 2) + return cli.NewExitError(err, -4) } - for _, stFld := range stCfg.Folders { - relativePath := strings.TrimPrefix(stFld.RawPath, ctx.Config.FileConf.ShareRootDir) - if relativePath == "" { - relativePath = stFld.RawPath - } - newFld := xdsconfig.NewFolderConfig(stFld.ID, - stFld.Label, - ctx.Config.FileConf.ShareRootDir, - strings.TrimRight(relativePath, "/"), - defaultSdk) - ctx.Config.Folders = ctx.Config.Folders.Update(xdsconfig.FoldersConfig{newFld}) + // FIXME: do we still need Builder notion ? if no cleanup + if ctx.Config.Builder, err = xdsconfig.NewBuilderConfig(ctx.SThg.MyID); err != nil { + return cli.NewExitError(err, -4) } + } - // Init model folder - ctx.MFolder = model.NewFolder(ctx.Config, ctx.SThg) + // Init model folder + ctx.MFolders = model.FoldersNew(ctx.Config, ctx.SThg) - default: - err = fmt.Errorf("Unsupported builder type") - return cli.NewExitError(err, 3) + // Load initial folders config from disk + if err := ctx.MFolders.LoadConfig(); err != nil { + return cli.NewExitError(err, -5) } // Init cross SDKs ctx.SDKs, err = crosssdk.Init(ctx.Config, ctx.Log) if err != nil { - return cli.NewExitError(err, 2) + return cli.NewExitError(err, -6) } // Create and start Web Server - ctx.WWWServer = webserver.New(ctx.Config, ctx.MFolder, ctx.SDKs, ctx.Log) + ctx.WWWServer = webserver.New(ctx.Config, ctx.MFolders, ctx.SDKs, ctx.Log) if err = ctx.WWWServer.Serve(); err != nil { ctx.Log.Println(err) - return cli.NewExitError(err, 3) + return cli.NewExitError(err, -7) } - return cli.NewExitError("Program exited ", 4) + return cli.NewExitError("Program exited ", -99) } // main @@ -271,6 +241,11 @@ func main() { Usage: "filename where logs will be redirected (default stdout)\n\t", EnvVar: "LOG_FILENAME", }, + cli.BoolFlag{ + Name: "no-folderconfig, nfc", + Usage: fmt.Sprintf("Do not read folder config file (%s)\n\t", xdsconfig.FoldersConfigFilename), + EnvVar: "NO_FOLDERCONFIG", + }, } // only one action: Web Server diff --git a/webapp/src/app/config/config.component.html b/webapp/src/app/config/config.component.html index d9229d5..5211c2d 100644 --- a/webapp/src/app/config/config.component.html +++ b/webapp/src/app/config/config.component.html @@ -71,13 +71,24 @@
- - + + +
+
+ +
+
+ + +
@@ -91,4 +102,4 @@
{{config$ | async | json}} -
\ No newline at end of file + diff --git a/webapp/src/app/config/config.component.ts b/webapp/src/app/config/config.component.ts index 7d9931e..0df707b 100644 --- a/webapp/src/app/config/config.component.ts +++ b/webapp/src/app/config/config.component.ts @@ -7,7 +7,8 @@ import 'rxjs/add/operator/map'; import 'rxjs/add/operator/filter'; import 'rxjs/add/operator/debounceTime'; -import { ConfigService, IConfig, IProject, ProjectType, IxdsAgentPackage } from "../services/config.service"; +import { ConfigService, IConfig, IProject, ProjectType, ProjectTypes, + IxdsAgentPackage } from "../services/config.service"; import { XDSServerService, IServerStatus, IXDSAgentInfo } from "../services/xdsserver.service"; import { XDSAgentService, IAgentStatus } from "../services/xdsagent.service"; import { SyncthingService, ISyncThingStatus } from "../services/syncthing.service"; @@ -33,6 +34,7 @@ export class ConfigComponent implements OnInit { curProj: number; userEditedLabel: boolean = false; xdsAgentPackages: IxdsAgentPackage[] = []; + projectTypes = ProjectTypes; // TODO replace by reactive FormControl + add validation syncToolUrl: string; @@ -45,8 +47,8 @@ export class ConfigComponent implements OnInit { }; addProjectForm: FormGroup; - pathCtrl = new FormControl("", Validators.required); - + pathCliCtrl = new FormControl("", Validators.required); + pathSvrCtrl = new FormControl("", Validators.required); constructor( private configSvr: ConfigService, @@ -57,11 +59,16 @@ export class ConfigComponent implements OnInit { private alert: AlertService, private fb: FormBuilder ) { - // FIXME implement multi project support + // Define types (first one is special/placeholder) + this.projectTypes.unshift({value: -1, display: "--Select a type--"}); + let selectedType = this.projectTypes[0].value; + this.curProj = 0; this.addProjectForm = fb.group({ - path: this.pathCtrl, + pathCli: this.pathCliCtrl, + pathSvr: this.pathSvrCtrl, label: ["", Validators.nullValidator], + type: [selectedType, Validators.pattern("[0-9]+")], }); } @@ -82,7 +89,7 @@ export class ConfigComponent implements OnInit { }); // Auto create label name - this.pathCtrl.valueChanges + this.pathCliCtrl.valueChanges .debounceTime(100) .filter(n => n) .map(n => "Project_" + n.split('/')[0]) @@ -91,6 +98,9 @@ export class ConfigComponent implements OnInit { this.addProjectForm.patchValue({ label: value }); } }); + + // Select 1 first type by default + // SEB this.typeCtrl.setValue({type: ProjectTypes[0].value}); } onKeyLabel(event: any) { @@ -118,21 +128,24 @@ export class ConfigComponent implements OnInit { } xdsAgentRestartConn() { - let aurl = this.xdsAgentUrl; + let aUrl = this.xdsAgentUrl; this.configSvr.syncToolURL = this.syncToolUrl; - this.configSvr.xdsAgentUrl = aurl; + this.configSvr.xdsAgentUrl = aUrl; this.configSvr.loadProjects(); } onSubmit() { let formVal = this.addProjectForm.value; + let type = formVal['type'].value; + let numType = Number(formVal['type']); this.configSvr.addProject({ label: formVal['label'], - path: formVal['path'], - type: ProjectType.SYNCTHING, + pathClient: formVal['pathCli'], + pathServer: formVal['pathSvr'], + type: numType, // FIXME: allow to set defaultSdkID from New Project config panel }); } -} \ No newline at end of file +} diff --git a/webapp/src/app/devel/deploy/deploy.component.ts b/webapp/src/app/devel/deploy/deploy.component.ts index 4dba256..e51b7f2 100644 --- a/webapp/src/app/devel/deploy/deploy.component.ts +++ b/webapp/src/app/devel/deploy/deploy.component.ts @@ -37,8 +37,8 @@ export class DeployComponent implements OnInit { ngOnInit() { this.deploying = false; - if (this.curProject && this.curProject.path) { - this.deployForm.patchValue({ wgtFile: this.curProject.path }); + if (this.curProject && this.curProject.pathClient) { + this.deployForm.patchValue({ wgtFile: this.curProject.pathClient }); } } @@ -60,4 +60,4 @@ export class DeployComponent implements OnInit { this.alert.error(msg); }); } -} \ No newline at end of file +} diff --git a/webapp/src/app/projects/projectCard.component.ts b/webapp/src/app/projects/projectCard.component.ts index 7a7fa21..23e10a6 100644 --- a/webapp/src/app/projects/projectCard.component.ts +++ b/webapp/src/app/projects/projectCard.component.ts @@ -19,14 +19,23 @@ import { ConfigService, IProject, ProjectType } from "../services/config.service {{ project.id }} -  Folder path - {{ project.path}} +  Sharing type + {{ project.type | readableType }} -  Synchronization type - {{ project.type | readableType }} +  Local path + {{ project.pathClient }} - + +  Server path + {{ project.pathServer }} + + `, @@ -53,11 +62,11 @@ export class ProjectCardComponent { }) export class ProjectReadableTypePipe implements PipeTransform { - transform(type: ProjectType): string { - switch (+type) { - case ProjectType.NATIVE: return "Native"; - case ProjectType.SYNCTHING: return "Cloud (Syncthing)"; - default: return String(type); + transform(type: ProjectType): string { + switch (type) { + case ProjectType.NATIVE_PATHMAP: return "Native (path mapping)"; + case ProjectType.SYNCTHING: return "Cloud (Syncthing)"; + default: return String(type); + } } - } -} \ No newline at end of file +} diff --git a/webapp/src/app/services/config.service.ts b/webapp/src/app/services/config.service.ts index 722c347..c65332f 100644 --- a/webapp/src/app/services/config.service.ts +++ b/webapp/src/app/services/config.service.ts @@ -13,17 +13,22 @@ import 'rxjs/add/observable/throw'; import 'rxjs/add/operator/mergeMap'; -import { XDSServerService, IXDSConfigProject } from "../services/xdsserver.service"; +import { XDSServerService, IXDSFolderConfig } from "../services/xdsserver.service"; import { XDSAgentService } from "../services/xdsagent.service"; import { SyncthingService, ISyncThingProject, ISyncThingStatus } from "../services/syncthing.service"; import { AlertService, IAlert } from "../services/alert.service"; import { UtilsService } from "../services/utils.service"; export enum ProjectType { - NATIVE = 1, + NATIVE_PATHMAP = 1, SYNCTHING = 2 } +export var ProjectTypes = [ + { value: ProjectType.NATIVE_PATHMAP, display: "Path mapping" }, + { value: ProjectType.SYNCTHING, display: "Cloud Sync" } +]; + export interface INativeProject { // TODO } @@ -31,7 +36,8 @@ export interface INativeProject { export interface IProject { id?: string; label: string; - path: string; + pathClient: string; + pathServer?: string; type: ProjectType; remotePrjDef?: INativeProject | ISyncThingProject; localPrjDef?: any; @@ -172,7 +178,7 @@ export class ConfigService { let zurl = this.confStore.xdsAgentPackages && this.confStore.xdsAgentPackages.filter(elem => elem.os === os); if (zurl && zurl.length) { msg += " Download XDS-Agent tarball for " + zurl[0].os + " host OS "; - msg += ""; + msg += ""; } msg += ""; this.alert.error(msg); @@ -213,8 +219,9 @@ export class ConfigService { let pp: IProject = { id: rPrj.id, label: rPrj.label, - path: rPrj.path, - type: ProjectType.SYNCTHING, // FIXME support other types + pathClient: rPrj.path, + pathServer: rPrj.dataPathMap.serverPath, + type: rPrj.type, remotePrjDef: Object.assign({}, rPrj), localPrjDef: Object.assign({}, lPrj[0]), }; @@ -272,57 +279,46 @@ export class ConfigService { addProject(prj: IProject) { // Substitute tilde with to user home path - prj.path = prj.path.trim(); - if (prj.path.charAt(0) === '~') { - prj.path = this.confStore.localSThg.tilde + prj.path.substring(1); + let pathCli = prj.pathClient.trim(); + if (pathCli.charAt(0) === '~') { + pathCli = this.confStore.localSThg.tilde + pathCli.substring(1); // Must be a full path (on Linux or Windows) - } else if (!((prj.path.charAt(0) === '/') || - (prj.path.charAt(1) === ':' && (prj.path.charAt(2) === '\\' || prj.path.charAt(2) === '/')))) { - prj.path = this.confStore.projectsRootDir + '/' + prj.path; - } - - if (prj.id == null) { - // FIXME - must be done on server side - let prefix = this.getLabelRootName() || new Date().toISOString(); - let splath = prj.path.split('/'); - prj.id = prefix + "_" + splath[splath.length - 1]; + } else if (!((pathCli.charAt(0) === '/') || + (pathCli.charAt(1) === ':' && (pathCli.charAt(2) === '\\' || pathCli.charAt(2) === '/')))) { + pathCli = this.confStore.projectsRootDir + '/' + pathCli; } - if (this._getProjectIdx(prj.id) !== -1) { - this.alert.warning("Project already exist (id=" + prj.id + ")", true); - return; - } - - // TODO - support others project types - if (prj.type !== ProjectType.SYNCTHING) { - this.alert.error('Project type not supported yet (type: ' + prj.type + ')'); - return; - } - - let sdkPrj: IXDSConfigProject = { - id: prj.id, - label: prj.label, - path: prj.path, - hostSyncThingID: this.confStore.localSThg.ID, + let xdsPrj: IXDSFolderConfig = { + id: "", + label: prj.label || "", + path: pathCli, + type: prj.type, defaultSdkID: prj.defaultSdkID, + dataPathMap: { + serverPath: prj.pathServer, + }, + dataCloudSync: { + syncThingID: this.confStore.localSThg.ID, + } }; - // Send config to XDS server let newPrj = prj; - this.xdsServerSvr.addProject(sdkPrj) + this.xdsServerSvr.addProject(xdsPrj) .subscribe(resStRemotePrj => { newPrj.remotePrjDef = resStRemotePrj; + newPrj.id = resStRemotePrj.id; // FIXME REWORK local ST config // move logic to server side tunneling-back by WS + let stData = resStRemotePrj.dataCloudSync; // Now setup local config let stLocPrj: ISyncThingProject = { - id: sdkPrj.id, - label: sdkPrj.label, - path: sdkPrj.path, - remoteSyncThingID: resStRemotePrj.builderSThgID + id: resStRemotePrj.id, + label: xdsPrj.label, + path: xdsPrj.path, + serverSyncThingID: stData.builderSThgID }; // Set local Syncthing config @@ -366,4 +362,4 @@ export class ConfigService { return this.confStore.projects.findIndex((item) => item.id === id); } -} \ No newline at end of file +} diff --git a/webapp/src/app/services/syncthing.service.ts b/webapp/src/app/services/syncthing.service.ts index 0e8c51c..aefb039 100644 --- a/webapp/src/app/services/syncthing.service.ts +++ b/webapp/src/app/services/syncthing.service.ts @@ -16,7 +16,7 @@ import 'rxjs/add/operator/retryWhen'; export interface ISyncThingProject { id: string; path: string; - remoteSyncThingID: string; + serverSyncThingID: string; label?: string; } @@ -180,7 +180,7 @@ export class SyncthingService { return this.getID() .flatMap(() => this._getConfig()) .flatMap((stCfg) => { - let newDevID = prj.remoteSyncThingID; + let newDevID = prj.serverSyncThingID; // Add new Device if needed let dev = stCfg.devices.filter(item => item.deviceID === newDevID); diff --git a/webapp/src/app/services/xdsserver.service.ts b/webapp/src/app/services/xdsserver.service.ts index 4d20fa4..b11fe9f 100644 --- a/webapp/src/app/services/xdsserver.service.ts +++ b/webapp/src/app/services/xdsserver.service.ts @@ -20,7 +20,8 @@ import 'rxjs/add/operator/mergeMap'; export interface IXDSConfigProject { id: string; path: string; - hostSyncThingID: string; + clientSyncThingID: string; + type: number; label?: string; defaultSdkID?: string; } @@ -31,15 +32,28 @@ interface IXDSBuilderConfig { syncThingID: string; } -interface IXDSFolderConfig { +export interface IXDSFolderConfig { id: string; label: string; path: string; type: number; - syncThingID: string; - builderSThgID?: string; status?: string; defaultSdkID: string; + + // FIXME better with union but tech pb with go code + //data?: IXDSPathMapConfig|IXDSCloudSyncConfig; + dataPathMap?:IXDSPathMapConfig; + dataCloudSync?:IXDSCloudSyncConfig; +} + +export interface IXDSPathMapConfig { + // TODO + serverPath: string; +} + +export interface IXDSCloudSyncConfig { + syncThingID: string; + builderSThgID?: string; } interface IXDSConfig { @@ -172,16 +186,8 @@ export class XDSServerService { return this._get('/folders'); } - addProject(cfg: IXDSConfigProject): Observable { - let folder: IXDSFolderConfig = { - id: cfg.id || null, - label: cfg.label || "", - path: cfg.path, - type: FOLDER_TYPE_CLOUDSYNC, - syncThingID: cfg.hostSyncThingID, - defaultSdkID: cfg.defaultSdkID || "", - }; - return this._post('/folder', folder); + addProject(cfg: IXDSFolderConfig): Observable { + return this._post('/folder', cfg); } deleteProject(id: string): Observable { @@ -244,7 +250,13 @@ export class XDSServerService { private _decodeError(err: any) { let e: string; - if (typeof err === "object") { + if (err instanceof Response) { + const body = err.json() || 'Server error'; + e = body.error || JSON.stringify(body); + if (!e || e === "") { + e = `${err.status} - ${err.statusText || 'Unknown error'}`; + } + } else if (typeof err === "object") { if (err.statusText) { e = err.statusText; } else if (err.error) { @@ -253,7 +265,7 @@ export class XDSServerService { e = JSON.stringify(err); } } else { - e = err.json().error || 'Server error'; + e = err.message ? err.message : err.toString(); } return Observable.throw(e); } -- cgit 1.2.3-korg From 4feef5296bf3aea331fdde4cd7b94ee2322a907e Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Wed, 16 Aug 2017 14:24:49 +0200 Subject: Moved project creation in a modal windows Signed-off-by: Sebastien Douheret --- .vscode/settings.json | 1 - lib/folder/folder-pathmap.go | 20 ++- lib/model/folders.go | 2 +- webapp/src/app/app.module.ts | 8 +- webapp/src/app/config/config.component.css | 2 +- webapp/src/app/config/config.component.html | 78 +++++------ webapp/src/app/config/config.component.ts | 69 ++-------- .../src/app/projects/projectAddModal.component.css | 24 ++++ .../app/projects/projectAddModal.component.html | 54 ++++++++ .../src/app/projects/projectAddModal.component.ts | 142 +++++++++++++++++++++ webapp/src/app/projects/projectCard.component.ts | 12 +- webapp/src/app/sdks/sdkAddModal.component.html | 23 ++++ webapp/src/app/sdks/sdkAddModal.component.ts | 24 ++++ webapp/src/app/services/alert.service.ts | 6 +- webapp/src/app/services/config.service.ts | 85 ++++++------ webapp/src/systemjs.config.js | 3 +- 16 files changed, 403 insertions(+), 150 deletions(-) create mode 100644 webapp/src/app/projects/projectAddModal.component.css create mode 100644 webapp/src/app/projects/projectAddModal.component.html create mode 100644 webapp/src/app/projects/projectAddModal.component.ts create mode 100644 webapp/src/app/sdks/sdkAddModal.component.html create mode 100644 webapp/src/app/sdks/sdkAddModal.component.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index 60fab57..7ccd637 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,7 +12,6 @@ "webapp/dist": true, "webapp/node_modules": true }, - // Specify paths/files to ignore. (Supports Globs) "cSpell.ignorePaths": [ "**/node_modules/**", diff --git a/lib/folder/folder-pathmap.go b/lib/folder/folder-pathmap.go index 8711df2..2ad8a93 100644 --- a/lib/folder/folder-pathmap.go +++ b/lib/folder/folder-pathmap.go @@ -7,18 +7,22 @@ import ( "path/filepath" common "github.com/iotbzh/xds-common/golib" + "github.com/iotbzh/xds-server/lib/xdsconfig" ) // IFOLDER interface implementation for native/path mapping folders // PathMap . type PathMap struct { - config FolderConfig + globalConfig *xdsconfig.Config + config FolderConfig } // NewFolderPathMap Create a new instance of PathMap -func NewFolderPathMap() *PathMap { - f := PathMap{} +func NewFolderPathMap(gc *xdsconfig.Config) *PathMap { + f := PathMap{ + globalConfig: gc, + } return &f } @@ -28,8 +32,13 @@ func (f *PathMap) Add(cfg FolderConfig) (*FolderConfig, error) { return nil, fmt.Errorf("ServerPath must be set") } - // Sanity check + // Use shareRootDir if ServerPath is a relative path dir := cfg.DataPathMap.ServerPath + if !filepath.IsAbs(dir) { + dir = filepath.Join(f.globalConfig.FileConf.ShareRootDir, dir) + } + + // Sanity check if !common.Exists(dir) { // try to create if not existing if err := os.MkdirAll(dir, 0755); err != nil { @@ -52,7 +61,8 @@ func (f *PathMap) Add(cfg FolderConfig) (*FolderConfig, error) { } f.config = cfg - f.config.RootPath = cfg.DataPathMap.ServerPath + f.config.RootPath = dir + f.config.DataPathMap.ServerPath = dir f.config.Status = StatusEnable return &f.config, nil diff --git a/lib/model/folders.go b/lib/model/folders.go index 3c2457c..02c3254 100644 --- a/lib/model/folders.go +++ b/lib/model/folders.go @@ -208,7 +208,7 @@ func (f *Folders) createUpdate(newF folder.FolderConfig, create bool) (*folder.F fld = f.SThg.NewFolderST(f.Conf) // PATH MAP case folder.TypePathMap: - fld = folder.NewFolderPathMap() + fld = folder.NewFolderPathMap(f.Conf) default: return nil, fmt.Errorf("Unsupported folder type") } diff --git a/webapp/src/app/app.module.ts b/webapp/src/app/app.module.ts index 4877f6e..10ff7a4 100644 --- a/webapp/src/app/app.module.ts +++ b/webapp/src/app/app.module.ts @@ -10,6 +10,7 @@ import { ModalModule } from 'ngx-bootstrap/modal'; import { AccordionModule } from 'ngx-bootstrap/accordion'; import { CarouselModule } from 'ngx-bootstrap/carousel'; import { PopoverModule } from 'ngx-bootstrap/popover'; +import { CollapseModule } from 'ngx-bootstrap/collapse'; import { BsDropdownModule } from 'ngx-bootstrap/dropdown'; // Import the application components and services. @@ -21,9 +22,11 @@ import { DlXdsAgentComponent, CapitalizePipe } from "./config/downloadXdsAgent.c import { ProjectCardComponent } from "./projects/projectCard.component"; import { ProjectReadableTypePipe } from "./projects/projectCard.component"; import { ProjectsListAccordionComponent } from "./projects/projectsListAccordion.component"; +import { ProjectAddModalComponent} from "./projects/projectAddModal.component"; import { SdkCardComponent } from "./sdks/sdkCard.component"; import { SdksListAccordionComponent } from "./sdks/sdksListAccordion.component"; import { SdkSelectDropdownComponent } from "./sdks/sdkSelectDropdown.component"; +import { SdkAddModalComponent} from "./sdks/sdkAddModal.component"; import { HomeComponent } from "./home/home.component"; import { DevelComponent } from "./devel/devel.component"; @@ -52,6 +55,7 @@ import { SdkService } from "./services/sdk.service"; AccordionModule.forRoot(), CarouselModule.forRoot(), PopoverModule.forRoot(), + CollapseModule.forRoot(), BsDropdownModule.forRoot(), ], declarations: [ @@ -67,9 +71,11 @@ import { SdkService } from "./services/sdk.service"; ProjectCardComponent, ProjectReadableTypePipe, ProjectsListAccordionComponent, + ProjectAddModalComponent, SdkCardComponent, SdksListAccordionComponent, SdkSelectDropdownComponent, + SdkAddModalComponent, ], providers: [ AppRoutingProviders, @@ -88,4 +94,4 @@ import { SdkService } from "./services/sdk.service"; bootstrap: [AppComponent] }) export class AppModule { -} \ No newline at end of file +} diff --git a/webapp/src/app/config/config.component.css b/webapp/src/app/config/config.component.css index f480857..208ce6f 100644 --- a/webapp/src/app/config/config.component.css +++ b/webapp/src/app/config/config.component.css @@ -23,4 +23,4 @@ tr.info>th { tr.info>td { vertical-align: middle; -} \ No newline at end of file +} diff --git a/webapp/src/app/config/config.component.html b/webapp/src/app/config/config.component.html index 5211c2d..6af7f0d 100644 --- a/webapp/src/app/config/config.component.html +++ b/webapp/src/app/config/config.component.html @@ -1,11 +1,18 @@
-
-

Global Configuration

-
- -
+
+

+ Global Configuration +
+ + + +
+

-
+
@@ -50,9 +57,19 @@
-

Cross SDKs Configuration

+

+ Cross SDKs +
+ + + +
+

-
+
@@ -61,43 +78,30 @@
-

Projects Configuration

-
-
-
-
-
- -
+

+ Projects +
+ -
- - -
-
- - -
-
- - -
-
- - -
+
- - +

+
+
+ + + + +
diff --git a/webapp/src/app/config/config.component.ts b/webapp/src/app/config/config.component.ts index 0df707b..b107e81 100644 --- a/webapp/src/app/config/config.component.ts +++ b/webapp/src/app/config/config.component.ts @@ -1,19 +1,16 @@ -import { Component, OnInit } from "@angular/core"; +import { Component, ViewChild, OnInit } from "@angular/core"; import { Observable } from 'rxjs/Observable'; import { FormControl, FormGroup, Validators, FormBuilder } from '@angular/forms'; +import { CollapseModule } from 'ngx-bootstrap/collapse'; -// Import RxJs required methods -import 'rxjs/add/operator/map'; -import 'rxjs/add/operator/filter'; -import 'rxjs/add/operator/debounceTime'; - -import { ConfigService, IConfig, IProject, ProjectType, ProjectTypes, - IxdsAgentPackage } from "../services/config.service"; +import { ConfigService, IConfig, IxdsAgentPackage } from "../services/config.service"; import { XDSServerService, IServerStatus, IXDSAgentInfo } from "../services/xdsserver.service"; import { XDSAgentService, IAgentStatus } from "../services/xdsagent.service"; import { SyncthingService, ISyncThingStatus } from "../services/syncthing.service"; import { AlertService } from "../services/alert.service"; import { ISdk, SdkService } from "../services/sdk.service"; +import { ProjectAddModalComponent } from "../projects/projectAddModal.component"; +import { SdkAddModalComponent } from "../sdks/sdkAddModal.component"; @Component({ templateUrl: './app/config/config.component.html', @@ -24,6 +21,8 @@ import { ISdk, SdkService } from "../services/sdk.service"; // and from http://plnkr.co/edit/vCdjZM?p=preview export class ConfigComponent implements OnInit { + @ViewChild('childProjectModal') childProjectModal: ProjectAddModalComponent; + @ViewChild('childSdkModal') childSdkModal: SdkAddModalComponent; config$: Observable; sdks$: Observable; @@ -34,22 +33,21 @@ export class ConfigComponent implements OnInit { curProj: number; userEditedLabel: boolean = false; xdsAgentPackages: IxdsAgentPackage[] = []; - projectTypes = ProjectTypes; + + gConfigIsCollapsed: boolean = true; + sdksIsCollapsed: boolean = true; + projectsIsCollapsed: boolean = false; // TODO replace by reactive FormControl + add validation syncToolUrl: string; xdsAgentUrl: string; xdsAgentRetry: string; - projectsRootDir: string; + projectsRootDir: string; // FIXME: should be remove when projectAddModal will always return full path showApplyBtn = { // Used to show/hide Apply buttons "retry": false, "rootDir": false, }; - addProjectForm: FormGroup; - pathCliCtrl = new FormControl("", Validators.required); - pathSvrCtrl = new FormControl("", Validators.required); - constructor( private configSvr: ConfigService, private xdsServerSvr: XDSServerService, @@ -57,19 +55,7 @@ export class ConfigComponent implements OnInit { private stSvr: SyncthingService, private sdkSvr: SdkService, private alert: AlertService, - private fb: FormBuilder ) { - // Define types (first one is special/placeholder) - this.projectTypes.unshift({value: -1, display: "--Select a type--"}); - let selectedType = this.projectTypes[0].value; - - this.curProj = 0; - this.addProjectForm = fb.group({ - pathCli: this.pathCliCtrl, - pathSvr: this.pathSvrCtrl, - label: ["", Validators.nullValidator], - type: [selectedType, Validators.pattern("[0-9]+")], - }); } ngOnInit() { @@ -88,23 +74,6 @@ export class ConfigComponent implements OnInit { this.xdsAgentPackages = cfg.xdsAgentPackages; }); - // Auto create label name - this.pathCliCtrl.valueChanges - .debounceTime(100) - .filter(n => n) - .map(n => "Project_" + n.split('/')[0]) - .subscribe(value => { - if (value && !this.userEditedLabel) { - this.addProjectForm.patchValue({ label: value }); - } - }); - - // Select 1 first type by default - // SEB this.typeCtrl.setValue({type: ProjectTypes[0].value}); - } - - onKeyLabel(event: any) { - this.userEditedLabel = (this.addProjectForm.value.label !== ""); } submitGlobConf(field: string) { @@ -134,18 +103,4 @@ export class ConfigComponent implements OnInit { this.configSvr.loadProjects(); } - onSubmit() { - let formVal = this.addProjectForm.value; - - let type = formVal['type'].value; - let numType = Number(formVal['type']); - this.configSvr.addProject({ - label: formVal['label'], - pathClient: formVal['pathCli'], - pathServer: formVal['pathSvr'], - type: numType, - // FIXME: allow to set defaultSdkID from New Project config panel - }); - } - } diff --git a/webapp/src/app/projects/projectAddModal.component.css b/webapp/src/app/projects/projectAddModal.component.css new file mode 100644 index 0000000..77f73a5 --- /dev/null +++ b/webapp/src/app/projects/projectAddModal.component.css @@ -0,0 +1,24 @@ +.table-borderless>tbody>tr>td, +.table-borderless>tbody>tr>th, +.table-borderless>tfoot>tr>td, +.table-borderless>tfoot>tr>th, +.table-borderless>thead>tr>td, +.table-borderless>thead>tr>th { + border: none; +} + +tr>th { + vertical-align: middle; +} + +tr>td { + vertical-align: middle; +} + +th label { + margin-bottom: 0; +} + +td input { + width: 100%; +} diff --git a/webapp/src/app/projects/projectAddModal.component.html b/webapp/src/app/projects/projectAddModal.component.html new file mode 100644 index 0000000..dc84985 --- /dev/null +++ b/webapp/src/app/projects/projectAddModal.component.html @@ -0,0 +1,54 @@ +
+ + + + + + + + + + + + + + + + + + +
+
+
+
+
+
+
+ +
+
+ +
+
+ +
+
+
diff --git a/webapp/src/app/projects/projectAddModal.component.ts b/webapp/src/app/projects/projectAddModal.component.ts new file mode 100644 index 0000000..47e9c89 --- /dev/null +++ b/webapp/src/app/projects/projectAddModal.component.ts @@ -0,0 +1,142 @@ +import { Component, Input, ViewChild, OnInit } from '@angular/core'; +import { Observable } from 'rxjs/Observable'; +import { ModalDirective } from 'ngx-bootstrap/modal'; +import { FormControl, FormGroup, Validators, FormBuilder, ValidatorFn, AbstractControl } from '@angular/forms'; + +// Import RxJs required methods +import 'rxjs/add/operator/map'; +import 'rxjs/add/operator/filter'; +import 'rxjs/add/operator/debounceTime'; + +import { AlertService, IAlert } from "../services/alert.service"; +import { + ConfigService, IConfig, IProject, ProjectType, ProjectTypes, + IxdsAgentPackage +} from "../services/config.service"; + + +@Component({ + selector: 'project-add-modal', + templateUrl: './app/projects/projectAddModal.component.html', + styleUrls: ['./app/projects/projectAddModal.component.css'] +}) +export class ProjectAddModalComponent { + @ViewChild('childProjectModal') public childProjectModal: ModalDirective; + @Input() title?: string; + + config$: Observable; + + cancelAction: boolean = false; + userEditedLabel: boolean = false; + projectTypes = ProjectTypes; + + addProjectForm: FormGroup; + typeCtrl: FormControl; + pathCliCtrl: FormControl; + pathSvrCtrl: FormControl; + + constructor( + private alert: AlertService, + private configSvr: ConfigService, + private fb: FormBuilder + ) { + // Define types (first one is special/placeholder) + this.projectTypes.unshift({ value: -1, display: "--Select a type--" }); + + this.typeCtrl = new FormControl(this.projectTypes[0].value, Validators.pattern("[0-9]+")); + this.pathCliCtrl = new FormControl("", Validators.required); + this.pathSvrCtrl = new FormControl({ value: "", disabled: true }, [Validators.required, Validators.minLength(1)]); + + this.addProjectForm = fb.group({ + type: this.typeCtrl, + pathCli: this.pathCliCtrl, + pathSvr: this.pathSvrCtrl, + label: ["", Validators.nullValidator], + }); + } + + ngOnInit() { + this.config$ = this.configSvr.conf; + + // Auto create label name + this.pathCliCtrl.valueChanges + .debounceTime(100) + .filter(n => n) + .map(n => "Project_" + n.split('/')[0]) + .subscribe(value => { + if (value && !this.userEditedLabel) { + this.addProjectForm.patchValue({ label: value }); + } + }); + + // Handle disabling of Server path + this.typeCtrl.valueChanges + .debounceTime(500) + .subscribe(valType => { + let dis = (valType === String(ProjectType.SYNCTHING)); + this.pathSvrCtrl.reset({ value: "", disabled: dis }); + }); + } + + show() { + this.cancelAction = false; + this.childProjectModal.show(); + } + + hide() { + this.childProjectModal.hide(); + } + + onKeyLabel(event: any) { + this.userEditedLabel = (this.addProjectForm.value.label !== ""); + } + + /* FIXME: change input to file type + + + onChangeLocalProject(e) { + if e.target.files.length < 1 { + console.log('SEB NO files'); + } + let dir = e.target.files[0].webkitRelativePath; + console.log("SEB files: " + dir); + let u = URL.createObjectURL(e.target.files[0]); + } + */ + onChangeLocalProject(e) { + } + + onSubmit() { + if (this.cancelAction) { + return; + } + + let formVal = this.addProjectForm.value; + + let type = formVal['type'].value; + let numType = Number(formVal['type']); + this.configSvr.addProject({ + label: formVal['label'], + pathClient: formVal['pathCli'], + pathServer: formVal['pathSvr'], + type: numType, + // FIXME: allow to set defaultSdkID from New Project config panel + }) + .subscribe(prj => { + this.alert.info("Project " + prj.label + " successfully created."); + this.hide(); + + // Reset Value for the next creation + this.addProjectForm.reset(); + let selectedType = this.projectTypes[0].value; + this.addProjectForm.patchValue({ type: selectedType }); + + }, + err => { + this.alert.error("Configuration ERROR: " + err, 60); + this.hide(); + }); + } + +} diff --git a/webapp/src/app/projects/projectCard.component.ts b/webapp/src/app/projects/projectCard.component.ts index 23e10a6..1b89fe7 100644 --- a/webapp/src/app/projects/projectCard.component.ts +++ b/webapp/src/app/projects/projectCard.component.ts @@ -1,5 +1,6 @@ import { Component, Input, Pipe, PipeTransform } from '@angular/core'; import { ConfigService, IProject, ProjectType } from "../services/config.service"; +import { AlertService } from "../services/alert.service"; @Component({ selector: 'project-card', @@ -46,12 +47,19 @@ export class ProjectCardComponent { @Input() project: IProject; - constructor(private configSvr: ConfigService) { + constructor( + private alert: AlertService, + private configSvr: ConfigService + ) { } delete(prj: IProject) { - this.configSvr.deleteProject(prj); + this.configSvr.deleteProject(prj) + .subscribe(res => { + }, err => { + this.alert.error("Delete local ERROR: " + err); + }); } } diff --git a/webapp/src/app/sdks/sdkAddModal.component.html b/webapp/src/app/sdks/sdkAddModal.component.html new file mode 100644 index 0000000..2c07fca --- /dev/null +++ b/webapp/src/app/sdks/sdkAddModal.component.html @@ -0,0 +1,23 @@ + diff --git a/webapp/src/app/sdks/sdkAddModal.component.ts b/webapp/src/app/sdks/sdkAddModal.component.ts new file mode 100644 index 0000000..b6c8eb2 --- /dev/null +++ b/webapp/src/app/sdks/sdkAddModal.component.ts @@ -0,0 +1,24 @@ +import { Component, Input, ViewChild } from '@angular/core'; +import { ModalDirective } from 'ngx-bootstrap/modal'; + +@Component({ + selector: 'sdk-add-modal', + templateUrl: './app/sdks/sdkAddModal.component.html', +}) +export class SdkAddModalComponent { + @ViewChild('sdkChildModal') public sdkChildModal: ModalDirective; + + @Input() title?: string; + + // TODO + constructor() { + } + + show() { + this.sdkChildModal.show(); + } + + hide() { + this.sdkChildModal.hide(); + } +} diff --git a/webapp/src/app/services/alert.service.ts b/webapp/src/app/services/alert.service.ts index 9dab36a..c3cae7a 100644 --- a/webapp/src/app/services/alert.service.ts +++ b/webapp/src/app/services/alert.service.ts @@ -30,8 +30,10 @@ export class AlertService { this.uid = 0; } - public error(msg: string) { - this.add({ type: "danger", msg: msg, dismissible: true }); + public error(msg: string, dismissTime?: number) { + this.add({ + type: "danger", msg: msg, dismissible: true, dismissTimeout: dismissTime + }); } public warning(msg: string, dismissible?: boolean) { diff --git a/webapp/src/app/services/config.service.ts b/webapp/src/app/services/config.service.ts index c65332f..3b51768 100644 --- a/webapp/src/app/services/config.service.ts +++ b/webapp/src/app/services/config.service.ts @@ -277,7 +277,7 @@ export class ConfigService { return id.slice(0, 15); } - addProject(prj: IProject) { + addProject(prj: IProject): Observable { // Substitute tilde with to user home path let pathCli = prj.pathClient.trim(); if (pathCli.charAt(0) === '~') { @@ -304,57 +304,58 @@ export class ConfigService { }; // Send config to XDS server let newPrj = prj; - this.xdsServerSvr.addProject(xdsPrj) - .subscribe(resStRemotePrj => { + return this.xdsServerSvr.addProject(xdsPrj) + .flatMap(resStRemotePrj => { newPrj.remotePrjDef = resStRemotePrj; newPrj.id = resStRemotePrj.id; + newPrj.pathClient = resStRemotePrj.path; - // FIXME REWORK local ST config - // move logic to server side tunneling-back by WS - let stData = resStRemotePrj.dataCloudSync; - - // Now setup local config - let stLocPrj: ISyncThingProject = { - id: resStRemotePrj.id, - label: xdsPrj.label, - path: xdsPrj.path, - serverSyncThingID: stData.builderSThgID - }; - - // Set local Syncthing config - this.stSvr.addProject(stLocPrj) - .subscribe(resStLocalPrj => { - newPrj.localPrjDef = resStLocalPrj; - - // FIXME: maybe reduce subject to only .project - //this.confSubject.next(Object.assign({}, this.confStore).project); - this.confStore.projects.push(Object.assign({}, newPrj)); - this.confSubject.next(Object.assign({}, this.confStore)); - }, - err => { - this.alert.error("Configuration local ERROR: " + err); - }); - }, - err => { - this.alert.error("Configuration remote ERROR: " + err); + if (newPrj.type === ProjectType.SYNCTHING) { + // FIXME REWORK local ST config + // move logic to server side tunneling-back by WS + let stData = resStRemotePrj.dataCloudSync; + + // Now setup local config + let stLocPrj: ISyncThingProject = { + id: resStRemotePrj.id, + label: xdsPrj.label, + path: xdsPrj.path, + serverSyncThingID: stData.builderSThgID + }; + + // Set local Syncthing config + return this.stSvr.addProject(stLocPrj); + + } else { + newPrj.pathServer = resStRemotePrj.dataPathMap.serverPath; + return Observable.of(null); + } + }) + .map(resStLocalPrj => { + newPrj.localPrjDef = resStLocalPrj; + + // FIXME: maybe reduce subject to only .project + //this.confSubject.next(Object.assign({}, this.confStore).project); + this.confStore.projects.push(Object.assign({}, newPrj)); + this.confSubject.next(Object.assign({}, this.confStore)); + + return newPrj; }); } - deleteProject(prj: IProject) { + deleteProject(prj: IProject): Observable { let idx = this._getProjectIdx(prj.id); + let delPrj = prj; if (idx === -1) { throw new Error("Invalid project id (id=" + prj.id + ")"); } - this.xdsServerSvr.deleteProject(prj.id) - .subscribe(res => { - this.stSvr.deleteProject(prj.id) - .subscribe(res => { - this.confStore.projects.splice(idx, 1); - }, err => { - this.alert.error("Delete local ERROR: " + err); - }); - }, err => { - this.alert.error("Delete remote ERROR: " + err); + return this.xdsServerSvr.deleteProject(prj.id) + .flatMap(res => { + return this.stSvr.deleteProject(prj.id); + }) + .map(res => { + this.confStore.projects.splice(idx, 1); + return delPrj; }); } diff --git a/webapp/src/systemjs.config.js b/webapp/src/systemjs.config.js index 19fe225..15c52ba 100644 --- a/webapp/src/systemjs.config.js +++ b/webapp/src/systemjs.config.js @@ -39,6 +39,7 @@ 'ngx-bootstrap/carousel': 'npm:ngx-bootstrap/bundles/ngx-bootstrap.umd.min.js', 'ngx-bootstrap/popover': 'npm:ngx-bootstrap/bundles/ngx-bootstrap.umd.min.js', 'ngx-bootstrap/dropdown': 'npm:ngx-bootstrap/bundles/ngx-bootstrap.umd.min.js', + 'ngx-bootstrap/collapse': 'npm:ngx-bootstrap/bundles/ngx-bootstrap.umd.min.js', // other libraries 'socket.io-client': 'npm:socket.io-client/dist/socket.io.min.js' }, @@ -65,4 +66,4 @@ } } }); -})(this); \ No newline at end of file +})(this); -- cgit 1.2.3-korg From 8f44cc7217ce48f3f94c8ea3f037cdf011c4493b Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Fri, 18 Aug 2017 01:04:02 +0200 Subject: Add folder synchronization status. Also add ability to force re-synchronization. --- .vscode/settings.json | 114 +++++----- lib/apiv1/apiv1.go | 5 + lib/apiv1/events.go | 147 +++++++++++++ lib/apiv1/folders.go | 16 +- lib/folder/folder-interface.go | 21 +- lib/folder/folder-pathmap.go | 17 ++ lib/model/folders.go | 105 ++++++--- lib/syncthing/folder-st.go | 83 ++++++- lib/syncthing/st.go | 10 + lib/syncthing/stEvent.go | 242 +++++++++++++++++++++ lib/syncthing/stfolder.go | 4 +- webapp/src/app/config/config.component.css | 4 + webapp/src/app/devel/build/build.component.html | 4 +- .../src/app/projects/projectAddModal.component.ts | 16 +- webapp/src/app/projects/projectCard.component.ts | 25 ++- .../projects/projectsListAccordion.component.ts | 17 +- webapp/src/app/services/config.service.ts | 101 ++++++--- webapp/src/app/services/xdsserver.service.ts | 28 ++- 18 files changed, 813 insertions(+), 146 deletions(-) create mode 100644 lib/apiv1/events.go create mode 100644 lib/syncthing/stEvent.go diff --git a/.vscode/settings.json b/.vscode/settings.json index 7ccd637..4f2a394 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,59 +1,59 @@ // Place your settings in this file to overwrite default and user settings. { - // Configure glob patterns for excluding files and folders. - "files.exclude": { - ".tmp": true, - ".git": true, - "glide.lock": true, - "vendor": true, - "debug": true, - "bin": true, - "tools": true, - "webapp/dist": true, - "webapp/node_modules": true - }, - // Specify paths/files to ignore. (Supports Globs) - "cSpell.ignorePaths": [ - "**/node_modules/**", - "**/vscode-extension/**", - "**/.git/**", - "**/vendor/**", - ".vscode", - "typings" - ], - // Words to add to dictionary for a workspace. - "cSpell.words": [ - "apiv", - "gonic", - "devel", - "csrffound", - "Syncthing", - "STID", - "ISTCONFIG", - "socketio", - "ldflags", - "SThg", - "Intf", - "dismissible", - "rpath", - "WSID", - "sess", - "IXDS", - "xdsconfig", - "xdsserver", - "mfolder", - "inotify", - "Inot", - "pname", - "pkill", - "sdkid", - "CLOUDSYNC", - "xdsagent", - "gdbserver", - "golib", - "eows", - "mfolders", - "IFOLDER", - "flds" - ] -} + // Configure glob patterns for excluding files and folders. + "files.exclude": { + ".tmp": true, + ".git": true, + "glide.lock": true, + "vendor": true, + "debug": true, + "bin": true, + "tools": true, + "webapp/dist": true, + "webapp/node_modules": true + }, + // Specify paths/files to ignore. (Supports Globs) + "cSpell.ignorePaths": [ + "**/node_modules/**", + "**/vscode-extension/**", + "**/.git/**", + "**/vendor/**", + ".vscode", + "typings" + ], + // Words to add to dictionary for a workspace. + "cSpell.words": [ + "apiv", + "gonic", + "devel", + "csrffound", + "Syncthing", + "STID", + "ISTCONFIG", + "socketio", + "ldflags", + "SThg", + "Intf", + "dismissible", + "rpath", + "WSID", + "sess", + "IXDS", + "xdsconfig", + "xdsserver", + "mfolder", + "inotify", + "Inot", + "pname", + "pkill", + "sdkid", + "CLOUDSYNC", + "xdsagent", + "gdbserver", + "golib", + "eows", + "mfolders", + "IFOLDER", + "flds" + ] +} \ No newline at end of file diff --git a/lib/apiv1/apiv1.go b/lib/apiv1/apiv1.go index f32e53b..262f513 100644 --- a/lib/apiv1/apiv1.go +++ b/lib/apiv1/apiv1.go @@ -42,6 +42,7 @@ func New(r *gin.Engine, sess *session.Sessions, cfg *xdsconfig.Config, mfolders s.apiRouter.GET("/folders", s.getFolders) s.apiRouter.GET("/folder/:id", s.getFolder) s.apiRouter.POST("/folder", s.addFolder) + s.apiRouter.POST("/folder/sync/:id", s.syncFolder) s.apiRouter.DELETE("/folder/:id", s.delFolder) s.apiRouter.GET("/sdks", s.getSdks) @@ -54,5 +55,9 @@ func New(r *gin.Engine, sess *session.Sessions, cfg *xdsconfig.Config, mfolders s.apiRouter.POST("/exec/:id", s.execCmd) s.apiRouter.POST("/signal", s.execSignalCmd) + s.apiRouter.GET("/events", s.eventsList) + s.apiRouter.POST("/events/register", s.eventsRegister) + s.apiRouter.POST("/events/unregister", s.eventsUnRegister) + return s } diff --git a/lib/apiv1/events.go b/lib/apiv1/events.go new file mode 100644 index 0000000..da8298c --- /dev/null +++ b/lib/apiv1/events.go @@ -0,0 +1,147 @@ +package apiv1 + +import ( + "net/http" + "time" + + "github.com/iotbzh/xds-server/lib/folder" + + "github.com/gin-gonic/gin" + common "github.com/iotbzh/xds-common/golib" +) + +// EventArgs is the parameters (json format) of /events/register command +type EventRegisterArgs struct { + Name string `json:"name"` + ProjectID string `json:"filterProjectID"` +} + +type EventUnRegisterArgs struct { + Name string `json:"name"` + ID int `json:"id"` +} + +// EventMsg Message send +type EventMsg struct { + Time string `json:"time"` + Type string `json:"type"` + Folder folder.FolderConfig `json:"folder"` +} + +// EventEvent Event send in WS when an internal event (eg. Syncthing event is received) +const EventEventAll = "event:all" +const EventEventType = "event:" // following by event type + +// eventsList Registering for events that will be send over a WS +func (s *APIService) eventsList(c *gin.Context) { + +} + +// eventsRegister Registering for events that will be send over a WS +func (s *APIService) eventsRegister(c *gin.Context) { + var args EventRegisterArgs + + if c.BindJSON(&args) != nil { + common.APIError(c, "Invalid arguments") + return + } + + sess := s.sessions.Get(c) + if sess == nil { + common.APIError(c, "Unknown sessions") + return + } + + evType := "FolderStateChanged" + if args.Name != evType { + common.APIError(c, "Unsupported event name") + return + } + + /* XXX - to be removed if no plan to support "generic" event + var cbFunc st.EventsCB + cbFunc = func(ev st.Event, data *st.EventsCBData) { + + evid, _ := strconv.Atoi((*data)["id"].(string)) + ssid := (*data)["sid"].(string) + so := s.sessions.IOSocketGet(ssid) + if so == nil { + s.log.Infof("Event %s not emitted - sid: %s", ev.Type, ssid) + + // Consider that client disconnected, so unregister this event + s.mfolders.SThg.Events.UnRegister(ev.Type, evid) + return + } + + msg := EventMsg{ + Time: ev.Time, + Type: ev.Type, + Data: ev.Data, + } + + if err := (*so).Emit(EventEventAll, msg); err != nil { + s.log.Errorf("WS Emit Event : %v", err) + } + + if err := (*so).Emit(EventEventType+ev.Type, msg); err != nil { + s.log.Errorf("WS Emit Event : %v", err) + } + } + + data := make(st.EventsCBData) + data["sid"] = sess.ID + + id, err := s.mfolders.SThg.Events.Register(args.Name, cbFunc, args.ProjectID, &data) + */ + + var cbFunc folder.EventCB + cbFunc = func(cfg *folder.FolderConfig, data *folder.EventCBData) { + ssid := (*data)["sid"].(string) + so := s.sessions.IOSocketGet(ssid) + if so == nil { + //s.log.Infof("Event %s not emitted - sid: %s", ev.Type, ssid) + + // Consider that client disconnected, so unregister this event + // SEB FIXMEs.mfolders.RegisterEventChange(ev.Type) + return + } + + msg := EventMsg{ + Time: time.Now().String(), + Type: evType, + Folder: *cfg, + } + + if err := (*so).Emit(EventEventType+evType, msg); err != nil { + s.log.Errorf("WS Emit Folder StateChanged event : %v", err) + } + } + data := make(folder.EventCBData) + data["sid"] = sess.ID + + err := s.mfolders.RegisterEventChange(args.ProjectID, &cbFunc, &data) + if err != nil { + common.APIError(c, err.Error()) + return + } + + c.JSON(http.StatusOK, gin.H{"status": "OK"}) +} + +// eventsRegister Registering for events that will be send over a WS +func (s *APIService) eventsUnRegister(c *gin.Context) { + var args EventUnRegisterArgs + + if c.BindJSON(&args) != nil || args.Name == "" || args.ID < 0 { + common.APIError(c, "Invalid arguments") + return + } + /* TODO + if err := s.mfolders.SThg.Events.UnRegister(args.Name, args.ID); err != nil { + common.APIError(c, err.Error()) + return + } + c.JSON(http.StatusOK, gin.H{"status": "OK"}) + */ + common.APIError(c, "Not implemented yet") +} diff --git a/lib/apiv1/folders.go b/lib/apiv1/folders.go index f957c6d..cf56c3f 100644 --- a/lib/apiv1/folders.go +++ b/lib/apiv1/folders.go @@ -43,6 +43,21 @@ func (s *APIService) addFolder(c *gin.Context) { c.JSON(http.StatusOK, newFld) } +// syncFolder force synchronization of folder files +func (s *APIService) syncFolder(c *gin.Context) { + id := c.Param("id") + + s.log.Debugln("Sync folder id: ", id) + + err := s.mfolders.ForceSync(id) + if err != nil { + common.APIError(c, err.Error()) + return + } + + c.JSON(http.StatusOK, "") +} + // delFolder deletes folder from server config func (s *APIService) delFolder(c *gin.Context) { id := c.Param("id") @@ -55,5 +70,4 @@ func (s *APIService) delFolder(c *gin.Context) { return } c.JSON(http.StatusOK, delEntry) - } diff --git a/lib/folder/folder-interface.go b/lib/folder/folder-interface.go index b76b3f3..c04cbd7 100644 --- a/lib/folder/folder-interface.go +++ b/lib/folder/folder-interface.go @@ -14,16 +14,24 @@ const ( StatusErrorConfig = "ErrorConfig" StatusDisable = "Disable" StatusEnable = "Enable" + StatusPause = "Pause" + StatusSyncing = "Syncing" ) +type EventCBData map[string]interface{} +type EventCB func(cfg *FolderConfig, data *EventCBData) + // IFOLDER Folder interface type IFOLDER interface { - Add(cfg FolderConfig) (*FolderConfig, error) // Add a new folder - GetConfig() FolderConfig // Get folder public configuration - GetFullPath(dir string) string // Get folder full path - Remove() error // Remove a folder - Sync() error // Force folder files synchronization - IsInSync() (bool, error) // Check if folder files are in-sync + NewUID(suffix string) string // Get a new folder UUID + Add(cfg FolderConfig) (*FolderConfig, error) // Add a new folder + GetConfig() FolderConfig // Get folder public configuration + GetFullPath(dir string) string // Get folder full path + Remove() error // Remove a folder + RegisterEventChange(cb *EventCB, data *EventCBData) error // Request events registration (sent through WS) + UnRegisterEventChange() error // Un-register events + Sync() error // Force folder files synchronization + IsInSync() (bool, error) // Check if folder files are in-sync } // FolderConfig is the config for one folder @@ -33,6 +41,7 @@ type FolderConfig struct { ClientPath string `json:"path"` Type FolderType `json:"type"` Status string `json:"status"` + IsInSync bool `json:"isInSync"` DefaultSdk string `json:"defaultSdk"` // Not exported fields from REST API point of view diff --git a/lib/folder/folder-pathmap.go b/lib/folder/folder-pathmap.go index 2ad8a93..f73f271 100644 --- a/lib/folder/folder-pathmap.go +++ b/lib/folder/folder-pathmap.go @@ -8,6 +8,7 @@ import ( common "github.com/iotbzh/xds-common/golib" "github.com/iotbzh/xds-server/lib/xdsconfig" + uuid "github.com/satori/go.uuid" ) // IFOLDER interface implementation for native/path mapping folders @@ -26,6 +27,11 @@ func NewFolderPathMap(gc *xdsconfig.Config) *PathMap { return &f } +// NewUID Get a UUID +func (f *PathMap) NewUID(suffix string) string { + return uuid.NewV1().String() + "_" + suffix +} + // Add a new folder func (f *PathMap) Add(cfg FolderConfig) (*FolderConfig, error) { if cfg.DataPathMap.ServerPath == "" { @@ -63,6 +69,7 @@ func (f *PathMap) Add(cfg FolderConfig) (*FolderConfig, error) { f.config = cfg f.config.RootPath = dir f.config.DataPathMap.ServerPath = dir + f.config.IsInSync = true f.config.Status = StatusEnable return &f.config, nil @@ -87,6 +94,16 @@ func (f *PathMap) Remove() error { return nil } +// RegisterEventChange requests registration for folder change event +func (f *PathMap) RegisterEventChange(cb *EventCB, data *EventCBData) error { + return nil +} + +// UnRegisterEventChange remove registered callback +func (f *PathMap) UnRegisterEventChange() error { + return nil +} + // Sync Force folder files synchronization func (f *PathMap) Sync() error { return nil diff --git a/lib/model/folders.go b/lib/model/folders.go index 02c3254..ed0078e 100644 --- a/lib/model/folders.go +++ b/lib/model/folders.go @@ -7,13 +7,13 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/Sirupsen/logrus" common "github.com/iotbzh/xds-common/golib" "github.com/iotbzh/xds-server/lib/folder" "github.com/iotbzh/xds-server/lib/syncthing" "github.com/iotbzh/xds-server/lib/xdsconfig" - uuid "github.com/satori/go.uuid" "github.com/syncthing/syncthing/lib/sync" ) @@ -24,6 +24,12 @@ type Folders struct { Log *logrus.Logger SThg *st.SyncThing folders map[string]*folder.IFOLDER + registerCB []RegisteredCB +} + +type RegisteredCB struct { + cb *folder.EventCB + data *folder.EventCBData } // Mutex to make add/delete atomic @@ -39,6 +45,7 @@ func FoldersNew(cfg *xdsconfig.Config, st *st.SyncThing) *Folders { Log: cfg.Log, SThg: st, folders: make(map[string]*folder.IFOLDER), + registerCB: []RegisteredCB{}, } } @@ -114,12 +121,15 @@ func (f *Folders) LoadConfig() error { // Update folders f.Log.Infof("Loading initial folders config: %d folders found", len(flds)) for _, fc := range flds { - if _, err := f.createUpdate(fc, false); err != nil { + if _, err := f.createUpdate(fc, false, true); err != nil { return err } } - return nil + // Save config on disk + err := f.SaveConfig() + + return err } // SaveConfig Save folders configuration to disk @@ -164,11 +174,11 @@ func (f *Folders) getConfigArrUnsafe() []folder.FolderConfig { // Add adds a new folder func (f *Folders) Add(newF folder.FolderConfig) (*folder.FolderConfig, error) { - return f.createUpdate(newF, true) + return f.createUpdate(newF, true, false) } // CreateUpdate creates or update a folder -func (f *Folders) createUpdate(newF folder.FolderConfig, create bool) (*folder.FolderConfig, error) { +func (f *Folders) createUpdate(newF folder.FolderConfig, create bool, initial bool) (*folder.FolderConfig, error) { fcMutex.Lock() defer fcMutex.Unlock() @@ -181,23 +191,7 @@ func (f *Folders) createUpdate(newF folder.FolderConfig, create bool) (*folder.F return nil, fmt.Errorf("ClientPath must be set") } - // Allocate a new UUID - if create { - newF.ID = uuid.NewV1().String() - } - if !create && newF.ID == "" { - return nil, fmt.Errorf("Cannot update folder with null ID") - } - - // Set default value if needed - if newF.Status == "" { - newF.Status = folder.StatusDisable - } - - if newF.Label == "" { - newF.Label = filepath.Base(newF.ClientPath) + "_" + newF.ID[0:8] - } - + // Create a new folder object var fld folder.IFOLDER switch newF.Type { // SYNCTHING @@ -213,6 +207,26 @@ func (f *Folders) createUpdate(newF folder.FolderConfig, create bool) (*folder.F return nil, fmt.Errorf("Unsupported folder type") } + // Set default value if needed + if newF.Status == "" { + newF.Status = folder.StatusDisable + } + if newF.Label == "" { + newF.Label = filepath.Base(newF.ClientPath) + "_" + newF.ID[0:8] + } + + // Allocate a new UUID + if create { + i := len(newF.Label) + if i > 20 { + i = 20 + } + newF.ID = fld.NewUID(newF.Label[:i]) + } + if !create && newF.ID == "" { + return nil, fmt.Errorf("Cannot update folder with null ID") + } + // Normalize path (needed for Windows path including bashlashes) newF.ClientPath = common.PathNormalize(newF.ClientPath) @@ -224,13 +238,31 @@ func (f *Folders) createUpdate(newF folder.FolderConfig, create bool) (*folder.F return newFolder, err } - // Register folder object + // Add to folders list f.folders[newF.ID] = &fld // Save config on disk - err = f.SaveConfig() + if !initial { + if err := f.SaveConfig(); err != nil { + return newFolder, err + } + } + + // Register event change callback + for _, rcb := range f.registerCB { + if err := fld.RegisterEventChange(rcb.cb, rcb.data); err != nil { + return newFolder, err + } + } + + // Force sync after creation + // (need to defer to be sure that WS events will arrive after HTTP creation reply) + go func() { + time.Sleep(time.Millisecond * 500) + fld.Sync() + }() - return newFolder, err + return newFolder, nil } // Delete deletes a specific folder @@ -260,6 +292,29 @@ func (f *Folders) Delete(id string) (folder.FolderConfig, error) { return fld, err } +// RegisterEventChange requests registration for folder event change +func (f *Folders) RegisterEventChange(id string, cb *folder.EventCB, data *folder.EventCBData) error { + + flds := make(map[string]*folder.IFOLDER) + if id != "" { + // Register to a specific folder + flds[id] = f.Get(id) + } else { + // Register to all folders + flds = f.folders + f.registerCB = append(f.registerCB, RegisteredCB{cb: cb, data: data}) + } + + for _, fld := range flds { + err := (*fld).RegisterEventChange(cb, data) + if err != nil { + return err + } + } + + return nil +} + // ForceSync Force the synchronization of a folder func (f *Folders) ForceSync(id string) error { fc := f.Get(id) diff --git a/lib/syncthing/folder-st.go b/lib/syncthing/folder-st.go index ffcd284..da27062 100644 --- a/lib/syncthing/folder-st.go +++ b/lib/syncthing/folder-st.go @@ -6,6 +6,7 @@ import ( "github.com/iotbzh/xds-server/lib/folder" "github.com/iotbzh/xds-server/lib/xdsconfig" + uuid "github.com/satori/go.uuid" "github.com/syncthing/syncthing/lib/config" ) @@ -13,10 +14,13 @@ import ( // STFolder . type STFolder struct { - globalConfig *xdsconfig.Config - st *SyncThing - fConfig folder.FolderConfig - stfConfig config.FolderConfiguration + globalConfig *xdsconfig.Config + st *SyncThing + fConfig folder.FolderConfig + stfConfig config.FolderConfiguration + eventIDs []int + eventChangeCB *folder.EventCB + eventChangeCBData *folder.EventCBData } // NewFolderST Create a new instance of STFolder @@ -27,6 +31,15 @@ func (s *SyncThing) NewFolderST(gc *xdsconfig.Config) *STFolder { } } +// NewUID Get a UUID +func (f *STFolder) NewUID(suffix string) string { + i := len(f.st.MyID) + if i > 15 { + i = 15 + } + return uuid.NewV1().String()[:14] + f.st.MyID[:i] + "_" + suffix +} + // Add a new folder func (f *STFolder) Add(cfg folder.FolderConfig) (*folder.FolderConfig, error) { @@ -59,6 +72,16 @@ func (f *STFolder) Add(cfg folder.FolderConfig) (*folder.FolderConfig, error) { return nil, err } + // Register to events to update folder status + for _, evName := range []string{EventStateChanged, EventFolderPaused} { + evID, err := f.st.Events.Register(evName, f.cbEventState, id, nil) + if err != nil { + return nil, err + } + f.eventIDs = append(f.eventIDs, evID) + } + + f.fConfig.IsInSync = false // will be updated later by events f.fConfig.Status = folder.StatusEnable } @@ -86,6 +109,20 @@ func (f *STFolder) Remove() error { return f.st.FolderDelete(f.stfConfig.ID) } +// RegisterEventChange requests registration for folder event change +func (f *STFolder) RegisterEventChange(cb *folder.EventCB, data *folder.EventCBData) error { + f.eventChangeCB = cb + f.eventChangeCBData = data + return nil +} + +// UnRegisterEventChange remove registered callback +func (f *STFolder) UnRegisterEventChange() error { + f.eventChangeCB = nil + f.eventChangeCBData = nil + return nil +} + // Sync Force folder files synchronization func (f *STFolder) Sync() error { return f.st.FolderScan(f.stfConfig.ID, "") @@ -93,5 +130,41 @@ func (f *STFolder) Sync() error { // IsInSync Check if folder files are in-sync func (f *STFolder) IsInSync() (bool, error) { - return f.st.IsFolderInSync(f.stfConfig.ID) + sts, err := f.st.IsFolderInSync(f.stfConfig.ID) + if err != nil { + return false, err + } + f.fConfig.IsInSync = sts + return sts, nil +} + +// callback use to update IsInSync status +func (f *STFolder) cbEventState(ev Event, data *EventsCBData) { + prevSync := f.fConfig.IsInSync + prevStatus := f.fConfig.Status + + switch ev.Type { + + case EventStateChanged: + to := ev.Data["to"] + switch to { + case "scanning", "syncing": + f.fConfig.Status = folder.StatusSyncing + case "idle": + f.fConfig.Status = folder.StatusEnable + } + f.fConfig.IsInSync = (to == "idle") + + case EventFolderPaused: + if f.fConfig.Status == folder.StatusEnable { + f.fConfig.Status = folder.StatusPause + } + f.fConfig.IsInSync = false + } + + if f.eventChangeCB != nil && + (prevSync != f.fConfig.IsInSync || prevStatus != f.fConfig.Status) { + cpConf := f.fConfig + (*f.eventChangeCB)(&cpConf, f.eventChangeCBData) + } } diff --git a/lib/syncthing/st.go b/lib/syncthing/st.go index 9bdb48f..10210a4 100644 --- a/lib/syncthing/st.go +++ b/lib/syncthing/st.go @@ -42,6 +42,7 @@ type SyncThing struct { conf *xdsconfig.Config client *common.HTTPClient log *logrus.Logger + Events *Events } // ExitChan Channel used for process exit @@ -126,6 +127,9 @@ func NewSyncThing(conf *xdsconfig.Config, log *logrus.Logger) *SyncThing { conf: conf, } + // Create Events monitoring + s.Events = s.NewEventListener() + return &s } @@ -316,6 +320,12 @@ func (s *SyncThing) Connect() error { s.client.SetLogger(s.log) s.MyID, err = s.IDGet() + if err != nil { + return fmt.Errorf("ERROR: cannot retrieve ID") + } + + // Start events monitoring + err = s.Events.Start() return err } diff --git a/lib/syncthing/stEvent.go b/lib/syncthing/stEvent.go new file mode 100644 index 0000000..bf2a809 --- /dev/null +++ b/lib/syncthing/stEvent.go @@ -0,0 +1,242 @@ +package st + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/Sirupsen/logrus" +) + +// Events . +type Events struct { + MonitorTime time.Duration + Debug bool + + stop chan bool + st *SyncThing + log *logrus.Logger + cbArr map[string][]cbMap +} + +type Event struct { + Type string `json:"type"` + Time time.Time `json:"time"` + Data map[string]string `json:"data"` +} + +type EventsCBData map[string]interface{} +type EventsCB func(ev Event, cbData *EventsCBData) + +const ( + EventFolderCompletion string = "FolderCompletion" + EventFolderSummary string = "FolderSummary" + EventFolderPaused string = "FolderPaused" + EventFolderResumed string = "FolderResumed" + EventFolderErrors string = "FolderErrors" + EventStateChanged string = "StateChanged" +) + +var EventsAll string = EventFolderCompletion + "|" + + EventFolderSummary + "|" + + EventFolderPaused + "|" + + EventFolderResumed + "|" + + EventFolderErrors + "|" + + EventStateChanged + +type STEvent struct { + // Per-subscription sequential event ID. Named "id" for backwards compatibility with the REST API + SubscriptionID int `json:"id"` + // Global ID of the event across all subscriptions + GlobalID int `json:"globalID"` + Time time.Time `json:"time"` + Type string `json:"type"` + Data map[string]interface{} `json:"data"` +} + +type cbMap struct { + id int + cb EventsCB + filterID string + data *EventsCBData +} + +// NewEventListener Create a new instance of Event listener +func (s *SyncThing) NewEventListener() *Events { + _, dbg := os.LookupEnv("XDS_DEBUG_STEVENTS") // set to add more debug log + return &Events{ + MonitorTime: 100, // in Milliseconds + Debug: dbg, + stop: make(chan bool, 1), + st: s, + log: s.log, + cbArr: make(map[string][]cbMap), + } +} + +// Start starts event monitoring loop +func (e *Events) Start() error { + go e.monitorLoop() + return nil +} + +// Stop stops event monitoring loop +func (e *Events) Stop() { + e.stop <- true +} + +// Register Add a listener on an event +func (e *Events) Register(evName string, cb EventsCB, filterID string, data *EventsCBData) (int, error) { + if evName == "" || !strings.Contains(EventsAll, evName) { + return -1, fmt.Errorf("Unknown event name") + } + if data == nil { + data = &EventsCBData{} + } + + cbList := []cbMap{} + if _, ok := e.cbArr[evName]; ok { + cbList = e.cbArr[evName] + } + + id := len(cbList) + (*data)["id"] = strconv.Itoa(id) + + e.cbArr[evName] = append(cbList, cbMap{id: id, cb: cb, filterID: filterID, data: data}) + + return id, nil +} + +// UnRegister Remove a listener event +func (e *Events) UnRegister(evName string, id int) error { + cbKey, ok := e.cbArr[evName] + if !ok { + return fmt.Errorf("No event registered to such name") + } + + // FIXME - NOT TESTED + if id >= len(cbKey) { + return fmt.Errorf("Invalid id") + } else if id == len(cbKey) { + e.cbArr[evName] = cbKey[:id-1] + } else { + e.cbArr[evName] = cbKey[id : id+1] + } + + return nil +} + +// GetEvents returns the Syncthing events +func (e *Events) getEvents(since int) ([]STEvent, error) { + var data []byte + ev := []STEvent{} + url := "events" + if since != -1 { + url += "?since=" + strconv.Itoa(since) + } + if err := e.st.client.HTTPGet(url, &data); err != nil { + return ev, err + } + err := json.Unmarshal(data, &ev) + return ev, err +} + +// Loop to monitor Syncthing events +func (e *Events) monitorLoop() { + e.log.Infof("Event monitoring running...") + since := 0 + for { + select { + case <-e.stop: + e.log.Infof("Event monitoring exited") + return + + case <-time.After(e.MonitorTime * time.Millisecond): + stEvArr, err := e.getEvents(since) + if err != nil { + e.log.Errorf("Syncthing Get Events: %v", err) + continue + } + // Process events + for _, stEv := range stEvArr { + since = stEv.SubscriptionID + if e.Debug { + e.log.Warnf("ST EVENT: %d %s\n %v", stEv.GlobalID, stEv.Type, stEv) + } + + cbKey, ok := e.cbArr[stEv.Type] + if !ok { + continue + } + + evData := Event{ + Type: stEv.Type, + Time: stEv.Time, + } + + // Decode Events + // FIXME: re-define data struct for each events + // instead of map of string and use JSON marshing/unmarshing + fID := "" + evData.Data = make(map[string]string) + switch stEv.Type { + + case EventFolderCompletion: + fID = convString(stEv.Data["folder"]) + evData.Data["completion"] = convFloat64(stEv.Data["completion"]) + + case EventFolderSummary: + fID = convString(stEv.Data["folder"]) + evData.Data["needBytes"] = convInt64(stEv.Data["needBytes"]) + evData.Data["state"] = convString(stEv.Data["state"]) + + case EventFolderPaused, EventFolderResumed: + fID = convString(stEv.Data["id"]) + evData.Data["label"] = convString(stEv.Data["label"]) + + case EventFolderErrors: + fID = convString(stEv.Data["folder"]) + // TODO decode array evData.Data["errors"] = convString(stEv.Data["errors"]) + + case EventStateChanged: + fID = convString(stEv.Data["folder"]) + evData.Data["from"] = convString(stEv.Data["from"]) + evData.Data["to"] = convString(stEv.Data["to"]) + + default: + e.log.Warnf("Unsupported event type") + } + + if fID != "" { + evData.Data["id"] = fID + } + + // Call all registered callbacks + for _, c := range cbKey { + if e.Debug { + e.log.Warnf("EVENT CB fID=%s, filterID=%s", fID, c.filterID) + } + // Call when filterID is not set or when it matches + if c.filterID == "" || (fID != "" && fID == c.filterID) { + c.cb(evData, c.data) + } + } + } + } + } +} + +func convString(d interface{}) string { + return d.(string) +} + +func convFloat64(d interface{}) string { + return strconv.FormatFloat(d.(float64), 'f', -1, 64) +} + +func convInt64(d interface{}) string { + return strconv.FormatInt(d.(int64), 10) +} diff --git a/lib/syncthing/stfolder.go b/lib/syncthing/stfolder.go index bbdcc43..70ac70a 100644 --- a/lib/syncthing/stfolder.go +++ b/lib/syncthing/stfolder.go @@ -191,13 +191,11 @@ func (s *SyncThing) FolderStatus(folderID string) (*FolderStatus, error) { // IsFolderInSync Returns true when folder is in sync func (s *SyncThing) IsFolderInSync(folderID string) (bool, error) { - // FIXME better to detected FolderCompletion event (/rest/events) - // See https://docs.syncthing.net/dev/events.html sts, err := s.FolderStatus(folderID) if err != nil { return false, err } - return sts.NeedBytes == 0, nil + return sts.NeedBytes == 0 && sts.State == "idle", nil } // FolderScan Request immediate folder scan. diff --git a/webapp/src/app/config/config.component.css b/webapp/src/app/config/config.component.css index 208ce6f..2bb3fea 100644 --- a/webapp/src/app/config/config.component.css +++ b/webapp/src/app/config/config.component.css @@ -24,3 +24,7 @@ tr.info>th { tr.info>td { vertical-align: middle; } + +.panel-heading { + background: aliceblue; +} diff --git a/webapp/src/app/devel/build/build.component.html b/webapp/src/app/devel/build/build.component.html index 7f85aa6..a66231c 100644 --- a/webapp/src/app/devel/build/build.component.html +++ b/webapp/src/app/devel/build/build.component.html @@ -18,7 +18,7 @@ Project root path - + Sub-path @@ -105,4 +105,4 @@ - \ No newline at end of file + diff --git a/webapp/src/app/projects/projectAddModal.component.ts b/webapp/src/app/projects/projectAddModal.component.ts index 47e9c89..7ef5b5e 100644 --- a/webapp/src/app/projects/projectAddModal.component.ts +++ b/webapp/src/app/projects/projectAddModal.component.ts @@ -62,7 +62,17 @@ export class ProjectAddModalComponent { this.pathCliCtrl.valueChanges .debounceTime(100) .filter(n => n) - .map(n => "Project_" + n.split('/')[0]) + .map(n => { + let last = n.split('/'); + let nm = n; + if (last.length > 0) { + nm = last.pop(); + if (nm === "" && last.length > 0) { + nm = last.pop(); + } + } + return "Project_" + nm; + }) .subscribe(value => { if (value && !this.userEditedLabel) { this.addProjectForm.patchValue({ label: value }); @@ -97,10 +107,10 @@ export class ProjectAddModalComponent { onChangeLocalProject(e) { if e.target.files.length < 1 { - console.log('SEB NO files'); + console.log('NO files'); } let dir = e.target.files[0].webkitRelativePath; - console.log("SEB files: " + dir); + console.log("files: " + dir); let u = URL.createObjectURL(e.target.files[0]); } */ diff --git a/webapp/src/app/projects/projectCard.component.ts b/webapp/src/app/projects/projectCard.component.ts index 1b89fe7..a7ca9a3 100644 --- a/webapp/src/app/projects/projectCard.component.ts +++ b/webapp/src/app/projects/projectCard.component.ts @@ -8,7 +8,9 @@ import { AlertService } from "../services/alert.service";
- +
@@ -27,16 +29,18 @@ import { AlertService } from "../services/alert.service";  Local path {{ project.pathClient }} - +  Server path {{ project.pathServer }} - `, @@ -53,7 +57,6 @@ export class ProjectCardComponent { ) { } - delete(prj: IProject) { this.configSvr.deleteProject(prj) .subscribe(res => { @@ -62,6 +65,14 @@ export class ProjectCardComponent { }); } + sync(prj: IProject) { + this.configSvr.syncProject(prj) + .subscribe(res => { + }, err => { + this.alert.error("ERROR: " + err); + }); + } + } // Remove APPS. prefix if translate has failed diff --git a/webapp/src/app/projects/projectsListAccordion.component.ts b/webapp/src/app/projects/projectsListAccordion.component.ts index 1b43cea..6e697f4 100644 --- a/webapp/src/app/projects/projectsListAccordion.component.ts +++ b/webapp/src/app/projects/projectsListAccordion.component.ts @@ -5,12 +5,25 @@ import { IProject } from "../services/config.service"; @Component({ selector: 'projects-list-accordion', template: ` +
{{ prj.label }} - +
+ + + +
diff --git a/webapp/src/app/services/config.service.ts b/webapp/src/app/services/config.service.ts index 3b51768..f5e353c 100644 --- a/webapp/src/app/services/config.service.ts +++ b/webapp/src/app/services/config.service.ts @@ -29,18 +29,15 @@ export var ProjectTypes = [ { value: ProjectType.SYNCTHING, display: "Cloud Sync" } ]; -export interface INativeProject { - // TODO -} - export interface IProject { id?: string; label: string; pathClient: string; pathServer?: string; type: ProjectType; - remotePrjDef?: INativeProject | ISyncThingProject; - localPrjDef?: any; + status?: string; + isInSync?: boolean; + serverPrjDef?: IXDSFolderConfig; isExpanded?: boolean; visible?: boolean; defaultSdkID?: string; @@ -139,6 +136,17 @@ export class ConfigService { ); this.confSubject.next(Object.assign({}, this.confStore)); }); + + // Update Project data + this.xdsServerSvr.FolderStateChange$.subscribe(prj => { + let i = this._getProjectIdx(prj.id); + if (i >= 0) { + // XXX for now, only isInSync and status may change + this.confStore.projects[i].isInSync = prj.isInSync; + this.confStore.projects[i].status = prj.status; + this.confSubject.next(Object.assign({}, this.confStore)); + } + }); } // Save config into cookie @@ -215,17 +223,8 @@ export class ConfigService { this.stSvr.getProjects().subscribe(localPrj => { remotePrj.forEach(rPrj => { let lPrj = localPrj.filter(item => item.id === rPrj.id); - if (lPrj.length > 0) { - let pp: IProject = { - id: rPrj.id, - label: rPrj.label, - pathClient: rPrj.path, - pathServer: rPrj.dataPathMap.serverPath, - type: rPrj.type, - remotePrjDef: Object.assign({}, rPrj), - localPrjDef: Object.assign({}, lPrj[0]), - }; - this.confStore.projects.push(pp); + if (lPrj.length > 0 || rPrj.type === ProjectType.NATIVE_PATHMAP) { + this._addProject(rPrj, true); } }); this.confSubject.next(Object.assign({}, this.confStore)); @@ -306,18 +305,15 @@ export class ConfigService { let newPrj = prj; return this.xdsServerSvr.addProject(xdsPrj) .flatMap(resStRemotePrj => { - newPrj.remotePrjDef = resStRemotePrj; - newPrj.id = resStRemotePrj.id; - newPrj.pathClient = resStRemotePrj.path; - - if (newPrj.type === ProjectType.SYNCTHING) { + xdsPrj = resStRemotePrj; + if (xdsPrj.type === ProjectType.SYNCTHING) { // FIXME REWORK local ST config // move logic to server side tunneling-back by WS - let stData = resStRemotePrj.dataCloudSync; + let stData = xdsPrj.dataCloudSync; // Now setup local config let stLocPrj: ISyncThingProject = { - id: resStRemotePrj.id, + id: xdsPrj.id, label: xdsPrj.label, path: xdsPrj.path, serverSyncThingID: stData.builderSThgID @@ -327,18 +323,11 @@ export class ConfigService { return this.stSvr.addProject(stLocPrj); } else { - newPrj.pathServer = resStRemotePrj.dataPathMap.serverPath; return Observable.of(null); } }) .map(resStLocalPrj => { - newPrj.localPrjDef = resStLocalPrj; - - // FIXME: maybe reduce subject to only .project - //this.confSubject.next(Object.assign({}, this.confStore).project); - this.confStore.projects.push(Object.assign({}, newPrj)); - this.confSubject.next(Object.assign({}, this.confStore)); - + this._addProject(xdsPrj); return newPrj; }); } @@ -351,7 +340,10 @@ export class ConfigService { } return this.xdsServerSvr.deleteProject(prj.id) .flatMap(res => { - return this.stSvr.deleteProject(prj.id); + if (prj.type === ProjectType.SYNCTHING) { + return this.stSvr.deleteProject(prj.id); + } + return Observable.of(null); }) .map(res => { this.confStore.projects.splice(idx, 1); @@ -359,8 +351,51 @@ export class ConfigService { }); } + syncProject(prj: IProject): Observable { + let idx = this._getProjectIdx(prj.id); + if (idx === -1) { + throw new Error("Invalid project id (id=" + prj.id + ")"); + } + return this.xdsServerSvr.syncProject(prj.id); + } + private _getProjectIdx(id: string): number { return this.confStore.projects.findIndex((item) => item.id === id); } + private _addProject(rPrj: IXDSFolderConfig, noNext?: boolean) { + + // Convert XDSFolderConfig to IProject + let pp: IProject = { + id: rPrj.id, + label: rPrj.label, + pathClient: rPrj.path, + pathServer: rPrj.dataPathMap.serverPath, + type: rPrj.type, + status: rPrj.status, + isInSync: rPrj.isInSync, + defaultSdkID: rPrj.defaultSdkID, + serverPrjDef: Object.assign({}, rPrj), // do a copy + }; + + // add new project + this.confStore.projects.push(pp); + + // sort project array + this.confStore.projects.sort((a, b) => { + if (a.label < b.label) { + return -1; + } + if (a.label > b.label) { + return 1; + } + return 0; + }); + + // FIXME: maybe reduce subject to only .project + //this.confSubject.next(Object.assign({}, this.confStore).project); + if (!noNext) { + this.confSubject.next(Object.assign({}, this.confStore)); + } + } } diff --git a/webapp/src/app/services/xdsserver.service.ts b/webapp/src/app/services/xdsserver.service.ts index b11fe9f..b69a196 100644 --- a/webapp/src/app/services/xdsserver.service.ts +++ b/webapp/src/app/services/xdsserver.service.ts @@ -38,12 +38,13 @@ export interface IXDSFolderConfig { path: string; type: number; status?: string; + isInSync?: boolean; defaultSdkID: string; // FIXME better with union but tech pb with go code //data?: IXDSPathMapConfig|IXDSCloudSyncConfig; - dataPathMap?:IXDSPathMapConfig; - dataCloudSync?:IXDSCloudSyncConfig; + dataPathMap?: IXDSPathMapConfig; + dataCloudSync?: IXDSCloudSyncConfig; } export interface IXDSPathMapConfig { @@ -106,8 +107,10 @@ export class XDSServerService { public CmdOutput$ = >new Subject(); public CmdExit$ = >new Subject(); + public FolderStateChange$ = >new Subject(); public Status$: Observable; + private baseUrl: string; private wsUrl: string; private _status = { WS_connected: false }; @@ -127,6 +130,7 @@ export class XDSServerService { } else { this.wsUrl = 'ws://' + re[1]; this._handleIoSocket(); + this._RegisterEvents(); } } @@ -172,6 +176,22 @@ export class XDSServerService { this.CmdExit$.next(Object.assign({}, data)); }); + this.socket.on('event:FolderStateChanged', ev => { + if (ev && ev.folder) { + this.FolderStateChange$.next(Object.assign({}, ev.folder)); + } + }); + } + + private _RegisterEvents() { + let ev = "FolderStateChanged"; + this._post('/events/register', { "name": ev }) + .subscribe( + res => { }, + error => { + this.alert.error("ERROR while registering events " + ev + ": ", error); + } + ); } getSdks(): Observable { @@ -194,6 +214,10 @@ export class XDSServerService { return this._delete('/folder/' + id); } + syncProject(id: string): Observable { + return this._post('/folder/sync/' + id, {}); + } + exec(prjID: string, dir: string, cmd: string, sdkid?: string, args?: string[], env?: string[]): Observable { return this._post('/exec', { -- cgit 1.2.3-korg From 88bd003f0e4d8e460e3cbd3583c94e6b3f82646f Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Tue, 22 Aug 2017 00:22:23 +0200 Subject: Used fix version of xds-common package Signed-off-by: Sebastien Douheret --- glide.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glide.yaml b/glide.yaml index 54cfc78..5d813f3 100644 --- a/glide.yaml +++ b/glide.yaml @@ -25,7 +25,7 @@ import: - package: github.com/satori/go.uuid version: ^1.1.0 - package: github.com/iotbzh/xds-common - version: master + version: 363bac39b844 subpackages: - golib/common - golib/eows -- cgit 1.2.3-korg From 6691c9f7c53dc765b5a333a2db84ce692bd2b44d Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Thu, 24 Aug 2017 14:20:14 +0200 Subject: Disabled/Greyed project when not usable. Not usable means sources are out of sync or project is not enable. --- webapp/src/app/devel/devel.component.css | 5 +++++ webapp/src/app/devel/devel.component.html | 7 +++++-- webapp/src/app/services/config.service.ts | 17 +++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/webapp/src/app/devel/devel.component.css b/webapp/src/app/devel/devel.component.css index 40d6fec..4b03dcb 100644 --- a/webapp/src/app/devel/devel.component.css +++ b/webapp/src/app/devel/devel.component.css @@ -12,3 +12,8 @@ .table-borderless>thead>tr>th { border: none; } + +a.dropdown-item.disabled { + pointer-events:none; + opacity:0.4; +} diff --git a/webapp/src/app/devel/devel.component.html b/webapp/src/app/devel/devel.component.html index feac413..0b90f8e 100644 --- a/webapp/src/app/devel/devel.component.html +++ b/webapp/src/app/devel/devel.component.html @@ -10,8 +10,11 @@ {{curPrj.label}} @@ -34,4 +37,4 @@ --> - \ No newline at end of file + diff --git a/webapp/src/app/services/config.service.ts b/webapp/src/app/services/config.service.ts index f5e353c..4501add 100644 --- a/webapp/src/app/services/config.service.ts +++ b/webapp/src/app/services/config.service.ts @@ -29,6 +29,14 @@ export var ProjectTypes = [ { value: ProjectType.SYNCTHING, display: "Cloud Sync" } ]; +export var ProjectStatus = { + ErrorConfig: "ErrorConfig", + Disable: "Disable", + Enable: "Enable", + Pause: "Pause", + Syncing: "Syncing" +}; + export interface IProject { id?: string; label: string; @@ -37,6 +45,7 @@ export interface IProject { type: ProjectType; status?: string; isInSync?: boolean; + isUsable?: boolean; serverPrjDef?: IXDSFolderConfig; isExpanded?: boolean; visible?: boolean; @@ -144,6 +153,7 @@ export class ConfigService { // XXX for now, only isInSync and status may change this.confStore.projects[i].isInSync = prj.isInSync; this.confStore.projects[i].status = prj.status; + this.confStore.projects[i].isUsable = this._isUsableProject(prj); this.confSubject.next(Object.assign({}, this.confStore)); } }); @@ -359,6 +369,12 @@ export class ConfigService { return this.xdsServerSvr.syncProject(prj.id); } + private _isUsableProject(p) { + return p && p.isInSync && + (p.status === ProjectStatus.Enable) && + (p.status !== ProjectStatus.Syncing); + } + private _getProjectIdx(id: string): number { return this.confStore.projects.findIndex((item) => item.id === id); } @@ -374,6 +390,7 @@ export class ConfigService { type: rPrj.type, status: rPrj.status, isInSync: rPrj.isInSync, + isUsable: this._isUsableProject(rPrj), defaultSdkID: rPrj.defaultSdkID, serverPrjDef: Object.assign({}, rPrj), // do a copy }; -- cgit 1.2.3-korg From 2fed63ed0087df0c79f7f30f7f397611381bfccd Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Thu, 24 Aug 2017 17:09:57 +0200 Subject: Fixed /exec command (don't prefix command with exec binary). adding 'exec' prevents to use "script" syntax: xds-exec -c xds-config.env -- "cd build && cmake .." But 'exec' binary is mandatory to allow to pass correctly signals to subprocess such as gdb. As workaround, exec is set for now on client side (eg. in xds-gdb). --- lib/apiv1/exec.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/apiv1/exec.go b/lib/apiv1/exec.go index 4a591be..6300dba 100644 --- a/lib/apiv1/exec.go +++ b/lib/apiv1/exec.go @@ -135,10 +135,13 @@ func (s *APIService) execCmd(c *gin.Context) { } } - // FIXME - SEB: exec prevents to use syntax: - // xds-exec -l debug -c xds-config.env -- "cd build && cmake .." cmd = append(cmd, "cd", folder.GetFullPath(args.RPath)) - cmd = append(cmd, "&&", "exec", args.Cmd) + // FIXME - add 'exec' prevents to use syntax: + // xds-exec -l debug -c xds-config.env -- "cd build && cmake .." + // but exec is mandatory to allow to pass correctly signals + // As workaround, exec is set for now on client side (eg. in xds-gdb) + //cmd = append(cmd, "&&", "exec", args.Cmd) + cmd = append(cmd, "&&", args.Cmd) // Process command arguments cmdArgs := make([]string, len(args.Args)+1) -- cgit 1.2.3-korg From 347bd1674bbf67ccb6209951a4bf8f2971715532 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Thu, 24 Aug 2017 21:23:40 +0200 Subject: Redirect HTTP and Gin server logs into a file (xds-server-verbose.log). Signed-off-by: Sebastien Douheret --- .vscode/launch.json | 19 +------------------ glide.yaml | 2 +- lib/syncthing/st.go | 5 ++++- lib/webserver/server.go | 8 ++++---- lib/xdsconfig/config.go | 8 +++++--- main.go | 36 ++++++++++++++++++++++++------------ 6 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 3637b39..5583251 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -16,7 +16,7 @@ "args": ["-log", "debug", "-c", "config.json.in"], "showLog": false }, -{ + { "name": "XDS-Server local dev", "type": "go", "request": "launch", @@ -31,23 +31,6 @@ }, "args": ["-log", "debug", "-c", "__config_local_dev.json"], "showLog": false - }, - { - "name": "XDS-Server IN DOCKER", - "type": "go", - "request": "launch", - "mode": "debug", - "port": 22000, - "host": "172.17.0.2", - "remotePath": "/xds/src/github.com/iotbzh/xds-server/bin/xds-server", - "program": "${workspaceRoot}", - "env": { - "GOPATH": "${workspaceRoot}/../../../..:${env:GOPATH}", - "ROOT_DIR": "${workspaceRoot}/../../../.." - }, - "args": [], - "showLog": true } - ] } diff --git a/glide.yaml b/glide.yaml index 5d813f3..e017281 100644 --- a/glide.yaml +++ b/glide.yaml @@ -25,7 +25,7 @@ import: - package: github.com/satori/go.uuid version: ^1.1.0 - package: github.com/iotbzh/xds-common - version: 363bac39b844 + version: 4b8e35b6786b subpackages: - golib/common - golib/eows diff --git a/lib/syncthing/st.go b/lib/syncthing/st.go index 10210a4..b622970 100644 --- a/lib/syncthing/st.go +++ b/lib/syncthing/st.go @@ -317,7 +317,10 @@ func (s *SyncThing) Connect() error { return fmt.Errorf("ERROR: cannot connect to Syncthing (null client)") } - s.client.SetLogger(s.log) + // Redirect HTTP log into a file + s.client.SetLogLevel(s.conf.Log.Level.String()) + s.client.LoggerPrefix = "SYNCTHING: " + s.client.LoggerOut = s.conf.LogVerboseOut s.MyID, err = s.IDGet() if err != nil { diff --git a/lib/webserver/server.go b/lib/webserver/server.go index 5183208..8639b66 100644 --- a/lib/webserver/server.go +++ b/lib/webserver/server.go @@ -46,10 +46,10 @@ func New(cfg *xdsconfig.Config, mfolders *model.Folders, sdks *crosssdk.SDKs, lo gin.SetMode(gin.ReleaseMode) } - // Redirect gin logs into logrus logger - gin.DefaultWriter = logr.Out - gin.DefaultErrorWriter = logr.Out - log.SetOutput(logr.Out) + // Redirect gin logs into another logger (LogVerboseOut may be stderr or a file) + gin.DefaultWriter = cfg.LogVerboseOut + gin.DefaultErrorWriter = cfg.LogVerboseOut + log.SetOutput(cfg.LogVerboseOut) // FIXME - fix pb about isTerminal=false when out is in VSC Debug Console diff --git a/lib/xdsconfig/config.go b/lib/xdsconfig/config.go index a3e5a7e..82ca97f 100644 --- a/lib/xdsconfig/config.go +++ b/lib/xdsconfig/config.go @@ -2,6 +2,7 @@ package xdsconfig import ( "fmt" + "io" "os" "github.com/Sirupsen/logrus" @@ -17,9 +18,10 @@ type Config struct { Builder BuilderConfig `json:"builder"` // Private (un-exported fields in REST GET /config route) - Options Options `json:"-"` - FileConf FileConfig `json:"-"` - Log *logrus.Logger `json:"-"` + Options Options `json:"-"` + FileConf FileConfig `json:"-"` + Log *logrus.Logger `json:"-"` + LogVerboseOut io.Writer `json:"-"` } // Options set at the command line diff --git a/main.go b/main.go index 65ab7a0..4fd49e9 100644 --- a/main.go +++ b/main.go @@ -117,22 +117,34 @@ func xdsApp(cliCtx *cli.Context) error { // Logs redirected into a file when logsDir is set logfilename := cliCtx.GlobalString("logfile") - if ctx.Config.FileConf.LogsDir != "" && logfilename != "stdout" { - if logfilename == "" { - logfilename = "xds-server.log" - } - // is it an absolute path ? - logFile := logfilename - if logfilename[0] == '.' || logfilename[0] != '/' { - logFile = filepath.Join(ctx.Config.FileConf.LogsDir, logfilename) + ctx.Config.LogVerboseOut = os.Stderr + if ctx.Config.FileConf.LogsDir != "" { + if logfilename != "stdout" { + if logfilename == "" { + logfilename = "xds-server.log" + } + // is it an absolute path ? + logFile := logfilename + if logfilename[0] == '.' || logfilename[0] != '/' { + logFile = filepath.Join(ctx.Config.FileConf.LogsDir, logfilename) + } + fmt.Printf("Logging file: %s\n", logFile) + fdL, err := os.OpenFile(logFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err != nil { + msgErr := fmt.Sprintf("Cannot create log file %s", logFile) + return cli.NewExitError(msgErr, int(syscall.EPERM)) + } + ctx.Log.Out = fdL } - fmt.Printf("Logging file: %s\n", logFile) - fdL, err := os.OpenFile(logFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + + logFileHTTPReq := filepath.Join(ctx.Config.FileConf.LogsDir, "xds-server-verbose.log") + fmt.Printf("Logging file for HTTP requests: %s\n", logFileHTTPReq) + fdLH, err := os.OpenFile(logFileHTTPReq, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) if err != nil { - msgErr := fmt.Sprintf("Cannot create log file %s", logFile) + msgErr := fmt.Sprintf("Cannot create log file %s", logFileHTTPReq) return cli.NewExitError(msgErr, int(syscall.EPERM)) } - ctx.Log.Out = fdL + ctx.Config.LogVerboseOut = fdLH } // Create syncthing instance when section "syncthing" is present in config.json -- cgit 1.2.3-korg From 3f0a0e6f9cf9e5963d89c6d1be515d8f43ee08bb Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Thu, 24 Aug 2017 21:45:32 +0200 Subject: Take care of ST connection lost in ST event monitor. --- lib/syncthing/st.go | 16 ++++++++++------ lib/syncthing/stEvent.go | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/lib/syncthing/st.go b/lib/syncthing/st.go index b622970..5086994 100644 --- a/lib/syncthing/st.go +++ b/lib/syncthing/st.go @@ -27,12 +27,13 @@ import ( // SyncThing . type SyncThing struct { - BaseURL string - APIKey string - Home string - STCmd *exec.Cmd - STICmd *exec.Cmd - MyID string + BaseURL string + APIKey string + Home string + STCmd *exec.Cmd + STICmd *exec.Cmd + MyID string + Connected bool // Private fields binDir string @@ -301,6 +302,7 @@ func (s *SyncThing) StopInotify() { // Connect Establish HTTP connection with Syncthing func (s *SyncThing) Connect() error { var err error + s.Connected = false s.client, err = common.HTTPNewClient(s.BaseURL, common.HTTPClientConfig{ URLPrefix: "/rest", @@ -327,6 +329,8 @@ func (s *SyncThing) Connect() error { return fmt.Errorf("ERROR: cannot retrieve ID") } + s.Connected = true + // Start events monitoring err = s.Events.Start() diff --git a/lib/syncthing/stEvent.go b/lib/syncthing/stEvent.go index bf2a809..9ca8b78 100644 --- a/lib/syncthing/stEvent.go +++ b/lib/syncthing/stEvent.go @@ -148,6 +148,8 @@ func (e *Events) getEvents(since int) ([]STEvent, error) { func (e *Events) monitorLoop() { e.log.Infof("Event monitoring running...") since := 0 + cntErrConn := 0 + cntErrRetry := 1 for { select { case <-e.stop: @@ -155,11 +157,32 @@ func (e *Events) monitorLoop() { return case <-time.After(e.MonitorTime * time.Millisecond): + + if !e.st.Connected { + cntErrConn++ + time.Sleep(time.Second) + if cntErrConn > cntErrRetry { + e.log.Error("ST Event monitor: ST connection down") + cntErrConn = 0 + cntErrRetry *= 2 + if _, err := e.getEvents(since); err == nil { + e.st.Connected = true + cntErrRetry = 1 + // XXX - should we reset since value ? + goto readEvent + } + } + continue + } + + readEvent: stEvArr, err := e.getEvents(since) if err != nil { e.log.Errorf("Syncthing Get Events: %v", err) + e.st.Connected = false continue } + // Process events for _, stEv := range stEvArr { since = stEv.SubscriptionID -- cgit 1.2.3-korg From 0a9826806b26fd179b739bd2c1ca9fcb114e321f Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Thu, 24 Aug 2017 23:18:01 +0200 Subject: Add IoT logo and change top bar color. --- webapp/assets/images/iot-bzh-logo-small.png | Bin 0 -> 14449 bytes webapp/src/app/app.component.css | 22 +++++-- webapp/src/app/app.component.html | 7 +- webapp/src/app/config/config.component.css | 5 ++ webapp/src/app/config/config.component.html | 9 +-- webapp/src/app/devel/build/build.component.css | 9 ++- webapp/src/app/devel/build/build.component.html | 83 +++++++++++++----------- webapp/src/app/devel/build/build.component.ts | 13 ++-- webapp/src/app/devel/devel.component.ts | 7 +- webapp/src/index.html | 5 +- 10 files changed, 98 insertions(+), 62 deletions(-) create mode 100644 webapp/assets/images/iot-bzh-logo-small.png diff --git a/webapp/assets/images/iot-bzh-logo-small.png b/webapp/assets/images/iot-bzh-logo-small.png new file mode 100644 index 0000000..2c3b2ae Binary files /dev/null and b/webapp/assets/images/iot-bzh-logo-small.png differ diff --git a/webapp/src/app/app.component.css b/webapp/src/app/app.component.css index 0ec4936..a47ad13 100644 --- a/webapp/src/app/app.component.css +++ b/webapp/src/app/app.component.css @@ -1,13 +1,18 @@ -.navbar-inverse { - background-color: #330066; +.navbar { + background-color: whitesmoke; } .navbar-brand { - background: #330066; - color: white; font-size: x-large; + font-variant: small-caps; + color: #5a28a1; } +a.navbar-brand { + margin-top: 5px; +} + + .navbar-nav ul li a { color: #fff; } @@ -15,3 +20,12 @@ .menu-text { color: #fff; } + +#logo-iot { + padding: 0 2px; + height: 60px; +} + +li>a { + color:#5a28a1; +} diff --git a/webapp/src/app/app.component.html b/webapp/src/app/app.component.html index b02dbe2..a889b12 100644 --- a/webapp/src/app/app.component.html +++ b/webapp/src/app/app.component.html @@ -1,4 +1,5 @@ -