Add logsDir setting and more

 - add logsDir setting in config.json
 - redirect Syncthing and Syncthing-inotify into log files
 - Use autogenerated Syncthing apikey if gui-apikey not set in config.json

10 files changed, 187 insertions, 79 deletions

diff --git a/README.md b/README.md
index 524ad14..ad8e6bb 100644
--- a/README.md
+++ b/README.md
@@ -51,10 +51,12 @@ Supported fields in configuration file are:
 ```json
 {
     "httpPort": "http port of agent REST interface",
+    "logsDir": "directory to store logs (eg. syncthing output)",
     "syncthing": {
         "binDir": "syncthing binaries directory (use xds-agent executable dir when not set)",
         "home": "syncthing home directory (usually .../syncthing-config)",
-        "gui-address": "syncthing gui url (default http://localhost:8384)"
+        "gui-address": "syncthing gui url (default http://localhost:8384)",
+        "gui-apikey": "syncthing api-key to use (default auto-generated)"
     }
 }
 ```
diff --git a/agent-config.json.in b/agent-config.json.in
index a0c0ad5..3ee736d 100644
--- a/agent-config.json.in
+++ b/agent-config.json.in
@@ -1,4 +1,5 @@
 {
+    "logsDir": "/tmp/xds-agent/logs",
     "syncthing": {
         "binDir": "./bin",
         "home": "${ROOT_DIR}/tmp/local_dev/syncthing-config",
diff --git a/lib/common/filepath.go b/lib/common/filepath.go
new file mode 100644
index 0000000..603c2a2
--- /dev/null
+++ b/lib/common/filepath.go
@@ -0,0 +1,15 @@
+package common
+
+import "os"
+
+// Exists returns whether the given file or directory exists or not
+func Exists(path string) bool {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true
+	}
+	if os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
diff --git a/lib/common/httpclient.go b/lib/common/httpclient.go
index 40d7bc2..72132bf 100644
--- a/lib/common/httpclient.go
+++ b/lib/common/httpclient.go
@@ -9,6 +9,8 @@ import (
 	"io/ioutil"
 	"net/http"
 	"strings"
+
+	"github.com/Sirupsen/logrus"
 )
 
 type HTTPClient struct {
@@ -20,6 +22,7 @@ type HTTPClient struct {
 	id         string
 	csrf       string
 	conf       HTTPClientConfig
+	logger     *logrus.Logger
 }
 
 type HTTPClientConfig struct {
@@ -30,6 +33,13 @@ type HTTPClientConfig struct {
 	CsrfDisable         bool
 }
 
+const (
+	logError   = 1
+	logWarning = 2
+	logInfo    = 3
+	logDebug   = 4
+)
+
 // Inspired by syncthing/cmd/cli
 
 const insecure = false
@@ -64,6 +74,30 @@ func HTTPNewClient(baseURL string, cfg HTTPClientConfig) (*HTTPClient, error) {
 	return &client, nil
 }
 
+// SetLogger Define the logger to use
+func (c *HTTPClient) SetLogger(log *logrus.Logger) {
+	c.logger = log
+}
+
+func (c *HTTPClient) log(level int, format string, args ...interface{}) {
+	if c.logger != nil {
+		switch level {
+		case logError:
+			c.logger.Errorf(format, args...)
+			break
+		case logWarning:
+			c.logger.Warningf(format, args...)
+			break
+		case logInfo:
+			c.logger.Infof(format, args...)
+			break
+		default:
+			c.logger.Debugf(format, args...)
+			break
+		}
+	}
+}
+
 // Send request to retrieve Client id and/or CSRF token
 func (c *HTTPClient) getCidAndCsrf() error {
 	request, err := http.NewRequest("GET", c.endpoint, nil)
@@ -171,6 +205,8 @@ func (c *HTTPClient) handleRequest(request *http.Request) (*http.Response, error
 		request.Header.Set("X-CSRF-Token-"+c.id[:5], c.csrf)
 	}
 
+	c.log(logDebug, "HTTP %s %v", request.Method, request.URL)
+
 	response, err := c.httpClient.Do(request)
 	if err != nil {
 		return nil, err
diff --git a/lib/session/session.go b/lib/session/session.go
index af05daa..b56f9ff 100644
--- a/lib/session/session.go
+++ b/lib/session/session.go
@@ -205,8 +205,8 @@ func (s *Sessions) monitorSessMap() {
 			s.log.Debugln("Stop monitorSessMap")
 			return
 		case <-time.After(sessionMonitorTime * time.Second):
-			s.log.Debugf("Sessions Map size: %d", len(s.sessMap))
 			if dbgFullTrace {
+				s.log.Debugf("Sessions Map size: %d", len(s.sessMap))
 				s.log.Debugf("Sessions Map : %v", s.sessMap)
 			}
 
diff --git a/lib/syncthing/st.go b/lib/syncthing/st.go
index e513876..5976e2f 100644
--- a/lib/syncthing/st.go
+++ b/lib/syncthing/st.go
@@ -2,9 +2,12 @@ package st
 
 import (
 	"encoding/json"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
+	"regexp"
 	"strings"
 	"syscall"
 	"time"
@@ -22,47 +25,65 @@ import (
 // SyncThing .
 type SyncThing struct {
 	BaseURL string
-	ApiKey  string
+	APIKey  string
 	Home    string
 	STCmd   *exec.Cmd
 	STICmd  *exec.Cmd
 
 	// Private fields
 	binDir      string
+	logsDir     string
 	exitSTChan  chan ExitChan
 	exitSTIChan chan ExitChan
 	client      *common.HTTPClient
 	log         *logrus.Logger
 }
 
-// Monitor process exit
+// ExitChan Channel used for process exit
 type ExitChan struct {
 	status int
 	err    error
 }
 
 // NewSyncThing creates a new instance of Syncthing
-//func NewSyncThing(url string, apiKey string, home string, log *logrus.Logger) *SyncThing {
-func NewSyncThing(conf *xdsconfig.SyncThingConf, log *logrus.Logger) *SyncThing {
-	url := conf.GuiAddress
-	apiKey := conf.GuiAPIKey
-	home := conf.Home
+func NewSyncThing(conf *xdsconfig.Config, log *logrus.Logger) *SyncThing {
+	var url, apiKey, home, binDir string
+	var err error
+
+	stCfg := conf.FileConf.SThgConf
+	if stCfg != nil {
+		url = stCfg.GuiAddress
+		apiKey = stCfg.GuiAPIKey
+		home = stCfg.Home
+		binDir = stCfg.BinDir
+	}
+
+	if url == "" {
+		url = "http://localhost:8384"
+	}
+	if url[0:7] != "http://" {
+		url = "http://" + url
+	}
+
+	if home == "" {
+		home = "/mnt/share"
+	}
+
+	if binDir == "" {
+		if binDir, err = filepath.Abs(filepath.Dir(os.Args[0])); err != nil {
+			binDir = "/usr/local/bin"
+		}
+	}
 
 	s := SyncThing{
 		BaseURL: url,
-		ApiKey:  apiKey,
+		APIKey:  apiKey,
 		Home:    home,
-		binDir:  conf.BinDir,
+		binDir:  binDir,
+		logsDir: conf.FileConf.LogsDir,
 		log:     log,
 	}
 
-	if s.BaseURL == "" {
-		s.BaseURL = "http://localhost:8384"
-	}
-	if s.BaseURL[0:7] != "http://" {
-		s.BaseURL = "http://" + s.BaseURL
-	}
-
 	return &s
 }
 
@@ -84,28 +105,43 @@ func (s *SyncThing) startProc(exeName string, args []string, env []string, eChan
 		cmd.Env = append(cmd.Env, ev)
 	}
 
+	// open log file
+	var outfile *os.File
+	logFilename := filepath.Join(s.logsDir, exeName+".log")
+	if s.logsDir != "" {
+		outfile, err := os.Create(logFilename)
+		if err != nil {
+			return nil, fmt.Errorf("Cannot create log file %s", logFilename)
+		}
+
+		cmdOut, err := cmd.StdoutPipe()
+		if err != nil {
+			return nil, fmt.Errorf("Pipe stdout error for : %s", err)
+		}
+
+		go io.Copy(outfile, cmdOut)
+	}
+
 	err = cmd.Start()
 	if err != nil {
 		return nil, err
 	}
 
 	*eChan = make(chan ExitChan, 1)
-	go func(c *exec.Cmd) {
+	go func(c *exec.Cmd, oF *os.File) {
 		status := 0
-		cmdOut, err := c.StdoutPipe()
-		if err == nil {
-			s.log.Errorf("Pipe stdout error for : %s", err)
-		} else if cmdOut != nil {
-			stdOutput, _ := ioutil.ReadAll(cmdOut)
-			fmt.Printf("STDOUT: %s\n", stdOutput)
-		}
 		sts, err := c.Process.Wait()
 		if !sts.Success() {
 			s := sts.Sys().(syscall.WaitStatus)
 			status = s.ExitStatus()
 		}
+		if oF != nil {
+			oF.Close()
+		}
+		s.log.Debugf("%s exited with status %d, err %v", exeName, status, err)
+
 		*eChan <- ExitChan{status, err}
-	}(cmd)
+	}(cmd, outfile)
 
 	return cmd, nil
 }
@@ -113,14 +149,19 @@ func (s *SyncThing) startProc(exeName string, args []string, env []string, eChan
 // Start Starts syncthing process
 func (s *SyncThing) Start() (*exec.Cmd, error) {
 	var err error
+
+	s.log.Infof(" ST home=%s", s.Home)
+	s.log.Infof(" ST  url=%s", s.BaseURL)
+
 	args := []string{
 		"--home=" + s.Home,
 		"-no-browser",
 		"--gui-address=" + s.BaseURL,
 	}
 
-	if s.ApiKey != "" {
-		args = append(args, "-gui-apikey=\""+s.ApiKey+"\"")
+	if s.APIKey != "" {
+		args = append(args, "-gui-apikey=\""+s.APIKey+"\"")
+		s.log.Infof(" ST apikey=%s", s.APIKey)
 	}
 	if s.log.Level == logrus.DebugLevel {
 		args = append(args, "-verbose")
@@ -132,38 +173,58 @@ func (s *SyncThing) Start() (*exec.Cmd, error) {
 
 	s.STCmd, err = s.startProc("syncthing", args, env, &s.exitSTChan)
 
+	// Use autogenerated apikey if not set by config.json
+	if s.APIKey == "" {
+		if fd, err := os.Open(filepath.Join(s.Home, "config.xml")); err == nil {
+			defer fd.Close()
+			if b, err := ioutil.ReadAll(fd); err == nil {
+				re := regexp.MustCompile("<apikey>(.*)</apikey>")
+				key := re.FindStringSubmatch(string(b))
+				if len(key) >= 1 {
+					s.APIKey = key[1]
+				}
+			}
+		}
+	}
+
 	return s.STCmd, err
 }
 
 // StartInotify Starts syncthing-inotify process
 func (s *SyncThing) StartInotify() (*exec.Cmd, error) {
 	var err error
+	exeName := "syncthing-inotify"
+
+	s.log.Infof(" STI  url=%s", s.BaseURL)
 
 	args := []string{
-		"--home=" + s.Home,
 		"-target=" + s.BaseURL,
 	}
+	if s.APIKey != "" {
+		args = append(args, "-api="+s.APIKey)
+		s.log.Infof("%s uses apikey=%s", exeName, s.APIKey)
+	}
 	if s.log.Level == logrus.DebugLevel {
 		args = append(args, "-verbosity=4")
 	}
 
 	env := []string{}
 
-	s.STICmd, err = s.startProc("syncthing-inotify", args, env, &s.exitSTIChan)
+	s.STICmd, err = s.startProc(exeName, args, env, &s.exitSTIChan)
 
 	return s.STICmd, err
 }
 
 func (s *SyncThing) stopProc(pname string, proc *os.Process, exit chan ExitChan) {
 	if err := proc.Signal(os.Interrupt); err != nil {
-		s.log.Errorf("Proc interrupt %s error: %s", pname, err.Error())
+		s.log.Infof("Proc interrupt %s error: %s", pname, err.Error())
 
 		select {
 		case <-exit:
 		case <-time.After(time.Second):
 			// A bigger bonk on the head.
 			if err := proc.Signal(os.Kill); err != nil {
-				s.log.Errorf("Proc term %s error: %s", pname, err.Error())
+				s.log.Infof("Proc term %s error: %s", pname, err.Error())
 			}
 			<-exit
 		}
@@ -207,6 +268,9 @@ func (s *SyncThing) Connect() error {
 	if s.client == nil {
 		return fmt.Errorf("ERROR: cannot connect to Syncthing (null client)")
 	}
+
+	s.client.SetLogger(s.log)
+
 	return nil
 }
 
diff --git a/lib/xdsconfig/config.go b/lib/xdsconfig/config.go
index 1f53cbd..efea5ba 100644
--- a/lib/xdsconfig/config.go
+++ b/lib/xdsconfig/config.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/codegangsta/cli"
+	"github.com/iotbzh/xds-server/lib/common"
 )
 
 // Config parameters (json format) of /config command
@@ -16,9 +17,9 @@ type Config struct {
 	VersionGitTag string `json:"gitTag"`
 
 	// Private / un-exported fields
-	HTTPPort string `json:"-"`
-	FileConf *FileConfig
-	log      *logrus.Logger
+	HTTPPort string         `json:"-"`
+	FileConf *FileConfig    `json:"-"`
+	Log      *logrus.Logger `json:"-"`
 }
 
 // Config default values
@@ -29,7 +30,7 @@ const (
 )
 
 // Init loads the configuration on start-up
-func Init(ctx *cli.Context, log *logrus.Logger) (Config, error) {
+func Init(ctx *cli.Context, log *logrus.Logger) (*Config, error) {
 	var err error
 
 	// Define default configuration
@@ -39,27 +40,26 @@ func Init(ctx *cli.Context, log *logrus.Logger) (Config, error) {
 		VersionGitTag: ctx.App.Metadata["git-tag"].(string),
 
 		HTTPPort: DefaultPort,
-		log:      log,
+		Log:      log,
 	}
 
 	// config file settings overwrite default config
 	c.FileConf, err = updateConfigFromFile(&c, ctx.GlobalString("config"))
 	if err != nil {
-		return Config{}, err
+		return nil, err
 	}
 
-	return c, nil
+	if c.FileConf.LogsDir != "" && !common.Exists(c.FileConf.LogsDir) {
+		if err := os.MkdirAll(c.FileConf.LogsDir, 0770); err != nil {
+			return nil, fmt.Errorf("Cannot create logs dir: %v", err)
+		}
+	}
+	c.Log.Infoln("Logs directory: ", c.FileConf.LogsDir)
+
+	return &c, nil
 }
 
 // UpdateAll Update the current configuration
 func (c *Config) UpdateAll(newCfg Config) error {
 	return fmt.Errorf("Not Supported")
 }
-
-func dirExists(path string) bool {
-	_, err := os.Stat(path)
-	if os.IsNotExist(err) {
-		return false
-	}
-	return true
-}
diff --git a/lib/xdsconfig/fileconfig.go b/lib/xdsconfig/fileconfig.go
index 0c4828c..5cf8db2 100644
--- a/lib/xdsconfig/fileconfig.go
+++ b/lib/xdsconfig/fileconfig.go
@@ -19,6 +19,7 @@ type SyncThingConf struct {
 
 type FileConfig struct {
 	HTTPPort string         `json:"httpPort"`
+	LogsDir  string         `json:"logsDir"`
 	SThgConf *SyncThingConf `json:"syncthing"`
 }
 
@@ -60,7 +61,7 @@ func updateConfigFromFile(c *Config, confFile string) (*FileConfig, error) {
 		return &fCfg, nil
 	}
 
-	c.log.Infof("Use config file: %s", *cFile)
+	c.Log.Infof("Use config file: %s", *cFile)
 
 	// TODO move on viper package to support comments in JSON and also
 	// bind with flags (command line options)
@@ -73,18 +74,22 @@ func updateConfigFromFile(c *Config, confFile string) (*FileConfig, error) {
 	}
 
 	// Support environment variables (IOW ${MY_ENV_VAR} syntax) in agent-config.json
-	// TODO: better to use reflect package to iterate on fields and be more generic
-	var rep string
-
-	if rep, err = resolveEnvVar(fCfg.SThgConf.BinDir); err != nil {
-		return nil, err
+	for _, field := range []*string{
+		&fCfg.LogsDir,
+		&fCfg.SThgConf.Home,
+		&fCfg.SThgConf.BinDir} {
+
+		rep, err := resolveEnvVar(*field)
+		if err != nil {
+			return nil, err
+		}
+		*field = path.Clean(rep)
 	}
-	fCfg.SThgConf.BinDir = path.Clean(rep)
 
-	if rep, err = resolveEnvVar(fCfg.SThgConf.Home); err != nil {
-		return nil, err
+	// Config file settings overwrite default config
+	if fCfg.HTTPPort != "" {
+		c.HTTPPort = fCfg.HTTPPort
 	}
-	fCfg.SThgConf.Home = path.Clean(rep)
 
 	return &fCfg, nil
 }
@@ -106,15 +111,3 @@ func resolveEnvVar(s string) (string, error) {
 
 	return res, nil
 }
-
-// exists returns whether the given file or directory exists or not
-func exists(path string) bool {
-	_, err := os.Stat(path)
-	if err == nil {
-		return true
-	}
-	if os.IsNotExist(err) {
-		return false
-	}
-	return true
-}
diff --git a/main.go b/main.go
index 93d13a2..0a0ad0a 100644
--- a/main.go
+++ b/main.go
@@ -7,8 +7,6 @@ import (
 	"os"
 	"time"
 
-	"fmt"
-
 	"github.com/Sirupsen/logrus"
 	"github.com/codegangsta/cli"
 	"github.com/iotbzh/xds-agent/lib/agent"
@@ -37,19 +35,19 @@ var AppSubVersion = "unknown-dev"
 
 // xdsAgent main routine
 func xdsAgent(cliCtx *cli.Context) error {
+	var err error
 
 	// Create Agent context
 	ctx := agent.NewAgent(cliCtx)
 
 	// Load config
-	cfg, err := xdsconfig.Init(cliCtx, ctx.Log)
+	ctx.Config, err = xdsconfig.Init(cliCtx, ctx.Log)
 	if err != nil {
 		return cli.NewExitError(err, 2)
 	}
-	ctx.Config = &cfg
 
 	// Start local instance of Syncthing and Syncthing-notify
-	ctx.SThg = st.NewSyncThing(ctx.Config.FileConf.SThgConf, ctx.Log)
+	ctx.SThg = st.NewSyncThing(ctx.Config, ctx.Log)
 
 	ctx.Log.Infof("Starting Syncthing...")
 	ctx.SThgCmd, err = ctx.SThg.Start()
@@ -72,12 +70,11 @@ func xdsAgent(cliCtx *cli.Context) error {
 		if err := ctx.SThg.Connect(); err == nil {
 			break
 		}
-		ctx.Log.Infof("Establishing connection to Syncthing (retry %d/5)", retry)
+		ctx.Log.Infof("Establishing connection to Syncthing (retry %d/10)", retry)
 		time.Sleep(time.Second)
 		retry--
 	}
-	if ctx.SThg == nil {
-		err = fmt.Errorf("ERROR: cannot connect to Syncthing (url: %s)", ctx.SThg.BaseURL)
+	if err != nil || retry == 0 {
 		return cli.NewExitError(err, 2)
 	}
 
diff --git a/scripts/get-syncthing.sh b/scripts/get-syncthing.sh
index 8bc5346..6eb9c5b 100755
--- a/scripts/get-syncthing.sh
+++ b/scripts/get-syncthing.sh
@@ -31,7 +31,7 @@ tarball="syncthing-linux-amd64-v${SYNCTHING_VERSION}.tar.gz" \
 	&& grep -E " ${tarball}\$" sha1sum.txt.asc | sha1sum -c - \
 	&& rm sha1sum.txt.asc \
 	&& tar -xvf "$tarball" --strip-components=1 "$(basename "$tarball" .tar.gz)"/syncthing \
-	&& mv syncthing ${DESTDIR}/syncthing
+	&& mv syncthing ${DESTDIR}/syncthing || exit 1
 
 
 echo "Get Syncthing-inotify..."