From dd6f08b10b1597f44e3dc25509ac9a45336b0914 Mon Sep 17 00:00:00 2001 From: Sebastien Douheret Date: Thu, 10 Aug 2017 12:19:34 +0200 Subject: Add folder interface and support native pathmap folder type. Signed-off-by: Sebastien Douheret --- lib/model/folder.go | 110 ----------------- lib/model/folders.go | 333 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 333 insertions(+), 110 deletions(-) delete mode 100644 lib/model/folder.go create mode 100644 lib/model/folders.go (limited to 'lib/model') diff --git a/lib/model/folder.go b/lib/model/folder.go deleted file mode 100644 index 56a46b1..0000000 --- a/lib/model/folder.go +++ /dev/null @@ -1,110 +0,0 @@ -package model - -import ( - "fmt" - - common "github.com/iotbzh/xds-common/golib" - "github.com/iotbzh/xds-server/lib/syncthing" - "github.com/iotbzh/xds-server/lib/xdsconfig" -) - -// Folder Represent a an XDS folder -type Folder struct { - Conf *xdsconfig.Config - SThg *st.SyncThing -} - -// NewFolder Create a new instance of Model Folder -func NewFolder(cfg *xdsconfig.Config, st *st.SyncThing) *Folder { - return &Folder{ - Conf: cfg, - SThg: st, - } -} - -// GetFolderFromID retrieves the Folder config from id -func (c *Folder) GetFolderFromID(id string) *xdsconfig.FolderConfig { - if idx := c.Conf.Folders.GetIdx(id); idx != -1 { - return &c.Conf.Folders[idx] - } - return nil -} - -// UpdateAll updates all the current configuration -func (c *Folder) UpdateAll(newCfg xdsconfig.Config) error { - return fmt.Errorf("Not Supported") - /* - if err := VerifyConfig(newCfg); err != nil { - return err - } - - // TODO: c.Builder = c.Builder.Update(newCfg.Builder) - c.Folders = c.Folders.Update(newCfg.Folders) - - // FIXME To be tested & improved error handling - for _, f := range c.Folders { - if err := c.SThg.FolderChange(st.FolderChangeArg{ - ID: f.ID, - Label: f.Label, - RelativePath: f.RelativePath, - SyncThingID: f.SyncThingID, - ShareRootDir: c.FileConf.ShareRootDir, - }); err != nil { - return err - } - } - - return nil - */ -} - -// UpdateFolder updates a specific folder into the current configuration -func (c *Folder) UpdateFolder(newFolder xdsconfig.FolderConfig) (xdsconfig.FolderConfig, error) { - // rootPath should not be empty - if newFolder.RootPath == "" { - newFolder.RootPath = c.Conf.FileConf.ShareRootDir - } - - // Sanity check of folder settings - if err := newFolder.Verify(); err != nil { - return xdsconfig.FolderConfig{}, err - } - - // Normalize path (needed for Windows path including bashlashes) - newFolder.RelativePath = common.PathNormalize(newFolder.RelativePath) - - // Update config folder - c.Conf.Folders = c.Conf.Folders.Update(xdsconfig.FoldersConfig{newFolder}) - - // Update Syncthing folder - err := c.SThg.FolderChange(newFolder) - - newFolder.BuilderSThgID = c.Conf.Builder.SyncThingID // FIXME - should be removed after local ST config rework - newFolder.Status = xdsconfig.FolderStatusEnable - - return newFolder, err -} - -// DeleteFolder deletes a specific folder -func (c *Folder) DeleteFolder(id string) (xdsconfig.FolderConfig, error) { - var fld xdsconfig.FolderConfig - var err error - - if err = c.SThg.FolderDelete(id); err != nil { - return fld, err - } - - c.Conf.Folders, fld, err = c.Conf.Folders.Delete(id) - - return fld, err -} - -// ForceSync Force the synchronization of a folder -func (c *Folder) ForceSync(id string) error { - return c.SThg.FolderScan(id, "") -} - -// IsFolderInSync Returns true when folder is in sync -func (c *Folder) IsFolderInSync(id string) (bool, error) { - return c.SThg.IsFolderInSync(id) -} diff --git a/lib/model/folders.go b/lib/model/folders.go new file mode 100644 index 0000000..3c2457c --- /dev/null +++ b/lib/model/folders.go @@ -0,0 +1,333 @@ +package model + +import ( + "encoding/xml" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/Sirupsen/logrus" + common "github.com/iotbzh/xds-common/golib" + "github.com/iotbzh/xds-server/lib/folder" + "github.com/iotbzh/xds-server/lib/syncthing" + "github.com/iotbzh/xds-server/lib/xdsconfig" + uuid "github.com/satori/go.uuid" + "github.com/syncthing/syncthing/lib/sync" +) + +// Folders Represent a an XDS folders +type Folders struct { + fileOnDisk string + Conf *xdsconfig.Config + Log *logrus.Logger + SThg *st.SyncThing + folders map[string]*folder.IFOLDER +} + +// Mutex to make add/delete atomic +var fcMutex = sync.NewMutex() +var ffMutex = sync.NewMutex() + +// FoldersNew Create a new instance of Model Folders +func FoldersNew(cfg *xdsconfig.Config, st *st.SyncThing) *Folders { + file, _ := xdsconfig.FoldersConfigFilenameGet() + return &Folders{ + fileOnDisk: file, + Conf: cfg, + Log: cfg.Log, + SThg: st, + folders: make(map[string]*folder.IFOLDER), + } +} + +// LoadConfig Load folders configuration from disk +func (f *Folders) LoadConfig() error { + var flds []folder.FolderConfig + var stFlds []folder.FolderConfig + + // load from disk + if f.Conf.Options.NoFolderConfig { + f.Log.Infof("Don't read folder config file (-no-folderconfig option is set)") + } else if f.fileOnDisk != "" { + f.Log.Infof("Use folder config file: %s", f.fileOnDisk) + err := foldersConfigRead(f.fileOnDisk, &flds) + if err != nil { + if strings.HasPrefix(err.Error(), "No folder config") { + f.Log.Warnf(err.Error()) + } else { + return err + } + } + } else { + f.Log.Warnf("Folders config filename not set") + } + + // Retrieve initial Syncthing config (just append don't overwrite existing ones) + if f.SThg != nil { + f.Log.Infof("Retrieve syncthing folder config") + if err := f.SThg.FolderLoadFromStConfig(&stFlds); err != nil { + // Don't exit on such error, just log it + f.Log.Errorf(err.Error()) + } + } + + // Merge syncthing folders into XDS folders + for _, stf := range stFlds { + found := false + for i, xf := range flds { + if xf.ID == stf.ID { + found = true + // sanity check + if xf.Type != folder.TypeCloudSync { + flds[i].Status = folder.StatusErrorConfig + } + break + } + } + // add it + if !found { + flds = append(flds, stf) + } + } + + // Detect ghost project + // (IOW existing in xds file config and not in syncthing database) + for i, xf := range flds { + // only for syncthing project + if xf.Type != folder.TypeCloudSync { + continue + } + found := false + for _, stf := range stFlds { + if stf.ID == xf.ID { + found = true + break + } + } + if !found { + flds[i].Status = folder.StatusErrorConfig + } + } + + // Update folders + f.Log.Infof("Loading initial folders config: %d folders found", len(flds)) + for _, fc := range flds { + if _, err := f.createUpdate(fc, false); err != nil { + return err + } + } + + return nil +} + +// SaveConfig Save folders configuration to disk +func (f *Folders) SaveConfig() error { + if f.fileOnDisk == "" { + return fmt.Errorf("Folders config filename not set") + } + + // FIXME: buffered save or avoid to write on disk each time + return foldersConfigWrite(f.fileOnDisk, f.getConfigArrUnsafe()) +} + +// Get returns the folder config or nil if not existing +func (f *Folders) Get(id string) *folder.IFOLDER { + if id == "" { + return nil + } + fc, exist := f.folders[id] + if !exist { + return nil + } + return fc +} + +// GetConfigArr returns the config of all folders as an array +func (f *Folders) GetConfigArr() []folder.FolderConfig { + fcMutex.Lock() + defer fcMutex.Unlock() + + return f.getConfigArrUnsafe() +} + +// getConfigArrUnsafe Same as GetConfigArr without mutex protection +func (f *Folders) getConfigArrUnsafe() []folder.FolderConfig { + var conf []folder.FolderConfig + + for _, v := range f.folders { + conf = append(conf, (*v).GetConfig()) + } + return conf +} + +// Add adds a new folder +func (f *Folders) Add(newF folder.FolderConfig) (*folder.FolderConfig, error) { + return f.createUpdate(newF, true) +} + +// CreateUpdate creates or update a folder +func (f *Folders) createUpdate(newF folder.FolderConfig, create bool) (*folder.FolderConfig, error) { + + fcMutex.Lock() + defer fcMutex.Unlock() + + // Sanity check + if _, exist := f.folders[newF.ID]; create && exist { + return nil, fmt.Errorf("ID already exists") + } + if newF.ClientPath == "" { + return nil, fmt.Errorf("ClientPath must be set") + } + + // Allocate a new UUID + if create { + newF.ID = uuid.NewV1().String() + } + if !create && newF.ID == "" { + return nil, fmt.Errorf("Cannot update folder with null ID") + } + + // Set default value if needed + if newF.Status == "" { + newF.Status = folder.StatusDisable + } + + if newF.Label == "" { + newF.Label = filepath.Base(newF.ClientPath) + "_" + newF.ID[0:8] + } + + var fld folder.IFOLDER + switch newF.Type { + // SYNCTHING + case folder.TypeCloudSync: + if f.SThg == nil { + return nil, fmt.Errorf("ClownSync type not supported (syncthing not initialized)") + } + fld = f.SThg.NewFolderST(f.Conf) + // PATH MAP + case folder.TypePathMap: + fld = folder.NewFolderPathMap() + default: + return nil, fmt.Errorf("Unsupported folder type") + } + + // Normalize path (needed for Windows path including bashlashes) + newF.ClientPath = common.PathNormalize(newF.ClientPath) + + // Add new folder + newFolder, err := fld.Add(newF) + if err != nil { + newF.Status = folder.StatusErrorConfig + log.Printf("ERROR Adding folder: %v\n", err) + return newFolder, err + } + + // Register folder object + f.folders[newF.ID] = &fld + + // Save config on disk + err = f.SaveConfig() + + return newFolder, err +} + +// Delete deletes a specific folder +func (f *Folders) Delete(id string) (folder.FolderConfig, error) { + var err error + + fcMutex.Lock() + defer fcMutex.Unlock() + + fld := folder.FolderConfig{} + fc, exist := f.folders[id] + if !exist { + return fld, fmt.Errorf("unknown id") + } + + fld = (*fc).GetConfig() + + if err = (*fc).Remove(); err != nil { + return fld, err + } + + delete(f.folders, id) + + // Save config on disk + err = f.SaveConfig() + + return fld, err +} + +// ForceSync Force the synchronization of a folder +func (f *Folders) ForceSync(id string) error { + fc := f.Get(id) + if fc == nil { + return fmt.Errorf("Unknown id") + } + return (*fc).Sync() +} + +// IsFolderInSync Returns true when folder is in sync +func (f *Folders) IsFolderInSync(id string) (bool, error) { + fc := f.Get(id) + if fc == nil { + return false, fmt.Errorf("Unknown id") + } + return (*fc).IsInSync() +} + +//*** Private functions *** + +// Use XML format and not json to be able to save/load all fields including +// ones that are masked in json (IOW defined with `json:"-"`) +type xmlFolders struct { + XMLName xml.Name `xml:"folders"` + Version string `xml:"version,attr"` + Folders []folder.FolderConfig `xml:"folders"` +} + +// foldersConfigRead reads folders config from disk +func foldersConfigRead(file string, folders *[]folder.FolderConfig) error { + if !common.Exists(file) { + return fmt.Errorf("No folder config file found (%s)", file) + } + + ffMutex.Lock() + defer ffMutex.Unlock() + + fd, err := os.Open(file) + defer fd.Close() + if err != nil { + return err + } + + data := xmlFolders{} + err = xml.NewDecoder(fd).Decode(&data) + if err == nil { + *folders = data.Folders + } + return err +} + +// foldersConfigWrite writes folders config on disk +func foldersConfigWrite(file string, folders []folder.FolderConfig) error { + ffMutex.Lock() + defer ffMutex.Unlock() + + fd, err := os.OpenFile(file, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + defer fd.Close() + if err != nil { + return err + } + + data := &xmlFolders{ + Version: "1", + Folders: folders, + } + + enc := xml.NewEncoder(fd) + enc.Indent("", " ") + return enc.Encode(data) +} -- cgit 1.2.3-korg