Initial download sync implementation

2 files changed, 293 insertions, 0 deletions

diff --git a/drive/download_sync.go b/drive/download_sync.go
new file mode 100644
index 0000000..c4bd977
--- /dev/null
+++ b/drive/download_sync.go
@@ -0,0 +1,216 @@
+package drive
+
+import (
+    "fmt"
+    "io"
+    "os"
+    "sort"
+    "time"
+    "path/filepath"
+    "google.golang.org/api/googleapi"
+    "google.golang.org/api/drive/v3"
+)
+
+type DownloadSyncArgs struct {
+    Out io.Writer
+    Progress io.Writer
+    RootId string
+    Path string
+    DeleteExtraneous bool
+}
+
+func (self *Drive) DownloadSync(args DownloadSyncArgs) error {
+    fmt.Fprintln(args.Out, "Starting sync...")
+    started := time.Now()
+
+    // Get remote root dir
+    rootDir, err := self.getSyncRoot(args.RootId)
+    if err != nil {
+        return err
+    }
+
+    fmt.Fprintln(args.Out, "Collecting local and remote file information...")
+    files, err := self.prepareSyncFiles(args.Path, rootDir)
+    if err != nil {
+        return err
+    }
+
+    fmt.Fprintf(args.Out, "Found %d local file(s) and %d remote file(s)\n", len(files.local), len(files.remote))
+
+    // Create missing directories
+    files, err = self.createMissingLocalDirs(files, args)
+    if err != nil {
+        return err
+    }
+
+    // Download missing files
+    err = self.downloadMissingFiles(files, args)
+    if err != nil {
+        return err
+    }
+
+    // Download files that has changed
+    err = self.downloadChangedFiles(files, args)
+    if err != nil {
+        return err
+    }
+
+    // Delete extraneous local files
+    if args.DeleteExtraneous {
+        err = self.deleteExtraneousLocalFiles(files, args)
+        if err != nil {
+            return err
+        }
+    }
+    fmt.Fprintf(args.Out, "Sync finished in %s\n", time.Since(started))
+
+    return nil
+}
+
+func (self *Drive) getSyncRoot(rootId string) (*drive.File, error) {
+    fields := []googleapi.Field{"id", "name", "mimeType", "appProperties"}
+    f, err := self.service.Files.Get(rootId).Fields(fields...).Do()
+    if err != nil {
+        return nil, fmt.Errorf("Failed to find root dir: %s", err)
+    }
+
+    // Ensure file is a directory
+    if !isDir(f) {
+        return nil, fmt.Errorf("Provided root id is not a directory")
+    }
+
+    // Ensure directory is a proper syncRoot
+    if _, ok := f.AppProperties["isSyncRoot"]; !ok {
+        return nil, fmt.Errorf("Provided id is not a sync root directory")
+    }
+
+    return f, nil
+}
+
+func (self *Drive) createMissingLocalDirs(files *syncFiles, args DownloadSyncArgs) (*syncFiles, error) {
+    missingDirs := files.filterMissingLocalDirs()
+    missingCount := len(missingDirs)
+
+    if missingCount > 0 {
+        fmt.Fprintf(args.Out, "\n%d local directories are missing\n", missingCount)
+    }
+
+    // Sort directories so that the dirs with the shortest path comes first
+    sort.Sort(byRemotePathLength(missingDirs))
+
+    for i, rf := range missingDirs {
+        path, err := filepath.Abs(filepath.Join(args.Path, rf.relPath))
+        if err != nil {
+            return nil, fmt.Errorf("Failed to determine local absolute path: %s", err)
+        }
+        fmt.Fprintf(args.Out, "[%04d/%04d] Creating directory: %s\n", i + 1, missingCount, path)
+        mkdir(path)
+    }
+
+    return files, nil
+}
+
+func (self *Drive) downloadMissingFiles(files *syncFiles, args DownloadSyncArgs) error {
+    missingFiles := files.filterMissingLocalFiles()
+    missingCount := len(missingFiles)
+
+    if missingCount > 0 {
+        fmt.Fprintf(args.Out, "\n%d local file(s) are missing\n", missingCount)
+    }
+
+    for i, rf := range missingFiles {
+        remotePath := filepath.Join(files.root.file.Name, rf.relPath)
+        localPath, err := filepath.Abs(filepath.Join(args.Path, rf.relPath))
+        if err != nil {
+            return fmt.Errorf("Failed to determine local absolute path: %s", err)
+        }
+        fmt.Fprintf(args.Out, "[%04d/%04d] Downloading %s -> %s\n", i + 1, missingCount, remotePath, localPath)
+        err = self.downloadRemoteFile(rf.file.Id, localPath, args)
+        if err != nil {
+            return err
+        }
+    }
+
+    return nil
+}
+
+func (self *Drive) downloadChangedFiles(files *syncFiles, args DownloadSyncArgs) error {
+    changedFiles := files.filterChangedRemoteFiles()
+    changedCount := len(changedFiles)
+
+    if changedCount > 0 {
+        fmt.Fprintf(args.Out, "\n%d remote file(s) has changed\n", changedCount)
+    }
+
+    for i, cf := range changedFiles {
+        remotePath := filepath.Join(files.root.file.Name, cf.remote.relPath)
+        localPath, err := filepath.Abs(filepath.Join(args.Path, cf.remote.relPath))
+        if err != nil {
+            return fmt.Errorf("Failed to determine local absolute path: %s", err)
+        }
+        fmt.Fprintf(args.Out, "[%04d/%04d] Downloading %s -> %s\n", i + 1, changedCount, remotePath, localPath)
+        err = self.downloadRemoteFile(cf.remote.file.Id, localPath, args)
+        if err != nil {
+            return err
+        }
+    }
+
+    return nil
+}
+
+func (self *Drive) downloadRemoteFile(id, fpath string, args DownloadSyncArgs) error {
+    res, err := self.service.Files.Get(id).Download()
+    if err != nil {
+        return fmt.Errorf("Failed to download file: %s", err)
+    }
+
+    // Close body on function exit
+    defer res.Body.Close()
+
+    // Wrap response body in progress reader
+    srcReader := getProgressReader(res.Body, args.Progress, res.ContentLength)
+
+    // Ensure any parent directories exists
+    if err = mkdir(fpath); err != nil {
+        return err
+    }
+
+    // Create new file
+    outFile, err := os.Create(fpath)
+    if err != nil {
+        return fmt.Errorf("Unable to create local file: %s", err)
+    }
+
+    // Close file on function exit
+    defer outFile.Close()
+
+    // Save file to disk
+    _, err = io.Copy(outFile, srcReader)
+    if err != nil {
+        return fmt.Errorf("Download was interrupted: %s", err)
+    }
+
+    return nil
+}
+
+func (self *Drive) deleteExtraneousLocalFiles(files *syncFiles, args DownloadSyncArgs) error {
+    extraneousFiles := files.filterExtraneousLocalFiles()
+    extraneousCount := len(extraneousFiles)
+
+    if extraneousCount > 0 {
+        fmt.Fprintf(args.Out, "\n%d local file(s) are extraneous\n", extraneousCount)
+    }
+
+    // Sort files so that the files with the longest path comes first
+    sort.Sort(sort.Reverse(byPathLength(extraneousFiles)))
+
+    for i, lf := range extraneousFiles {
+        fmt.Fprintf(args.Out, "[%04d/%04d] Deleting %s\n", i + 1, extraneousCount, lf.absPath)
+        err := os.Remove(lf.absPath)
+        if err != nil {
+            return fmt.Errorf("Failed to delete local file: %s", err)
+        }
+    }
+
+    return nil
+}
diff --git a/drive/upload_sync.go b/drive/upload_sync.go
index 3e157ed..081f739 100644
--- a/drive/upload_sync.go
+++ b/drive/upload_sync.go
@@ -481,6 +481,18 @@ func (self *syncFiles) filterMissingRemoteDirs() []*localFile {
     return files
 }
 
+func (self *syncFiles) filterMissingLocalDirs() []*remoteFile {
+    var files []*remoteFile
+
+    for _, rf := range self.remote {
+        if isDir(rf.file) && !self.existsLocal(rf) {
+            files = append(files, rf)
+        }
+    }
+
+    return files
+}
+
 func (self *syncFiles) filterMissingRemoteFiles() []*localFile {
     var files []*localFile
 
@@ -493,6 +505,18 @@ func (self *syncFiles) filterMissingRemoteFiles() []*localFile {
     return files
 }
 
+func (self *syncFiles) filterMissingLocalFiles() []*remoteFile {
+    var files []*remoteFile
+
+    for _, rf := range self.remote {
+        if !isDir(rf.file) && !self.existsLocal(rf) {
+            files = append(files, rf)
+        }
+    }
+
+    return files
+}
+
 func (self *syncFiles) filterChangedLocalFiles() []*changedFile {
     var files []*changedFile
 
@@ -520,6 +544,33 @@ func (self *syncFiles) filterChangedLocalFiles() []*changedFile {
     return files
 }
 
+func (self *syncFiles) filterChangedRemoteFiles() []*changedFile {
+    var files []*changedFile
+
+    for _, rf := range self.remote {
+        // Skip directories
+        if isDir(rf.file) {
+            continue
+        }
+
+        // Skip local files that don't exist
+        lf, found := self.findLocalByPath(rf.relPath)
+        if !found {
+            continue
+        }
+
+        // Add files where remote md5 sum does not match local
+        if rf.file.Md5Checksum != md5sum(lf.absPath) {
+            files = append(files, &changedFile{
+                local: lf,
+                remote: rf,
+            })
+        }
+    }
+
+    return files
+}
+
 func (self *syncFiles) filterExtraneousRemoteFiles() []*remoteFile {
     var files []*remoteFile
 
@@ -532,6 +583,18 @@ func (self *syncFiles) filterExtraneousRemoteFiles() []*remoteFile {
     return files
 }
 
+func (self *syncFiles) filterExtraneousLocalFiles() []*localFile {
+    var files []*localFile
+
+    for _, lf := range self.local {
+        if !self.existsRemote(lf) {
+            files = append(files, lf)
+        }
+    }
+
+    return files
+}
+
 func (self *syncFiles) existsRemote(lf *localFile) bool {
     _, found := self.findRemoteByPath(lf.relPath)
     return found
@@ -579,3 +642,17 @@ func (self byPathLength) Swap(i, j int) {
 func (self byPathLength) Less(i, j int) bool {
     return pathLength(self[i].relPath) < pathLength(self[j].relPath)
 }
+
+type byRemotePathLength []*remoteFile
+
+func (self byRemotePathLength) Len() int {
+    return len(self)
+}
+
+func (self byRemotePathLength) Swap(i, j int) {
+    self[i], self[j] = self[j], self[i]
+}
+
+func (self byRemotePathLength) Less(i, j int) bool {
+    return pathLength(self[i].relPath) < pathLength(self[j].relPath)
+}