Retry file upload on backend error

3 files changed, 36 insertions, 3 deletions

diff --git a/drive/errors.go b/drive/errors.go
new file mode 100644
index 0000000..703dae5
--- /dev/null
+++ b/drive/errors.go
@@ -0,0 +1,22 @@
+package drive
+
+import (
+    "google.golang.org/api/googleapi"
+    "time"
+)
+
+const MaxBackendErrorRetries = 5
+
+func isBackendError(err error) bool {
+    if err == nil {
+        return false
+    }
+
+    ae, ok := err.(*googleapi.Error)
+    return ok && ae.Code >= 500 && ae.Code <= 599
+}
+
+func exponentialBackoffSleep(try int) {
+    seconds := pow(2, try)
+    time.Sleep(time.Duration(seconds) * time.Second)
+}
diff --git a/drive/sync_upload.go b/drive/sync_upload.go
index 445ed42..c4df1e8 100644
--- a/drive/sync_upload.go
+++ b/drive/sync_upload.go
@@ -185,7 +185,7 @@ func (self *Drive) uploadMissingFiles(files *syncFiles, args UploadSyncArgs) err
             continue
         }
 
-        err := self.uploadMissingFile(parent.file.Id, lf, args)
+        err := self.uploadMissingFile(parent.file.Id, lf, args, 0)
         if err != nil {
             return err
         }
@@ -245,7 +245,7 @@ func (self *Drive) deleteExtraneousRemoteFiles(files *syncFiles, args UploadSync
     return nil
 }
 
-func (self *Drive) uploadMissingFile(parentId string, lf *LocalFile, args UploadSyncArgs) error {
+func (self *Drive) uploadMissingFile(parentId string, lf *LocalFile, args UploadSyncArgs, try int) error {
     srcFile, err := os.Open(lf.absPath)
     if err != nil {
         return fmt.Errorf("Failed to open file: %s", err)
@@ -269,7 +269,13 @@ func (self *Drive) uploadMissingFile(parentId string, lf *LocalFile, args Upload
 
     _, err = self.service.Files.Create(dstFile).Fields("id", "name", "size", "md5Checksum").Media(srcReader, chunkSize).Do()
     if err != nil {
-        return fmt.Errorf("Failed to upload file: %s", err)
+        if isBackendError(err) && try < MaxBackendErrorRetries {
+            exponentialBackoffSleep(try)
+            try++
+            self.uploadMissingFile(parentId, lf, args, try)
+        } else {
+            return fmt.Errorf("Failed to upload file: %s", err)
+        }
     }
 
     return nil
diff --git a/drive/util.go b/drive/util.go
index 2116f9b..481a48e 100644
--- a/drive/util.go
+++ b/drive/util.go
@@ -143,3 +143,8 @@ func parentFilePath(path string) string {
     dir, _ := filepath.Split(path)
     return filepath.Dir(dir)
 }
+
+func pow(x int, y int) int {
+    f := math.Pow(float64(x), float64(y))
+    return int(f)
+}