diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | README.md | 82 | ||||
| -rw-r--r-- | compare.go | 2 | ||||
| -rw-r--r-- | drive/download.go | 6 | ||||
| -rw-r--r-- | drive/errors.go | 20 | ||||
| -rw-r--r-- | drive/revision_download.go | 7 | ||||
| -rw-r--r-- | drive/sync_download.go | 9 | ||||
| -rw-r--r-- | drive/sync_upload.go | 17 | ||||
| -rw-r--r-- | drive/timeout_reader.go | 44 | ||||
| -rw-r--r-- | drive/update.go | 6 | ||||
| -rw-r--r-- | drive/upload.go | 16 | ||||
| -rw-r--r-- | gdrive.go | 47 | ||||
| -rw-r--r-- | handlers_drive.go | 14 | ||||
| -rw-r--r-- | handlers_meta.go | 2 |
14 files changed, 199 insertions, 74 deletions
@@ -7,3 +7,4 @@ _release/bin Session.vim .netrwhist gdrive +gdrive.sh @@ -8,6 +8,7 @@ gdrive is a command line utility for interacting with Google Drive. ## Prerequisites None, binaries are statically linked. If you want to compile from source you need the [go toolchain](http://golang.org/doc/install). +Version 1.5 or higher. ## Installation Download `gdrive` from one of the links below. On unix systems @@ -26,33 +27,33 @@ You will be prompted for a new verification code if the folder does not exist. ### Downloads | Filename | Version | Description | Shasum | |:-----------------------|:--------|:-------------------|:-----------------------------------------| -| [gdrive-osx-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnZUM0bXRBeUpYQ00&export=download) | 2.0.1 | OS X 64-bit | 180bc98408c7ec6deac6a66bbd9c307c4348ae6f | -| [gdrive-osx-386](https://docs.google.com/uc?id=0B3X9GlR6EmbnM05hQXhvWm9EOHM&export=download) | 2.0.1 | OS X 32-bit | 366ee217d4327a1855245d8c4a1204f4831eb979 | -| [gdrive-osx-arm](https://docs.google.com/uc?id=0B3X9GlR6EmbnelYxRU5LbEVfVzQ&export=download) | 2.0.1 | OS X arm | cdc31f83e50560a7f8fbf8a25b9c87c945d1407f | -| [gdrive-linux-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnWksyTEtCM0VfaFE&export=download) | 2.0.1 | Linux 64-bit | c636778c4a2c76e47ac731c142f4219a19c30263 | -| [gdrive-linux-386](https://docs.google.com/uc?id=0B3X9GlR6EmbnbndOUW50ZVllZ3M&export=download) | 2.0.1 | Linux 32-bit | 0968993e4a70a594e0f315034640fd811977e4f1 | -| [gdrive-linux-rpi](https://docs.google.com/uc?id=0B3X9GlR6EmbnbVRIelM5SG5zcmM&export=download) | 2.0.1 | Linux Raspberry Pi | 7865a1e96e70791aa0f33cb758e6cda7886be240 | -| [gdrive-linux-arm64](https://docs.google.com/uc?id=0B3X9GlR6EmbnNkxrQ3VzOGNMNWc&export=download) | 2.0.1 | Linux arm 64-bit | 00b293e4501da64e956d32b7c1589a014b541abe | -| [gdrive-linux-arm](https://docs.google.com/uc?id=0B3X9GlR6EmbnLW51cVJrazVLOG8&export=download) | 2.0.1 | Linux arm 32-bit | 3577a6462dafc47823d5ed053a978af84a99c5af | -| [gdrive-linux-mips64](https://docs.google.com/uc?id=0B3X9GlR6EmbnTS1fejFtaDMyLU0&export=download) | 2.0.1 | Linux mips 64-bit | e1e5992a5635467b84f149435544d980e99f30c6 | -| [gdrive-linux-mips64le](https://docs.google.com/uc?id=0B3X9GlR6EmbnV3pJU0ZBTEg0a2M&export=download) | 2.0.1 | Linux mips 64-bit le | 8273d53fc21b6028de781958c6f224f41c0a92db | -| [gdrive-linux-ppc64](https://docs.google.com/uc?id=0B3X9GlR6EmbnOXNPM01ITHl3NUk&export=download) | 2.0.1 | Linux PPC 64-bit | fc1409b9960ae4a209331e40a4cd9b10a789f8e6 | -| [gdrive-linux-ppc64le](https://docs.google.com/uc?id=0B3X9GlR6EmbnRVBCRlpXNl95bW8&export=download) | 2.0.1 | Linux PPC 64-bit le | 7a09ed4b43c31198efdf4e4a7da1e196cd3cd54f | -| [gdrive-windows-386.exe](https://docs.google.com/uc?id=0B3X9GlR6EmbncnpncGpZX0pULUU&export=download) | 2.0.1 | Window 32-bit | 134480ca113d03b91dbfa43326704b57f07ca547 | -| [gdrive-windows-x64.exe](https://docs.google.com/uc?id=0B3X9GlR6EmbncWNLOS1KYWhLVFE&export=download) | 2.0.1 | Windows 64-bit | c68df6e77aa7fa412bfe318ab270e1245a24966b | -| [gdrive-dragonfly-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnX1FiZzFaN1hRekk&export=download) | 2.0.1 | DragonFly BSD 64-bit | 0116d291a859152a4af842254eb88102c18f9ad6 | -| [gdrive-freebsd-x64](https://docs.google.com/uc?id=0B3X9GlR6Embnck8wR0ozV3J2WHM&export=download) | 2.0.1 | FreeBSD 64-bit | 0ab7d8509efcbbf41a4c684f9942c749706ea8ab | -| [gdrive-freebsd-386](https://docs.google.com/uc?id=0B3X9GlR6EmbnVUtwX010YUlENWs&export=download) | 2.0.1 | FreeBSD 32-bit | 4e1ac2aeeed59df1d8636641b796aad54ade42d2 | -| [gdrive-freebsd-arm](https://docs.google.com/uc?id=0B3X9GlR6EmbncXV3SU9NZDc3X3M&export=download) | 2.0.1 | FreeBSD arm | 1220f7f75579b28205d302f1e8ad0faeefc5d188 | -| [gdrive-netbsd-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnUk5JMVFiS0pKZW8&export=download) | 2.0.1 | NetBSD 64-bit | dba9e9f57b6ed3c370961e95efaea1ed0ea4429f | -| [gdrive-netbsd-386](https://docs.google.com/uc?id=0B3X9GlR6EmbnLWZRTy1LX1kxUkE&export=download) | 2.0.1 | NetBSD 32-bit | 8303ec4f0f7ba2acecc4509e0e73f8bf1eb2cd68 | -| [gdrive-netbsd-arm](https://docs.google.com/uc?id=0B3X9GlR6EmbnQUNFOWxBYmdjUEk&export=download) | 2.0.1 | NetBSD arm | 84aeb602e0cfbb09a35fff97f2af5990673f9bee | -| [gdrive-openbsd-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnYk50VlVuNUhuaVE&export=download) | 2.0.1 | OpenBSD 64-bit | 729794ef7cfc320cab84fd78e3113f5fdd476ac6 | -| [gdrive-openbsd-386](https://docs.google.com/uc?id=0B3X9GlR6EmbnOExCLVNYakJiSUk&export=download) | 2.0.1 | OpenBSD 32-bit | a1f45252d86ae238ce17b49c226b218aec805a02 | -| [gdrive-openbsd-arm](https://docs.google.com/uc?id=0B3X9GlR6EmbnN1htLTBSSElRUDQ&export=download) | 2.0.1 | OpenBSD arm | 5a732b15a4d36e61768388e323807e9a2fccb4bc | -| [gdrive-solaris-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnSEpJSDBidHNDNkE&export=download) | 2.0.1 | Solaris 64-bit | 82dc342873693b37302fc8f3cb97a667bae6e41c | -| [gdrive-plan9-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnZWMzR3pyZlZ0cFU&export=download) | 2.0.1 | Plan9 64-bit | e92e4a25517116c4fd466142baab03e0b9d11772 | -| [gdrive-plan9-386](https://docs.google.com/uc?id=0B3X9GlR6EmbnOXV4d0dNeGZCR00&export=download) | 2.0.1 | Plan9 32-bit | efdfced751ca43995ad28dc1aa96b29a8c237433 | +| [gdrive-osx-x64](https://docs.google.com/uc?id=0B3X9GlR6Embnb010SnpUV0s2ZkU&export=download) | 2.1.0 | OS X 64-bit | 297ccf3c945b364b5d306cef335ba44b0900e927 | +| [gdrive-osx-386](https://docs.google.com/uc?id=0B3X9GlR6EmbnTjByNlNvZVNRTjQ&export=download) | 2.1.0 | OS X 32-bit | c64714676a5b028aeeaf09e5f3b84d363e0ec7ed | +| [gdrive-osx-arm](https://docs.google.com/uc?id=0B3X9GlR6EmbnbURvYnVyVmNNX2M&export=download) | 2.1.0 | OS X arm | eb23b7bb5a072497372bd253e8fc8353bec8a64c | +| [gdrive-linux-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnQ0FtZmJJUXEyRTA&export=download) | 2.1.0 | Linux 64-bit | 4fd8391b300cac45963e53da44dcfe68da08d843 | +| [gdrive-linux-386](https://docs.google.com/uc?id=0B3X9GlR6EmbnLV92dHBpTkFhTEU&export=download) | 2.1.0 | Linux 32-bit | de9f49565fc62552fe862f08f84694ab4653adc2 | +| [gdrive-linux-rpi](https://docs.google.com/uc?id=0B3X9GlR6EmbnVXNLanp4ZFRRbzg&export=download) | 2.1.0 | Linux Raspberry Pi | e26e9ca3df3d08f970a276782ac5e92731c85467 | +| [gdrive-linux-arm64](https://docs.google.com/uc?id=0B3X9GlR6EmbnamliN0Rld01oRVk&export=download) | 2.1.0 | Linux arm 64-bit | 3d670905e13edf96d43c9f97293bdba62c740926 | +| [gdrive-linux-arm](https://docs.google.com/uc?id=0B3X9GlR6EmbnRjBaMVVLalN4cTA&export=download) | 2.1.0 | Linux arm 32-bit | 5b1036e0ef479ce228f7c32d1adfdc3840d71d10 | +| [gdrive-linux-mips64](https://docs.google.com/uc?id=0B3X9GlR6Embna2lzdEJ6blFzSzQ&export=download) | 2.1.0 | Linux mips 64-bit | 334bbd74b87fd1d05550e366724fe8e3c9e61ca4 | +| [gdrive-linux-mips64le](https://docs.google.com/uc?id=0B3X9GlR6EmbnWFk4Q3ZVZ1g3ZHM&export=download) | 2.1.0 | Linux mips 64-bit le | bb6961a2c03c074e6d34a1ec280cc69f5d5002f5 | +| [gdrive-linux-ppc64](https://docs.google.com/uc?id=0B3X9GlR6EmbnS09XMzhfRXBnUzA&export=download) | 2.1.0 | Linux PPC 64-bit | 70a1ac5be9ba819da5cf7a8dbd513805a26509ac | +| [gdrive-linux-ppc64le](https://docs.google.com/uc?id=0B3X9GlR6EmbneDJ2b3hqbVlNZnc&export=download) | 2.1.0 | Linux PPC 64-bit le | f426817ee4824b83b978f82f8e72eac6db92f2d1 | +| [gdrive-windows-386.exe](https://docs.google.com/uc?id=0B3X9GlR6EmbnV3RNeFVUQjZvS2c&export=download) | 2.1.0 | Window 32-bit | 1429200631b598543eddc3df3487117cad95adbb | +| [gdrive-windows-x64.exe](https://docs.google.com/uc?id=0B3X9GlR6EmbnNFRSSW1GaFBSRk0&export=download) | 2.1.0 | Windows 64-bit | 16ccab7c66b144e5806daeb2ba50d567b51504ca | +| [gdrive-dragonfly-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnelNIdmRMMGpVa2s&export=download) | 2.1.0 | DragonFly BSD 64-bit | dc214a24e59f68d99ca62757d99099051f83804a | +| [gdrive-freebsd-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnQkN0NnUwZ0tKLXM&export=download) | 2.1.0 | FreeBSD 64-bit | 93a5581652f9c01c47fb6c16e8ae655182f265da | +| [gdrive-freebsd-386](https://docs.google.com/uc?id=0B3X9GlR6EmbnNU5rbXBzeEhhOTA&export=download) | 2.1.0 | FreeBSD 32-bit | b9a3ee1e0fdbb5fa970942ab89b354ee863a5758 | +| [gdrive-freebsd-arm](https://docs.google.com/uc?id=0B3X9GlR6EmbnVHpUbVFzZzNqeW8&export=download) | 2.1.0 | FreeBSD arm | 7f5d1dedaa98501932ea368f2baba240da0b00d8 | +| [gdrive-netbsd-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnbGJobnBQR0dtV2c&export=download) | 2.1.0 | NetBSD 64-bit | 2a088dbd1e149204eb71a47ade109816983fe53f | +| [gdrive-netbsd-386](https://docs.google.com/uc?id=0B3X9GlR6EmbneWszMnl5RGZnYWs&export=download) | 2.1.0 | NetBSD 32-bit | a2c231b91839171a58da780657c445d4a1430537 | +| [gdrive-netbsd-arm](https://docs.google.com/uc?id=0B3X9GlR6EmbnVVhOWG9UUUhWNVU&export=download) | 2.1.0 | NetBSD arm | ac8a6354f8a8346c2bf84585e14f4a2cc69451db | +| [gdrive-openbsd-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnSy1JdFlHdUYyaGs&export=download) | 2.1.0 | OpenBSD 64-bit | 54be1d38b9014c6a8de5d71233cd6f208c27ac1c | +| [gdrive-openbsd-386](https://docs.google.com/uc?id=0B3X9GlR6EmbnRWhIZFRCNE1OdWc&export=download) | 2.1.0 | OpenBSD 32-bit | c2e08a9c7242de6d6ffa01598425fea0550076b8 | +| [gdrive-openbsd-arm](https://docs.google.com/uc?id=0B3X9GlR6EmbnWnAzMTNZanp2UEE&export=download) | 2.1.0 | OpenBSD arm | 22cd413c2705012b2ac78e64cc9f2b5bfa96dbea | +| [gdrive-solaris-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnWEtENmQ5dDJtTHc&export=download) | 2.1.0 | Solaris 64-bit | 2da03dfcc818a0bd3588ad850349a5a2554913fb | +| [gdrive-plan9-x64](https://docs.google.com/uc?id=0B3X9GlR6EmbnRmVyelhxLUUySjA&export=download) | 2.1.0 | Plan9 64-bit | 7b498ce0f416a3e8c1e17f603d21a3e84c1a9283 | +| [gdrive-plan9-386](https://docs.google.com/uc?id=0B3X9GlR6EmbnckdHZVdRZ0dZTU0&export=download) | 2.1.0 | Plan9 32-bit | cccd9ba86774bc6bd70f092158e2fcafa94601c0 | ## Compile from source ```bash @@ -147,12 +148,13 @@ global: --access-token <accessToken> Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users) options: - -f, --force Overwrite existing file - -r, --recursive Download directory recursively, documents will be skipped - --path <path> Download path - --delete Delete remote file when download is successful - --no-progress Hide progress - --stdout Write file content to stdout + -f, --force Overwrite existing file + -r, --recursive Download directory recursively, documents will be skipped + --path <path> Download path + --delete Delete remote file when download is successful + --no-progress Hide progress + --stdout Write file content to stdout + --timeout <timeout> Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: 300 ``` #### Download all files and directories matching query @@ -188,6 +190,7 @@ options: --mime <mime> Force mime type --share Share file --delete Delete local file when upload is successful + --timeout <timeout> Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: 300 --chunksize <chunksize> Set chunk size in bytes, default: 8388608 ``` @@ -205,6 +208,7 @@ options: --chunksize <chunksize> Set chunk size in bytes, default: 8388608 --mime <mime> Force mime type --share Share file + --timeout <timeout> Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: 300 --no-progress Hide progress ``` @@ -222,6 +226,7 @@ options: --name <name> Filename --no-progress Hide progress --mime <mime> Force mime type + --timeout <timeout> Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: 300 --chunksize <chunksize> Set chunk size in bytes, default: 8388608 ``` @@ -346,6 +351,7 @@ options: --delete-extraneous Delete extraneous local files --dry-run Show what would have been transferred --no-progress Hide progress + --timeout <timeout> Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: 300 ``` #### Sync local directory to drive @@ -364,6 +370,7 @@ options: --delete-extraneous Delete extraneous remote files --dry-run Show what would have been transferred --no-progress Hide progress + --timeout <timeout> Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: 300 --chunksize <chunksize> Set chunk size in bytes, default: 8388608 ``` @@ -409,10 +416,11 @@ global: --access-token <accessToken> Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users) options: - -f, --force Overwrite existing file - --no-progress Hide progress - --stdout Write file content to stdout - --path <path> Download path + -f, --force Overwrite existing file + --no-progress Hide progress + --stdout Write file content to stdout + --path <path> Download path + --timeout <timeout> Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: 300 ``` #### Delete file revision @@ -1,8 +1,8 @@ package main import ( - "github.com/prasmussen/gdrive/drive" "encoding/json" + "github.com/prasmussen/gdrive/drive" "os" ) diff --git a/drive/download.go b/drive/download.go index 0856258..ec1af8a 100644 --- a/drive/download.go +++ b/drive/download.go @@ -19,6 +19,7 @@ type DownloadArgs struct { Recursive bool Delete bool Stdout bool + Timeout time.Duration } func (self *Drive) Download(args DownloadArgs) error { @@ -120,10 +121,13 @@ func (self *Drive) downloadRecursive(args DownloadArgs) error { func (self *Drive) downloadBinary(f *drive.File, args DownloadArgs) (int64, int64, error) { // Get timeout reader wrapper and context - timeoutReaderWrapper, ctx := getTimeoutReaderWrapperContext() + timeoutReaderWrapper, ctx := getTimeoutReaderWrapperContext(args.Timeout) res, err := self.service.Files.Get(f.Id).Context(ctx).Download() if err != nil { + if isTimeoutError(err) { + return 0, 0, fmt.Errorf("Failed to download file: timeout, no data was transferred for %v", args.Timeout) + } return 0, 0, fmt.Errorf("Failed to download file: %s", err) } diff --git a/drive/errors.go b/drive/errors.go index e7631f7..f70f0f0 100644 --- a/drive/errors.go +++ b/drive/errors.go @@ -1,11 +1,16 @@ package drive import ( + "golang.org/x/net/context" "google.golang.org/api/googleapi" "time" ) -const MaxBackendErrorRetries = 5 +const MaxErrorRetries = 5 + +func isBackendOrRateLimitError(err error) bool { + return isBackendError(err) || isRateLimitError(err) +} func isBackendError(err error) bool { if err == nil { @@ -16,6 +21,19 @@ func isBackendError(err error) bool { return ok && ae.Code >= 500 && ae.Code <= 599 } +func isRateLimitError(err error) bool { + if err == nil { + return false + } + + ae, ok := err.(*googleapi.Error) + return ok && ae.Code == 403 +} + +func isTimeoutError(err error) bool { + return err == context.Canceled +} + func exponentialBackoffSleep(try int) { seconds := pow(2, try) time.Sleep(time.Duration(seconds) * time.Second) diff --git a/drive/revision_download.go b/drive/revision_download.go index 04055fa..6bed886 100644 --- a/drive/revision_download.go +++ b/drive/revision_download.go @@ -5,6 +5,7 @@ import ( "io" "io/ioutil" "path/filepath" + "time" ) type DownloadRevisionArgs struct { @@ -15,6 +16,7 @@ type DownloadRevisionArgs struct { Path string Force bool Stdout bool + Timeout time.Duration } func (self *Drive) DownloadRevision(args DownloadRevisionArgs) (err error) { @@ -30,10 +32,13 @@ func (self *Drive) DownloadRevision(args DownloadRevisionArgs) (err error) { } // Get timeout reader wrapper and context - timeoutReaderWrapper, ctx := getTimeoutReaderWrapperContext() + timeoutReaderWrapper, ctx := getTimeoutReaderWrapperContext(args.Timeout) res, err := getRev.Context(ctx).Download() if err != nil { + if isTimeoutError(err) { + return fmt.Errorf("Failed to download file: timeout, no data was transferred for %v", args.Timeout) + } return fmt.Errorf("Failed to download file: %s", err) } diff --git a/drive/sync_download.go b/drive/sync_download.go index 04b50b9..58cd49c 100644 --- a/drive/sync_download.go +++ b/drive/sync_download.go @@ -19,6 +19,7 @@ type DownloadSyncArgs struct { Path string DryRun bool DeleteExtraneous bool + Timeout time.Duration Resolution ConflictResolution Comparer FileComparer } @@ -188,14 +189,16 @@ func (self *Drive) downloadRemoteFile(id, fpath string, args DownloadSyncArgs, t } // Get timeout reader wrapper and context - timeoutReaderWrapper, ctx := getTimeoutReaderWrapperContext() + timeoutReaderWrapper, ctx := getTimeoutReaderWrapperContext(args.Timeout) res, err := self.service.Files.Get(id).Context(ctx).Download() if err != nil { - if isBackendError(err) && try < MaxBackendErrorRetries { + if isBackendOrRateLimitError(err) && try < MaxErrorRetries { exponentialBackoffSleep(try) try++ return self.downloadRemoteFile(id, fpath, args, try) + } else if isTimeoutError(err) { + return fmt.Errorf("Failed to download file: timeout, no data was transferred for %v", args.Timeout) } else { return fmt.Errorf("Failed to download file: %s", err) } @@ -228,7 +231,7 @@ func (self *Drive) downloadRemoteFile(id, fpath string, args DownloadSyncArgs, t _, err = io.Copy(outFile, reader) if err != nil { outFile.Close() - if try < MaxBackendErrorRetries { + if try < MaxErrorRetries { exponentialBackoffSleep(try) try++ return self.downloadRemoteFile(id, fpath, args, try) diff --git a/drive/sync_upload.go b/drive/sync_upload.go index 0d5c208..261497d 100644 --- a/drive/sync_upload.go +++ b/drive/sync_upload.go @@ -20,6 +20,7 @@ type UploadSyncArgs struct { DryRun bool DeleteExtraneous bool ChunkSize int64 + Timeout time.Duration Resolution ConflictResolution Comparer FileComparer } @@ -269,7 +270,7 @@ func (self *Drive) createMissingRemoteDir(args createMissingRemoteDirArgs) (*dri f, err := self.service.Files.Create(dstFile).Do() if err != nil { - if isBackendError(err) && args.try < MaxBackendErrorRetries { + if isBackendOrRateLimitError(err) && args.try < MaxErrorRetries { exponentialBackoffSleep(args.try) args.try++ return self.createMissingRemoteDir(args) @@ -308,14 +309,16 @@ func (self *Drive) uploadMissingFile(parentId string, lf *LocalFile, args Upload progressReader := getProgressReader(srcFile, args.Progress, lf.info.Size()) // Wrap reader in timeout reader - reader, ctx := getTimeoutReaderContext(progressReader) + reader, ctx := getTimeoutReaderContext(progressReader, args.Timeout) _, err = self.service.Files.Create(dstFile).Fields("id", "name", "size", "md5Checksum").Context(ctx).Media(reader, chunkSize).Do() if err != nil { - if isBackendError(err) && try < MaxBackendErrorRetries { + if isBackendOrRateLimitError(err) && try < MaxErrorRetries { exponentialBackoffSleep(try) try++ return self.uploadMissingFile(parentId, lf, args, try) + } else if isTimeoutError(err) { + return fmt.Errorf("Failed to upload file: timeout, no data was transferred for %v", args.Timeout) } else { return fmt.Errorf("Failed to upload file: %s", err) } @@ -347,14 +350,16 @@ func (self *Drive) updateChangedFile(cf *changedFile, args UploadSyncArgs, try i progressReader := getProgressReader(srcFile, args.Progress, cf.local.info.Size()) // Wrap reader in timeout reader - reader, ctx := getTimeoutReaderContext(progressReader) + reader, ctx := getTimeoutReaderContext(progressReader, args.Timeout) _, err = self.service.Files.Update(cf.remote.file.Id, dstFile).Context(ctx).Media(reader, chunkSize).Do() if err != nil { - if isBackendError(err) && try < MaxBackendErrorRetries { + if isBackendOrRateLimitError(err) && try < MaxErrorRetries { exponentialBackoffSleep(try) try++ return self.updateChangedFile(cf, args, try) + } else if isTimeoutError(err) { + return fmt.Errorf("Failed to upload file: timeout, no data was transferred for %v", args.Timeout) } else { return fmt.Errorf("Failed to update file: %s", err) } @@ -370,7 +375,7 @@ func (self *Drive) deleteRemoteFile(rf *RemoteFile, args UploadSyncArgs, try int err := self.service.Files.Delete(rf.file.Id).Do() if err != nil { - if isBackendError(err) && try < MaxBackendErrorRetries { + if isBackendOrRateLimitError(err) && try < MaxErrorRetries { exponentialBackoffSleep(try) try++ return self.deleteRemoteFile(rf, args, try) diff --git a/drive/timeout_reader.go b/drive/timeout_reader.go index 9930c12..67fd5b0 100644 --- a/drive/timeout_reader.go +++ b/drive/timeout_reader.go @@ -7,39 +7,51 @@ import ( "time" ) -const MaxIdleTimeout = time.Second * 120 const TimeoutTimerInterval = time.Second * 10 type timeoutReaderWrapper func(io.Reader) io.Reader -func getTimeoutReaderWrapperContext() (timeoutReaderWrapper, context.Context) { +func getTimeoutReaderWrapperContext(timeout time.Duration) (timeoutReaderWrapper, context.Context) { ctx, cancel := context.WithCancel(context.TODO()) wrapper := func(r io.Reader) io.Reader { - return getTimeoutReader(r, cancel) + // Return untouched reader if timeout is 0 + if timeout == 0 { + return r + } + + return getTimeoutReader(r, cancel, timeout) } return wrapper, ctx } -func getTimeoutReaderContext(r io.Reader) (io.Reader, context.Context) { +func getTimeoutReaderContext(r io.Reader, timeout time.Duration) (io.Reader, context.Context) { ctx, cancel := context.WithCancel(context.TODO()) - return getTimeoutReader(r, cancel), ctx + + // Return untouched reader if timeout is 0 + if timeout == 0 { + return r, ctx + } + + return getTimeoutReader(r, cancel, timeout), ctx } -func getTimeoutReader(r io.Reader, cancel context.CancelFunc) io.Reader { +func getTimeoutReader(r io.Reader, cancel context.CancelFunc, timeout time.Duration) io.Reader { return &TimeoutReader{ - reader: r, - cancel: cancel, - mutex: &sync.Mutex{}, + reader: r, + cancel: cancel, + mutex: &sync.Mutex{}, + maxIdleTimeout: timeout, } } type TimeoutReader struct { - reader io.Reader - cancel context.CancelFunc - lastActivity time.Time - timer *time.Timer - mutex *sync.Mutex - done bool + reader io.Reader + cancel context.CancelFunc + lastActivity time.Time + timer *time.Timer + mutex *sync.Mutex + maxIdleTimeout time.Duration + done bool } func (self *TimeoutReader) Read(p []byte) (int, error) { @@ -90,7 +102,7 @@ func (self *TimeoutReader) timeout() { return } - if time.Since(self.lastActivity) > MaxIdleTimeout { + if time.Since(self.lastActivity) > self.maxIdleTimeout { self.cancel() self.mutex.Unlock() return diff --git a/drive/update.go b/drive/update.go index 156eb2f..2ab684e 100644 --- a/drive/update.go +++ b/drive/update.go @@ -20,6 +20,7 @@ type UpdateArgs struct { Mime string Recursive bool ChunkSize int64 + Timeout time.Duration } func (self *Drive) Update(args UpdateArgs) error { @@ -57,13 +58,16 @@ func (self *Drive) Update(args UpdateArgs) error { progressReader := getProgressReader(srcFile, args.Progress, srcFileInfo.Size()) // Wrap reader in timeout reader - reader, ctx := getTimeoutReaderContext(progressReader) + reader, ctx := getTimeoutReaderContext(progressReader, args.Timeout) fmt.Fprintf(args.Out, "Uploading %s\n", args.Path) started := time.Now() f, err := self.service.Files.Update(args.Id, dstFile).Fields("id", "name", "size").Context(ctx).Media(reader, chunkSize).Do() if err != nil { + if isTimeoutError(err) { + return fmt.Errorf("Failed to upload file: timeout, no data was transferred for %v", args.Timeout) + } return fmt.Errorf("Failed to upload file: %s", err) } diff --git a/drive/upload.go b/drive/upload.go index c42bebd..dbf068c 100644 --- a/drive/upload.go +++ b/drive/upload.go @@ -22,6 +22,7 @@ type UploadArgs struct { Share bool Delete bool ChunkSize int64 + Timeout time.Duration } func (self *Drive) Upload(args UploadArgs) error { @@ -89,10 +90,12 @@ func (self *Drive) uploadRecursive(args UploadArgs) error { if info.IsDir() { args.Name = "" return self.uploadDirectory(args) - } else { + } else if info.Mode().IsRegular() { _, _, err := self.uploadFile(args) return err } + + return nil } func (self *Drive) uploadDirectory(args UploadArgs) error { @@ -173,13 +176,16 @@ func (self *Drive) uploadFile(args UploadArgs) (*drive.File, int64, error) { progressReader := getProgressReader(srcFile, args.Progress, srcFileInfo.Size()) // Wrap reader in timeout reader - reader, ctx := getTimeoutReaderContext(progressReader) + reader, ctx := getTimeoutReaderContext(progressReader, args.Timeout) fmt.Fprintf(args.Out, "Uploading %s\n", args.Path) started := time.Now() f, err := self.service.Files.Create(dstFile).Fields("id", "name", "size", "md5Checksum", "webContentLink").Context(ctx).Media(reader, chunkSize).Do() if err != nil { + if isTimeoutError(err) { + return nil, 0, fmt.Errorf("Failed to upload file: timeout, no data was transferred for %v", args.Timeout) + } return nil, 0, fmt.Errorf("Failed to upload file: %s", err) } @@ -198,6 +204,7 @@ type UploadStreamArgs struct { Share bool ChunkSize int64 Progress io.Writer + Timeout time.Duration } func (self *Drive) UploadStream(args UploadStreamArgs) error { @@ -223,13 +230,16 @@ func (self *Drive) UploadStream(args UploadStreamArgs) error { progressReader := getProgressReader(args.In, args.Progress, 0) // Wrap reader in timeout reader - reader, ctx := getTimeoutReaderContext(progressReader) + reader, ctx := getTimeoutReaderContext(progressReader, args.Timeout) fmt.Fprintf(args.Out, "Uploading %s\n", dstFile.Name) started := time.Now() f, err := self.service.Files.Create(dstFile).Fields("id", "name", "size", "webContentLink").Context(ctx).Media(reader, chunkSize).Do() if err != nil { + if isTimeoutError(err) { + return fmt.Errorf("Failed to upload file: timeout, no data was transferred for %v", args.Timeout) + } return fmt.Errorf("Failed to upload file: %s", err) } @@ -1,19 +1,20 @@ package main import ( - "github.com/prasmussen/gdrive/cli" "fmt" + "github.com/prasmussen/gdrive/cli" "os" ) const Name = "gdrive" -const Version = "2.0.1" +const Version = "2.1.0" const DefaultMaxFiles = 30 const DefaultMaxChanges = 100 const DefaultNameWidth = 40 const DefaultPathWidth = 60 const DefaultUploadChunkSize = 8 * 1024 * 1024 +const DefaultTimeout = 5 * 60 const DefaultQuery = "trashed = false and 'me' in owners" const DefaultShareRole = "reader" const DefaultShareType = "anyone" @@ -134,6 +135,12 @@ func main() { Description: "Write file content to stdout", OmitValue: true, }, + cli.IntFlag{ + Name: "timeout", + Patterns: []string{"--timeout"}, + Description: fmt.Sprintf("Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: %d", DefaultTimeout), + DefaultValue: DefaultTimeout, + }, ), }, }, @@ -217,6 +224,12 @@ func main() { OmitValue: true, }, cli.IntFlag{ + Name: "timeout", + Patterns: []string{"--timeout"}, + Description: fmt.Sprintf("Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: %d", DefaultTimeout), + DefaultValue: DefaultTimeout, + }, + cli.IntFlag{ Name: "chunksize", Patterns: []string{"--chunksize"}, Description: fmt.Sprintf("Set chunk size in bytes, default: %d", DefaultUploadChunkSize), @@ -254,6 +267,12 @@ func main() { Description: "Share file", OmitValue: true, }, + cli.IntFlag{ + Name: "timeout", + Patterns: []string{"--timeout"}, + Description: fmt.Sprintf("Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: %d", DefaultTimeout), + DefaultValue: DefaultTimeout, + }, cli.BoolFlag{ Name: "noProgress", Patterns: []string{"--no-progress"}, @@ -292,6 +311,12 @@ func main() { Description: "Force mime type", }, cli.IntFlag{ + Name: "timeout", + Patterns: []string{"--timeout"}, + Description: fmt.Sprintf("Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: %d", DefaultTimeout), + DefaultValue: DefaultTimeout, + }, + cli.IntFlag{ Name: "chunksize", Patterns: []string{"--chunksize"}, Description: fmt.Sprintf("Set chunk size in bytes, default: %d", DefaultUploadChunkSize), @@ -494,6 +519,12 @@ func main() { Description: "Hide progress", OmitValue: true, }, + cli.IntFlag{ + Name: "timeout", + Patterns: []string{"--timeout"}, + Description: fmt.Sprintf("Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: %d", DefaultTimeout), + DefaultValue: DefaultTimeout, + }, ), }, }, @@ -541,6 +572,12 @@ func main() { OmitValue: true, }, cli.IntFlag{ + Name: "timeout", + Patterns: []string{"--timeout"}, + Description: fmt.Sprintf("Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: %d", DefaultTimeout), + DefaultValue: DefaultTimeout, + }, + cli.IntFlag{ Name: "chunksize", Patterns: []string{"--chunksize"}, Description: fmt.Sprintf("Set chunk size in bytes, default: %d", DefaultUploadChunkSize), @@ -647,6 +684,12 @@ func main() { Patterns: []string{"--path"}, Description: "Download path", }, + cli.IntFlag{ + Name: "timeout", + Patterns: []string{"--timeout"}, + Description: fmt.Sprintf("Set timeout in seconds, use 0 for no timeout. Timeout is reached when no data is transferred in set amount of seconds, default: %d", DefaultTimeout), + DefaultValue: DefaultTimeout, + }, ), }, }, diff --git a/handlers_drive.go b/handlers_drive.go index f0386ce..8db7329 100644 --- a/handlers_drive.go +++ b/handlers_drive.go @@ -1,15 +1,16 @@ package main import ( + "fmt" "github.com/prasmussen/gdrive/auth" "github.com/prasmussen/gdrive/cli" "github.com/prasmussen/gdrive/drive" - "fmt" "io" "io/ioutil" "net/http" "os" "path/filepath" + "time" ) const ClientId = "367116221053-7n0vf5akeru7on6o2fjinrecpdoe99eg.apps.googleusercontent.com" @@ -57,6 +58,7 @@ func downloadHandler(ctx cli.Context) { Recursive: args.Bool("recursive"), Stdout: args.Bool("stdout"), Progress: progressWriter(args.Bool("noProgress")), + Timeout: durationInSeconds(args.Int64("timeout")), }) checkErr(err) } @@ -84,6 +86,7 @@ func downloadSyncHandler(ctx cli.Context) { RootId: args.String("fileId"), DryRun: args.Bool("dryRun"), DeleteExtraneous: args.Bool("deleteExtraneous"), + Timeout: durationInSeconds(args.Int64("timeout")), Resolution: conflictResolution(args), Comparer: NewCachedMd5Comparer(cachePath), }) @@ -100,6 +103,7 @@ func downloadRevisionHandler(ctx cli.Context) { Stdout: args.Bool("stdout"), Path: args.String("path"), Progress: progressWriter(args.Bool("noProgress")), + Timeout: durationInSeconds(args.Int64("timeout")), }) checkErr(err) } @@ -118,6 +122,7 @@ func uploadHandler(ctx cli.Context) { Share: args.Bool("share"), Delete: args.Bool("delete"), ChunkSize: args.Int64("chunksize"), + Timeout: durationInSeconds(args.Int64("timeout")), }) checkErr(err) } @@ -132,6 +137,7 @@ func uploadStdinHandler(ctx cli.Context) { Mime: args.String("mime"), Share: args.Bool("share"), ChunkSize: args.Int64("chunksize"), + Timeout: durationInSeconds(args.Int64("timeout")), Progress: progressWriter(args.Bool("noProgress")), }) checkErr(err) @@ -148,6 +154,7 @@ func uploadSyncHandler(ctx cli.Context) { DryRun: args.Bool("dryRun"), DeleteExtraneous: args.Bool("deleteExtraneous"), ChunkSize: args.Int64("chunksize"), + Timeout: durationInSeconds(args.Int64("timeout")), Resolution: conflictResolution(args), Comparer: NewCachedMd5Comparer(cachePath), }) @@ -165,6 +172,7 @@ func updateHandler(ctx cli.Context) { Mime: args.String("mime"), Progress: progressWriter(args.Bool("noProgress")), ChunkSize: args.Int64("chunksize"), + Timeout: durationInSeconds(args.Int64("timeout")), }) checkErr(err) } @@ -385,6 +393,10 @@ func progressWriter(discard bool) io.Writer { return os.Stderr } +func durationInSeconds(seconds int64) time.Duration { + return time.Second * time.Duration(seconds) +} + func conflictResolution(args cli.Arguments) drive.ConflictResolution { keepLocal := args.Bool("keepLocal") keepRemote := args.Bool("keepRemote") diff --git a/handlers_meta.go b/handlers_meta.go index bfd5b8f..2e1c95c 100644 --- a/handlers_meta.go +++ b/handlers_meta.go @@ -1,8 +1,8 @@ package main import ( - "github.com/prasmussen/gdrive/cli" "fmt" + "github.com/prasmussen/gdrive/cli" "os" "runtime" "strings" |