diff options
| -rw-r--r-- | compare.go | 67 | ||||
| -rw-r--r-- | handlers_drive.go | 8 | ||||
| -rw-r--r-- | util.go | 32 |
3 files changed, 95 insertions, 12 deletions
@@ -1,27 +1,74 @@ package main import ( - "fmt" "os" - "io" - "crypto/md5" + "encoding/json" "./drive" ) +const MinCacheFileSize = 5 * 1024 * 1024 + type Md5Comparer struct {} func (self Md5Comparer) Changed(local *drive.LocalFile, remote *drive.RemoteFile) bool { return remote.Md5() != md5sum(local.AbsPath()) } -func md5sum(path string) string { - h := md5.New() +type CachedFileInfo struct { + Size int64 `json:"size"` + Modified int64 `json:"modified"` + Md5 string `json:"md5"` +} + +func NewCachedMd5Comparer(path string) CachedMd5Comparer { + cache := map[string]*CachedFileInfo{} + f, err := os.Open(path) - if err != nil { - return "" + if err == nil { + json.NewDecoder(f).Decode(&cache) + } + f.Close() + return CachedMd5Comparer{path, cache} +} + +type CachedMd5Comparer struct { + path string + cache map[string]*CachedFileInfo +} + +func (self CachedMd5Comparer) Changed(local *drive.LocalFile, remote *drive.RemoteFile) bool { + return remote.Md5() != self.md5(local) +} + +func (self CachedMd5Comparer) md5(local *drive.LocalFile) string { + // See if file exist in cache + cached, found := self.cache[local.AbsPath()] + + // If found and modification time and size has not changed, return cached md5 + if found && local.Modified().UnixNano() == cached.Modified && local.Size() == cached.Size { + return cached.Md5 + } + + // Calculate new md5 sum + md5 := md5sum(local.AbsPath()) + + // Cache file info if file meets size criteria + if local.Size() > MinCacheFileSize { + self.cacheAdd(local, md5) + self.persist() } - defer f.Close() - io.Copy(h, f) - return fmt.Sprintf("%x", h.Sum(nil)) + return md5 +} + +func (self CachedMd5Comparer) cacheAdd(lf *drive.LocalFile, md5 string) { + self.cache[lf.AbsPath()] = &CachedFileInfo{ + Size: lf.Size(), + Modified: lf.Modified().UnixNano(), + Md5: md5, + } +} + +func (self CachedMd5Comparer) persist() { + writeJson(self.path, self.cache) } diff --git a/handlers_drive.go b/handlers_drive.go index 029b47e..9de27cc 100644 --- a/handlers_drive.go +++ b/handlers_drive.go @@ -5,6 +5,7 @@ import ( "os" "io" "io/ioutil" + "path/filepath" "./cli" "./auth" "./drive" @@ -13,6 +14,7 @@ import ( const ClientId = "367116221053-7n0vf5akeru7on6o2fjinrecpdoe99eg.apps.googleusercontent.com" const ClientSecret = "1qsNodXNaWq1mQuBjUjmvhoO" const TokenFilename = "token_v2.json" +const DefaultCacheFileName = "file_cache.json" func listHandler(ctx cli.Context) { @@ -58,6 +60,7 @@ func downloadHandler(ctx cli.Context) { func downloadSyncHandler(ctx cli.Context) { args := ctx.Args() + cachePath := filepath.Join(args.String("configDir"), DefaultCacheFileName) err := newDrive(args).DownloadSync(drive.DownloadSyncArgs{ Out: os.Stdout, Progress: progressWriter(args.Bool("noProgress")), @@ -65,7 +68,7 @@ func downloadSyncHandler(ctx cli.Context) { RootId: args.String("id"), DryRun: args.Bool("dryRun"), DeleteExtraneous: args.Bool("deleteExtraneous"), - Comparer: Md5Comparer{}, + Comparer: NewCachedMd5Comparer(cachePath), }) checkErr(err) } @@ -115,6 +118,7 @@ func uploadStdinHandler(ctx cli.Context) { func uploadSyncHandler(ctx cli.Context) { args := ctx.Args() + cachePath := filepath.Join(args.String("configDir"), DefaultCacheFileName) err := newDrive(args).UploadSync(drive.UploadSyncArgs{ Out: os.Stdout, Progress: progressWriter(args.Bool("noProgress")), @@ -123,7 +127,7 @@ func uploadSyncHandler(ctx cli.Context) { DryRun: args.Bool("dryRun"), DeleteExtraneous: args.Bool("deleteExtraneous"), ChunkSize: args.Int64("chunksize"), - Comparer: Md5Comparer{}, + Comparer: NewCachedMd5Comparer(cachePath), }) checkErr(err) } @@ -4,7 +4,10 @@ import ( "runtime" "path/filepath" "fmt" + "encoding/json" "os" + "io" + "crypto/md5" ) func GetDefaultConfigDir() string { @@ -56,3 +59,32 @@ func checkErr(err error) { os.Exit(1) } } + +func writeJson(path string, data interface{}) error { + tmpFile := path + ".tmp" + f, err := os.Create(tmpFile) + if err != nil { + return err + } + + err = json.NewEncoder(f).Encode(data) + f.Close() + if err != nil { + os.Remove(tmpFile) + return err + } + + return os.Rename(tmpFile, path) +} + +func md5sum(path string) string { + h := md5.New() + f, err := os.Open(path) + if err != nil { + return "" + } + defer f.Close() + + io.Copy(h, f) + return fmt.Sprintf("%x", h.Sum(nil)) +} |