123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179 |
- package vfs
- import (
- "context"
- "fmt"
- "os"
- "path"
- "sort"
- "strings"
- "sync"
- "sync/atomic"
- "time"
- "github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/dirtree"
- "github.com/rclone/rclone/fs/list"
- "github.com/rclone/rclone/fs/log"
- "github.com/rclone/rclone/fs/operations"
- "github.com/rclone/rclone/fs/walk"
- "github.com/rclone/rclone/vfs/vfscommon"
- "golang.org/x/text/unicode/norm"
- )
- // Dir represents a directory entry
- type Dir struct {
- vfs *VFS // read only
- inode uint64 // read only: inode number
- f fs.Fs // read only
- cleanupTimer *time.Timer // read only: timer to call cacheCleanup
- mu sync.RWMutex // protects the following
- parent *Dir // parent, nil for root
- path string
- entry fs.Directory
- read time.Time // time directory entry last read
- items map[string]Node // directory entries - can be empty but not nil
- virtual map[string]vState // virtual directory entries - may be nil
- sys atomic.Value // user defined info to be attached here
- modTimeMu sync.Mutex // protects the following
- modTime time.Time
- _hasVirtual atomic.Bool // shows if the directory has virtual entries
- }
- //go:generate stringer -type=vState
- // vState describes the state of the virtual directory entries
- type vState byte
- const (
- vOK vState = iota // Not virtual
- vAddFile // added file
- vAddDir // added directory
- vDel // removed file or directory
- )
- func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
- d := &Dir{
- vfs: vfs,
- f: f,
- parent: parent,
- entry: fsDir,
- path: fsDir.Remote(),
- modTime: fsDir.ModTime(context.TODO()),
- inode: newInode(),
- items: make(map[string]Node),
- }
- d.cleanupTimer = time.AfterFunc(vfs.Opt.DirCacheTime*2, d.cacheCleanup)
- d.setHasVirtual(false)
- return d
- }
- func (d *Dir) cacheCleanup() {
- defer func() {
- // We should never panic here
- _ = recover()
- }()
- when := time.Now()
- d.mu.Lock()
- _, stale := d._age(when)
- d.mu.Unlock()
- if stale {
- d.ForgetAll()
- }
- }
- // String converts it to printable
- func (d *Dir) String() string {
- if d == nil {
- return "<nil *Dir>"
- }
- d.mu.RLock()
- defer d.mu.RUnlock()
- return d.path + "/"
- }
- // Dumps the directory tree to the string builder with the given indent
- //
- //lint:ignore U1000 false positive when running staticcheck,
- //nolint:unused // Don't include unused when running golangci-lint
- func (d *Dir) dumpIndent(out *strings.Builder, indent string) {
- if d == nil {
- fmt.Fprintf(out, "%s<nil *Dir>\n", indent)
- return
- }
- d.mu.RLock()
- defer d.mu.RUnlock()
- fmt.Fprintf(out, "%sPath: %s\n", indent, d.path)
- fmt.Fprintf(out, "%sEntry: %v\n", indent, d.entry)
- fmt.Fprintf(out, "%sRead: %v\n", indent, d.read)
- fmt.Fprintf(out, "%s- items %d\n", indent, len(d.items))
- // Sort?
- for leaf, node := range d.items {
- switch x := node.(type) {
- case *Dir:
- fmt.Fprintf(out, "%s %s/ - %v\n", indent, leaf, x)
- // check the parent is correct
- if x.parent != d {
- fmt.Fprintf(out, "%s PARENT POINTER WRONG\n", indent)
- }
- x.dumpIndent(out, indent+"\t")
- case *File:
- fmt.Fprintf(out, "%s %s - %v\n", indent, leaf, x)
- default:
- panic("bad dir entry")
- }
- }
- fmt.Fprintf(out, "%s- virtual %d\n", indent, len(d.virtual))
- for leaf, state := range d.virtual {
- fmt.Fprintf(out, "%s %s - %v\n", indent, leaf, state)
- }
- }
- // Dumps a nicely formatted directory tree to a string
- //
- //lint:ignore U1000 false positive when running staticcheck,
- //nolint:unused // Don't include unused when running golangci-lint
- func (d *Dir) dump() string {
- var out strings.Builder
- d.dumpIndent(&out, "")
- return out.String()
- }
- // IsFile returns false for Dir - satisfies Node interface
- func (d *Dir) IsFile() bool {
- return false
- }
- // IsDir returns true for Dir - satisfies Node interface
- func (d *Dir) IsDir() bool {
- return true
- }
- // Mode bits of the directory - satisfies Node interface
- func (d *Dir) Mode() (mode os.FileMode) {
- return d.vfs.Opt.DirPerms
- }
- // Name (base) of the directory - satisfies Node interface
- func (d *Dir) Name() (name string) {
- d.mu.RLock()
- name = path.Base(d.path)
- d.mu.RUnlock()
- if name == "." {
- name = "/"
- }
- return name
- }
- // Path of the directory - satisfies Node interface
- func (d *Dir) Path() (name string) {
- d.mu.RLock()
- defer d.mu.RUnlock()
- return d.path
- }
- // Sys returns underlying data source (can be nil) - satisfies Node interface
- func (d *Dir) Sys() interface{} {
- return d.sys.Load()
- }
- // SetSys sets the underlying data source (can be nil) - satisfies Node interface
- func (d *Dir) SetSys(x interface{}) {
- d.sys.Store(x)
- }
- // Inode returns the inode number - satisfies Node interface
- func (d *Dir) Inode() uint64 {
- return d.inode
- }
- // Node returns the Node associated with this - satisfies Noder interface
- func (d *Dir) Node() Node {
- return d
- }
- // hasVirtual returns whether the directory has virtual entries
- func (d *Dir) hasVirtual() bool {
- return d._hasVirtual.Load()
- }
- // setHasVirtual sets the hasVirtual flag for the directory
- func (d *Dir) setHasVirtual(hasVirtual bool) {
- d._hasVirtual.Store(hasVirtual)
- }
- // ForgetAll forgets directory entries for this directory and any children.
- //
- // It does not invalidate or clear the cache of the parent directory.
- //
- // It returns true if the directory or any of its children had virtual entries
- // so could not be forgotten. Children which didn't have virtual entries and
- // children with virtual entries will be forgotten even if true is returned.
- func (d *Dir) ForgetAll() (hasVirtual bool) {
- d.mu.RLock()
- fs.Debugf(d.path, "forgetting directory cache")
- for _, node := range d.items {
- if dir, ok := node.(*Dir); ok {
- if dir.ForgetAll() {
- d.setHasVirtual(true)
- }
- }
- }
- d.mu.RUnlock()
- d.mu.Lock()
- defer d.mu.Unlock()
- // Purge any unnecessary virtual entries
- d._purgeVirtual()
- d.read = time.Time{}
- // Check if this dir has virtual entries
- if len(d.virtual) != 0 {
- d.setHasVirtual(true)
- }
- // Don't clear directory entries if there are virtual entries in this
- // directory or any children
- if !d.hasVirtual() {
- d.items = make(map[string]Node)
- d.cleanupTimer.Stop()
- }
- return d.hasVirtual()
- }
- // forgetDirPath clears the cache for itself and all subdirectories if
- // they match the given path. The path is specified relative from the
- // directory it is called from.
- //
- // It does not invalidate or clear the cache of the parent directory.
- func (d *Dir) forgetDirPath(relativePath string) {
- dir := d.cachedDir(relativePath)
- if dir == nil {
- return
- }
- dir.ForgetAll()
- }
- // invalidateDir invalidates the directory cache for absPath relative to the root
- func (d *Dir) invalidateDir(absPath string) {
- node := d.vfs.root.cachedNode(absPath)
- if dir, ok := node.(*Dir); ok {
- dir.mu.Lock()
- if !dir.read.IsZero() {
- fs.Debugf(dir.path, "invalidating directory cache")
- dir.read = time.Time{}
- }
- dir.mu.Unlock()
- }
- }
- // changeNotify invalidates the directory cache for the relativePath
- // passed in.
- //
- // if entryType is a directory it invalidates the parent of the directory too.
- func (d *Dir) changeNotify(relativePath string, entryType fs.EntryType) {
- defer log.Trace(d.path, "relativePath=%q, type=%v", relativePath, entryType)("")
- d.mu.RLock()
- absPath := path.Join(d.path, relativePath)
- d.mu.RUnlock()
- d.invalidateDir(vfscommon.FindParent(absPath))
- if entryType == fs.EntryDirectory {
- d.invalidateDir(absPath)
- }
- }
- // ForgetPath clears the cache for itself and all subdirectories if
- // they match the given path. The path is specified relative from the
- // directory it is called from. The cache of the parent directory is
- // marked as stale, but not cleared otherwise.
- // It is not possible to traverse the directory tree upwards, i.e.
- // you cannot clear the cache for the Dir's ancestors or siblings.
- func (d *Dir) ForgetPath(relativePath string, entryType fs.EntryType) {
- defer log.Trace(d.path, "relativePath=%q, type=%v", relativePath, entryType)("")
- d.mu.RLock()
- absPath := path.Join(d.path, relativePath)
- d.mu.RUnlock()
- if absPath != "" {
- d.invalidateDir(vfscommon.FindParent(absPath))
- }
- if entryType == fs.EntryDirectory {
- d.forgetDirPath(relativePath)
- }
- }
- // walk runs a function on all cached directories. It will be called
- // on a directory's children first.
- //
- // The mutex will be held for the directory when fun is called
- func (d *Dir) walk(fun func(*Dir)) {
- d.mu.Lock()
- defer d.mu.Unlock()
- for _, node := range d.items {
- if dir, ok := node.(*Dir); ok {
- dir.walk(fun)
- }
- }
- fun(d)
- }
- // countActiveWriters returns the number of writers active in this
- // directory and any subdirectories.
- func (d *Dir) countActiveWriters() (writers int) {
- d.walk(func(d *Dir) {
- // NB d.mu is held by walk() here
- fs.Debugf(d.path, "Looking for writers")
- for leaf, item := range d.items {
- fs.Debugf(leaf, "reading active writers")
- if file, ok := item.(*File); ok {
- n := file.activeWriters()
- if n != 0 {
- fs.Debugf(file, "active writers %d", n)
- }
- writers += n
- }
- }
- })
- return writers
- }
- // age returns the duration since the last time the directory contents
- // was read and the content is considered stale. age will be 0 and
- // stale true if the last read time is empty.
- // age must be called with d.mu held.
- func (d *Dir) _age(when time.Time) (age time.Duration, stale bool) {
- if d.read.IsZero() {
- return age, true
- }
- age = when.Sub(d.read)
- stale = age > d.vfs.Opt.DirCacheTime
- return
- }
- // renameTree renames the directories under this directory
- //
- // path should be the desired path
- func (d *Dir) renameTree(dirPath string) {
- d.mu.Lock()
- defer d.mu.Unlock()
- // Make sure the path is correct for each node
- if d.path != dirPath {
- fs.Debugf(d.path, "Renaming to %q", dirPath)
- d.path = dirPath
- d.entry = fs.NewDirCopy(context.TODO(), d.entry).SetRemote(dirPath)
- }
- // Do the same to any child directories and files
- for leaf, node := range d.items {
- switch x := node.(type) {
- case *Dir:
- x.renameTree(path.Join(dirPath, leaf))
- case *File:
- x.renameDir(dirPath)
- default:
- panic("bad dir entry")
- }
- }
- }
- // rename should be called after the directory is renamed
- //
- // Reset the directory to new state, discarding all the objects and
- // reading everything again
- func (d *Dir) rename(newParent *Dir, fsDir fs.Directory) {
- d.ForgetAll()
- d.modTimeMu.Lock()
- d.modTime = fsDir.ModTime(context.TODO())
- d.modTimeMu.Unlock()
- d.mu.Lock()
- oldPath := d.path
- d.parent = newParent
- d.entry = fsDir
- d.path = fsDir.Remote()
- newPath := d.path
- d.read = time.Time{}
- d.mu.Unlock()
- // Rename any remaining items in the tree that we couldn't forget
- d.renameTree(d.path)
- // Rename in the cache
- if d.vfs.cache != nil && d.vfs.cache.DirExists(oldPath) {
- if err := d.vfs.cache.DirRename(oldPath, newPath); err != nil {
- fs.Infof(d, "Dir.Rename failed in Cache: %v", err)
- }
- }
- }
- // addObject adds a new object or directory to the directory
- //
- // The name passed in is marked as virtual as it hasn't been read from a remote
- // directory listing.
- //
- // note that we add new objects rather than updating old ones
- func (d *Dir) addObject(node Node) {
- d.mu.Lock()
- leaf := node.Name()
- d.items[leaf] = node
- if d.virtual == nil {
- d.virtual = make(map[string]vState)
- }
- vAdd := vAddFile
- if node.IsDir() {
- vAdd = vAddDir
- }
- d.virtual[leaf] = vAdd
- d.setHasVirtual(true)
- fs.Debugf(d.path, "Added virtual directory entry %v: %q", vAdd, leaf)
- d.mu.Unlock()
- }
- // AddVirtual adds a virtual object of name and size to the directory
- //
- // This will be replaced with a real object when it is read back from the
- // remote.
- //
- // This is used to add directory entries while things are uploading
- func (d *Dir) AddVirtual(leaf string, size int64, isDir bool) {
- var node Node
- d.mu.RLock()
- dPath := d.path
- _, found := d.items[leaf]
- d.mu.RUnlock()
- if found {
- // Don't overwrite existing objects
- return
- }
- if isDir {
- remote := path.Join(dPath, leaf)
- entry := fs.NewDir(remote, time.Now())
- node = newDir(d.vfs, d.f, d, entry)
- } else {
- f := newFile(d, dPath, nil, leaf)
- f.setSize(size)
- node = f
- }
- d.addObject(node)
- }
- // delObject removes an object from the directory
- //
- // The name passed in is marked as virtual as the delete it hasn't been read
- // from a remote directory listing.
- func (d *Dir) delObject(leaf string) {
- d.mu.Lock()
- delete(d.items, leaf)
- if d.virtual == nil {
- d.virtual = make(map[string]vState)
- }
- d.virtual[leaf] = vDel
- d.setHasVirtual(true)
- fs.Debugf(d.path, "Added virtual directory entry %v: %q", vDel, leaf)
- d.mu.Unlock()
- }
- // DelVirtual removes an object from the directory listing
- //
- // It marks it as removed until it has confirmed the object is missing when the
- // directory entries are re-read in.
- //
- // This is used to remove directory entries after things have been deleted or
- // renamed but before we've had confirmation from the backend.
- func (d *Dir) DelVirtual(leaf string) {
- d.delObject(leaf)
- }
- // read the directory and sets d.items - must be called with the lock held
- func (d *Dir) _readDir() error {
- when := time.Now()
- if age, stale := d._age(when); stale {
- if age != 0 {
- fs.Debugf(d.path, "Re-reading directory (%v old)", age)
- }
- } else {
- return nil
- }
- entries, err := list.DirSorted(context.TODO(), d.f, false, d.path)
- if err == fs.ErrorDirNotFound {
- // We treat directory not found as empty because we
- // create directories on the fly
- } else if err != nil {
- return err
- }
- if d.vfs.Opt.BlockNormDupes { // do this only if requested, as it will have a performance hit
- ci := fs.GetConfig(context.TODO())
- // sort entries such that NFD comes before NFC of same name
- sort.Slice(entries, func(i, j int) bool {
- if entries[i] != entries[j] && fs.DirEntryType(entries[i]) == fs.DirEntryType(entries[j]) && norm.NFC.String(entries[i].Remote()) == norm.NFC.String(entries[j].Remote()) {
- if norm.NFD.IsNormalString(entries[i].Remote()) && !norm.NFD.IsNormalString(entries[j].Remote()) {
- return true
- }
- }
- return entries.Less(i, j)
- })
- // detect dupes, remove them from the list and log an error
- normalizedNames := make(map[string]struct{}, entries.Len())
- filteredEntries := make(fs.DirEntries, 0)
- for _, e := range entries {
- normName := fmt.Sprintf("%s-%T", operations.ToNormal(e.Remote(), !ci.NoUnicodeNormalization, (ci.IgnoreCaseSync || d.vfs.Opt.CaseInsensitive)), e) // include type to track objects and dirs separately
- _, found := normalizedNames[normName]
- if found {
- fs.Errorf(e.Remote(), "duplicate normalized names detected - skipping")
- continue
- }
- normalizedNames[normName] = struct{}{}
- filteredEntries = append(filteredEntries, e)
- }
- entries = filteredEntries
- }
- err = d._readDirFromEntries(entries, nil, time.Time{})
- if err != nil {
- return err
- }
- d.read = when
- d.cleanupTimer.Reset(d.vfs.Opt.DirCacheTime * 2)
- return nil
- }
- // update d.items for each dir in the DirTree below this one and
- // set the last read time - must be called with the lock held
- func (d *Dir) _readDirFromDirTree(dirTree dirtree.DirTree, when time.Time) error {
- return d._readDirFromEntries(dirTree[d.path], dirTree, when)
- }
- // Remove the virtual directory entry leaf
- func (d *Dir) _deleteVirtual(name string) {
- virtualState, ok := d.virtual[name]
- if !ok {
- return
- }
- delete(d.virtual, name)
- if len(d.virtual) == 0 {
- d.virtual = nil
- d.setHasVirtual(false)
- }
- fs.Debugf(d.path, "Removed virtual directory entry %v: %q", virtualState, name)
- }
- // Purge virtual entries assuming the directory has just been re-read
- //
- // Remove all the entries except:
- //
- // 1) vDirAdd on remotes which can't have empty directories. These will remain
- // virtual as long as the directory is empty. When the directory becomes real
- // (ie files are added) the virtual directory will be removed. This means that
- // directories will disappear when the last file is deleted which is probably
- // OK.
- //
- // 2) vFileAdd that are being written or uploaded
- func (d *Dir) _purgeVirtual() {
- canHaveEmptyDirectories := d.f.Features().CanHaveEmptyDirectories
- for name, virtualState := range d.virtual {
- switch virtualState {
- case vAddDir:
- if canHaveEmptyDirectories {
- // if remote can have empty directories then a
- // new dir will be read in the listing
- d._deleteVirtual(name)
- //} else {
- // leave the empty directory marker
- }
- case vAddFile:
- // Delete all virtual file adds that have finished uploading
- node, ok := d.items[name]
- if !ok {
- // if the object has disappeared somehow then remove the virtual
- d._deleteVirtual(name)
- continue
- }
- f, ok := node.(*File)
- if !ok {
- // if the object isn't a file then remove the virtual as it is wrong
- d._deleteVirtual(name)
- continue
- }
- if f.writingInProgress() {
- // if writing in progress then leave virtual
- continue
- }
- if d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal && d.vfs.cache.InUse(f.Path()) {
- // if object in use or dirty then leave virtual
- continue
- }
- d._deleteVirtual(name)
- default:
- d._deleteVirtual(name)
- }
- }
- }
- // Manage the virtuals in a listing
- //
- // This keeps a record of the names listed in this directory so far
- type manageVirtuals map[string]struct{}
- // Create a new manageVirtuals and purge the d.virtuals of any entries which can
- // be removed.
- //
- // must be called with the Dir lock held
- func (d *Dir) _newManageVirtuals() manageVirtuals {
- tv := make(manageVirtuals)
- d._purgeVirtual()
- return tv
- }
- // This should be called for every entry added to the directory
- //
- // It returns true if this entry should be skipped.
- //
- // must be called with the Dir lock held
- func (mv manageVirtuals) add(d *Dir, name string) bool {
- // Keep a record of all names listed
- mv[name] = struct{}{}
- // Remove virtuals if possible
- switch d.virtual[name] {
- case vAddFile, vAddDir:
- // item was added to the dir but since it is found in a
- // listing is no longer virtual
- d._deleteVirtual(name)
- case vDel:
- // item is deleted from the dir so skip it
- return true
- case vOK:
- }
- return false
- }
- // This should be called after the directory entry is read to update d.items
- // with virtual entries
- //
- // must be called with the Dir lock held
- func (mv manageVirtuals) end(d *Dir) {
- // delete unused d.items
- for name := range d.items {
- if _, ok := mv[name]; !ok {
- // name was previously in the directory but wasn't found
- // in the current listing
- switch d.virtual[name] {
- case vAddFile, vAddDir:
- // virtually added so leave virtual item
- default:
- // otherwise delete it
- delete(d.items, name)
- }
- }
- }
- // delete unused d.virtual~s
- for name, virtualState := range d.virtual {
- if _, ok := mv[name]; !ok {
- // name exists as a virtual but isn't in the current
- // listing so if it is a virtual delete we can remove it
- // as it is no longer needed.
- if virtualState == vDel {
- d._deleteVirtual(name)
- }
- }
- }
- }
- // update d.items and if dirTree is not nil update each dir in the DirTree below this one and
- // set the last read time - must be called with the lock held
- func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree, when time.Time) error {
- var err error
- mv := d._newManageVirtuals()
- for _, entry := range entries {
- name := path.Base(entry.Remote())
- if name == "." || name == ".." {
- continue
- }
- node := d.items[name]
- if mv.add(d, name) {
- continue
- }
- switch item := entry.(type) {
- case fs.Object:
- obj := item
- // Reuse old file value if it exists
- if file, ok := node.(*File); node != nil && ok {
- file.setObjectNoUpdate(obj)
- } else {
- node = newFile(d, d.path, obj, name)
- }
- case fs.Directory:
- // Reuse old dir value if it exists
- if node == nil || !node.IsDir() {
- node = newDir(d.vfs, d.f, d, item)
- }
- dir := node.(*Dir)
- dir.mu.Lock()
- dir.modTime = item.ModTime(context.TODO())
- if dirTree != nil {
- err = dir._readDirFromDirTree(dirTree, when)
- if err != nil {
- dir.read = time.Time{}
- } else {
- dir.read = when
- dir.cleanupTimer.Reset(d.vfs.Opt.DirCacheTime * 2)
- }
- }
- dir.mu.Unlock()
- if err != nil {
- return err
- }
- default:
- err = fmt.Errorf("unknown type %T", item)
- fs.Errorf(d, "readDir error: %v", err)
- return err
- }
- d.items[name] = node
- }
- mv.end(d)
- return nil
- }
- // readDirTree forces a refresh of the complete directory tree
- func (d *Dir) readDirTree() error {
- d.mu.RLock()
- f, path := d.f, d.path
- d.mu.RUnlock()
- when := time.Now()
- fs.Debugf(path, "Reading directory tree")
- dt, err := walk.NewDirTree(context.TODO(), f, path, false, -1)
- if err != nil {
- return err
- }
- d.mu.Lock()
- defer d.mu.Unlock()
- d.read = time.Time{}
- err = d._readDirFromDirTree(dt, when)
- if err != nil {
- return err
- }
- fs.Debugf(d.path, "Reading directory tree done in %s", time.Since(when))
- d.read = when
- d.cleanupTimer.Reset(d.vfs.Opt.DirCacheTime * 2)
- return nil
- }
- // readDir forces a refresh of the directory
- func (d *Dir) readDir() error {
- d.mu.Lock()
- defer d.mu.Unlock()
- d.read = time.Time{}
- return d._readDir()
- }
- // stat a single item in the directory
- //
- // returns ENOENT if not found.
- // returns a custom error if directory on a case-insensitive file system
- // contains files with names that differ only by case.
- func (d *Dir) stat(leaf string) (Node, error) {
- d.mu.Lock()
- defer d.mu.Unlock()
- err := d._readDir()
- if err != nil {
- return nil, err
- }
- item, ok := d.items[leaf]
- ci := fs.GetConfig(context.TODO())
- normUnicode := !ci.NoUnicodeNormalization
- normCase := ci.IgnoreCaseSync || d.vfs.Opt.CaseInsensitive
- if !ok && (normUnicode || normCase) {
- leafNormalized := operations.ToNormal(leaf, normUnicode, normCase) // this handles both case and unicode normalization
- for name, node := range d.items {
- if operations.ToNormal(name, normUnicode, normCase) == leafNormalized {
- if ok {
- // duplicate normalized match is an error
- return nil, fmt.Errorf("duplicate filename %q detected with case/unicode normalization settings", leaf)
- }
- // found a normalized match
- ok = true
- item = node
- }
- }
- }
- if !ok {
- return nil, ENOENT
- }
- return item, nil
- }
- // Check to see if a directory is empty
- func (d *Dir) isEmpty() (bool, error) {
- d.mu.Lock()
- defer d.mu.Unlock()
- err := d._readDir()
- if err != nil {
- return false, err
- }
- return len(d.items) == 0, nil
- }
- // ModTime returns the modification time of the directory
- func (d *Dir) ModTime() time.Time {
- d.modTimeMu.Lock()
- defer d.modTimeMu.Unlock()
- // fs.Debugf(d.path, "Dir.ModTime %v", d.modTime)
- return d.modTime
- }
- // Size of the directory
- func (d *Dir) Size() int64 {
- return 0
- }
- // SetModTime sets the modTime for this dir
- func (d *Dir) SetModTime(modTime time.Time) error {
- if d.vfs.Opt.ReadOnly {
- return EROFS
- }
- d.modTimeMu.Lock()
- d.modTime = modTime
- d.modTimeMu.Unlock()
- return nil
- }
- func (d *Dir) cachedDir(relativePath string) (dir *Dir) {
- dir, _ = d.cachedNode(relativePath).(*Dir)
- return
- }
- func (d *Dir) cachedNode(relativePath string) Node {
- segments := strings.Split(strings.Trim(relativePath, "/"), "/")
- var node Node = d
- for _, s := range segments {
- if s == "" {
- continue
- }
- if dir, ok := node.(*Dir); ok {
- dir.mu.Lock()
- node = dir.items[s]
- dir.mu.Unlock()
- if node != nil {
- continue
- }
- }
- return nil
- }
- return node
- }
- // Stat looks up a specific entry in the receiver.
- //
- // Stat should return a Node corresponding to the entry. If the
- // name does not exist in the directory, Stat should return ENOENT.
- //
- // Stat need not to handle the names "." and "..".
- func (d *Dir) Stat(name string) (node Node, err error) {
- // fs.Debugf(path, "Dir.Stat")
- node, err = d.stat(name)
- if err != nil {
- if err != ENOENT {
- fs.Errorf(d, "Dir.Stat error: %v", err)
- }
- return nil, err
- }
- // fs.Debugf(path, "Dir.Stat OK")
- return node, nil
- }
- // ReadDirAll reads the contents of the directory sorted
- func (d *Dir) ReadDirAll() (items Nodes, err error) {
- // fs.Debugf(d.path, "Dir.ReadDirAll")
- d.mu.Lock()
- err = d._readDir()
- if err != nil {
- fs.Debugf(d.path, "Dir.ReadDirAll error: %v", err)
- d.mu.Unlock()
- return nil, err
- }
- for _, item := range d.items {
- items = append(items, item)
- }
- d.mu.Unlock()
- sort.Sort(items)
- // fs.Debugf(d.path, "Dir.ReadDirAll OK with %d entries", len(items))
- return items, nil
- }
- // accessModeMask masks off the read modes from the flags
- const accessModeMask = (os.O_RDONLY | os.O_WRONLY | os.O_RDWR)
- // Open the directory according to the flags provided
- func (d *Dir) Open(flags int) (fd Handle, err error) {
- rdwrMode := flags & accessModeMask
- if rdwrMode != os.O_RDONLY {
- fs.Errorf(d, "Can only open directories read only")
- return nil, EPERM
- }
- return newDirHandle(d), nil
- }
- // Create makes a new file node
- func (d *Dir) Create(name string, flags int) (*File, error) {
- // fs.Debugf(path, "Dir.Create")
- // Return existing node if one exists
- node, err := d.stat(name)
- switch err {
- case ENOENT:
- // not found, carry on
- case nil:
- // found so check what it is
- if node.IsFile() {
- return node.(*File), err
- }
- return nil, EEXIST // EISDIR would be better but we don't have that
- default:
- // a different error - report
- fs.Errorf(d, "Dir.Create stat failed: %v", err)
- return nil, err
- }
- // node doesn't exist so create it
- if d.vfs.Opt.ReadOnly {
- return nil, EROFS
- }
- if err = d.SetModTime(time.Now()); err != nil {
- fs.Errorf(d, "Dir.Create failed to set modtime on parent dir: %v", err)
- return nil, err
- }
- // This gets added to the directory when the file is opened for write
- return newFile(d, d.Path(), nil, name), nil
- }
- // Mkdir creates a new directory
- func (d *Dir) Mkdir(name string) (*Dir, error) {
- if d.vfs.Opt.ReadOnly {
- return nil, EROFS
- }
- path := path.Join(d.path, name)
- node, err := d.stat(name)
- switch err {
- case ENOENT:
- // not found, carry on
- case nil:
- // found so check what it is
- if node.IsDir() {
- return node.(*Dir), err
- }
- return nil, EEXIST
- default:
- // a different error - report
- fs.Errorf(d, "Dir.Mkdir failed to read directory: %v", err)
- return nil, err
- }
- // fs.Debugf(path, "Dir.Mkdir")
- err = d.f.Mkdir(context.TODO(), path)
- if err != nil {
- fs.Errorf(d, "Dir.Mkdir failed to create directory: %v", err)
- return nil, err
- }
- fsDir := fs.NewDir(path, time.Now())
- dir := newDir(d.vfs, d.f, d, fsDir)
- d.addObject(dir)
- if err = d.SetModTime(time.Now()); err != nil {
- fs.Errorf(d, "Dir.Mkdir failed to set modtime on parent dir: %v", err)
- return nil, err
- }
- // fs.Debugf(path, "Dir.Mkdir OK")
- return dir, nil
- }
- // Remove the directory
- func (d *Dir) Remove() error {
- if d.vfs.Opt.ReadOnly {
- return EROFS
- }
- // Check directory is empty first
- empty, err := d.isEmpty()
- if err != nil {
- fs.Errorf(d, "Dir.Remove dir error: %v", err)
- return err
- }
- if !empty {
- fs.Errorf(d, "Dir.Remove not empty")
- return ENOTEMPTY
- }
- // remove directory
- err = d.f.Rmdir(context.TODO(), d.path)
- if err != nil {
- fs.Errorf(d, "Dir.Remove failed to remove directory: %v", err)
- return err
- }
- // Remove the item from the parent directory listing
- if d.parent != nil {
- d.parent.delObject(d.Name())
- }
- return nil
- }
- // RemoveAll removes the directory and any contents recursively
- func (d *Dir) RemoveAll() error {
- if d.vfs.Opt.ReadOnly {
- return EROFS
- }
- // Remove contents of the directory
- nodes, err := d.ReadDirAll()
- if err != nil {
- fs.Errorf(d, "Dir.RemoveAll failed to read directory: %v", err)
- return err
- }
- for _, node := range nodes {
- err = node.RemoveAll()
- if err != nil {
- fs.Errorf(node.Path(), "Dir.RemoveAll failed to remove: %v", err)
- return err
- }
- }
- return d.Remove()
- }
- // DirEntry returns the underlying fs.DirEntry
- func (d *Dir) DirEntry() (entry fs.DirEntry) {
- return d.entry
- }
- // RemoveName removes the entry with the given name from the receiver,
- // which must be a directory. The entry to be removed may correspond
- // to a file (unlink) or to a directory (rmdir).
- func (d *Dir) RemoveName(name string) error {
- if d.vfs.Opt.ReadOnly {
- return EROFS
- }
- // fs.Debugf(path, "Dir.Remove")
- node, err := d.stat(name)
- if err != nil {
- fs.Errorf(d, "Dir.Remove error: %v", err)
- return err
- }
- if err = d.SetModTime(time.Now()); err != nil {
- fs.Errorf(d, "Dir.Remove failed to set modtime on parent dir: %v", err)
- return err
- }
- return node.Remove()
- }
- // Rename the file
- func (d *Dir) Rename(oldName, newName string, destDir *Dir) error {
- // fs.Debugf(d, "BEFORE\n%s", d.dump())
- if d.vfs.Opt.ReadOnly {
- return EROFS
- }
- oldPath := path.Join(d.path, oldName)
- newPath := path.Join(destDir.path, newName)
- // fs.Debugf(oldPath, "Dir.Rename to %q", newPath)
- oldNode, err := d.stat(oldName)
- if err != nil {
- fs.Errorf(oldPath, "Dir.Rename error: %v", err)
- return err
- }
- switch x := oldNode.DirEntry().(type) {
- case nil:
- if oldFile, ok := oldNode.(*File); ok {
- if err = oldFile.rename(context.TODO(), destDir, newName); err != nil {
- fs.Errorf(oldPath, "Dir.Rename error: %v", err)
- return err
- }
- } else {
- fs.Errorf(oldPath, "Dir.Rename can't rename open file that is not a vfs.File")
- return EPERM
- }
- case fs.Object:
- if oldFile, ok := oldNode.(*File); ok {
- if err = oldFile.rename(context.TODO(), destDir, newName); err != nil {
- fs.Errorf(oldPath, "Dir.Rename error: %v", err)
- return err
- }
- } else {
- err := fmt.Errorf("Fs %q can't rename file that is not a vfs.File", d.f)
- fs.Errorf(oldPath, "Dir.Rename error: %v", err)
- return err
- }
- case fs.Directory:
- features := d.f.Features()
- if features.DirMove == nil && features.Move == nil && features.Copy == nil {
- err := fmt.Errorf("Fs %q can't rename directories (no DirMove, Move or Copy)", d.f)
- fs.Errorf(oldPath, "Dir.Rename error: %v", err)
- return err
- }
- srcRemote := x.Remote()
- dstRemote := newPath
- err = operations.DirMove(context.TODO(), d.f, srcRemote, dstRemote)
- if err != nil {
- fs.Errorf(oldPath, "Dir.Rename error: %v", err)
- return err
- }
- newDir := fs.NewDirCopy(context.TODO(), x).SetRemote(newPath)
- // Update the node with the new details
- if oldNode != nil {
- if oldDir, ok := oldNode.(*Dir); ok {
- fs.Debugf(x, "Updating dir with %v %p", newDir, oldDir)
- oldDir.rename(destDir, newDir)
- }
- }
- default:
- err = fmt.Errorf("unknown type %T", oldNode)
- fs.Errorf(d.path, "Dir.Rename error: %v", err)
- return err
- }
- // Show moved - delete from old dir and add to new
- d.delObject(oldName)
- destDir.addObject(oldNode)
- if err = d.SetModTime(time.Now()); err != nil {
- fs.Errorf(d, "Dir.Rename failed to set modtime on parent dir: %v", err)
- return err
- }
- // fs.Debugf(newPath, "Dir.Rename renamed from %q", oldPath)
- // fs.Debugf(d, "AFTER\n%s", d.dump())
- return nil
- }
- // Sync the directory
- //
- // Note that we don't do anything except return OK
- func (d *Dir) Sync() error {
- return nil
- }
- // VFS returns the instance of the VFS
- func (d *Dir) VFS() *VFS {
- // No locking required
- return d.vfs
- }
- // Fs returns the Fs that the Dir is on
- func (d *Dir) Fs() fs.Fs {
- // No locking required
- return d.f
- }
- // Truncate changes the size of the named file.
- func (d *Dir) Truncate(size int64) error {
- return ENOSYS
- }
|