123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610 |
- package vfs
- import (
- "context"
- "fmt"
- "os"
- "sort"
- "testing"
- "time"
- "unsafe"
- "github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/operations"
- "github.com/rclone/rclone/fstest"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- )
- func dirCreate(t *testing.T) (r *fstest.Run, vfs *VFS, dir *Dir, item fstest.Item) {
- r, vfs = newTestVFS(t)
- file1 := r.WriteObject(context.Background(), "dir/file1", "file1 contents", t1)
- r.CheckRemoteItems(t, file1)
- node, err := vfs.Stat("dir")
- require.NoError(t, err)
- require.True(t, node.IsDir())
- return r, vfs, node.(*Dir), file1
- }
- func TestDirMethods(t *testing.T) {
- _, vfs, dir, _ := dirCreate(t)
- // String
- assert.Equal(t, "dir/", dir.String())
- assert.Equal(t, "<nil *Dir>", (*Dir)(nil).String())
- // IsDir
- assert.Equal(t, true, dir.IsDir())
- // IsFile
- assert.Equal(t, false, dir.IsFile())
- // Mode
- assert.Equal(t, os.FileMode(vfs.Opt.DirPerms), dir.Mode())
- // Name
- assert.Equal(t, "dir", dir.Name())
- // Path
- assert.Equal(t, "dir", dir.Path())
- // Sys
- assert.Equal(t, nil, dir.Sys())
- // SetSys
- dir.SetSys(42)
- assert.Equal(t, 42, dir.Sys())
- // Inode
- assert.NotEqual(t, uint64(0), dir.Inode())
- // Node
- assert.Equal(t, dir, dir.Node())
- // ModTime
- assert.WithinDuration(t, t1, dir.ModTime(), 100*365*24*60*60*time.Second)
- // Size
- assert.Equal(t, int64(0), dir.Size())
- // Sync
- assert.NoError(t, dir.Sync())
- // DirEntry
- assert.Equal(t, dir.entry, dir.DirEntry())
- // VFS
- assert.Equal(t, vfs, dir.VFS())
- }
- func TestDirForgetAll(t *testing.T) {
- _, vfs, dir, file1 := dirCreate(t)
- // Make sure / and dir are in cache
- _, err := vfs.Stat(file1.Path)
- require.NoError(t, err)
- root, err := vfs.Root()
- require.NoError(t, err)
- assert.Equal(t, 1, len(root.items))
- assert.Equal(t, 1, len(dir.items))
- assert.False(t, root.read.IsZero())
- assert.False(t, dir.read.IsZero())
- dir.ForgetAll()
- assert.Equal(t, 1, len(root.items))
- assert.Equal(t, 0, len(dir.items))
- assert.False(t, root.read.IsZero())
- assert.True(t, dir.read.IsZero())
- root.ForgetAll()
- assert.Equal(t, 0, len(root.items))
- assert.Equal(t, 0, len(dir.items))
- assert.True(t, root.read.IsZero())
- }
- func TestDirForgetPath(t *testing.T) {
- _, vfs, dir, file1 := dirCreate(t)
- // Make sure / and dir are in cache
- _, err := vfs.Stat(file1.Path)
- require.NoError(t, err)
- root, err := vfs.Root()
- require.NoError(t, err)
- assert.Equal(t, 1, len(root.items))
- assert.Equal(t, 1, len(dir.items))
- assert.False(t, root.read.IsZero())
- assert.False(t, dir.read.IsZero())
- root.ForgetPath("dir/notfound", fs.EntryObject)
- assert.Equal(t, 1, len(root.items))
- assert.Equal(t, 1, len(dir.items))
- assert.False(t, root.read.IsZero())
- assert.True(t, dir.read.IsZero())
- root.ForgetPath("dir", fs.EntryDirectory)
- assert.Equal(t, 1, len(root.items))
- assert.Equal(t, 0, len(dir.items))
- assert.True(t, root.read.IsZero())
- root.ForgetPath("not/in/cache", fs.EntryDirectory)
- assert.Equal(t, 1, len(root.items))
- assert.Equal(t, 0, len(dir.items))
- }
- func TestDirWalk(t *testing.T) {
- r, vfs, _, file1 := dirCreate(t)
- file2 := r.WriteObject(context.Background(), "fil/a/b/c", "super long file", t1)
- r.CheckRemoteItems(t, file1, file2)
- root, err := vfs.Root()
- require.NoError(t, err)
- // Forget the cache since we put another object in
- root.ForgetAll()
- // Read the directories in
- _, err = vfs.Stat("dir")
- require.NoError(t, err)
- _, err = vfs.Stat("fil/a/b")
- require.NoError(t, err)
- fil, err := vfs.Stat("fil")
- require.NoError(t, err)
- var result []string
- fn := func(d *Dir) {
- result = append(result, d.path)
- }
- result = nil
- root.walk(fn)
- sort.Strings(result) // sort as there is a map traversal involved
- assert.Equal(t, []string{"", "dir", "fil", "fil/a", "fil/a/b"}, result)
- assert.Nil(t, root.cachedDir("not found"))
- if dir := root.cachedDir("dir"); assert.NotNil(t, dir) {
- result = nil
- dir.walk(fn)
- assert.Equal(t, []string{"dir"}, result)
- }
- if dir := root.cachedDir("fil"); assert.NotNil(t, dir) {
- result = nil
- dir.walk(fn)
- assert.Equal(t, []string{"fil/a/b", "fil/a", "fil"}, result)
- }
- if dir := fil.(*Dir); assert.NotNil(t, dir) {
- result = nil
- dir.walk(fn)
- assert.Equal(t, []string{"fil/a/b", "fil/a", "fil"}, result)
- }
- if dir := root.cachedDir("fil/a"); assert.NotNil(t, dir) {
- result = nil
- dir.walk(fn)
- assert.Equal(t, []string{"fil/a/b", "fil/a"}, result)
- }
- if dir := fil.(*Dir).cachedDir("a"); assert.NotNil(t, dir) {
- result = nil
- dir.walk(fn)
- assert.Equal(t, []string{"fil/a/b", "fil/a"}, result)
- }
- if dir := root.cachedDir("fil/a"); assert.NotNil(t, dir) {
- result = nil
- dir.walk(fn)
- assert.Equal(t, []string{"fil/a/b", "fil/a"}, result)
- }
- if dir := root.cachedDir("fil/a/b"); assert.NotNil(t, dir) {
- result = nil
- dir.walk(fn)
- assert.Equal(t, []string{"fil/a/b"}, result)
- }
- }
- func TestDirSetModTime(t *testing.T) {
- _, vfs, dir, _ := dirCreate(t)
- err := dir.SetModTime(t1)
- require.NoError(t, err)
- assert.WithinDuration(t, t1, dir.ModTime(), time.Second)
- err = dir.SetModTime(t2)
- require.NoError(t, err)
- assert.WithinDuration(t, t2, dir.ModTime(), time.Second)
- vfs.Opt.ReadOnly = true
- err = dir.SetModTime(t2)
- assert.Equal(t, EROFS, err)
- }
- func TestDirStat(t *testing.T) {
- _, _, dir, _ := dirCreate(t)
- node, err := dir.Stat("file1")
- require.NoError(t, err)
- _, ok := node.(*File)
- assert.True(t, ok)
- assert.Equal(t, int64(14), node.Size())
- assert.Equal(t, "file1", node.Name())
- _, err = dir.Stat("not found")
- assert.Equal(t, ENOENT, err)
- }
- // This lists dir and checks the listing is as expected
- func checkListing(t *testing.T, dir *Dir, want []string) {
- var got []string
- nodes, err := dir.ReadDirAll()
- require.NoError(t, err)
- for _, node := range nodes {
- got = append(got, fmt.Sprintf("%s,%d,%v", node.Name(), node.Size(), node.IsDir()))
- }
- assert.Equal(t, want, got)
- }
- func TestDirReadDirAll(t *testing.T) {
- r, vfs := newTestVFS(t)
- file1 := r.WriteObject(context.Background(), "dir/file1", "file1 contents", t1)
- file2 := r.WriteObject(context.Background(), "dir/file2", "file2- contents", t2)
- file3 := r.WriteObject(context.Background(), "dir/subdir/file3", "file3-- contents", t3)
- r.CheckRemoteItems(t, file1, file2, file3)
- node, err := vfs.Stat("dir")
- require.NoError(t, err)
- dir := node.(*Dir)
- checkListing(t, dir, []string{"file1,14,false", "file2,15,false", "subdir,0,true"})
- node, err = vfs.Stat("")
- require.NoError(t, err)
- root := node.(*Dir)
- checkListing(t, root, []string{"dir,0,true"})
- node, err = vfs.Stat("dir/subdir")
- require.NoError(t, err)
- subdir := node.(*Dir)
- checkListing(t, subdir, []string{"file3,16,false"})
- t.Run("Virtual", func(t *testing.T) {
- // Add some virtual entries and check what happens
- dir.AddVirtual("virtualFile", 17, false)
- dir.AddVirtual("virtualDir", 0, true)
- // Remove some existing entries
- dir.DelVirtual("file2")
- dir.DelVirtual("subdir")
- checkListing(t, dir, []string{"file1,14,false", "virtualDir,0,true", "virtualFile,17,false"})
- // Now action the deletes and uploads
- _ = r.WriteObject(context.Background(), "dir/virtualFile", "virtualFile contents", t1)
- _ = r.WriteObject(context.Background(), "dir/virtualDir/testFile", "testFile contents", t1)
- o, err := r.Fremote.NewObject(context.Background(), "dir/file2")
- require.NoError(t, err)
- require.NoError(t, o.Remove(context.Background()))
- require.NoError(t, operations.Purge(context.Background(), r.Fremote, "dir/subdir"))
- // Force a directory reload...
- dir.invalidateDir("dir")
- checkListing(t, dir, []string{"file1,14,false", "virtualDir,0,true", "virtualFile,20,false"})
- // check no virtuals left
- dir.mu.Lock()
- assert.Nil(t, dir.virtual)
- dir.mu.Unlock()
- // Add some virtual entries and check what happens
- dir.AddVirtual("virtualFile2", 100, false)
- dir.AddVirtual("virtualDir2", 0, true)
- // Remove some existing entries
- dir.DelVirtual("file1")
- checkListing(t, dir, []string{"virtualDir,0,true", "virtualDir2,0,true", "virtualFile,20,false", "virtualFile2,100,false"})
- // Force a directory reload...
- dir.invalidateDir("dir")
- want := []string{"file1,14,false", "virtualDir,0,true", "virtualDir2,0,true", "virtualFile,20,false", "virtualFile2,100,false"}
- features := r.Fremote.Features()
- if features.CanHaveEmptyDirectories {
- // snip out virtualDir2 which will only be present if can't have empty dirs
- want = append(want[:2], want[3:]...)
- }
- checkListing(t, dir, want)
- // Check that forgetting the root doesn't invalidate the virtual entries
- root.ForgetAll()
- checkListing(t, dir, want)
- })
- }
- func TestDirOpen(t *testing.T) {
- _, _, dir, _ := dirCreate(t)
- fd, err := dir.Open(os.O_RDONLY)
- require.NoError(t, err)
- _, ok := fd.(*DirHandle)
- assert.True(t, ok)
- require.NoError(t, fd.Close())
- _, err = dir.Open(os.O_WRONLY)
- assert.Equal(t, EPERM, err)
- }
- func TestDirCreate(t *testing.T) {
- _, vfs, dir, _ := dirCreate(t)
- origModTime := dir.ModTime()
- time.Sleep(100 * time.Millisecond) // for low rez Windows timers
- file, err := dir.Create("potato", os.O_WRONLY|os.O_CREATE)
- require.NoError(t, err)
- assert.Equal(t, int64(0), file.Size())
- assert.True(t, dir.ModTime().After(origModTime))
- fd, err := file.Open(os.O_WRONLY | os.O_CREATE)
- require.NoError(t, err)
- // FIXME Note that this fails with the current implementation
- // until the file has been opened.
- // file2, err := vfs.Stat("dir/potato")
- // require.NoError(t, err)
- // assert.Equal(t, file, file2)
- n, err := fd.Write([]byte("hello"))
- require.NoError(t, err)
- assert.Equal(t, 5, n)
- require.NoError(t, fd.Close())
- file2, err := vfs.Stat("dir/potato")
- require.NoError(t, err)
- assert.Equal(t, int64(5), file2.Size())
- // Try creating the file again - make sure we get the same file node
- file3, err := dir.Create("potato", os.O_RDWR|os.O_CREATE)
- require.NoError(t, err)
- assert.Equal(t, int64(5), file3.Size())
- assert.Equal(t, fmt.Sprintf("%p", file), fmt.Sprintf("%p", file3), "didn't return same node")
- // Test read only fs creating new
- vfs.Opt.ReadOnly = true
- _, err = dir.Create("sausage", os.O_WRONLY|os.O_CREATE)
- assert.Equal(t, EROFS, err)
- }
- func TestDirMkdir(t *testing.T) {
- r, vfs, dir, file1 := dirCreate(t)
- _, err := dir.Mkdir("file1")
- assert.Error(t, err)
- origModTime := dir.ModTime()
- time.Sleep(100 * time.Millisecond) // for low rez Windows timers
- sub, err := dir.Mkdir("sub")
- assert.NoError(t, err)
- assert.True(t, dir.ModTime().After(origModTime))
- // check the vfs
- checkListing(t, dir, []string{"file1,14,false", "sub,0,true"})
- checkListing(t, sub, []string(nil))
- // check the underlying r.Fremote
- fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"dir", "dir/sub"}, r.Fremote.Precision())
- vfs.Opt.ReadOnly = true
- _, err = dir.Mkdir("sausage")
- assert.Equal(t, EROFS, err)
- }
- func TestDirMkdirSub(t *testing.T) {
- r, vfs, dir, file1 := dirCreate(t)
- _, err := dir.Mkdir("file1")
- assert.Error(t, err)
- sub, err := dir.Mkdir("sub")
- assert.NoError(t, err)
- subsub, err := sub.Mkdir("subsub")
- assert.NoError(t, err)
- // check the vfs
- checkListing(t, dir, []string{"file1,14,false", "sub,0,true"})
- checkListing(t, sub, []string{"subsub,0,true"})
- checkListing(t, subsub, []string(nil))
- // check the underlying r.Fremote
- fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"dir", "dir/sub", "dir/sub/subsub"}, r.Fremote.Precision())
- vfs.Opt.ReadOnly = true
- _, err = dir.Mkdir("sausage")
- assert.Equal(t, EROFS, err)
- }
- func TestDirRemove(t *testing.T) {
- r, vfs, dir, _ := dirCreate(t)
- // check directory is there
- node, err := vfs.Stat("dir")
- require.NoError(t, err)
- assert.True(t, node.IsDir())
- err = dir.Remove()
- assert.Equal(t, ENOTEMPTY, err)
- // Delete the sub file
- node, err = vfs.Stat("dir/file1")
- require.NoError(t, err)
- err = node.Remove()
- require.NoError(t, err)
- // Remove the now empty directory
- err = dir.Remove()
- require.NoError(t, err)
- // check directory is not there
- _, err = vfs.Stat("dir")
- assert.Equal(t, ENOENT, err)
- // check the vfs
- root, err := vfs.Root()
- require.NoError(t, err)
- checkListing(t, root, []string(nil))
- // check the underlying r.Fremote
- fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, r.Fremote.Precision())
- // read only check
- vfs.Opt.ReadOnly = true
- err = dir.Remove()
- assert.Equal(t, EROFS, err)
- }
- func TestDirRemoveAll(t *testing.T) {
- r, vfs, dir, _ := dirCreate(t)
- // Remove the directory and contents
- err := dir.RemoveAll()
- require.NoError(t, err)
- // check the vfs
- root, err := vfs.Root()
- require.NoError(t, err)
- checkListing(t, root, []string(nil))
- // check the underlying r.Fremote
- fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, r.Fremote.Precision())
- // read only check
- vfs.Opt.ReadOnly = true
- err = dir.RemoveAll()
- assert.Equal(t, EROFS, err)
- }
- func TestDirRemoveName(t *testing.T) {
- r, vfs, dir, _ := dirCreate(t)
- origModTime := dir.ModTime()
- time.Sleep(100 * time.Millisecond) // for low rez Windows timers
- err := dir.RemoveName("file1")
- require.NoError(t, err)
- assert.True(t, dir.ModTime().After(origModTime))
- checkListing(t, dir, []string(nil))
- root, err := vfs.Root()
- require.NoError(t, err)
- checkListing(t, root, []string{"dir,0,true"})
- // check the underlying r.Fremote
- fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"dir"}, r.Fremote.Precision())
- // read only check
- vfs.Opt.ReadOnly = true
- err = dir.RemoveName("potato")
- assert.Equal(t, EROFS, err)
- }
- func TestDirRename(t *testing.T) {
- r, vfs, dir, file1 := dirCreate(t)
- features := r.Fremote.Features()
- if features.DirMove == nil && features.Move == nil && features.Copy == nil {
- t.Skip("can't rename directories")
- }
- file3 := r.WriteObject(context.Background(), "dir/file3", "file3 contents!", t1)
- fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file3}, []string{"dir"}, r.Fremote.Precision())
- root, err := vfs.Root()
- require.NoError(t, err)
- err = dir.Rename("not found", "tuba", dir)
- assert.Equal(t, ENOENT, err)
- // Rename a directory
- err = root.Rename("dir", "dir2", root)
- assert.NoError(t, err)
- checkListing(t, root, []string{"dir2,0,true"})
- checkListing(t, dir, []string{"file1,14,false", "file3,15,false"})
- // check the underlying r.Fremote
- file1.Path = "dir2/file1"
- file3.Path = "dir2/file3"
- fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file3}, []string{"dir2"}, r.Fremote.Precision())
- // refetch dir
- node, err := vfs.Stat("dir2")
- assert.NoError(t, err)
- dir = node.(*Dir)
- // Rename a file
- origModTime := dir.ModTime()
- time.Sleep(100 * time.Millisecond) // for low rez Windows timers
- err = dir.Rename("file1", "file2", root)
- assert.NoError(t, err)
- assert.True(t, dir.ModTime().After(origModTime))
- checkListing(t, root, []string{"dir2,0,true", "file2,14,false"})
- checkListing(t, dir, []string{"file3,15,false"})
- // check the underlying r.Fremote
- file1.Path = "file2"
- fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file3}, []string{"dir2"}, r.Fremote.Precision())
- // Rename a file on top of another file
- err = root.Rename("file2", "file3", dir)
- assert.NoError(t, err)
- checkListing(t, root, []string{"dir2,0,true"})
- checkListing(t, dir, []string{"file3,14,false"})
- // check the underlying r.Fremote
- file1.Path = "dir2/file3"
- fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"dir2"}, r.Fremote.Precision())
- // rename an empty directory
- _, err = root.Mkdir("empty directory")
- assert.NoError(t, err)
- checkListing(t, root, []string{
- "dir2,0,true",
- "empty directory,0,true",
- })
- err = root.Rename("empty directory", "renamed empty directory", root)
- assert.NoError(t, err)
- checkListing(t, root, []string{
- "dir2,0,true",
- "renamed empty directory,0,true",
- })
- // ...we don't check the underlying f.Fremote because on
- // bucket-based remotes the directory won't be there
- // read only check
- vfs.Opt.ReadOnly = true
- err = dir.Rename("potato", "tuba", dir)
- assert.Equal(t, EROFS, err)
- // Rename a dir, check that key was correctly renamed in dir.parent.items
- vfs.Opt.ReadOnly = false
- _, ok := dir.parent.items["dir2"]
- assert.True(t, ok, "dir.parent.items should have 'dir2' key before rename")
- _, ok = dir.parent.items["dir3"]
- assert.False(t, ok, "dir.parent.items should not have 'dir3' key before rename")
- dir.renameTree("dir3") // rename dir2 to dir3
- _, ok = dir.parent.items["dir2"]
- assert.False(t, ok, "dir.parent.items should not have 'dir2' key after rename")
- d, ok := dir.parent.items["dir3"]
- assert.True(t, ok, fmt.Sprintf("expected to find 'dir3' key in dir.parent.items after rename, got %v", dir.parent.items))
- assert.Equal(t, dir, d, `expected renamed dir to match value of dir.parent.items["dir3"]`)
- }
- func TestDirStructSize(t *testing.T) {
- t.Logf("Dir struct has size %d bytes", unsafe.Sizeof(Dir{}))
- }
|