123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394 |
- /*
- This provides Run for use in creating test suites
- To use this declare a TestMain
- // TestMain drives the tests
- func TestMain(m *testing.M) {
- fstest.TestMain(m)
- }
- And then make and destroy a Run in each test
- func TestMkdir(t *testing.T) {
- r := fstest.NewRun(t)
- defer r.Finalise()
- // test stuff
- }
- This will make r.Fremote and r.Flocal for a remote and a local
- remote. The remote is determined by the -remote flag passed in.
- */
- package fstest
- import (
- "bytes"
- "context"
- "flag"
- "fmt"
- "log"
- "os"
- "path"
- "path/filepath"
- "sort"
- "testing"
- "time"
- "github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/cache"
- "github.com/rclone/rclone/fs/fserrors"
- "github.com/rclone/rclone/fs/hash"
- "github.com/rclone/rclone/fs/object"
- "github.com/rclone/rclone/fs/walk"
- "github.com/rclone/rclone/lib/file"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- )
- // Run holds the remotes for a test run
- type Run struct {
- LocalName string
- Flocal fs.Fs
- Fremote fs.Fs
- FremoteName string
- Precision time.Duration
- cleanRemote func()
- mkdir map[string]bool // whether the remote has been made yet for the fs name
- Logf, Fatalf func(text string, args ...interface{})
- }
- // TestMain drives the tests
- func TestMain(m *testing.M) {
- flag.Parse()
- if !*Individual {
- oneRun = newRun()
- }
- rc := m.Run()
- if !*Individual {
- oneRun.Finalise()
- }
- os.Exit(rc)
- }
- // oneRun holds the master run data if individual is not set
- var oneRun *Run
- // newRun initialise the remote and local for testing and returns a
- // run object.
- //
- // r.Flocal is an empty local Fs
- // r.Fremote is an empty remote Fs
- //
- // Finalise() will tidy them away when done.
- func newRun() *Run {
- r := &Run{
- Logf: log.Printf,
- Fatalf: log.Fatalf,
- mkdir: make(map[string]bool),
- }
- Initialise()
- var err error
- r.Fremote, r.FremoteName, r.cleanRemote, err = RandomRemote()
- if err != nil {
- r.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
- }
- r.LocalName, err = os.MkdirTemp("", "rclone")
- if err != nil {
- r.Fatalf("Failed to create temp dir: %v", err)
- }
- r.LocalName = filepath.ToSlash(r.LocalName)
- r.Flocal, err = fs.NewFs(context.Background(), r.LocalName)
- if err != nil {
- r.Fatalf("Failed to make %q: %v", r.LocalName, err)
- }
- r.Precision = fs.GetModifyWindow(context.Background(), r.Fremote, r.Flocal)
- return r
- }
- // run f(), retrying it until it returns with no error or the limit
- // expires and it calls t.Fatalf
- func retry(t *testing.T, what string, f func() error) {
- var err error
- for try := 1; try <= *ListRetries; try++ {
- err = f()
- if err == nil {
- return
- }
- t.Logf("%s failed - try %d/%d: %v", what, try, *ListRetries, err)
- time.Sleep(time.Second)
- }
- t.Logf("%s failed: %v", what, err)
- }
- // newRunIndividual initialise the remote and local for testing and
- // returns a run object. Pass in true to make individual tests or
- // false to use the global one.
- //
- // r.Flocal is an empty local Fs
- // r.Fremote is an empty remote Fs
- //
- // Finalise() will tidy them away when done.
- func newRunIndividual(t *testing.T, individual bool) *Run {
- ctx := context.Background()
- var r *Run
- if individual {
- r = newRun()
- } else {
- // If not individual, use the global one with the clean method overridden
- r = new(Run)
- *r = *oneRun
- r.cleanRemote = func() {
- var toDelete []string
- err := walk.ListR(ctx, r.Fremote, "", true, -1, walk.ListAll, func(entries fs.DirEntries) error {
- for _, entry := range entries {
- switch x := entry.(type) {
- case fs.Object:
- retry(t, fmt.Sprintf("removing file %q", x.Remote()), func() error { return x.Remove(ctx) })
- case fs.Directory:
- toDelete = append(toDelete, x.Remote())
- }
- }
- return nil
- })
- if err == fs.ErrorDirNotFound {
- return
- }
- require.NoError(t, err)
- sort.Strings(toDelete)
- for i := len(toDelete) - 1; i >= 0; i-- {
- dir := toDelete[i]
- retry(t, fmt.Sprintf("removing dir %q", dir), func() error {
- return r.Fremote.Rmdir(ctx, dir)
- })
- }
- // Check remote is empty
- CheckListingWithPrecision(t, r.Fremote, []Item{}, []string{}, r.Fremote.Precision())
- // Clear the remote cache
- cache.Clear()
- }
- }
- r.Logf = t.Logf
- r.Fatalf = t.Fatalf
- r.Logf("Remote %q, Local %q, Modify Window %q", r.Fremote, r.Flocal, fs.GetModifyWindow(ctx, r.Fremote))
- t.Cleanup(r.Finalise)
- return r
- }
- // NewRun initialise the remote and local for testing and returns a
- // run object. Call this from the tests.
- //
- // r.Flocal is an empty local Fs
- // r.Fremote is an empty remote Fs
- func NewRun(t *testing.T) *Run {
- return newRunIndividual(t, *Individual)
- }
- // NewRunIndividual as per NewRun but makes an individual remote for this test
- func NewRunIndividual(t *testing.T) *Run {
- return newRunIndividual(t, true)
- }
- // RenameFile renames a file in local
- func (r *Run) RenameFile(item Item, newpath string) Item {
- oldFilepath := path.Join(r.LocalName, item.Path)
- newFilepath := path.Join(r.LocalName, newpath)
- if err := os.Rename(oldFilepath, newFilepath); err != nil {
- r.Fatalf("Failed to rename file from %q to %q: %v", item.Path, newpath, err)
- }
- item.Path = newpath
- return item
- }
- // WriteFile writes a file to local
- func (r *Run) WriteFile(filePath, content string, t time.Time) Item {
- item := NewItem(filePath, content, t)
- // FIXME make directories?
- filePath = path.Join(r.LocalName, filePath)
- dirPath := path.Dir(filePath)
- err := file.MkdirAll(dirPath, 0770)
- if err != nil {
- r.Fatalf("Failed to make directories %q: %v", dirPath, err)
- }
- err = os.WriteFile(filePath, []byte(content), 0600)
- if err != nil {
- r.Fatalf("Failed to write file %q: %v", filePath, err)
- }
- err = os.Chtimes(filePath, t, t)
- if err != nil {
- r.Fatalf("Failed to chtimes file %q: %v", filePath, err)
- }
- return item
- }
- // ForceMkdir creates the remote
- func (r *Run) ForceMkdir(ctx context.Context, f fs.Fs) {
- err := f.Mkdir(ctx, "")
- if err != nil {
- r.Fatalf("Failed to mkdir %q: %v", f, err)
- }
- r.mkdir[f.String()] = true
- }
- // Mkdir creates the remote if it hasn't been created already
- func (r *Run) Mkdir(ctx context.Context, f fs.Fs) {
- if !r.mkdir[f.String()] {
- r.ForceMkdir(ctx, f)
- }
- }
- // WriteObjectTo writes an object to the fs, remote passed in
- func (r *Run) WriteObjectTo(ctx context.Context, f fs.Fs, remote, content string, modTime time.Time, useUnchecked bool) Item {
- put := f.Put
- if useUnchecked {
- put = f.Features().PutUnchecked
- if put == nil {
- r.Fatalf("Fs doesn't support PutUnchecked")
- }
- }
- r.Mkdir(ctx, f)
- // calculate all hashes f supports for content
- hash, err := hash.NewMultiHasherTypes(f.Hashes())
- if err != nil {
- r.Fatalf("Failed to make new multi hasher: %v", err)
- }
- _, err = hash.Write([]byte(content))
- if err != nil {
- r.Fatalf("Failed to make write to hash: %v", err)
- }
- hashSums := hash.Sums()
- const maxTries = 10
- for tries := 1; ; tries++ {
- in := bytes.NewBufferString(content)
- objinfo := object.NewStaticObjectInfo(remote, modTime, int64(len(content)), true, hashSums, nil)
- _, err := put(ctx, in, objinfo)
- if err == nil {
- break
- }
- // Retry if err returned a retry error
- if fserrors.IsRetryError(err) && tries < maxTries {
- r.Logf("Retry Put of %q to %v: %d/%d (%v)", remote, f, tries, maxTries, err)
- time.Sleep(2 * time.Second)
- continue
- }
- r.Fatalf("Failed to put %q to %q: %v", remote, f, err)
- }
- return NewItem(remote, content, modTime)
- }
- // WriteObject writes an object to the remote
- func (r *Run) WriteObject(ctx context.Context, remote, content string, modTime time.Time) Item {
- return r.WriteObjectTo(ctx, r.Fremote, remote, content, modTime, false)
- }
- // WriteUncheckedObject writes an object to the remote not checking for duplicates
- func (r *Run) WriteUncheckedObject(ctx context.Context, remote, content string, modTime time.Time) Item {
- return r.WriteObjectTo(ctx, r.Fremote, remote, content, modTime, true)
- }
- // WriteBoth calls WriteObject and WriteFile with the same arguments
- func (r *Run) WriteBoth(ctx context.Context, remote, content string, modTime time.Time) Item {
- r.WriteFile(remote, content, modTime)
- return r.WriteObject(ctx, remote, content, modTime)
- }
- // CheckWithDuplicates does a test but allows duplicates
- func (r *Run) CheckWithDuplicates(t *testing.T, items ...Item) {
- var want, got []string
- // construct a []string of desired items
- for _, item := range items {
- want = append(want, fmt.Sprintf("%q %d", item.Path, item.Size))
- }
- sort.Strings(want)
- // do the listing
- objs, _, err := walk.GetAll(context.Background(), r.Fremote, "", true, -1)
- if err != nil && err != fs.ErrorDirNotFound {
- t.Fatalf("Error listing: %v", err)
- }
- // construct a []string of actual items
- for _, o := range objs {
- got = append(got, fmt.Sprintf("%q %d", o.Remote(), o.Size()))
- }
- sort.Strings(got)
- assert.Equal(t, want, got)
- }
- // CheckLocalItems checks the local fs with proper precision
- // to see if it has the expected items.
- func (r *Run) CheckLocalItems(t *testing.T, items ...Item) {
- CheckItemsWithPrecision(t, r.Flocal, r.Precision, items...)
- }
- // CheckRemoteItems checks the remote fs with proper precision
- // to see if it has the expected items.
- func (r *Run) CheckRemoteItems(t *testing.T, items ...Item) {
- CheckItemsWithPrecision(t, r.Fremote, r.Precision, items...)
- }
- // CheckLocalListing checks the local fs with proper precision
- // to see if it has the expected contents.
- //
- // If expectedDirs is non nil then we check those too. Note that no
- // directories returned is also OK as some remotes don't return
- // directories.
- func (r *Run) CheckLocalListing(t *testing.T, items []Item, expectedDirs []string) {
- CheckListingWithPrecision(t, r.Flocal, items, expectedDirs, r.Precision)
- }
- // CheckRemoteListing checks the remote fs with proper precision
- // to see if it has the expected contents.
- //
- // If expectedDirs is non nil then we check those too. Note that no
- // directories returned is also OK as some remotes don't return
- // directories.
- func (r *Run) CheckRemoteListing(t *testing.T, items []Item, expectedDirs []string) {
- CheckListingWithPrecision(t, r.Fremote, items, expectedDirs, r.Precision)
- }
- // CheckDirectoryModTimes checks that the directory names in r.Flocal has the correct modtime compared to r.Fremote
- func (r *Run) CheckDirectoryModTimes(t *testing.T, names ...string) {
- if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil {
- fs.Debugf(r.Fremote, "Skipping modtime test as remote does not support DirSetModTime or MkdirMetadata")
- return
- }
- ctx := context.Background()
- for _, name := range names {
- wantT := NewDirectory(ctx, t, r.Flocal, name).ModTime(ctx)
- got := NewDirectory(ctx, t, r.Fremote, name)
- CheckDirModTime(ctx, t, r.Fremote, got, wantT)
- }
- }
- // Clean the temporary directory
- func (r *Run) cleanTempDir() {
- err := os.RemoveAll(r.LocalName)
- if err != nil {
- r.Logf("Failed to clean temporary directory %q: %v", r.LocalName, err)
- }
- }
- // Finalise cleans the remote and local
- func (r *Run) Finalise() {
- // r.Logf("Cleaning remote %q", r.Fremote)
- r.cleanRemote()
- // r.Logf("Cleaning local %q", r.LocalName)
- r.cleanTempDir()
- // Clear the remote cache
- cache.Clear()
- }
|