123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575 |
- // Package cmd implements the rclone command
- //
- // It is in a sub package so it's internals can be reused elsewhere
- package cmd
- // FIXME only attach the remote flags when using a remote???
- // would probably mean bringing all the flags in to here? Or define some flagsets in fs...
- import (
- "context"
- "errors"
- "fmt"
- "log"
- "os"
- "os/exec"
- "path"
- "regexp"
- "runtime"
- "runtime/pprof"
- "strconv"
- "strings"
- "sync"
- "time"
- "github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/accounting"
- "github.com/rclone/rclone/fs/cache"
- "github.com/rclone/rclone/fs/config/configfile"
- "github.com/rclone/rclone/fs/config/configflags"
- "github.com/rclone/rclone/fs/config/flags"
- "github.com/rclone/rclone/fs/filter"
- "github.com/rclone/rclone/fs/filter/filterflags"
- "github.com/rclone/rclone/fs/fserrors"
- "github.com/rclone/rclone/fs/fspath"
- fslog "github.com/rclone/rclone/fs/log"
- "github.com/rclone/rclone/fs/rc/rcflags"
- "github.com/rclone/rclone/fs/rc/rcserver"
- fssync "github.com/rclone/rclone/fs/sync"
- "github.com/rclone/rclone/lib/atexit"
- "github.com/rclone/rclone/lib/buildinfo"
- "github.com/rclone/rclone/lib/exitcode"
- "github.com/rclone/rclone/lib/terminal"
- "github.com/spf13/cobra"
- "github.com/spf13/pflag"
- )
- // Globals
- var (
- // Flags
- cpuProfile = flags.StringP("cpuprofile", "", "", "Write cpu profile to file", "Debugging")
- memProfile = flags.StringP("memprofile", "", "", "Write memory profile to file", "Debugging")
- statsInterval = flags.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable)", "Logging")
- dataRateUnit = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes' per second", "Logging")
- version bool
- // Errors
- errorCommandNotFound = errors.New("command not found")
- errorUncategorized = errors.New("uncategorized error")
- errorNotEnoughArguments = errors.New("not enough arguments")
- errorTooManyArguments = errors.New("too many arguments")
- )
- // ShowVersion prints the version to stdout
- func ShowVersion() {
- osVersion, osKernel := buildinfo.GetOSVersion()
- if osVersion == "" {
- osVersion = "unknown"
- }
- if osKernel == "" {
- osKernel = "unknown"
- }
- linking, tagString := buildinfo.GetLinkingAndTags()
- arch := buildinfo.GetArch()
- fmt.Printf("rclone %s\n", fs.Version)
- fmt.Printf("- os/version: %s\n", osVersion)
- fmt.Printf("- os/kernel: %s\n", osKernel)
- fmt.Printf("- os/type: %s\n", runtime.GOOS)
- fmt.Printf("- os/arch: %s\n", arch)
- fmt.Printf("- go/version: %s\n", runtime.Version())
- fmt.Printf("- go/linking: %s\n", linking)
- fmt.Printf("- go/tags: %s\n", tagString)
- }
- // NewFsFile creates an Fs from a name but may point to a file.
- //
- // It returns a string with the file name if points to a file
- // otherwise "".
- func NewFsFile(remote string) (fs.Fs, string) {
- _, fsPath, err := fspath.SplitFs(remote)
- if err != nil {
- err = fs.CountError(err)
- log.Fatalf("Failed to create file system for %q: %v", remote, err)
- }
- f, err := cache.Get(context.Background(), remote)
- switch err {
- case fs.ErrorIsFile:
- cache.Pin(f) // pin indefinitely since it was on the CLI
- return f, path.Base(fsPath)
- case nil:
- cache.Pin(f) // pin indefinitely since it was on the CLI
- return f, ""
- default:
- err = fs.CountError(err)
- log.Fatalf("Failed to create file system for %q: %v", remote, err)
- }
- return nil, ""
- }
- // newFsFileAddFilter creates an src Fs from a name
- //
- // This works the same as NewFsFile however it adds filters to the Fs
- // to limit it to a single file if the remote pointed to a file.
- func newFsFileAddFilter(remote string) (fs.Fs, string) {
- fi := filter.GetConfig(context.Background())
- f, fileName := NewFsFile(remote)
- if fileName != "" {
- if !fi.InActive() {
- err := fmt.Errorf("can't limit to single files when using filters: %v", remote)
- err = fs.CountError(err)
- log.Fatalf(err.Error())
- }
- // Limit transfers to this file
- err := fi.AddFile(fileName)
- if err != nil {
- err = fs.CountError(err)
- log.Fatalf("Failed to limit to single file %q: %v", remote, err)
- }
- }
- return f, fileName
- }
- // NewFsSrc creates a new src fs from the arguments.
- //
- // The source can be a file or a directory - if a file then it will
- // limit the Fs to a single file.
- func NewFsSrc(args []string) fs.Fs {
- fsrc, _ := newFsFileAddFilter(args[0])
- return fsrc
- }
- // newFsDir creates an Fs from a name
- //
- // This must point to a directory
- func newFsDir(remote string) fs.Fs {
- f, err := cache.Get(context.Background(), remote)
- if err != nil {
- err = fs.CountError(err)
- log.Fatalf("Failed to create file system for %q: %v", remote, err)
- }
- cache.Pin(f) // pin indefinitely since it was on the CLI
- return f
- }
- // NewFsDir creates a new Fs from the arguments
- //
- // The argument must point a directory
- func NewFsDir(args []string) fs.Fs {
- fdst := newFsDir(args[0])
- return fdst
- }
- // NewFsSrcDst creates a new src and dst fs from the arguments
- func NewFsSrcDst(args []string) (fs.Fs, fs.Fs) {
- fsrc, _ := newFsFileAddFilter(args[0])
- fdst := newFsDir(args[1])
- return fsrc, fdst
- }
- // NewFsSrcFileDst creates a new src and dst fs from the arguments
- //
- // The source may be a file, in which case the source Fs and file name is returned
- func NewFsSrcFileDst(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs) {
- fsrc, srcFileName = NewFsFile(args[0])
- fdst = newFsDir(args[1])
- return fsrc, srcFileName, fdst
- }
- // NewFsSrcDstFiles creates a new src and dst fs from the arguments
- // If src is a file then srcFileName and dstFileName will be non-empty
- func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs, dstFileName string) {
- fsrc, srcFileName = newFsFileAddFilter(args[0])
- // If copying a file...
- dstRemote := args[1]
- // If file exists then srcFileName != "", however if the file
- // doesn't exist then we assume it is a directory...
- if srcFileName != "" {
- var err error
- dstRemote, dstFileName, err = fspath.Split(dstRemote)
- if err != nil {
- log.Fatalf("Parsing %q failed: %v", args[1], err)
- }
- if dstRemote == "" {
- dstRemote = "."
- }
- if dstFileName == "" {
- log.Fatalf("%q is a directory", args[1])
- }
- }
- fdst, err := cache.Get(context.Background(), dstRemote)
- switch err {
- case fs.ErrorIsFile:
- _ = fs.CountError(err)
- log.Fatalf("Source doesn't exist or is a directory and destination is a file")
- case nil:
- default:
- _ = fs.CountError(err)
- log.Fatalf("Failed to create file system for destination %q: %v", dstRemote, err)
- }
- cache.Pin(fdst) // pin indefinitely since it was on the CLI
- return
- }
- // NewFsDstFile creates a new dst fs with a destination file name from the arguments
- func NewFsDstFile(args []string) (fdst fs.Fs, dstFileName string) {
- dstRemote, dstFileName, err := fspath.Split(args[0])
- if err != nil {
- log.Fatalf("Parsing %q failed: %v", args[0], err)
- }
- if dstRemote == "" {
- dstRemote = "."
- }
- if dstFileName == "" {
- log.Fatalf("%q is a directory", args[0])
- }
- fdst = newFsDir(dstRemote)
- return
- }
- // ShowStats returns true if the user added a `--stats` flag to the command line.
- //
- // This is called by Run to override the default value of the
- // showStats passed in.
- func ShowStats() bool {
- statsIntervalFlag := pflag.Lookup("stats")
- return statsIntervalFlag != nil && statsIntervalFlag.Changed
- }
- // Run the function with stats and retries if required
- func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
- ci := fs.GetConfig(context.Background())
- var cmdErr error
- stopStats := func() {}
- if !showStats && ShowStats() {
- showStats = true
- }
- if ci.Progress {
- stopStats = startProgress()
- } else if showStats {
- stopStats = StartStats()
- }
- SigInfoHandler()
- for try := 1; try <= ci.Retries; try++ {
- cmdErr = f()
- cmdErr = fs.CountError(cmdErr)
- lastErr := accounting.GlobalStats().GetLastError()
- if cmdErr == nil {
- cmdErr = lastErr
- }
- if !Retry || !accounting.GlobalStats().Errored() {
- if try > 1 {
- fs.Errorf(nil, "Attempt %d/%d succeeded", try, ci.Retries)
- }
- break
- }
- if accounting.GlobalStats().HadFatalError() {
- fs.Errorf(nil, "Fatal error received - not attempting retries")
- break
- }
- if accounting.GlobalStats().Errored() && !accounting.GlobalStats().HadRetryError() {
- fs.Errorf(nil, "Can't retry any of the errors - not attempting retries")
- break
- }
- if retryAfter := accounting.GlobalStats().RetryAfter(); !retryAfter.IsZero() {
- d := time.Until(retryAfter)
- if d > 0 {
- fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d)
- time.Sleep(d)
- }
- }
- if lastErr != nil {
- fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, ci.Retries, accounting.GlobalStats().GetErrors(), lastErr)
- } else {
- fs.Errorf(nil, "Attempt %d/%d failed with %d errors", try, ci.Retries, accounting.GlobalStats().GetErrors())
- }
- if try < ci.Retries {
- accounting.GlobalStats().ResetErrors()
- }
- if ci.RetriesInterval > 0 {
- time.Sleep(ci.RetriesInterval)
- }
- }
- stopStats()
- if showStats && (accounting.GlobalStats().Errored() || *statsInterval > 0) {
- accounting.GlobalStats().Log()
- }
- fs.Debugf(nil, "%d go routines active\n", runtime.NumGoroutine())
- if ci.Progress && ci.ProgressTerminalTitle {
- // Clear terminal title
- terminal.WriteTerminalTitle("")
- }
- // dump all running go-routines
- if ci.Dump&fs.DumpGoRoutines != 0 {
- err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
- if err != nil {
- fs.Errorf(nil, "Failed to dump goroutines: %v", err)
- }
- }
- // dump open files
- if ci.Dump&fs.DumpOpenFiles != 0 {
- c := exec.Command("lsof", "-p", strconv.Itoa(os.Getpid()))
- c.Stdout = os.Stdout
- c.Stderr = os.Stderr
- err := c.Run()
- if err != nil {
- fs.Errorf(nil, "Failed to list open files: %v", err)
- }
- }
- // clear cache and shutdown backends
- cache.Clear()
- if lastErr := accounting.GlobalStats().GetLastError(); cmdErr == nil {
- cmdErr = lastErr
- }
- // Log the final error message and exit
- if cmdErr != nil {
- nerrs := accounting.GlobalStats().GetErrors()
- if nerrs <= 1 {
- log.Printf("Failed to %s: %v", cmd.Name(), cmdErr)
- } else {
- log.Printf("Failed to %s with %d errors: last error was: %v", cmd.Name(), nerrs, cmdErr)
- }
- }
- resolveExitCode(cmdErr)
- }
- // CheckArgs checks there are enough arguments and prints a message if not
- func CheckArgs(MinArgs, MaxArgs int, cmd *cobra.Command, args []string) {
- if len(args) < MinArgs {
- _ = cmd.Usage()
- _, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments minimum: you provided %d non flag arguments: %q\n", cmd.Name(), MinArgs, len(args), args)
- resolveExitCode(errorNotEnoughArguments)
- } else if len(args) > MaxArgs {
- _ = cmd.Usage()
- _, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum: you provided %d non flag arguments: %q\n", cmd.Name(), MaxArgs, len(args), args)
- resolveExitCode(errorTooManyArguments)
- }
- }
- // StartStats prints the stats every statsInterval
- //
- // It returns a func which should be called to stop the stats.
- func StartStats() func() {
- if *statsInterval <= 0 {
- return func() {}
- }
- stopStats := make(chan struct{})
- var wg sync.WaitGroup
- wg.Add(1)
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(*statsInterval)
- for {
- select {
- case <-ticker.C:
- accounting.GlobalStats().Log()
- case <-stopStats:
- ticker.Stop()
- return
- }
- }
- }()
- return func() {
- close(stopStats)
- wg.Wait()
- }
- }
- // initConfig is run by cobra after initialising the flags
- func initConfig() {
- ctx := context.Background()
- ci := fs.GetConfig(ctx)
- // Start the logger
- fslog.InitLogging()
- // Finish parsing any command line flags
- configflags.SetFlags(ci)
- // Load the config
- configfile.Install()
- // Start accounting
- accounting.Start(ctx)
- // Configure console
- if ci.NoConsole {
- // Hide the console window
- terminal.HideConsole()
- } else {
- // Enable color support on stdout if possible.
- // This enables virtual terminal processing on Windows 10,
- // adding native support for ANSI/VT100 escape sequences.
- terminal.EnableColorsStdout()
- }
- // Load filters
- err := filterflags.Reload(ctx)
- if err != nil {
- log.Fatalf("Failed to load filters: %v", err)
- }
- // Write the args for debug purposes
- fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args)
- // Inform user about systemd log support now that we have a logger
- if fslog.Opt.LogSystemdSupport {
- fs.Debugf("rclone", "systemd logging support activated")
- }
- // Start the remote control server if configured
- _, err = rcserver.Start(context.Background(), &rcflags.Opt)
- if err != nil {
- log.Fatalf("Failed to start remote control: %v", err)
- }
- // Setup CPU profiling if desired
- if *cpuProfile != "" {
- fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile)
- f, err := os.Create(*cpuProfile)
- if err != nil {
- err = fs.CountError(err)
- log.Fatal(err)
- }
- err = pprof.StartCPUProfile(f)
- if err != nil {
- err = fs.CountError(err)
- log.Fatal(err)
- }
- atexit.Register(func() {
- pprof.StopCPUProfile()
- err := f.Close()
- if err != nil {
- err = fs.CountError(err)
- log.Fatal(err)
- }
- })
- }
- // Setup memory profiling if desired
- if *memProfile != "" {
- atexit.Register(func() {
- fs.Infof(nil, "Saving Memory profile %q\n", *memProfile)
- f, err := os.Create(*memProfile)
- if err != nil {
- err = fs.CountError(err)
- log.Fatal(err)
- }
- err = pprof.WriteHeapProfile(f)
- if err != nil {
- err = fs.CountError(err)
- log.Fatal(err)
- }
- err = f.Close()
- if err != nil {
- err = fs.CountError(err)
- log.Fatal(err)
- }
- })
- }
- if m, _ := regexp.MatchString("^(bits|bytes)$", *dataRateUnit); !m {
- fs.Errorf(nil, "Invalid unit passed to --stats-unit. Defaulting to bytes.")
- ci.DataRateUnit = "bytes"
- } else {
- ci.DataRateUnit = *dataRateUnit
- }
- }
- func resolveExitCode(err error) {
- ci := fs.GetConfig(context.Background())
- atexit.Run()
- if err == nil {
- if ci.ErrorOnNoTransfer {
- if accounting.GlobalStats().GetTransfers() == 0 {
- os.Exit(exitcode.NoFilesTransferred)
- }
- }
- os.Exit(exitcode.Success)
- }
- switch {
- case errors.Is(err, fs.ErrorDirNotFound):
- os.Exit(exitcode.DirNotFound)
- case errors.Is(err, fs.ErrorObjectNotFound):
- os.Exit(exitcode.FileNotFound)
- case errors.Is(err, errorUncategorized):
- os.Exit(exitcode.UncategorizedError)
- case errors.Is(err, accounting.ErrorMaxTransferLimitReached):
- os.Exit(exitcode.TransferExceeded)
- case errors.Is(err, fssync.ErrorMaxDurationReached):
- os.Exit(exitcode.DurationExceeded)
- case fserrors.ShouldRetry(err):
- os.Exit(exitcode.RetryError)
- case fserrors.IsNoRetryError(err), fserrors.IsNoLowLevelRetryError(err):
- os.Exit(exitcode.NoRetryError)
- case fserrors.IsFatalError(err):
- os.Exit(exitcode.FatalError)
- default:
- os.Exit(exitcode.UsageError)
- }
- }
- var backendFlags map[string]struct{}
- // AddBackendFlags creates flags for all the backend options
- func AddBackendFlags() {
- backendFlags = map[string]struct{}{}
- for _, fsInfo := range fs.Registry {
- done := map[string]struct{}{}
- for i := range fsInfo.Options {
- opt := &fsInfo.Options[i]
- // Skip if done already (e.g. with Provider options)
- if _, doneAlready := done[opt.Name]; doneAlready {
- continue
- }
- done[opt.Name] = struct{}{}
- // Make a flag from each option
- name := opt.FlagName(fsInfo.Prefix)
- found := pflag.CommandLine.Lookup(name) != nil
- if !found {
- // Take first line of help only
- help := strings.TrimSpace(opt.Help)
- if nl := strings.IndexRune(help, '\n'); nl >= 0 {
- help = help[:nl]
- }
- help = strings.TrimRight(strings.TrimSpace(help), ".!?")
- if opt.IsPassword {
- help += " (obscured)"
- }
- flag := pflag.CommandLine.VarPF(opt, name, opt.ShortOpt, help)
- flags.SetDefaultFromEnv(pflag.CommandLine, name)
- if _, isBool := opt.Default.(bool); isBool {
- flag.NoOptDefVal = "true"
- }
- // Hide on the command line if requested
- if opt.Hide&fs.OptionHideCommandLine != 0 {
- flag.Hidden = true
- }
- backendFlags[name] = struct{}{}
- } else {
- fs.Errorf(nil, "Not adding duplicate flag --%s", name)
- }
- // flag.Hidden = true
- }
- }
- }
- // Main runs rclone interpreting flags and commands out of os.Args
- func Main() {
- setupRootCommand(Root)
- AddBackendFlags()
- if err := Root.Execute(); err != nil {
- if strings.HasPrefix(err.Error(), "unknown command") && selfupdateEnabled {
- Root.PrintErrf("You could use '%s selfupdate' to get latest features.\n\n", Root.CommandPath())
- }
- log.Fatalf("Fatal error: %v", err)
- }
- }
|