123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569 |
- // Define the internal rc functions
- package rc
- import (
- "context"
- "fmt"
- "net/http"
- "os"
- "os/exec"
- "runtime"
- "strings"
- "time"
- "github.com/coreos/go-semver/semver"
- "github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/config/obscure"
- "github.com/rclone/rclone/lib/atexit"
- "github.com/rclone/rclone/lib/buildinfo"
- "github.com/rclone/rclone/lib/debug"
- )
- func init() {
- Add(Call{
- Path: "rc/noopauth",
- AuthRequired: true,
- Fn: rcNoop,
- Title: "Echo the input to the output parameters requiring auth",
- Help: `
- This echoes the input parameters to the output parameters for testing
- purposes. It can be used to check that rclone is still alive and to
- check that parameter passing is working properly.`,
- })
- Add(Call{
- Path: "rc/noop",
- Fn: rcNoop,
- Title: "Echo the input to the output parameters",
- Help: `
- This echoes the input parameters to the output parameters for testing
- purposes. It can be used to check that rclone is still alive and to
- check that parameter passing is working properly.`,
- })
- }
- // Echo the input to the output parameters
- func rcNoop(ctx context.Context, in Params) (out Params, err error) {
- return in, nil
- }
- func init() {
- Add(Call{
- Path: "rc/error",
- Fn: rcError,
- Title: "This returns an error",
- Help: `
- This returns an error with the input as part of its error string.
- Useful for testing error handling.`,
- })
- }
- // Return an error regardless
- func rcError(ctx context.Context, in Params) (out Params, err error) {
- return nil, fmt.Errorf("arbitrary error on input %+v", in)
- }
- func init() {
- Add(Call{
- Path: "rc/list",
- Fn: rcList,
- Title: "List all the registered remote control commands",
- Help: `
- This lists all the registered remote control commands as a JSON map in
- the commands response.`,
- })
- }
- // List the registered commands
- func rcList(ctx context.Context, in Params) (out Params, err error) {
- out = make(Params)
- out["commands"] = Calls.List()
- return out, nil
- }
- func init() {
- Add(Call{
- Path: "core/pid",
- Fn: rcPid,
- Title: "Return PID of current process",
- Help: `
- This returns PID of current process.
- Useful for stopping rclone process.`,
- })
- }
- // Return PID of current process
- func rcPid(ctx context.Context, in Params) (out Params, err error) {
- out = make(Params)
- out["pid"] = os.Getpid()
- return out, nil
- }
- func init() {
- Add(Call{
- Path: "core/memstats",
- Fn: rcMemStats,
- Title: "Returns the memory statistics",
- Help: `
- This returns the memory statistics of the running program. What the values mean
- are explained in the go docs: https://golang.org/pkg/runtime/#MemStats
- The most interesting values for most people are:
- - HeapAlloc - this is the amount of memory rclone is actually using
- - HeapSys - this is the amount of memory rclone has obtained from the OS
- - Sys - this is the total amount of memory requested from the OS
- - It is virtual memory so may include unused memory
- `,
- })
- }
- // Return the memory statistics
- func rcMemStats(ctx context.Context, in Params) (out Params, err error) {
- out = make(Params)
- var m runtime.MemStats
- runtime.ReadMemStats(&m)
- out["Alloc"] = m.Alloc
- out["TotalAlloc"] = m.TotalAlloc
- out["Sys"] = m.Sys
- out["Mallocs"] = m.Mallocs
- out["Frees"] = m.Frees
- out["HeapAlloc"] = m.HeapAlloc
- out["HeapSys"] = m.HeapSys
- out["HeapIdle"] = m.HeapIdle
- out["HeapInuse"] = m.HeapInuse
- out["HeapReleased"] = m.HeapReleased
- out["HeapObjects"] = m.HeapObjects
- out["StackInuse"] = m.StackInuse
- out["StackSys"] = m.StackSys
- out["MSpanInuse"] = m.MSpanInuse
- out["MSpanSys"] = m.MSpanSys
- out["MCacheInuse"] = m.MCacheInuse
- out["MCacheSys"] = m.MCacheSys
- out["BuckHashSys"] = m.BuckHashSys
- out["GCSys"] = m.GCSys
- out["OtherSys"] = m.OtherSys
- return out, nil
- }
- func init() {
- Add(Call{
- Path: "core/gc",
- Fn: rcGc,
- Title: "Runs a garbage collection.",
- Help: `
- This tells the go runtime to do a garbage collection run. It isn't
- necessary to call this normally, but it can be useful for debugging
- memory problems.
- `,
- })
- }
- // Do a garbage collection run
- func rcGc(ctx context.Context, in Params) (out Params, err error) {
- runtime.GC()
- return nil, nil
- }
- func init() {
- Add(Call{
- Path: "core/version",
- Fn: rcVersion,
- Title: "Shows the current version of rclone and the go runtime.",
- Help: `
- This shows the current version of go and the go runtime:
- - version - rclone version, e.g. "v1.53.0"
- - decomposed - version number as [major, minor, patch]
- - isGit - boolean - true if this was compiled from the git version
- - isBeta - boolean - true if this is a beta version
- - os - OS in use as according to Go
- - arch - cpu architecture in use according to Go
- - goVersion - version of Go runtime in use
- - linking - type of rclone executable (static or dynamic)
- - goTags - space separated build tags or "none"
- `,
- })
- }
- // Return version info
- func rcVersion(ctx context.Context, in Params) (out Params, err error) {
- version, err := semver.NewVersion(fs.Version[1:])
- if err != nil {
- return nil, err
- }
- linking, tagString := buildinfo.GetLinkingAndTags()
- out = Params{
- "version": fs.Version,
- "decomposed": version.Slice(),
- "isGit": strings.HasSuffix(fs.Version, "-DEV"),
- "isBeta": version.PreRelease != "",
- "os": runtime.GOOS,
- "arch": runtime.GOARCH,
- "goVersion": runtime.Version(),
- "linking": linking,
- "goTags": tagString,
- }
- return out, nil
- }
- func init() {
- Add(Call{
- Path: "core/obscure",
- Fn: rcObscure,
- Title: "Obscures a string passed in.",
- Help: `
- Pass a clear string and rclone will obscure it for the config file:
- - clear - string
- Returns:
- - obscured - string
- `,
- })
- }
- // Return obscured string
- func rcObscure(ctx context.Context, in Params) (out Params, err error) {
- clear, err := in.GetString("clear")
- if err != nil {
- return nil, err
- }
- obscured, err := obscure.Obscure(clear)
- if err != nil {
- return nil, err
- }
- out = Params{
- "obscured": obscured,
- }
- return out, nil
- }
- func init() {
- Add(Call{
- Path: "core/quit",
- Fn: rcQuit,
- Title: "Terminates the app.",
- Help: `
- (Optional) Pass an exit code to be used for terminating the app:
- - exitCode - int
- `,
- })
- }
- // Terminates app
- func rcQuit(ctx context.Context, in Params) (out Params, err error) {
- code, err := in.GetInt64("exitCode")
- if IsErrParamInvalid(err) {
- return nil, err
- }
- if IsErrParamNotFound(err) {
- code = 0
- }
- exitCode := int(code)
- go func(exitCode int) {
- time.Sleep(time.Millisecond * 1500)
- atexit.Run()
- os.Exit(exitCode)
- }(exitCode)
- return nil, nil
- }
- func init() {
- Add(Call{
- Path: "debug/set-mutex-profile-fraction",
- Fn: rcSetMutexProfileFraction,
- Title: "Set runtime.SetMutexProfileFraction for mutex profiling.",
- Help: `
- SetMutexProfileFraction controls the fraction of mutex contention
- events that are reported in the mutex profile. On average 1/rate
- events are reported. The previous rate is returned.
- To turn off profiling entirely, pass rate 0. To just read the current
- rate, pass rate < 0. (For n>1 the details of sampling may change.)
- Once this is set you can look use this to profile the mutex contention:
- go tool pprof http://localhost:5572/debug/pprof/mutex
- Parameters:
- - rate - int
- Results:
- - previousRate - int
- `,
- })
- }
- func rcSetMutexProfileFraction(ctx context.Context, in Params) (out Params, err error) {
- rate, err := in.GetInt64("rate")
- if err != nil {
- return nil, err
- }
- previousRate := runtime.SetMutexProfileFraction(int(rate))
- out = make(Params)
- out["previousRate"] = previousRate
- return out, nil
- }
- func init() {
- Add(Call{
- Path: "debug/set-block-profile-rate",
- Fn: rcSetBlockProfileRate,
- Title: "Set runtime.SetBlockProfileRate for blocking profiling.",
- Help: `
- SetBlockProfileRate controls the fraction of goroutine blocking events
- that are reported in the blocking profile. The profiler aims to sample
- an average of one blocking event per rate nanoseconds spent blocked.
- To include every blocking event in the profile, pass rate = 1. To turn
- off profiling entirely, pass rate <= 0.
- After calling this you can use this to see the blocking profile:
- go tool pprof http://localhost:5572/debug/pprof/block
- Parameters:
- - rate - int
- `,
- })
- }
- func rcSetBlockProfileRate(ctx context.Context, in Params) (out Params, err error) {
- rate, err := in.GetInt64("rate")
- if err != nil {
- return nil, err
- }
- runtime.SetBlockProfileRate(int(rate))
- return nil, nil
- }
- func init() {
- Add(Call{
- Path: "debug/set-soft-memory-limit",
- Fn: rcSetSoftMemoryLimit,
- Title: "Call runtime/debug.SetMemoryLimit for setting a soft memory limit for the runtime.",
- Help: `
- SetMemoryLimit provides the runtime with a soft memory limit.
- The runtime undertakes several processes to try to respect this memory limit, including
- adjustments to the frequency of garbage collections and returning memory to the underlying
- system more aggressively. This limit will be respected even if GOGC=off (or, if SetGCPercent(-1) is executed).
- The input limit is provided as bytes, and includes all memory mapped, managed, and not
- released by the Go runtime. Notably, it does not account for space used by the Go binary
- and memory external to Go, such as memory managed by the underlying system on behalf of
- the process, or memory managed by non-Go code inside the same process.
- Examples of excluded memory sources include: OS kernel memory held on behalf of the process,
- memory allocated by C code, and memory mapped by syscall.Mmap (because it is not managed by the Go runtime).
- A zero limit or a limit that's lower than the amount of memory used by the Go runtime may cause
- the garbage collector to run nearly continuously. However, the application may still make progress.
- The memory limit is always respected by the Go runtime, so to effectively disable this behavior,
- set the limit very high. math.MaxInt64 is the canonical value for disabling the limit, but values
- much greater than the available memory on the underlying system work just as well.
- See https://go.dev/doc/gc-guide for a detailed guide explaining the soft memory limit in more detail,
- as well as a variety of common use-cases and scenarios.
- SetMemoryLimit returns the previously set memory limit. A negative input does not adjust the limit,
- and allows for retrieval of the currently set memory limit.
- Parameters:
- - mem-limit - int
- `,
- })
- }
- func rcSetSoftMemoryLimit(ctx context.Context, in Params) (out Params, err error) {
- memLimit, err := in.GetInt64("mem-limit")
- if err != nil {
- return nil, err
- }
- oldMemLimit := debug.SetMemoryLimit(memLimit)
- out = Params{
- "existing-mem-limit": oldMemLimit,
- }
- return out, nil
- }
- func init() {
- Add(Call{
- Path: "debug/set-gc-percent",
- Fn: rcSetGCPercent,
- Title: "Call runtime/debug.SetGCPercent for setting the garbage collection target percentage.",
- Help: `
- SetGCPercent sets the garbage collection target percentage: a collection is triggered
- when the ratio of freshly allocated data to live data remaining after the previous collection
- reaches this percentage. SetGCPercent returns the previous setting. The initial setting is the
- value of the GOGC environment variable at startup, or 100 if the variable is not set.
- This setting may be effectively reduced in order to maintain a memory limit.
- A negative percentage effectively disables garbage collection, unless the memory limit is reached.
- See https://pkg.go.dev/runtime/debug#SetMemoryLimit for more details.
- Parameters:
- - gc-percent - int
- `,
- })
- }
- func rcSetGCPercent(ctx context.Context, in Params) (out Params, err error) {
- gcPercent, err := in.GetInt64("gc-percent")
- if err != nil {
- return nil, err
- }
- oldGCPercent := debug.SetGCPercent(int(gcPercent))
- out = Params{
- "existing-gc-percent": oldGCPercent,
- }
- return out, nil
- }
- func init() {
- Add(Call{
- Path: "core/command",
- AuthRequired: true,
- Fn: rcRunCommand,
- NeedsRequest: true,
- NeedsResponse: true,
- Title: "Run a rclone terminal command over rc.",
- Help: `This takes the following parameters:
- - command - a string with the command name.
- - arg - a list of arguments for the backend command.
- - opt - a map of string to string of options.
- - returnType - one of ("COMBINED_OUTPUT", "STREAM", "STREAM_ONLY_STDOUT", "STREAM_ONLY_STDERR").
- - Defaults to "COMBINED_OUTPUT" if not set.
- - The STREAM returnTypes will write the output to the body of the HTTP message.
- - The COMBINED_OUTPUT will write the output to the "result" parameter.
- Returns:
- - result - result from the backend command.
- - Only set when using returnType "COMBINED_OUTPUT".
- - error - set if rclone exits with an error code.
- - returnType - one of ("COMBINED_OUTPUT", "STREAM", "STREAM_ONLY_STDOUT", "STREAM_ONLY_STDERR").
- Example:
- rclone rc core/command command=ls -a mydrive:/ -o max-depth=1
- rclone rc core/command -a ls -a mydrive:/ -o max-depth=1
- Returns:
- ` + "```" + `
- {
- "error": false,
- "result": "<Raw command line output>"
- }
- OR
- {
- "error": true,
- "result": "<Raw command line output>"
- }
- ` + "```" + `
- `,
- })
- }
- // rcRunCommand runs an rclone command with the given args and flags
- func rcRunCommand(ctx context.Context, in Params) (out Params, err error) {
- command, err := in.GetString("command")
- if err != nil {
- command = ""
- }
- var opt = map[string]string{}
- err = in.GetStructMissingOK("opt", &opt)
- if err != nil {
- return nil, err
- }
- var arg = []string{}
- err = in.GetStructMissingOK("arg", &arg)
- if err != nil {
- return nil, err
- }
- returnType, err := in.GetString("returnType")
- if err != nil {
- returnType = "COMBINED_OUTPUT"
- }
- var httpResponse http.ResponseWriter
- httpResponse, err = in.GetHTTPResponseWriter()
- if err != nil {
- return nil, fmt.Errorf("response object is required\n%w", err)
- }
- var allArgs = []string{}
- if command != "" {
- // Add the command e.g.: ls to the args
- allArgs = append(allArgs, command)
- }
- // Add all from arg
- allArgs = append(allArgs, arg...)
- // Add flags to args for e.g. --max-depth 1 comes in as { max-depth 1 }.
- // Convert it to [ max-depth, 1 ] and append to args list
- for key, value := range opt {
- if len(key) == 1 {
- allArgs = append(allArgs, "-"+key)
- } else {
- allArgs = append(allArgs, "--"+key)
- }
- allArgs = append(allArgs, value)
- }
- // Get the path for the current executable which was used to run rclone.
- ex, err := os.Executable()
- if err != nil {
- return nil, err
- }
- cmd := exec.CommandContext(ctx, ex, allArgs...)
- if returnType == "COMBINED_OUTPUT" {
- // Run the command and get the output for error and stdout combined.
- out, err := cmd.CombinedOutput()
- if err != nil {
- return Params{
- "result": string(out),
- "error": true,
- }, nil
- }
- return Params{
- "result": string(out),
- "error": false,
- }, nil
- } else if returnType == "STREAM_ONLY_STDOUT" {
- cmd.Stdout = httpResponse
- } else if returnType == "STREAM_ONLY_STDERR" {
- cmd.Stderr = httpResponse
- } else if returnType == "STREAM" {
- cmd.Stdout = httpResponse
- cmd.Stderr = httpResponse
- } else {
- return nil, fmt.Errorf("unknown returnType %q", returnType)
- }
- err = cmd.Run()
- return nil, err
- }
|