123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625 |
- // Package fichier provides an interface to the 1Fichier storage system.
- package fichier
- import (
- "context"
- "errors"
- "fmt"
- "io"
- "net/http"
- "strconv"
- "strings"
- "time"
- "github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/config"
- "github.com/rclone/rclone/fs/config/configmap"
- "github.com/rclone/rclone/fs/config/configstruct"
- "github.com/rclone/rclone/fs/fshttp"
- "github.com/rclone/rclone/fs/hash"
- "github.com/rclone/rclone/lib/dircache"
- "github.com/rclone/rclone/lib/encoder"
- "github.com/rclone/rclone/lib/pacer"
- "github.com/rclone/rclone/lib/rest"
- )
- const (
- rootID = "0"
- apiBaseURL = "https://api.1fichier.com/v1"
- minSleep = 400 * time.Millisecond // api is extremely rate limited now
- maxSleep = 5 * time.Second
- decayConstant = 2 // bigger for slower decay, exponential
- attackConstant = 0 // start with max sleep
- )
- func init() {
- fs.Register(&fs.RegInfo{
- Name: "fichier",
- Description: "1Fichier",
- NewFs: NewFs,
- Options: []fs.Option{{
- Help: "Your API Key, get it from https://1fichier.com/console/params.pl.",
- Name: "api_key",
- Sensitive: true,
- }, {
- Help: "If you want to download a shared folder, add this parameter.",
- Name: "shared_folder",
- Advanced: true,
- }, {
- Help: "If you want to download a shared file that is password protected, add this parameter.",
- Name: "file_password",
- Advanced: true,
- IsPassword: true,
- }, {
- Help: "If you want to list the files in a shared folder that is password protected, add this parameter.",
- Name: "folder_password",
- Advanced: true,
- IsPassword: true,
- }, {
- Help: "Set if you wish to use CDN download links.",
- Name: "cdn",
- Default: false,
- Advanced: true,
- }, {
- Name: config.ConfigEncoding,
- Help: config.ConfigEncodingHelp,
- Advanced: true,
- // Characters that need escaping
- //
- // '\\': '\', // FULLWIDTH REVERSE SOLIDUS
- // '<': '<', // FULLWIDTH LESS-THAN SIGN
- // '>': '>', // FULLWIDTH GREATER-THAN SIGN
- // '"': '"', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
- // '\'': ''', // FULLWIDTH APOSTROPHE
- // '$': '$', // FULLWIDTH DOLLAR SIGN
- // '`': '`', // FULLWIDTH GRAVE ACCENT
- //
- // Leading space and trailing space
- Default: (encoder.Display |
- encoder.EncodeBackSlash |
- encoder.EncodeSingleQuote |
- encoder.EncodeBackQuote |
- encoder.EncodeDoubleQuote |
- encoder.EncodeLtGt |
- encoder.EncodeDollar |
- encoder.EncodeLeftSpace |
- encoder.EncodeRightSpace |
- encoder.EncodeInvalidUtf8),
- }},
- })
- }
- // Options defines the configuration for this backend
- type Options struct {
- APIKey string `config:"api_key"`
- SharedFolder string `config:"shared_folder"`
- FilePassword string `config:"file_password"`
- FolderPassword string `config:"folder_password"`
- CDN bool `config:"cdn"`
- Enc encoder.MultiEncoder `config:"encoding"`
- }
- // Fs is the interface a cloud storage system must provide
- type Fs struct {
- root string
- name string
- features *fs.Features
- opt Options
- dirCache *dircache.DirCache
- baseClient *http.Client
- pacer *fs.Pacer
- rest *rest.Client
- }
- // FindLeaf finds a directory of name leaf in the folder with ID pathID
- func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
- folderID, err := strconv.Atoi(pathID)
- if err != nil {
- return "", false, err
- }
- folders, err := f.listFolders(ctx, folderID)
- if err != nil {
- return "", false, err
- }
- for _, folder := range folders.SubFolders {
- if folder.Name == leaf {
- pathIDOut := strconv.Itoa(folder.ID)
- return pathIDOut, true, nil
- }
- }
- return "", false, nil
- }
- // CreateDir makes a directory with pathID as parent and name leaf
- func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
- folderID, err := strconv.Atoi(pathID)
- if err != nil {
- return "", err
- }
- resp, err := f.makeFolder(ctx, leaf, folderID)
- if err != nil {
- return "", err
- }
- return strconv.Itoa(resp.FolderID), err
- }
- // Name of the remote (as passed into NewFs)
- func (f *Fs) Name() string {
- return f.name
- }
- // Root of the remote (as passed into NewFs)
- func (f *Fs) Root() string {
- return f.root
- }
- // String returns a description of the FS
- func (f *Fs) String() string {
- return fmt.Sprintf("1Fichier root '%s'", f.root)
- }
- // Precision of the ModTimes in this Fs
- func (f *Fs) Precision() time.Duration {
- return fs.ModTimeNotSupported
- }
- // Hashes returns the supported hash types of the filesystem
- func (f *Fs) Hashes() hash.Set {
- return hash.Set(hash.Whirlpool)
- }
- // Features returns the optional features of this Fs
- func (f *Fs) Features() *fs.Features {
- return f.features
- }
- // NewFs makes a new Fs object from the path
- //
- // The path is of the form remote:path
- //
- // Remotes are looked up in the config file. If the remote isn't
- // found then NotFoundInConfigFile will be returned.
- //
- // On Windows avoid single character remote names as they can be mixed
- // up with drive letters.
- func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
- opt := new(Options)
- err := configstruct.Set(config, opt)
- if err != nil {
- return nil, err
- }
- // If using a Shared Folder override root
- if opt.SharedFolder != "" {
- root = ""
- }
- //workaround for wonky parser
- root = strings.Trim(root, "/")
- f := &Fs{
- name: name,
- root: root,
- opt: *opt,
- pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
- baseClient: &http.Client{},
- }
- f.features = (&fs.Features{
- DuplicateFiles: true,
- CanHaveEmptyDirectories: true,
- ReadMimeType: true,
- }).Fill(ctx, f)
- client := fshttp.NewClient(ctx)
- f.rest = rest.NewClient(client).SetRoot(apiBaseURL)
- f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
- f.dirCache = dircache.New(root, rootID, f)
- // Find the current root
- err = f.dirCache.FindRoot(ctx, false)
- if err != nil {
- // Assume it is a file
- newRoot, remote := dircache.SplitPath(root)
- tempF := *f
- tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
- tempF.root = newRoot
- // Make new Fs which is the parent
- err = tempF.dirCache.FindRoot(ctx, false)
- if err != nil {
- // No root so return old f
- return f, nil
- }
- _, err := tempF.NewObject(ctx, remote)
- if err != nil {
- if err == fs.ErrorObjectNotFound {
- // File doesn't exist so return old f
- return f, nil
- }
- return nil, err
- }
- f.features.Fill(ctx, &tempF)
- // XXX: update the old f here instead of returning tempF, since
- // `features` were already filled with functions having *f as a receiver.
- // See https://github.com/rclone/rclone/issues/2182
- f.dirCache = tempF.dirCache
- f.root = tempF.root
- // return an error with an fs which points to the parent
- return f, fs.ErrorIsFile
- }
- return f, nil
- }
- // List the objects and directories in dir into entries. The
- // entries can be returned in any order but should be for a
- // complete directory.
- //
- // dir should be "" to list the root, and should not have
- // trailing slashes.
- //
- // This should return ErrDirNotFound if the directory isn't
- // found.
- func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
- if f.opt.SharedFolder != "" {
- return f.listSharedFiles(ctx, f.opt.SharedFolder)
- }
- dirContent, err := f.listDir(ctx, dir)
- if err != nil {
- return nil, err
- }
- return dirContent, nil
- }
- // NewObject finds the Object at remote. If it can't be found
- // it returns the error ErrorObjectNotFound.
- func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
- leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
- if err != nil {
- if err == fs.ErrorDirNotFound {
- return nil, fs.ErrorObjectNotFound
- }
- return nil, err
- }
- folderID, err := strconv.Atoi(directoryID)
- if err != nil {
- return nil, err
- }
- files, err := f.listFiles(ctx, folderID)
- if err != nil {
- return nil, err
- }
- for _, file := range files.Items {
- if file.Filename == leaf {
- path, ok := f.dirCache.GetInv(directoryID)
- if !ok {
- return nil, errors.New("cannot find dir in dircache")
- }
- return f.newObjectFromFile(ctx, path, file), nil
- }
- }
- return nil, fs.ErrorObjectNotFound
- }
- // Put in to the remote path with the modTime given of the given size
- //
- // When called from outside an Fs by rclone, src.Size() will always be >= 0.
- // But for unknown-sized objects (indicated by src.Size() == -1), Put should either
- // return an error or upload it properly (rather than e.g. calling panic).
- //
- // May create the object even if it returns an error - if so
- // will return the object and the error, otherwise will return
- // nil and the error
- func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
- existingObj, err := f.NewObject(ctx, src.Remote())
- switch err {
- case nil:
- return existingObj, existingObj.Update(ctx, in, src, options...)
- case fs.ErrorObjectNotFound:
- // Not found so create it
- return f.PutUnchecked(ctx, in, src, options...)
- default:
- return nil, err
- }
- }
- // putUnchecked uploads the object with the given name and size
- //
- // This will create a duplicate if we upload a new file without
- // checking to see if there is one already - use Put() for that.
- func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) {
- if size > int64(300e9) {
- return nil, errors.New("File too big, can't upload")
- } else if size == 0 {
- return nil, fs.ErrorCantUploadEmptyFiles
- }
- nodeResponse, err := f.getUploadNode(ctx)
- if err != nil {
- return nil, err
- }
- leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
- if err != nil {
- return nil, err
- }
- _, err = f.uploadFile(ctx, in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL, options...)
- if err != nil {
- return nil, err
- }
- fileUploadResponse, err := f.endUpload(ctx, nodeResponse.ID, nodeResponse.URL)
- if err != nil {
- return nil, err
- }
- if len(fileUploadResponse.Links) == 0 {
- return nil, errors.New("upload response not found")
- } else if len(fileUploadResponse.Links) > 1 {
- fs.Debugf(remote, "Multiple upload responses found, using the first")
- }
- link := fileUploadResponse.Links[0]
- fileSize, err := strconv.ParseInt(link.Size, 10, 64)
- if err != nil {
- return nil, err
- }
- return &Object{
- fs: f,
- remote: remote,
- file: File{
- CDN: 0,
- Checksum: link.Whirlpool,
- ContentType: "",
- Date: time.Now().Format("2006-01-02 15:04:05"),
- Filename: link.Filename,
- Pass: 0,
- Size: fileSize,
- URL: link.Download,
- },
- }, nil
- }
- // PutUnchecked uploads the object
- //
- // This will create a duplicate if we upload a new file without
- // checking to see if there is one already - use Put() for that.
- func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
- return f.putUnchecked(ctx, in, src.Remote(), src.Size(), options...)
- }
- // Mkdir makes the directory (container, bucket)
- //
- // Shouldn't return an error if it already exists
- func (f *Fs) Mkdir(ctx context.Context, dir string) error {
- _, err := f.dirCache.FindDir(ctx, dir, true)
- return err
- }
- // Rmdir removes the directory (container, bucket) if empty
- //
- // Return an error if it doesn't exist or isn't empty
- func (f *Fs) Rmdir(ctx context.Context, dir string) error {
- directoryID, err := f.dirCache.FindDir(ctx, dir, false)
- if err != nil {
- return err
- }
- folderID, err := strconv.Atoi(directoryID)
- if err != nil {
- return err
- }
- _, err = f.removeFolder(ctx, dir, folderID)
- if err != nil {
- return err
- }
- f.dirCache.FlushDir(dir)
- return nil
- }
- // Move src to this remote using server side move operations.
- func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
- srcObj, ok := src.(*Object)
- if !ok {
- fs.Debugf(src, "Can't move - not same remote type")
- return nil, fs.ErrorCantMove
- }
- srcFs := srcObj.fs
- // Find current directory ID
- srcLeaf, srcDirectoryID, err := srcFs.dirCache.FindPath(ctx, srcObj.remote, false)
- if err != nil {
- return nil, err
- }
- // Create temporary object
- dstObj, dstLeaf, dstDirectoryID, err := f.createObject(ctx, remote)
- if err != nil {
- return nil, err
- }
- // If it is in the correct directory, just rename it
- var url string
- if srcDirectoryID == dstDirectoryID {
- // No rename needed
- if srcLeaf == dstLeaf {
- return src, nil
- }
- resp, err := f.renameFile(ctx, srcObj.file.URL, dstLeaf)
- if err != nil {
- return nil, fmt.Errorf("couldn't rename file: %w", err)
- }
- if resp.Status != "OK" {
- return nil, fmt.Errorf("couldn't rename file: %s", resp.Message)
- }
- url = resp.URLs[0].URL
- } else {
- dstFolderID, err := strconv.Atoi(dstDirectoryID)
- if err != nil {
- return nil, err
- }
- rename := dstLeaf
- // No rename needed
- if srcLeaf == dstLeaf {
- rename = ""
- }
- resp, err := f.moveFile(ctx, srcObj.file.URL, dstFolderID, rename)
- if err != nil {
- return nil, fmt.Errorf("couldn't move file: %w", err)
- }
- if resp.Status != "OK" {
- return nil, fmt.Errorf("couldn't move file: %s", resp.Message)
- }
- url = resp.URLs[0]
- }
- file, err := f.readFileInfo(ctx, url)
- if err != nil {
- return nil, errors.New("couldn't read file data")
- }
- dstObj.setMetaData(*file)
- return dstObj, nil
- }
- // DirMove moves src, srcRemote to this remote at dstRemote
- // using server-side move operations.
- //
- // Will only be called if src.Fs().Name() == f.Name()
- //
- // If it isn't possible then return fs.ErrorCantDirMove.
- //
- // If destination exists then return fs.ErrorDirExists.
- //
- // This is complicated by the fact that we can't use moveDir to move
- // to a different directory AND rename at the same time as it can
- // overwrite files in the source directory.
- func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
- srcFs, ok := src.(*Fs)
- if !ok {
- fs.Debugf(srcFs, "Can't move directory - not same remote type")
- return fs.ErrorCantDirMove
- }
- srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
- if err != nil {
- return err
- }
- srcIDnumeric, err := strconv.Atoi(srcID)
- if err != nil {
- return err
- }
- dstDirectoryIDnumeric, err := strconv.Atoi(dstDirectoryID)
- if err != nil {
- return err
- }
- var resp *MoveDirResponse
- resp, err = f.moveDir(ctx, srcIDnumeric, dstLeaf, dstDirectoryIDnumeric)
- if err != nil {
- return fmt.Errorf("couldn't rename leaf: %w", err)
- }
- if resp.Status != "OK" {
- return fmt.Errorf("couldn't rename leaf: %s", resp.Message)
- }
- srcFs.dirCache.FlushDir(srcRemote)
- return nil
- }
- // Copy src to this remote using server side move operations.
- func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
- srcObj, ok := src.(*Object)
- if !ok {
- fs.Debugf(src, "Can't move - not same remote type")
- return nil, fs.ErrorCantMove
- }
- // Create temporary object
- dstObj, leaf, directoryID, err := f.createObject(ctx, remote)
- if err != nil {
- return nil, err
- }
- folderID, err := strconv.Atoi(directoryID)
- if err != nil {
- return nil, err
- }
- resp, err := f.copyFile(ctx, srcObj.file.URL, folderID, leaf)
- if err != nil {
- return nil, fmt.Errorf("couldn't move file: %w", err)
- }
- if resp.Status != "OK" {
- return nil, fmt.Errorf("couldn't move file: %s", resp.Message)
- }
- file, err := f.readFileInfo(ctx, resp.URLs[0].ToURL)
- if err != nil {
- return nil, errors.New("couldn't read file data")
- }
- dstObj.setMetaData(*file)
- return dstObj, nil
- }
- // About gets quota information
- func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
- opts := rest.Opts{
- Method: "POST",
- Path: "/user/info.cgi",
- ContentType: "application/json",
- }
- var accountInfo AccountInfo
- var resp *http.Response
- err = f.pacer.Call(func() (bool, error) {
- resp, err = f.rest.CallJSON(ctx, &opts, nil, &accountInfo)
- return shouldRetry(ctx, resp, err)
- })
- if err != nil {
- return nil, fmt.Errorf("failed to read user info: %w", err)
- }
- // FIXME max upload size would be useful to use in Update
- usage = &fs.Usage{
- Used: fs.NewUsageValue(accountInfo.ColdStorage), // bytes in use
- Total: fs.NewUsageValue(accountInfo.AvailableColdStorage), // bytes total
- Free: fs.NewUsageValue(accountInfo.AvailableColdStorage - accountInfo.ColdStorage), // bytes free
- }
- return usage, nil
- }
- // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
- func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
- o, err := f.NewObject(ctx, remote)
- if err != nil {
- return "", err
- }
- return o.(*Object).file.URL, nil
- }
- // Check the interfaces are satisfied
- var (
- _ fs.Fs = (*Fs)(nil)
- _ fs.Mover = (*Fs)(nil)
- _ fs.DirMover = (*Fs)(nil)
- _ fs.Copier = (*Fs)(nil)
- _ fs.PublicLinker = (*Fs)(nil)
- _ fs.PutUncheckeder = (*Fs)(nil)
- _ dircache.DirCacher = (*Fs)(nil)
- )
|