12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162 |
- // Package jottacloud provides an interface to the Jottacloud storage system.
- package jottacloud
- import (
- "bytes"
- "context"
- "crypto/md5"
- "encoding/base64"
- "encoding/hex"
- "encoding/json"
- "encoding/xml"
- "errors"
- "fmt"
- "io"
- "math/rand"
- "net/http"
- "net/url"
- "os"
- "path"
- "strconv"
- "strings"
- "time"
- "github.com/rclone/rclone/backend/jottacloud/api"
- "github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/accounting"
- "github.com/rclone/rclone/fs/config"
- "github.com/rclone/rclone/fs/config/configmap"
- "github.com/rclone/rclone/fs/config/configstruct"
- "github.com/rclone/rclone/fs/config/obscure"
- "github.com/rclone/rclone/fs/fserrors"
- "github.com/rclone/rclone/fs/fshttp"
- "github.com/rclone/rclone/fs/hash"
- "github.com/rclone/rclone/fs/walk"
- "github.com/rclone/rclone/lib/encoder"
- "github.com/rclone/rclone/lib/oauthutil"
- "github.com/rclone/rclone/lib/pacer"
- "github.com/rclone/rclone/lib/rest"
- "golang.org/x/oauth2"
- )
- // Globals
- const (
- minSleep = 10 * time.Millisecond
- maxSleep = 2 * time.Second
- decayConstant = 2 // bigger for slower decay, exponential
- defaultDevice = "Jotta"
- defaultMountpoint = "Archive"
- jfsURL = "https://jfs.jottacloud.com/jfs/"
- apiURL = "https://api.jottacloud.com/"
- wwwURL = "https://www.jottacloud.com/"
- cachePrefix = "rclone-jcmd5-"
- configDevice = "device"
- configMountpoint = "mountpoint"
- configTokenURL = "tokenURL"
- configClientID = "client_id"
- configClientSecret = "client_secret"
- configUsername = "username"
- configVersion = 1
- defaultTokenURL = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
- defaultClientID = "jottacli"
- legacyTokenURL = "https://api.jottacloud.com/auth/v1/token"
- legacyRegisterURL = "https://api.jottacloud.com/auth/v1/register"
- legacyClientID = "nibfk8biu12ju7hpqomr8b1e40"
- legacyEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
- legacyConfigVersion = 0
- teliaseCloudTokenURL = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/token"
- teliaseCloudAuthURL = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/auth"
- teliaseCloudClientID = "desktop"
- telianoCloudTokenURL = "https://sky-auth.telia.no/auth/realms/get/protocol/openid-connect/token"
- telianoCloudAuthURL = "https://sky-auth.telia.no/auth/realms/get/protocol/openid-connect/auth"
- telianoCloudClientID = "desktop"
- tele2CloudTokenURL = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/token"
- tele2CloudAuthURL = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/auth"
- tele2CloudClientID = "desktop"
- onlimeCloudTokenURL = "https://cloud-auth.onlime.dk/auth/realms/onlime_wl/protocol/openid-connect/token"
- onlimeCloudAuthURL = "https://cloud-auth.onlime.dk/auth/realms/onlime_wl/protocol/openid-connect/auth"
- onlimeCloudClientID = "desktop"
- )
- // Register with Fs
- func init() {
- // needs to be done early so we can use oauth during config
- fs.Register(&fs.RegInfo{
- Name: "jottacloud",
- Description: "Jottacloud",
- NewFs: NewFs,
- Config: Config,
- MetadataInfo: &fs.MetadataInfo{
- Help: `Jottacloud has limited support for metadata, currently an extended set of timestamps.`,
- System: map[string]fs.MetadataHelp{
- "btime": {
- Help: "Time of file birth (creation), read from rclone metadata",
- Type: "RFC 3339",
- Example: "2006-01-02T15:04:05.999999999Z07:00",
- },
- "mtime": {
- Help: "Time of last modification, read from rclone metadata",
- Type: "RFC 3339",
- Example: "2006-01-02T15:04:05.999999999Z07:00",
- },
- "utime": {
- Help: "Time of last upload, when current revision was created, generated by backend",
- Type: "RFC 3339",
- Example: "2006-01-02T15:04:05.999999999Z07:00",
- ReadOnly: true,
- },
- "content-type": {
- Help: "MIME type, also known as media type",
- Type: "string",
- Example: "text/plain",
- ReadOnly: true,
- },
- },
- },
- Options: append(oauthutil.SharedOptions, []fs.Option{{
- Name: "md5_memory_limit",
- Help: "Files bigger than this will be cached on disk to calculate the MD5 if required.",
- Default: fs.SizeSuffix(10 * 1024 * 1024),
- Advanced: true,
- }, {
- Name: "trashed_only",
- Help: "Only show files that are in the trash.\n\nThis will show trashed files in their original directory structure.",
- Default: false,
- Advanced: true,
- }, {
- Name: "hard_delete",
- Help: "Delete files permanently rather than putting them into the trash.",
- Default: false,
- Advanced: true,
- }, {
- Name: "upload_resume_limit",
- Help: "Files bigger than this can be resumed if the upload fail's.",
- Default: fs.SizeSuffix(10 * 1024 * 1024),
- Advanced: true,
- }, {
- Name: "no_versions",
- Help: "Avoid server side versioning by deleting files and recreating files instead of overwriting them.",
- Default: false,
- Advanced: true,
- }, {
- Name: config.ConfigEncoding,
- Help: config.ConfigEncodingHelp,
- Advanced: true,
- // Encode invalid UTF-8 bytes as xml doesn't handle them properly.
- //
- // Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|'
- Default: (encoder.Display |
- encoder.EncodeWin | // :?"*<>|
- encoder.EncodeInvalidUtf8),
- }}...),
- })
- }
- // Config runs the backend configuration protocol
- func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
- switch config.State {
- case "":
- return fs.ConfigChooseExclusiveFixed("auth_type_done", "config_type", `Select authentication type.`, []fs.OptionExample{{
- Value: "standard",
- Help: "Standard authentication.\nUse this if you're a normal Jottacloud user.",
- }, {
- Value: "legacy",
- Help: "Legacy authentication.\nThis is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.",
- }, {
- Value: "telia_se",
- Help: "Telia Cloud authentication.\nUse this if you are using Telia Cloud (Sweden).",
- }, {
- Value: "telia_no",
- Help: "Telia Sky authentication.\nUse this if you are using Telia Sky (Norway).",
- }, {
- Value: "tele2",
- Help: "Tele2 Cloud authentication.\nUse this if you are using Tele2 Cloud.",
- }, {
- Value: "onlime",
- Help: "Onlime Cloud authentication.\nUse this if you are using Onlime Cloud.",
- }})
- case "auth_type_done":
- // Jump to next state according to config chosen
- return fs.ConfigGoto(config.Result)
- case "standard": // configure a jottacloud backend using the modern JottaCli token based authentication
- m.Set("configVersion", fmt.Sprint(configVersion))
- return fs.ConfigInput("standard_token", "config_login_token", "Personal login token.\nGenerate here: https://www.jottacloud.com/web/secure")
- case "standard_token":
- loginToken := config.Result
- m.Set(configClientID, defaultClientID)
- m.Set(configClientSecret, "")
- srv := rest.NewClient(fshttp.NewClient(ctx))
- token, tokenEndpoint, err := doTokenAuth(ctx, srv, loginToken)
- if err != nil {
- return nil, fmt.Errorf("failed to get oauth token: %w", err)
- }
- m.Set(configTokenURL, tokenEndpoint)
- err = oauthutil.PutToken(name, m, &token, true)
- if err != nil {
- return nil, fmt.Errorf("error while saving token: %w", err)
- }
- return fs.ConfigGoto("choose_device")
- case "legacy": // configure a jottacloud backend using legacy authentication
- m.Set("configVersion", fmt.Sprint(legacyConfigVersion))
- return fs.ConfigConfirm("legacy_api", false, "config_machine_specific", `Do you want to create a machine specific API key?
- Rclone has it's own Jottacloud API KEY which works fine as long as one
- only uses rclone on a single machine. When you want to use rclone with
- this account on more than one machine it's recommended to create a
- machine specific API key. These keys can NOT be shared between
- machines.`)
- case "legacy_api":
- srv := rest.NewClient(fshttp.NewClient(ctx))
- if config.Result == "true" {
- deviceRegistration, err := registerDevice(ctx, srv)
- if err != nil {
- return nil, fmt.Errorf("failed to register device: %w", err)
- }
- m.Set(configClientID, deviceRegistration.ClientID)
- m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
- fs.Debugf(nil, "Got clientID %q and clientSecret %q", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
- }
- return fs.ConfigInput("legacy_username", "config_username", "Username (e-mail address)")
- case "legacy_username":
- m.Set(configUsername, config.Result)
- return fs.ConfigPassword("legacy_password", "config_password", "Password (only used in setup, will not be stored)")
- case "legacy_password":
- m.Set("password", config.Result)
- m.Set("auth_code", "")
- return fs.ConfigGoto("legacy_do_auth")
- case "legacy_auth_code":
- authCode := strings.ReplaceAll(config.Result, "-", "") // remove any "-" contained in the code so we have a 6 digit number
- m.Set("auth_code", authCode)
- return fs.ConfigGoto("legacy_do_auth")
- case "legacy_do_auth":
- username, _ := m.Get(configUsername)
- password, _ := m.Get("password")
- password = obscure.MustReveal(password)
- authCode, _ := m.Get("auth_code")
- srv := rest.NewClient(fshttp.NewClient(ctx))
- clientID, ok := m.Get(configClientID)
- if !ok {
- clientID = legacyClientID
- }
- clientSecret, ok := m.Get(configClientSecret)
- if !ok {
- clientSecret = legacyEncryptedClientSecret
- }
- oauthConfig := &oauth2.Config{
- Endpoint: oauth2.Endpoint{
- AuthURL: legacyTokenURL,
- },
- ClientID: clientID,
- ClientSecret: obscure.MustReveal(clientSecret),
- }
- token, err := doLegacyAuth(ctx, srv, oauthConfig, username, password, authCode)
- if err == errAuthCodeRequired {
- return fs.ConfigInput("legacy_auth_code", "config_auth_code", "Verification Code\nThis account uses 2 factor authentication you will receive a verification code via SMS.")
- }
- m.Set("password", "")
- m.Set("auth_code", "")
- if err != nil {
- return nil, fmt.Errorf("failed to get oauth token: %w", err)
- }
- err = oauthutil.PutToken(name, m, &token, true)
- if err != nil {
- return nil, fmt.Errorf("error while saving token: %w", err)
- }
- return fs.ConfigGoto("choose_device")
- case "telia_se": // telia_se cloud config
- m.Set("configVersion", fmt.Sprint(configVersion))
- m.Set(configClientID, teliaseCloudClientID)
- m.Set(configTokenURL, teliaseCloudTokenURL)
- return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
- OAuth2Config: &oauth2.Config{
- Endpoint: oauth2.Endpoint{
- AuthURL: teliaseCloudAuthURL,
- TokenURL: teliaseCloudTokenURL,
- },
- ClientID: teliaseCloudClientID,
- Scopes: []string{"openid", "jotta-default", "offline_access"},
- RedirectURL: oauthutil.RedirectLocalhostURL,
- },
- })
- case "telia_no": // telia_no cloud config
- m.Set("configVersion", fmt.Sprint(configVersion))
- m.Set(configClientID, telianoCloudClientID)
- m.Set(configTokenURL, telianoCloudTokenURL)
- return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
- OAuth2Config: &oauth2.Config{
- Endpoint: oauth2.Endpoint{
- AuthURL: telianoCloudAuthURL,
- TokenURL: telianoCloudTokenURL,
- },
- ClientID: telianoCloudClientID,
- Scopes: []string{"openid", "jotta-default", "offline_access"},
- RedirectURL: oauthutil.RedirectLocalhostURL,
- },
- })
- case "tele2": // tele2 cloud config
- m.Set("configVersion", fmt.Sprint(configVersion))
- m.Set(configClientID, tele2CloudClientID)
- m.Set(configTokenURL, tele2CloudTokenURL)
- return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
- OAuth2Config: &oauth2.Config{
- Endpoint: oauth2.Endpoint{
- AuthURL: tele2CloudAuthURL,
- TokenURL: tele2CloudTokenURL,
- },
- ClientID: tele2CloudClientID,
- Scopes: []string{"openid", "jotta-default", "offline_access"},
- RedirectURL: oauthutil.RedirectLocalhostURL,
- },
- })
- case "onlime": // onlime cloud config
- m.Set("configVersion", fmt.Sprint(configVersion))
- m.Set(configClientID, onlimeCloudClientID)
- m.Set(configTokenURL, onlimeCloudTokenURL)
- return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
- OAuth2Config: &oauth2.Config{
- Endpoint: oauth2.Endpoint{
- AuthURL: onlimeCloudAuthURL,
- TokenURL: onlimeCloudTokenURL,
- },
- ClientID: onlimeCloudClientID,
- Scopes: []string{"openid", "jotta-default", "offline_access"},
- RedirectURL: oauthutil.RedirectLocalhostURL,
- },
- })
- case "choose_device":
- return fs.ConfigConfirm("choose_device_query", false, "config_non_standard", `Use a non-standard device/mountpoint?
- Choosing no, the default, will let you access the storage used for the archive
- section of the official Jottacloud client. If you instead want to access the
- sync or the backup section, for example, you must choose yes.`)
- case "choose_device_query":
- if config.Result != "true" {
- m.Set(configDevice, "")
- m.Set(configMountpoint, "")
- return fs.ConfigGoto("end")
- }
- oAuthClient, _, err := getOAuthClient(ctx, name, m)
- if err != nil {
- return nil, err
- }
- jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL)
- apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
- cust, err := getCustomerInfo(ctx, apiSrv)
- if err != nil {
- return nil, err
- }
- acc, err := getDriveInfo(ctx, jfsSrv, cust.Username)
- if err != nil {
- return nil, err
- }
- deviceNames := make([]string, len(acc.Devices))
- for i, dev := range acc.Devices {
- if i > 0 && dev.Name == defaultDevice {
- // Insert the special Jotta device as first entry, making it the default choice.
- copy(deviceNames[1:i+1], deviceNames[0:i])
- deviceNames[0] = dev.Name
- } else {
- deviceNames[i] = dev.Name
- }
- }
- help := fmt.Sprintf(`The device to use. In standard setup the built-in %s device is used,
- which contains predefined mountpoints for archive, sync etc. All other devices
- are treated as backup devices by the official Jottacloud client. You may create
- a new by entering a unique name.`, defaultDevice)
- return fs.ConfigChoose("choose_device_result", "config_device", help, len(deviceNames), func(i int) (string, string) {
- return deviceNames[i], ""
- })
- case "choose_device_result":
- device := config.Result
- oAuthClient, _, err := getOAuthClient(ctx, name, m)
- if err != nil {
- return nil, err
- }
- jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL)
- apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
- cust, err := getCustomerInfo(ctx, apiSrv)
- if err != nil {
- return nil, err
- }
- acc, err := getDriveInfo(ctx, jfsSrv, cust.Username)
- if err != nil {
- return nil, err
- }
- isNew := true
- for _, dev := range acc.Devices {
- if strings.EqualFold(dev.Name, device) { // If device name exists with different casing we prefer the existing (not sure if and how the api handles the opposite)
- device = dev.Name // Prefer same casing as existing, e.g. if user entered "jotta" we use the standard casing "Jotta" instead
- isNew = false
- break
- }
- }
- var dev *api.JottaDevice
- if isNew {
- fs.Debugf(nil, "Creating new device: %s", device)
- dev, err = createDevice(ctx, jfsSrv, path.Join(cust.Username, device))
- if err != nil {
- return nil, err
- }
- }
- m.Set(configDevice, device)
- if !isNew {
- dev, err = getDeviceInfo(ctx, jfsSrv, path.Join(cust.Username, device))
- if err != nil {
- return nil, err
- }
- }
- var help string
- if device == defaultDevice {
- // With built-in Jotta device the mountpoint choice is exclusive,
- // we do not want to risk any problems by creating new mountpoints on it.
- help = fmt.Sprintf(`The mountpoint to use on the built-in device %s.
- The standard setup is to use the %s mountpoint. Most other mountpoints
- have very limited support in rclone and should generally be avoided.`, defaultDevice, defaultMountpoint)
- return fs.ConfigChooseExclusive("choose_device_mountpoint", "config_mountpoint", help, len(dev.MountPoints), func(i int) (string, string) {
- return dev.MountPoints[i].Name, ""
- })
- }
- help = fmt.Sprintf(`The mountpoint to use on the non-standard device %s.
- You may create a new by entering a unique name.`, device)
- return fs.ConfigChoose("choose_device_mountpoint", "config_mountpoint", help, len(dev.MountPoints), func(i int) (string, string) {
- return dev.MountPoints[i].Name, ""
- })
- case "choose_device_mountpoint":
- mountpoint := config.Result
- oAuthClient, _, err := getOAuthClient(ctx, name, m)
- if err != nil {
- return nil, err
- }
- jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL)
- apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
- cust, err := getCustomerInfo(ctx, apiSrv)
- if err != nil {
- return nil, err
- }
- device, _ := m.Get(configDevice)
- dev, err := getDeviceInfo(ctx, jfsSrv, path.Join(cust.Username, device))
- if err != nil {
- return nil, err
- }
- isNew := true
- for _, mnt := range dev.MountPoints {
- if strings.EqualFold(mnt.Name, mountpoint) {
- mountpoint = mnt.Name
- isNew = false
- break
- }
- }
- if isNew {
- if device == defaultDevice {
- return nil, fmt.Errorf("custom mountpoints not supported on built-in %s device: %w", defaultDevice, err)
- }
- fs.Debugf(nil, "Creating new mountpoint: %s", mountpoint)
- _, err := createMountPoint(ctx, jfsSrv, path.Join(cust.Username, device, mountpoint))
- if err != nil {
- return nil, err
- }
- }
- m.Set(configMountpoint, mountpoint)
- return fs.ConfigGoto("end")
- case "end":
- // All the config flows end up here in case we need to carry on with something
- return nil, nil
- }
- return nil, fmt.Errorf("unknown state %q", config.State)
- }
- // Options defines the configuration for this backend
- type Options struct {
- Device string `config:"device"`
- Mountpoint string `config:"mountpoint"`
- MD5MemoryThreshold fs.SizeSuffix `config:"md5_memory_limit"`
- TrashedOnly bool `config:"trashed_only"`
- HardDelete bool `config:"hard_delete"`
- NoVersions bool `config:"no_versions"`
- UploadThreshold fs.SizeSuffix `config:"upload_resume_limit"`
- Enc encoder.MultiEncoder `config:"encoding"`
- }
- // Fs represents a remote jottacloud
- type Fs struct {
- name string
- root string
- user string
- opt Options
- features *fs.Features
- fileEndpoint string
- allocateEndpoint string
- jfsSrv *rest.Client
- apiSrv *rest.Client
- pacer *fs.Pacer
- tokenRenewer *oauthutil.Renew // renew the token on expiry
- }
- // Object describes a jottacloud object
- //
- // Will definitely have info but maybe not meta
- type Object struct {
- fs *Fs
- remote string
- hasMetaData bool
- size int64
- createTime time.Time
- modTime time.Time
- updateTime time.Time
- md5 string
- mimeType string
- }
- // ------------------------------------------------------------
- // Name of the remote (as passed into NewFs)
- func (f *Fs) Name() string {
- return f.name
- }
- // Root of the remote (as passed into NewFs)
- func (f *Fs) Root() string {
- return f.root
- }
- // String converts this Fs to a string
- func (f *Fs) String() string {
- return fmt.Sprintf("jottacloud root '%s'", f.root)
- }
- // Features returns the optional features of this Fs
- func (f *Fs) Features() *fs.Features {
- return f.features
- }
- // joinPath joins two path/url elements
- //
- // Does not perform clean on the result like path.Join does,
- // which breaks urls by changing prefix "https://" into "https:/".
- func joinPath(base string, rel string) string {
- if rel == "" {
- return base
- }
- if strings.HasSuffix(base, "/") {
- return base + strings.TrimPrefix(rel, "/")
- }
- if strings.HasPrefix(rel, "/") {
- return strings.TrimSuffix(base, "/") + rel
- }
- return base + "/" + rel
- }
- // retryErrorCodes is a slice of error codes that we will retry
- var retryErrorCodes = []int{
- 429, // Too Many Requests.
- 500, // Internal Server Error
- 502, // Bad Gateway
- 503, // Service Unavailable
- 504, // Gateway Timeout
- 509, // Bandwidth Limit Exceeded
- }
- // shouldRetry returns a boolean as to whether this resp and err
- // deserve to be retried. It returns the err as a convenience
- func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
- if fserrors.ContextError(ctx, &err) {
- return false, err
- }
- return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
- }
- // registerDevice register a new device for use with the jottacloud API
- func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegistrationResponse, err error) {
- // random generator to generate random device names
- seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
- randonDeviceNamePartLength := 21
- randomDeviceNamePart := make([]byte, randonDeviceNamePartLength)
- charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
- for i := range randomDeviceNamePart {
- randomDeviceNamePart[i] = charset[seededRand.Intn(len(charset))]
- }
- randomDeviceName := "rclone-" + string(randomDeviceNamePart)
- fs.Debugf(nil, "Trying to register device '%s'", randomDeviceName)
- values := url.Values{}
- values.Set("device_id", randomDeviceName)
- opts := rest.Opts{
- Method: "POST",
- RootURL: legacyRegisterURL,
- ContentType: "application/x-www-form-urlencoded",
- ExtraHeaders: map[string]string{"Authorization": "Bearer c2xrZmpoYWRsZmFramhkc2xma2phaHNkbGZramhhc2xkZmtqaGFzZGxrZmpobGtq"},
- Parameters: values,
- }
- var deviceRegistration *api.DeviceRegistrationResponse
- _, err = srv.CallJSON(ctx, &opts, nil, &deviceRegistration)
- return deviceRegistration, err
- }
- var errAuthCodeRequired = errors.New("auth code required")
- // doLegacyAuth runs the actual token request for V1 authentication
- //
- // Call this first with blank authCode. If errAuthCodeRequired is
- // returned then call it again with an authCode
- func doLegacyAuth(ctx context.Context, srv *rest.Client, oauthConfig *oauth2.Config, username, password, authCode string) (token oauth2.Token, err error) {
- // prepare out token request with username and password
- values := url.Values{}
- values.Set("grant_type", "PASSWORD")
- values.Set("password", password)
- values.Set("username", username)
- values.Set("client_id", oauthConfig.ClientID)
- values.Set("client_secret", oauthConfig.ClientSecret)
- opts := rest.Opts{
- Method: "POST",
- RootURL: oauthConfig.Endpoint.AuthURL,
- ContentType: "application/x-www-form-urlencoded",
- Parameters: values,
- }
- if authCode != "" {
- opts.ExtraHeaders = make(map[string]string)
- opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
- }
- // do the first request
- var jsonToken api.TokenJSON
- resp, err := srv.CallJSON(ctx, &opts, nil, &jsonToken)
- if err != nil && authCode == "" {
- // if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
- if resp != nil {
- if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
- return token, errAuthCodeRequired
- }
- }
- }
- token.AccessToken = jsonToken.AccessToken
- token.RefreshToken = jsonToken.RefreshToken
- token.TokenType = jsonToken.TokenType
- token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
- return token, err
- }
- // doTokenAuth runs the actual token request for V2 authentication
- func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 string) (token oauth2.Token, tokenEndpoint string, err error) {
- loginTokenBytes, err := base64.RawURLEncoding.DecodeString(loginTokenBase64)
- if err != nil {
- return token, "", err
- }
- // decode login token
- var loginToken api.LoginToken
- decoder := json.NewDecoder(bytes.NewReader(loginTokenBytes))
- err = decoder.Decode(&loginToken)
- if err != nil {
- return token, "", err
- }
- // retrieve endpoint urls
- opts := rest.Opts{
- Method: "GET",
- RootURL: loginToken.WellKnownLink,
- }
- var wellKnown api.WellKnown
- _, err = apiSrv.CallJSON(ctx, &opts, nil, &wellKnown)
- if err != nil {
- return token, "", err
- }
- // prepare out token request with username and password
- values := url.Values{}
- values.Set("client_id", defaultClientID)
- values.Set("grant_type", "password")
- values.Set("password", loginToken.AuthToken)
- values.Set("scope", "openid offline_access")
- values.Set("username", loginToken.Username)
- values.Encode()
- opts = rest.Opts{
- Method: "POST",
- RootURL: wellKnown.TokenEndpoint,
- ContentType: "application/x-www-form-urlencoded",
- Body: strings.NewReader(values.Encode()),
- }
- // do the first request
- var jsonToken api.TokenJSON
- _, err = apiSrv.CallJSON(ctx, &opts, nil, &jsonToken)
- if err != nil {
- return token, "", err
- }
- token.AccessToken = jsonToken.AccessToken
- token.RefreshToken = jsonToken.RefreshToken
- token.TokenType = jsonToken.TokenType
- token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
- return token, wellKnown.TokenEndpoint, err
- }
- // getCustomerInfo queries general information about the account
- func getCustomerInfo(ctx context.Context, apiSrv *rest.Client) (info *api.CustomerInfo, err error) {
- opts := rest.Opts{
- Method: "GET",
- Path: "account/v1/customer",
- }
- _, err = apiSrv.CallJSON(ctx, &opts, nil, &info)
- if err != nil {
- return nil, fmt.Errorf("couldn't get customer info: %w", err)
- }
- return info, nil
- }
- // getDriveInfo queries general information about the account and the available devices and mountpoints.
- func getDriveInfo(ctx context.Context, srv *rest.Client, username string) (info *api.DriveInfo, err error) {
- opts := rest.Opts{
- Method: "GET",
- Path: username,
- }
- _, err = srv.CallXML(ctx, &opts, nil, &info)
- if err != nil {
- return nil, fmt.Errorf("couldn't get drive info: %w", err)
- }
- return info, nil
- }
- // getDeviceInfo queries Information about a jottacloud device
- func getDeviceInfo(ctx context.Context, srv *rest.Client, path string) (info *api.JottaDevice, err error) {
- opts := rest.Opts{
- Method: "GET",
- Path: urlPathEscape(path),
- }
- _, err = srv.CallXML(ctx, &opts, nil, &info)
- if err != nil {
- return nil, fmt.Errorf("couldn't get device info: %w", err)
- }
- return info, nil
- }
- // createDevice makes a device
- func createDevice(ctx context.Context, srv *rest.Client, path string) (info *api.JottaDevice, err error) {
- opts := rest.Opts{
- Method: "POST",
- Path: urlPathEscape(path),
- Parameters: url.Values{},
- }
- opts.Parameters.Set("type", "WORKSTATION")
- _, err = srv.CallXML(ctx, &opts, nil, &info)
- if err != nil {
- return nil, fmt.Errorf("couldn't create device: %w", err)
- }
- return info, nil
- }
- // createMountPoint makes a mount point
- func createMountPoint(ctx context.Context, srv *rest.Client, path string) (info *api.JottaMountPoint, err error) {
- opts := rest.Opts{
- Method: "POST",
- Path: urlPathEscape(path),
- }
- _, err = srv.CallXML(ctx, &opts, nil, &info)
- if err != nil {
- return nil, fmt.Errorf("couldn't create mountpoint: %w", err)
- }
- return info, nil
- }
- // setEndpoints generates the API endpoints
- func (f *Fs) setEndpoints() {
- if f.opt.Device == "" {
- f.opt.Device = defaultDevice
- }
- if f.opt.Mountpoint == "" {
- f.opt.Mountpoint = defaultMountpoint
- }
- f.fileEndpoint = path.Join(f.user, f.opt.Device, f.opt.Mountpoint)
- f.allocateEndpoint = path.Join("/jfs", f.opt.Device, f.opt.Mountpoint)
- }
- // readMetaDataForPath reads the metadata from the path
- func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.JottaFile, err error) {
- opts := rest.Opts{
- Method: "GET",
- Path: f.filePath(path),
- }
- var result api.JottaFile
- var resp *http.Response
- err = f.pacer.Call(func() (bool, error) {
- resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result)
- return shouldRetry(ctx, resp, err)
- })
- if apiErr, ok := err.(*api.Error); ok {
- // does not exist
- if apiErr.StatusCode == http.StatusNotFound {
- return nil, fs.ErrorObjectNotFound
- }
- }
- if err != nil {
- return nil, fmt.Errorf("read metadata failed: %w", err)
- }
- if result.XMLName.Local == "folder" {
- return nil, fs.ErrorIsDir
- } else if result.XMLName.Local != "file" {
- return nil, fs.ErrorNotAFile
- }
- return &result, nil
- }
- // errorHandler parses a non 2xx error response into an error
- func errorHandler(resp *http.Response) error {
- // Decode error response
- errResponse := new(api.Error)
- err := rest.DecodeXML(resp, &errResponse)
- if err != nil {
- fs.Debugf(nil, "Couldn't decode error response: %v", err)
- }
- if errResponse.Message == "" {
- errResponse.Message = resp.Status
- }
- if errResponse.StatusCode == 0 {
- errResponse.StatusCode = resp.StatusCode
- }
- return errResponse
- }
- // Jottacloud wants '+' to be URL encoded even though the RFC states it's not reserved
- func urlPathEscape(in string) string {
- return strings.ReplaceAll(rest.URLPathEscape(in), "+", "%2B")
- }
- // filePathRaw returns an unescaped file path (f.root, file)
- // Optionally made absolute by prefixing with "/", typically required when used
- // as request parameter instead of the path (which is relative to some root url).
- func (f *Fs) filePathRaw(file string, absolute bool) string {
- prefix := ""
- if absolute {
- prefix = "/"
- }
- return path.Join(prefix, f.fileEndpoint, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
- }
- // filePath returns an escaped file path (f.root, file)
- func (f *Fs) filePath(file string) string {
- return urlPathEscape(f.filePathRaw(file, false))
- }
- // allocatePathRaw returns an unescaped allocate file path (f.root, file)
- // Optionally made absolute by prefixing with "/", typically required when used
- // as request parameter instead of the path (which is relative to some root url).
- func (f *Fs) allocatePathRaw(file string, absolute bool) string {
- prefix := ""
- if absolute {
- prefix = "/"
- }
- return path.Join(prefix, f.allocateEndpoint, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
- }
- // Jottacloud requires the grant_type 'refresh_token' string
- // to be uppercase and throws a 400 Bad Request if we use the
- // lower case used by the oauth2 module
- //
- // This filter catches all refresh requests, reads the body,
- // changes the case and then sends it on
- func grantTypeFilter(req *http.Request) {
- if legacyTokenURL == req.URL.String() {
- // read the entire body
- refreshBody, err := io.ReadAll(req.Body)
- if err != nil {
- return
- }
- _ = req.Body.Close()
- // make the refresh token upper case
- refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
- // set the new ReadCloser (with a dummy Close())
- req.Body = io.NopCloser(bytes.NewReader(refreshBody))
- }
- }
- func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuthClient *http.Client, ts *oauthutil.TokenSource, err error) {
- // Check config version
- var ver int
- version, ok := m.Get("configVersion")
- if ok {
- ver, err = strconv.Atoi(version)
- if err != nil {
- return nil, nil, errors.New("failed to parse config version")
- }
- ok = (ver == configVersion) || (ver == legacyConfigVersion)
- }
- if !ok {
- return nil, nil, errors.New("outdated config - please reconfigure this backend")
- }
- baseClient := fshttp.NewClient(ctx)
- oauthConfig := &oauth2.Config{
- Endpoint: oauth2.Endpoint{
- AuthURL: defaultTokenURL,
- TokenURL: defaultTokenURL,
- },
- }
- if ver == configVersion {
- oauthConfig.ClientID = defaultClientID
- // if custom endpoints are set use them else stick with defaults
- if tokenURL, ok := m.Get(configTokenURL); ok {
- oauthConfig.Endpoint.TokenURL = tokenURL
- // jottacloud is weird. we need to use the tokenURL as authURL
- oauthConfig.Endpoint.AuthURL = tokenURL
- }
- } else if ver == legacyConfigVersion {
- clientID, ok := m.Get(configClientID)
- if !ok {
- clientID = legacyClientID
- }
- clientSecret, ok := m.Get(configClientSecret)
- if !ok {
- clientSecret = legacyEncryptedClientSecret
- }
- oauthConfig.ClientID = clientID
- oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
- oauthConfig.Endpoint.TokenURL = legacyTokenURL
- oauthConfig.Endpoint.AuthURL = legacyTokenURL
- // add the request filter to fix token refresh
- if do, ok := baseClient.Transport.(interface {
- SetRequestFilter(f func(req *http.Request))
- }); ok {
- do.SetRequestFilter(grantTypeFilter)
- } else {
- fs.Debugf(name+":", "Couldn't add request filter - uploads will fail")
- }
- }
- // Create OAuth Client
- oAuthClient, ts, err = oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
- if err != nil {
- return nil, nil, fmt.Errorf("failed to configure Jottacloud oauth client: %w", err)
- }
- return oAuthClient, ts, nil
- }
- // NewFs constructs an Fs from the path, container:path
- func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
- // Parse config into Options struct
- opt := new(Options)
- err := configstruct.Set(m, opt)
- if err != nil {
- return nil, err
- }
- oAuthClient, ts, err := getOAuthClient(ctx, name, m)
- if err != nil {
- return nil, err
- }
- rootIsDir := strings.HasSuffix(root, "/")
- root = strings.Trim(root, "/")
- f := &Fs{
- name: name,
- root: root,
- opt: *opt,
- jfsSrv: rest.NewClient(oAuthClient).SetRoot(jfsURL),
- apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL),
- pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
- }
- f.features = (&fs.Features{
- CaseInsensitive: true,
- CanHaveEmptyDirectories: true,
- ReadMimeType: true,
- WriteMimeType: false,
- ReadMetadata: true,
- WriteMetadata: true,
- UserMetadata: false,
- }).Fill(ctx, f)
- f.jfsSrv.SetErrorHandler(errorHandler)
- if opt.TrashedOnly { // we cannot support showing Trashed Files when using ListR right now
- f.features.ListR = nil
- }
- // Renew the token in the background
- f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
- _, err := f.readMetaDataForPath(ctx, "")
- if err == fs.ErrorNotAFile || err == fs.ErrorIsDir {
- err = nil
- }
- return err
- })
- cust, err := getCustomerInfo(ctx, f.apiSrv)
- if err != nil {
- return nil, err
- }
- f.user = cust.Username
- f.setEndpoints()
- if root != "" && !rootIsDir {
- // Check to see if the root actually an existing file
- remote := path.Base(root)
- f.root = path.Dir(root)
- if f.root == "." {
- f.root = ""
- }
- _, err := f.NewObject(context.TODO(), remote)
- if err != nil {
- if errors.Is(err, fs.ErrorObjectNotFound) || errors.Is(err, fs.ErrorNotAFile) || errors.Is(err, fs.ErrorIsDir) {
- // File doesn't exist so return old f
- f.root = root
- return f, nil
- }
- return nil, err
- }
- // return an error with an fs which points to the parent
- return f, fs.ErrorIsFile
- }
- return f, nil
- }
- // Return an Object from a path
- //
- // If it can't be found it returns the error fs.ErrorObjectNotFound.
- func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.JottaFile) (fs.Object, error) {
- o := &Object{
- fs: f,
- remote: remote,
- }
- var err error
- if info != nil {
- if !f.validFile(info) {
- return nil, fs.ErrorObjectNotFound
- }
- err = o.setMetaData(info) // sets the info
- } else {
- err = o.readMetaData(ctx, false) // reads info and meta, returning an error
- }
- if err != nil {
- return nil, err
- }
- return o, nil
- }
- // NewObject finds the Object at remote. If it can't be found
- // it returns the error fs.ErrorObjectNotFound.
- func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
- return f.newObjectWithInfo(ctx, remote, nil)
- }
- // CreateDir makes a directory
- func (f *Fs) CreateDir(ctx context.Context, path string) (jf *api.JottaFolder, err error) {
- // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf)
- var resp *http.Response
- opts := rest.Opts{
- Method: "POST",
- Path: f.filePath(path),
- Parameters: url.Values{},
- }
- opts.Parameters.Set("mkDir", "true")
- err = f.pacer.Call(func() (bool, error) {
- resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &jf)
- return shouldRetry(ctx, resp, err)
- })
- if err != nil {
- //fmt.Printf("...Error %v\n", err)
- return nil, err
- }
- // fmt.Printf("...Id %q\n", *info.Id)
- return jf, nil
- }
- // List the objects and directories in dir into entries. The
- // entries can be returned in any order but should be for a
- // complete directory.
- //
- // dir should be "" to list the root, and should not have
- // trailing slashes.
- //
- // This should return ErrDirNotFound if the directory isn't
- // found.
- func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
- opts := rest.Opts{
- Method: "GET",
- Path: f.filePath(dir),
- }
- var resp *http.Response
- var result api.JottaFolder
- err = f.pacer.Call(func() (bool, error) {
- resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result)
- return shouldRetry(ctx, resp, err)
- })
- if err != nil {
- if apiErr, ok := err.(*api.Error); ok {
- // does not exist
- if apiErr.StatusCode == http.StatusNotFound {
- return nil, fs.ErrorDirNotFound
- }
- }
- return nil, fmt.Errorf("couldn't list files: %w", err)
- }
- if !f.validFolder(&result) {
- return nil, fs.ErrorDirNotFound
- }
- for i := range result.Folders {
- item := &result.Folders[i]
- if f.validFolder(item) {
- remote := path.Join(dir, f.opt.Enc.ToStandardName(item.Name))
- d := fs.NewDir(remote, time.Time(item.ModifiedAt))
- entries = append(entries, d)
- }
- }
- for i := range result.Files {
- item := &result.Files[i]
- if f.validFile(item) {
- remote := path.Join(dir, f.opt.Enc.ToStandardName(item.Name))
- if o, err := f.newObjectWithInfo(ctx, remote, item); err == nil {
- entries = append(entries, o)
- }
- }
- }
- return entries, nil
- }
- func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback func(fs.DirEntry) error) error {
- type stats struct {
- Folders int `xml:"folders"`
- Files int `xml:"files"`
- }
- var expected, actual stats
- type xmlFile struct {
- Path string `xml:"path"`
- Name string `xml:"filename"`
- Checksum string `xml:"md5"`
- Size int64 `xml:"size"`
- Modified api.Rfc3339Time `xml:"modified"` // Note: Liststream response includes 3 decimal milliseconds, but we ignore them since there is second precision everywhere else
- Created api.Rfc3339Time `xml:"created"`
- }
- type xmlFolder struct {
- Path string `xml:"path"`
- }
- addFolder := func(path string) error {
- return callback(fs.NewDir(filesystem.opt.Enc.ToStandardPath(path), time.Time{}))
- }
- addFile := func(f *xmlFile) error {
- return callback(&Object{
- hasMetaData: true,
- fs: filesystem,
- remote: filesystem.opt.Enc.ToStandardPath(path.Join(f.Path, f.Name)),
- size: f.Size,
- md5: f.Checksum,
- createTime: time.Time(f.Created),
- modTime: time.Time(f.Modified),
- })
- }
- // liststream paths are /mountpoint/root/path
- // so the returned paths should have /mountpoint/root/ trimmed
- // as the caller is expecting path.
- pathPrefix := filesystem.opt.Enc.FromStandardPath(path.Join("/", filesystem.opt.Mountpoint, filesystem.root))
- trimPathPrefix := func(p string) string {
- p = strings.TrimPrefix(p, pathPrefix)
- p = strings.TrimPrefix(p, "/")
- return p
- }
- uniqueFolders := map[string]bool{}
- decoder := xml.NewDecoder(r)
- for {
- t, err := decoder.Token()
- if err != nil {
- if err != io.EOF {
- return err
- }
- break
- }
- switch se := t.(type) {
- case xml.StartElement:
- switch se.Name.Local {
- case "file":
- var f xmlFile
- if err := decoder.DecodeElement(&f, &se); err != nil {
- return err
- }
- f.Path = trimPathPrefix(f.Path)
- actual.Files++
- if !uniqueFolders[f.Path] {
- uniqueFolders[f.Path] = true
- actual.Folders++
- if err := addFolder(f.Path); err != nil {
- return err
- }
- }
- if err := addFile(&f); err != nil {
- return err
- }
- case "folder":
- var f xmlFolder
- if err := decoder.DecodeElement(&f, &se); err != nil {
- return err
- }
- f.Path = trimPathPrefix(f.Path)
- uniqueFolders[f.Path] = true
- actual.Folders++
- if err := addFolder(f.Path); err != nil {
- return err
- }
- case "stats":
- if err := decoder.DecodeElement(&expected, &se); err != nil {
- return err
- }
- }
- }
- }
- if expected.Folders != actual.Folders ||
- expected.Files != actual.Files {
- return fmt.Errorf("invalid result from listStream: expected[%#v] != actual[%#v]", expected, actual)
- }
- return nil
- }
- // ListR lists the objects and directories of the Fs starting
- // from dir recursively into out.
- //
- // dir should be "" to start from the root, and should not
- // have trailing slashes.
- func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
- opts := rest.Opts{
- Method: "GET",
- Path: f.filePath(dir),
- Parameters: url.Values{},
- }
- opts.Parameters.Set("mode", "liststream")
- list := walk.NewListRHelper(callback)
- var resp *http.Response
- err = f.pacer.Call(func() (bool, error) {
- resp, err = f.jfsSrv.Call(ctx, &opts)
- if err != nil {
- return shouldRetry(ctx, resp, err)
- }
- err = parseListRStream(ctx, resp.Body, f, func(d fs.DirEntry) error {
- if d.Remote() == dir {
- return nil
- }
- return list.Add(d)
- })
- _ = resp.Body.Close()
- return shouldRetry(ctx, resp, err)
- })
- if err != nil {
- if apiErr, ok := err.(*api.Error); ok {
- // does not exist
- if apiErr.StatusCode == http.StatusNotFound {
- return fs.ErrorDirNotFound
- }
- }
- return fmt.Errorf("couldn't list files: %w", err)
- }
- if err != nil {
- return err
- }
- return list.Flush()
- }
- // Creates from the parameters passed in a half finished Object which
- // must have setMetaData called on it
- //
- // Used to create new objects
- func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object) {
- // Temporary Object under construction
- o = &Object{
- fs: f,
- remote: remote,
- size: size,
- modTime: modTime,
- }
- return o
- }
- // Put the object
- //
- // Copy the reader in to the new object which is returned.
- //
- // The new object may have been created if an error is returned
- func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
- o := f.createObject(src.Remote(), src.ModTime(ctx), src.Size())
- return o, o.Update(ctx, in, src, options...)
- }
- // mkParentDir makes the parent of the native path dirPath if
- // necessary and any directories above that
- func (f *Fs) mkParentDir(ctx context.Context, dirPath string) error {
- // defer log.Trace(dirPath, "")("")
- // chop off trailing / if it exists
- parent := path.Dir(strings.TrimSuffix(dirPath, "/"))
- if parent == "." {
- parent = ""
- }
- return f.Mkdir(ctx, parent)
- }
- // Mkdir creates the container if it doesn't exist
- func (f *Fs) Mkdir(ctx context.Context, dir string) error {
- _, err := f.CreateDir(ctx, dir)
- return err
- }
- // purgeCheck removes the root directory, if check is set then it
- // refuses to do so if it has anything in
- func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error) {
- root := path.Join(f.root, dir)
- if root == "" {
- return errors.New("can't purge root directory")
- }
- // check that the directory exists
- entries, err := f.List(ctx, dir)
- if err != nil {
- return err
- }
- if check {
- if len(entries) != 0 {
- return fs.ErrorDirectoryNotEmpty
- }
- }
- opts := rest.Opts{
- Method: "POST",
- Path: f.filePath(dir),
- Parameters: url.Values{},
- NoResponse: true,
- }
- if f.opt.HardDelete {
- opts.Parameters.Set("rmDir", "true")
- } else {
- opts.Parameters.Set("dlDir", "true")
- }
- var resp *http.Response
- err = f.pacer.Call(func() (bool, error) {
- resp, err = f.jfsSrv.Call(ctx, &opts)
- return shouldRetry(ctx, resp, err)
- })
- if err != nil {
- return fmt.Errorf("couldn't purge directory: %w", err)
- }
- return nil
- }
- // Rmdir deletes the root folder
- //
- // Returns an error if it isn't empty
- func (f *Fs) Rmdir(ctx context.Context, dir string) error {
- return f.purgeCheck(ctx, dir, true)
- }
- // Precision return the precision of this Fs
- func (f *Fs) Precision() time.Duration {
- return time.Second
- }
- // Purge deletes all the files and the container
- func (f *Fs) Purge(ctx context.Context, dir string) error {
- return f.purgeCheck(ctx, dir, false)
- }
- // createOrUpdate tries to make remote file match without uploading.
- // If the remote file exists, and has matching size and md5, only
- // timestamps are updated. If the file does not exist or does does
- // not match size and md5, but matching content can be constructed
- // from deduplication, the file will be updated/created. If the file
- // is currently in trash, but can be made to match, it will be
- // restored. Returns ErrorObjectNotFound if upload will be necessary
- // to get a matching remote file.
- func (f *Fs) createOrUpdate(ctx context.Context, file string, createTime time.Time, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
- opts := rest.Opts{
- Method: "POST",
- Path: f.filePath(file),
- Parameters: url.Values{},
- ExtraHeaders: make(map[string]string),
- }
- opts.Parameters.Set("cphash", "true")
- opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
- opts.ExtraHeaders["JMd5"] = md5
- opts.ExtraHeaders["JCreated"] = api.JottaTime(createTime).String()
- opts.ExtraHeaders["JModified"] = api.JottaTime(modTime).String()
- var resp *http.Response
- err = f.pacer.Call(func() (bool, error) {
- resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &info)
- return shouldRetry(ctx, resp, err)
- })
- if apiErr, ok := err.(*api.Error); ok {
- // does not exist, i.e. not matching size and md5, and not possible to make it by deduplication
- if apiErr.StatusCode == http.StatusNotFound {
- return nil, fs.ErrorObjectNotFound
- }
- }
- return info, nil
- }
- // copyOrMoves copies or moves directories or files depending on the method parameter
- func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *api.JottaFile, err error) {
- opts := rest.Opts{
- Method: "POST",
- Path: src,
- Parameters: url.Values{},
- }
- opts.Parameters.Set(method, f.filePathRaw(dest, true))
- var resp *http.Response
- err = f.pacer.Call(func() (bool, error) {
- resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &info)
- return shouldRetry(ctx, resp, err)
- })
- if err != nil {
- return nil, err
- }
- return info, nil
- }
- // Copy src to this remote using server-side copy operations.
- //
- // This is stored with the remote path given.
- //
- // It returns the destination Object and a possible error.
- //
- // Will only be called if src.Fs().Name() == f.Name()
- //
- // If it isn't possible then return fs.ErrorCantCopy
- func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
- srcObj, ok := src.(*Object)
- if !ok {
- fs.Debugf(src, "Can't copy - not same remote type")
- return nil, fs.ErrorCantMove
- }
- meta, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
- if err != nil {
- return nil, err
- }
- if err := f.mkParentDir(ctx, remote); err != nil {
- return nil, err
- }
- info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
- if err == nil {
- var createTime time.Time
- var createTimeMeta bool
- var modTime time.Time
- var modTimeMeta bool
- if meta != nil {
- createTime, createTimeMeta = srcObj.parseFsMetadataTime(meta, "btime")
- if !createTimeMeta {
- createTime = srcObj.createTime
- }
- modTime, modTimeMeta = srcObj.parseFsMetadataTime(meta, "mtime")
- if !modTimeMeta {
- modTime = srcObj.modTime
- }
- }
- if bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
- // Workaround necessary when destination was a trashed file, to avoid the copied file also being in trash (bug in api?)
- fs.Debugf(src, "Server-side copied to trashed destination, restoring")
- info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
- } else if createTimeMeta || modTimeMeta {
- info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
- }
- }
- if err != nil {
- return nil, fmt.Errorf("couldn't copy file: %w", err)
- }
- return f.newObjectWithInfo(ctx, remote, info)
- //return f.newObjectWithInfo(remote, &result)
- }
- // Move src to this remote using server-side move operations.
- //
- // This is stored with the remote path given.
- //
- // It returns the destination Object and a possible error.
- //
- // Will only be called if src.Fs().Name() == f.Name()
- //
- // If it isn't possible then return fs.ErrorCantMove
- func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
- srcObj, ok := src.(*Object)
- if !ok {
- fs.Debugf(src, "Can't move - not same remote type")
- return nil, fs.ErrorCantMove
- }
- meta, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
- if err != nil {
- return nil, err
- }
- if err := f.mkParentDir(ctx, remote); err != nil {
- return nil, err
- }
- info, err := f.copyOrMove(ctx, "mv", srcObj.filePath(), remote)
- if err == nil && meta != nil {
- createTime, createTimeMeta := srcObj.parseFsMetadataTime(meta, "btime")
- if !createTimeMeta {
- createTime = srcObj.createTime
- }
- modTime, modTimeMeta := srcObj.parseFsMetadataTime(meta, "mtime")
- if !modTimeMeta {
- modTime = srcObj.modTime
- }
- if createTimeMeta || modTimeMeta {
- info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
- }
- }
- if err != nil {
- return nil, fmt.Errorf("couldn't move file: %w", err)
- }
- return f.newObjectWithInfo(ctx, remote, info)
- //return f.newObjectWithInfo(remote, result)
- }
- // DirMove moves src, srcRemote to this remote at dstRemote
- // using server-side move operations.
- //
- // Will only be called if src.Fs().Name() == f.Name()
- //
- // If it isn't possible then return fs.ErrorCantDirMove
- //
- // If destination exists then return fs.ErrorDirExists
- func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
- srcFs, ok := src.(*Fs)
- if !ok {
- fs.Debugf(srcFs, "Can't move directory - not same remote type")
- return fs.ErrorCantDirMove
- }
- srcPath := path.Join(srcFs.root, srcRemote)
- dstPath := path.Join(f.root, dstRemote)
- // Refuse to move to or from the root
- if srcPath == "" || dstPath == "" {
- fs.Debugf(src, "DirMove error: Can't move root")
- return errors.New("can't move root directory")
- }
- //fmt.Printf("Move src: %s (FullPath %s), dst: %s (FullPath: %s)\n", srcRemote, srcPath, dstRemote, dstPath)
- var err error
- _, err = f.List(ctx, dstRemote)
- if err == fs.ErrorDirNotFound {
- // OK
- } else if err != nil {
- return err
- } else {
- return fs.ErrorDirExists
- }
- _, err = f.copyOrMove(ctx, "mvDir", path.Join(f.fileEndpoint, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote)
- if err != nil {
- return fmt.Errorf("couldn't move directory: %w", err)
- }
- return nil
- }
- // PublicLink generates a public link to the remote path (usually readable by anyone)
- func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
- opts := rest.Opts{
- Method: "GET",
- Path: f.filePath(remote),
- Parameters: url.Values{},
- }
- if unlink {
- opts.Parameters.Set("mode", "disableShare")
- } else {
- opts.Parameters.Set("mode", "enableShare")
- }
- var resp *http.Response
- var result api.JottaFile
- err = f.pacer.Call(func() (bool, error) {
- resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result)
- return shouldRetry(ctx, resp, err)
- })
- if apiErr, ok := err.(*api.Error); ok {
- // does not exist
- if apiErr.StatusCode == http.StatusNotFound {
- return "", fs.ErrorObjectNotFound
- }
- }
- if err != nil {
- if unlink {
- return "", fmt.Errorf("couldn't remove public link: %w", err)
- }
- return "", fmt.Errorf("couldn't create public link: %w", err)
- }
- if unlink {
- if result.PublicURI != "" {
- return "", fmt.Errorf("couldn't remove public link - %q", result.PublicURI)
- }
- return "", nil
- }
- if result.PublicURI == "" {
- return "", errors.New("couldn't create public link - no uri received")
- }
- if result.PublicSharePath != "" {
- webLink := joinPath(wwwURL, result.PublicSharePath)
- fs.Debugf(nil, "Web link: %s", webLink)
- } else {
- fs.Debugf(nil, "No web link received")
- }
- directLink := joinPath(wwwURL, fmt.Sprintf("opin/io/downloadPublic/%s/%s", f.user, result.PublicURI))
- fs.Debugf(nil, "Direct link: %s", directLink)
- return directLink, nil
- }
- // About gets quota information
- func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
- info, err := getDriveInfo(ctx, f.jfsSrv, f.user)
- if err != nil {
- return nil, err
- }
- usage := &fs.Usage{
- Used: fs.NewUsageValue(info.Usage),
- }
- if info.Capacity > 0 {
- usage.Total = fs.NewUsageValue(info.Capacity)
- usage.Free = fs.NewUsageValue(info.Capacity - info.Usage)
- }
- return usage, nil
- }
- // UserInfo fetches info about the current user
- func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err error) {
- cust, err := getCustomerInfo(ctx, f.apiSrv)
- if err != nil {
- return nil, err
- }
- return map[string]string{
- "Username": cust.Username,
- "Email": cust.Email,
- "Name": cust.Name,
- "AccountType": cust.AccountType,
- "SubscriptionType": cust.SubscriptionType,
- }, nil
- }
- // CleanUp empties the trash
- func (f *Fs) CleanUp(ctx context.Context) error {
- opts := rest.Opts{
- Method: "POST",
- Path: "files/v1/purge_trash",
- }
- var info api.TrashResponse
- _, err := f.apiSrv.CallJSON(ctx, &opts, nil, &info)
- if err != nil {
- return fmt.Errorf("couldn't empty trash: %w", err)
- }
- return nil
- }
- // Shutdown shutdown the fs
- func (f *Fs) Shutdown(ctx context.Context) error {
- f.tokenRenewer.Shutdown()
- return nil
- }
- // Hashes returns the supported hash sets.
- func (f *Fs) Hashes() hash.Set {
- return hash.Set(hash.MD5)
- }
- // ---------------------------------------------
- // Fs returns the parent Fs
- func (o *Object) Fs() fs.Info {
- return o.fs
- }
- // Return a string version
- func (o *Object) String() string {
- if o == nil {
- return "<nil>"
- }
- return o.remote
- }
- // Remote returns the remote path
- func (o *Object) Remote() string {
- return o.remote
- }
- // filePath returns an escaped file path (f.root, remote)
- func (o *Object) filePath() string {
- return o.fs.filePath(o.remote)
- }
- // Hash returns the MD5 of an object returning a lowercase hex string
- func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
- if t != hash.MD5 {
- return "", hash.ErrUnsupported
- }
- return o.md5, nil
- }
- // Size returns the size of an object in bytes
- func (o *Object) Size() int64 {
- ctx := context.TODO()
- err := o.readMetaData(ctx, false)
- if err != nil {
- fs.Logf(o, "Failed to read metadata: %v", err)
- return 0
- }
- return o.size
- }
- // MimeType of an Object if known, "" otherwise
- func (o *Object) MimeType(ctx context.Context) string {
- return o.mimeType
- }
- // validFile checks if info indicates file is valid
- func (f *Fs) validFile(info *api.JottaFile) bool {
- if info.State != "COMPLETED" {
- return false // File is incomplete or corrupt
- }
- if !info.Deleted {
- return !f.opt.TrashedOnly // Regular file; return false if TrashedOnly, else true
- }
- return f.opt.TrashedOnly // Deleted file; return true if TrashedOnly, else false
- }
- // validFolder checks if info indicates folder is valid
- func (f *Fs) validFolder(info *api.JottaFolder) bool {
- // Returns true if folder is not deleted.
- // If TrashedOnly option then always returns true, because a folder not
- // in trash must be traversed to get to files/subfolders that are.
- return !bool(info.Deleted) || f.opt.TrashedOnly
- }
- // setMetaData sets the metadata from info
- func (o *Object) setMetaData(info *api.JottaFile) (err error) {
- o.hasMetaData = true
- o.size = info.Size
- o.md5 = info.MD5
- o.mimeType = info.MimeType
- o.createTime = time.Time(info.CreatedAt)
- o.modTime = time.Time(info.ModifiedAt)
- o.updateTime = time.Time(info.UpdatedAt)
- return nil
- }
- // readMetaData reads and updates the metadata for an object
- func (o *Object) readMetaData(ctx context.Context, force bool) (err error) {
- if o.hasMetaData && !force {
- return nil
- }
- info, err := o.fs.readMetaDataForPath(ctx, o.remote)
- if err != nil {
- return err
- }
- if !o.fs.validFile(info) {
- return fs.ErrorObjectNotFound
- }
- return o.setMetaData(info)
- }
- // parseFsMetadataTime parses a time string from fs.Metadata with key
- func (o *Object) parseFsMetadataTime(m fs.Metadata, key string) (t time.Time, ok bool) {
- value, ok := m[key]
- if ok {
- var err error
- t, err = time.Parse(time.RFC3339Nano, value) // metadata stores RFC3339Nano timestamps
- if err != nil {
- fs.Debugf(o, "failed to parse metadata %s: %q: %v", key, value, err)
- ok = false
- }
- }
- return t, ok
- }
- // ModTime returns the modification time of the object
- //
- // It attempts to read the objects mtime and if that isn't present the
- // LastModified returned in the http headers
- func (o *Object) ModTime(ctx context.Context) time.Time {
- err := o.readMetaData(ctx, false)
- if err != nil {
- fs.Logf(o, "Failed to read metadata: %v", err)
- return time.Now()
- }
- return o.modTime
- }
- // SetModTime sets the modification time of the local fs object
- func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
- // make sure metadata is available, we need its current size and md5
- err := o.readMetaData(ctx, false)
- if err != nil {
- fs.Logf(o, "Failed to read metadata: %v", err)
- return err
- }
- // request check/update with existing metadata and new modtime
- // (note that if size/md5 does not match, the file content will
- // also be modified if deduplication is possible, i.e. it is
- // important to use correct/latest values)
- _, err = o.fs.createOrUpdate(ctx, o.remote, o.createTime, modTime, o.size, o.md5)
- if err != nil {
- if err == fs.ErrorObjectNotFound {
- // file was modified (size/md5 changed) between readMetaData and createOrUpdate?
- return errors.New("metadata did not match")
- }
- return err
- }
- // update local metadata
- o.modTime = modTime
- return nil
- }
- // Storable returns a boolean showing whether this object storable
- func (o *Object) Storable() bool {
- return true
- }
- // Open an object for read
- func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
- fs.FixRangeOption(options, o.size)
- var resp *http.Response
- opts := rest.Opts{
- Method: "GET",
- Path: o.filePath(),
- Parameters: url.Values{},
- Options: options,
- }
- opts.Parameters.Set("mode", "bin")
- err = o.fs.pacer.Call(func() (bool, error) {
- resp, err = o.fs.jfsSrv.Call(ctx, &opts)
- return shouldRetry(ctx, resp, err)
- })
- if err != nil {
- return nil, err
- }
- return resp.Body, err
- }
- // Read the md5 of in returning a reader which will read the same contents
- //
- // The cleanup function should be called when out is finished with
- // regardless of whether this function returned an error or not.
- func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader, cleanup func(), err error) {
- // we need an MD5
- md5Hasher := md5.New()
- // use the teeReader to write to the local file AND calculate the MD5 while doing so
- teeReader := io.TeeReader(in, md5Hasher)
- // nothing to clean up by default
- cleanup = func() {}
- // don't cache small files on disk to reduce wear of the disk
- if size > threshold {
- var tempFile *os.File
- // create the cache file
- tempFile, err = os.CreateTemp("", cachePrefix)
- if err != nil {
- return
- }
- _ = os.Remove(tempFile.Name()) // Delete the file - may not work on Windows
- // clean up the file after we are done downloading
- cleanup = func() {
- // the file should normally already be close, but just to make sure
- _ = tempFile.Close()
- _ = os.Remove(tempFile.Name()) // delete the cache file after we are done - may be deleted already
- }
- // copy the ENTIRE file to disc and calculate the MD5 in the process
- if _, err = io.Copy(tempFile, teeReader); err != nil {
- return
- }
- // jump to the start of the local file so we can pass it along
- if _, err = tempFile.Seek(0, 0); err != nil {
- return
- }
- // replace the already read source with a reader of our cached file
- out = tempFile
- } else {
- // that's a small file, just read it into memory
- var inData []byte
- inData, err = io.ReadAll(teeReader)
- if err != nil {
- return
- }
- // set the reader to our read memory block
- out = bytes.NewReader(inData)
- }
- return hex.EncodeToString(md5Hasher.Sum(nil)), out, cleanup, nil
- }
- // Update the object with the contents of the io.Reader, modTime and size
- //
- // If existing is set then it updates the object rather than creating a new one.
- //
- // The new object may have been created if an error is returned
- func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
- if o.fs.opt.NoVersions {
- err := o.readMetaData(ctx, false)
- if err == nil {
- // if the object exists delete it
- err = o.remove(ctx, true)
- if err != nil && err != fs.ErrorObjectNotFound {
- // if delete failed then report that, unless it was because the file did not exist after all
- return fmt.Errorf("failed to remove old object: %w", err)
- }
- } else if err != fs.ErrorObjectNotFound {
- // if the object does not exist we can just continue but if the error is something different we should report that
- return err
- }
- }
- o.fs.tokenRenewer.Start()
- defer o.fs.tokenRenewer.Stop()
- size := src.Size()
- md5String, err := src.Hash(ctx, hash.MD5)
- if err != nil || md5String == "" {
- // unwrap the accounting from the input, we use wrap to put it
- // back on after the buffering
- var wrap accounting.WrapFn
- in, wrap = accounting.UnWrap(in)
- var cleanup func()
- md5String, in, cleanup, err = readMD5(in, size, int64(o.fs.opt.MD5MemoryThreshold))
- defer cleanup()
- if err != nil {
- return fmt.Errorf("failed to calculate MD5: %w", err)
- }
- // Wrap the accounting back onto the stream
- in = wrap(in)
- }
- // Fetch metadata if --metadata is in use
- meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
- if err != nil {
- return fmt.Errorf("failed to read metadata from source object: %w", err)
- }
- var createdTime string
- var modTime string
- if meta != nil {
- if t, ok := o.parseFsMetadataTime(meta, "btime"); ok {
- createdTime = api.Rfc3339Time(t).String() // jottacloud api wants RFC3339 timestamps
- }
- if t, ok := o.parseFsMetadataTime(meta, "mtime"); ok {
- modTime = api.Rfc3339Time(t).String()
- }
- }
- if modTime == "" { // prefer mtime in meta as Modified time, fallback to source ModTime
- modTime = api.Rfc3339Time(src.ModTime(ctx)).String()
- }
- if createdTime == "" { // if no Created time set same as Modified
- createdTime = modTime
- }
- // use the api to allocate the file first and get resume / deduplication info
- var resp *http.Response
- opts := rest.Opts{
- Method: "POST",
- Path: "files/v1/allocate",
- Options: options,
- ExtraHeaders: make(map[string]string),
- }
- // the allocate request
- var request = api.AllocateFileRequest{
- Bytes: size,
- Created: createdTime,
- Modified: modTime,
- Md5: md5String,
- Path: o.fs.allocatePathRaw(o.remote, true),
- }
- // send it
- var response api.AllocateFileResponse
- err = o.fs.pacer.CallNoRetry(func() (bool, error) {
- resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response)
- return shouldRetry(ctx, resp, err)
- })
- if err != nil {
- return err
- }
- // If the file state is INCOMPLETE and CORRUPT, we must upload it.
- // Else, if the file state is COMPLETE, we don't need to upload it because
- // the content is already there, possibly it was created with deduplication,
- // and also any metadata changes are already performed by the allocate request.
- if response.State != "COMPLETED" {
- // how much do we still have to upload?
- remainingBytes := size - response.ResumePos
- opts = rest.Opts{
- Method: "POST",
- RootURL: response.UploadURL,
- ContentLength: &remainingBytes,
- ContentType: "application/octet-stream",
- Body: in,
- ExtraHeaders: make(map[string]string),
- }
- if response.ResumePos != 0 {
- opts.ExtraHeaders["Range"] = "bytes=" + strconv.FormatInt(response.ResumePos, 10) + "-" + strconv.FormatInt(size-1, 10)
- }
- // copy the already uploaded bytes into the trash :)
- var result api.UploadResponse
- _, err = io.CopyN(io.Discard, in, response.ResumePos)
- if err != nil {
- return err
- }
- // send the remaining bytes
- _, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result)
- if err != nil {
- return err
- }
- // Upload response contains main metadata properties (size, md5 and modTime)
- // which could be set back to the object, but it does not contain the
- // necessary information to set the createTime and updateTime properties,
- // so must therefore perform a read instead.
- }
- // in any case we must update the object meta data
- return o.readMetaData(ctx, true)
- }
- func (o *Object) remove(ctx context.Context, hard bool) error {
- opts := rest.Opts{
- Method: "POST",
- Path: o.filePath(),
- Parameters: url.Values{},
- NoResponse: true,
- }
- if hard {
- opts.Parameters.Set("rm", "true")
- } else {
- opts.Parameters.Set("dl", "true")
- }
- err := o.fs.pacer.Call(func() (bool, error) {
- resp, err := o.fs.jfsSrv.CallXML(ctx, &opts, nil, nil)
- return shouldRetry(ctx, resp, err)
- })
- if apiErr, ok := err.(*api.Error); ok {
- // attempting to hard delete will fail if path does not exist, but standard delete will succeed
- if apiErr.StatusCode == http.StatusNotFound {
- return fs.ErrorObjectNotFound
- }
- }
- return err
- }
- // Remove an object
- func (o *Object) Remove(ctx context.Context) error {
- return o.remove(ctx, o.fs.opt.HardDelete)
- }
- // Metadata returns metadata for an object
- //
- // It should return nil if there is no Metadata
- func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
- err = o.readMetaData(ctx, false)
- if err != nil {
- fs.Logf(o, "Failed to read metadata: %v", err)
- return nil, err
- }
- metadata.Set("btime", o.createTime.Format(time.RFC3339Nano)) // metadata timestamps should be RFC3339Nano
- metadata.Set("mtime", o.modTime.Format(time.RFC3339Nano))
- metadata.Set("utime", o.updateTime.Format(time.RFC3339Nano))
- metadata.Set("content-type", o.mimeType)
- return metadata, nil
- }
- // Check the interfaces are satisfied
- var (
- _ fs.Fs = (*Fs)(nil)
- _ fs.Purger = (*Fs)(nil)
- _ fs.Copier = (*Fs)(nil)
- _ fs.Mover = (*Fs)(nil)
- _ fs.DirMover = (*Fs)(nil)
- _ fs.ListRer = (*Fs)(nil)
- _ fs.PublicLinker = (*Fs)(nil)
- _ fs.Abouter = (*Fs)(nil)
- _ fs.UserInfoer = (*Fs)(nil)
- _ fs.CleanUpper = (*Fs)(nil)
- _ fs.Shutdowner = (*Fs)(nil)
- _ fs.Object = (*Object)(nil)
- _ fs.MimeTyper = (*Object)(nil)
- _ fs.Metadataer = (*Object)(nil)
- )
|