pixiv_down/source/app/cmds/bookmarked.d

/*
 * pixiv_down - CLI-based downloading tool for https://www.pixiv.net.
 * Copyright (C) 2024  Mio
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, version 3 of the License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
module app.cmds.bookmarked;

import pd.configuration: Config;

import std.stdio;

public void displayBookmarkedHelp()
{
   stderr.writefln(
      "pixiv_down bookmarked - download bookmarked works.\n" ~
      "\nUsage:\tpixiv_down bookmarked [options]\n" ~
      "\nThis command allows you to download all of your bookmarked works.\n" ~
      "A list containing the works that are no longer available (because\n" ~
      "they've been removed or made private) can be found in a file called\n" ~
      "'pixiv_down-missing.txt' after running the `bookmarked' command.\n" ~
      "\nOptions:\n" ~
      "   -h, --help             \tDisplay this help message and exit.\n" ~
      "   -s, --skip OFFSET      \tSkip downloading the first OFFSET creators.\n" ~
      "   --novels               \tDownload bookmarked novels instead of artworks.\n" ~
      "   --private              \tDownload your privately bookmarked works instead.\n" ~
      "   --remove-invalid       \tRemove bookmarks for works that are no longer available.\n" ~
      "   --remove-from-file FILE\tRemove bookmarks for works listed in FILE.\n" ~
      "\nThe --remove-invalid and --remove-from-file options DO NOT remove any\n" ~
      "files from your system, they only remove the \"bookmarked\" status on\n" ~
      "pixiv.  The FILE for --remove-from-file is expected to be the generated\n" ~
      "pixiv_down-missing.txt.");
}

/**
 * Download bookmarked content.
 *
 * The expected format of *args* is: `["pixiv_down", "bookmarked", ...options]`
 *
 * Params:
 *   args = The arguments passed to pixiv_down.
 *   config = pixiv_down configuration
 * Returns: 0 on success, non-zero on error
 */
public int bookmarkedHandle(string[] args, in Config config)
{
   import std.file: exists;
   import std.getopt: getopt, GetOptException;

   Options options;

   try {
      auto helpInformation = getopt(args,
         "private|p", &options.privateRequested,
         "skip|s", &options.offset,
         "remove-invalid", &options.removeInvalid,
         "novels", &options.novelsRequested,
         "remove-from-file", &options.removalFilePath);

      if (!options.removeInvalid) {
         // --remove-invalid is a flag. Not providing it means it's false.
         options.removeInvalid = config.bookmarked.alwaysRemoveInvalid;
      }

      if (helpInformation.helpWanted) {
         displayBookmarkedHelp();
         return 0;
      }
   } catch (GetOptException e) {
      stderr.writefln("pixiv_down bookmarked: %s", e.msg);
      stderr.writefln("Run 'pixiv_down bookmarked --help' for more information.");
      return 1;
   }

   if ((options.removalFilePath != string.init) && (exists(options.removalFilePath))) {
      removeBookmarksFromFile(options, config);
      return 0;
   }

   fetchAndDownloadBookmarks(options, config);
   return 0;
}

private:

import pd.pixiv;
import pd.pixiv_downloader;
import std.experimental.logger;
import app.util;
import mlib.term;

struct Options
{
   bool privateRequested;
   bool removeInvalid;
   bool novelsRequested;
   long offset;
   string removalFilePath;
}

void fetchAndDownloadBookmarks(in Options options, in Config config)
{
   long totalIDs;
   long processedIDs = options.offset;
   long numberOfMissingIDs = 0;

   const csrfToken = fetchCSRFToken(config.sessionid);

   do {
      trace("fetching user bookmarks...");
      Bookmarks bookmarks = fetchUserBookmarks(
         options.novelsRequested ? "novels" : "illusts",
         options.privateRequested, processedIDs, config);
      totalIDs = bookmarks.total;

      /* Break early incase someone has manually removed bookmarks
       * while we've been downloading. */
      if (totalIDs <= processedIDs) {
         warningf("totalIDs (≈%d) has changed to be less than processedIDs (%d)", totalIDs,
            processedIDs);
         break;
      }

      string[] missingIDs = downloadBookmarks(bookmarks, csrfToken, options, config);

      numberOfMissingIDs += missingIDs.length;
      if (missingIDs.length > 0) {
         writeMissingIDs(missingIDs, options.removeInvalid);
      }
      processedIDs += bookmarks.works.length;

      sleep(5, 10);
      Term.goUpAndClearLine(Yes.useStderr);
      writefln("-- Downloaded %d of %d bookmarks. --", processedIDs, totalIDs);
   } while (processedIDs < totalIDs);

   trace("finished downloading bookmarks");

   if (numberOfMissingIDs > 0) {
      info("missing IDs were found");

      writefln("Warning: %d works are no longer available.", numberOfMissingIDs);
      writeln( "        You can find a list of work IDs at 'pixiv_down-missing.txt'.");
      if (false == options.removeInvalid) {
         writeln("\nTip: Use the '--remove-invalid' flag to un-bookmark unavailable works.");
      }
   }

   writeln("Finished downloading bookmarked works.");
}

/// Returns an array containing all Missing IDs.
string[] downloadBookmarks(Bookmarks bookmarks, in string csrfToken, in Options options, in Config config)
{
   import std.format : format;

   string[] missingIDs;
   const type = options.novelsRequested ? "novels" : "illusts";

   foreach(bookmark; bookmarks.works) {
      if (bookmark.isMasked) {
         warningf("masked bookmark ID:%s reason:%s", bookmark.id, bookmark.maskReason);
         missingIDs ~= format("%s\t%s\t%s", bookmark.id, bookmark.bookmarkData.id, type);
         if (options.removeInvalid) {
            const success = postBookmarksDelete(bookmark.bookmarkData.id, csrfToken, type, config);
            if (success) {
               info("successfully removed masked bookmark");
            }
            sleep(1, 3, false);
         }
         continue;
      }

      if (options.novelsRequested) {
         NovelInfo novelInfo = fetchNovelInfo(bookmark.id, config);
         downloadNovel(novelInfo, config);
      } else {
         ArtworkInfo artworkInfo = fetchArtworkInfo(bookmark.id, config);
         downloadArtwork(artworkInfo, config);
      }
      sleep(4, 9);
      Term.goUpAndClearLine(Yes.useStderr);
   }

   return missingIDs;
}

void writeMissingIDs(string[] ids, bool removeInvalid)
{
   import std.datetime.systime : Clock;
   import app.pd_version: VCS_TAG;
   import app.logger: PIXIV_DOWN_VERSION_STRING;

   static bool loggedHeader = false;
   string openMode = loggedHeader ? "a+" : "w+";
   File idFile = File("pixiv_down-missing.txt", openMode);

   if (false == loggedHeader) {
      idFile.writefln("# List created on %s by pixiv_down/%s (%s)", Clock.currTime.toSimpleString(),
         PIXIV_DOWN_VERSION_STRING, VCS_TAG);

      if (removeInvalid) {
         idFile.writefln("# The following IDs were un-bookmarked because they have been");
         idFile.writeln("# removed from pixiv and pixiv_down was run with --remove-invalid");
      } else {
         idFile.writefln("# The following IDs were found to be removed from pixiv.");
         idFile.writeln("# You can run 'pixiv_down bookmarked --remove-invalid' to un-bookmark them.");
      }

      loggedHeader = true;
   }

   foreach(id; ids) {
      idFile.writeln(id);
   }
}

void removeBookmarksFromFile(in Options options, in Config config)
{
   import std.algorithm.searching : any;
   import std.string : split, stripRight;
   import pd.pixiv : postBookmarksDelete;

   const csrfToken = fetchCSRFToken(config.sessionid);

   bool encounteredErrors = false;
   auto missingIDs = File(options.removalFilePath, "r");
   foreach(string line; missingIDs.byLineCopy(No.keepTerminator)) {
      // Skip empty lines and comment lines
      if (line.length == 0 || line[0] == '#') {
         continue;
      }
      // <work_id>\t<bookmark_id>\t<work_type>
      const segments = line.split("\t");
      if (segments.length != 3) {
         continue;
      }

      const workID = segments[0];
      const bookmarkID = segments[1];
      const workType = segments[2];
      if (any!"a < '0' || a > '9'"(workID)) {
         stderr.writefln("Failed to remove bookmark: Invalid work ID: ", workID);
         continue;
      }
      if (any!"a < '0' || a > '9'"(bookmarkID)) {
         stderr.writefln("Failed to remove bookmark: Invalid bookmark ID: ", bookmarkID);
         errorf("Invalid bookmark ID: %s (workID = %s, workType = %s)", bookmarkID, workID,
            workType);
         continue;
      }
      if (workType != "novels" && workType != "illusts") {
         stderr.writefln("Failed to remove bookmark: Invalid type: ", workType);
         errorf("Invalid bookmark type: %s (workID = %s, bookmarkID = %s)", workType, workID,
            bookmarkID);
      }

      const success = postBookmarksDelete(bookmarkID, csrfToken, workType, config);
      if (!success) {
         encounteredErrors = true;
         stderr.writefln("Failed to remove bookmark for ID %s", workID);
         errorf("Failed to unbookmark bookmarkID %s (workID = %s, workType = %s)", bookmarkID,
            workID, workType);
      }
      sleep(2, 7);
   }

   if (encounteredErrors) {
      stderr.writeln("There were some errors removing bookmarks.");
   }
}