ChangeLog

@@ -1,3 +1,19 @@
+2024-08-01  mio  <stigma@disroot.org>
+
+	Fix autotools-based compilation for GDC12
+
+	Bump UserAgent to Firefox 126
+
+	Add bookmarked 'remove-from-file' option
+
+	Potential fixes for CSRF Token fetching
+	Instead of requesting the home page (www.pixiv.net), we now request
+	the 'bookmarks' page of the authenticated account.  I think there may
+	have been some issues with certain IPs making the homepage request,
+	since it seemed to encounter a Cloudflare verification more often then
+	not.  We also only send a request to fetch the CSRF token if the 'bookmarked'
+	command is invoked, not for every command except for help.
+
 2024-07-27  mio  <stigma@disroot.org>
 
 	Add GNU Guix package definition for development

VERSION

@@ -1 +1 @@
-v2024.07
+v2024.07+

configure.ac

@@ -19,6 +19,20 @@ AS_IF([test x"$DC" = x"none"],
 
 AM_CONDITIONAL(IS_GDC, test x$DC = xgdc)
 AM_CONDITIONAL(IS_LDC, test x$DC = xldc2)
+AM_CONDITIONAL(IS_DMD, test x$DC = xdmd)
+
+dnl This should be moved to custom M4 script.
+GDC_VERSION=""
+GDC_VERSION_MAJOR=""
+AS_IF([test x"$DC" = x"gdc"],
+	AC_MSG_CHECKING([if gdc is version 12])
+	GDC_VERSION=$($DC -dumpversion)
+	GDC_VERSION_MAJOR=$(echo $GDC_VERSION | cut -d'.' -f1)
+	AS_IF([test "x$GDC_VERSION_MAJOR" = "x12"],
+		AC_MSG_RESULT([yes]),
+		AC_MSG_RESULT([no])))
+
+AM_CONDITIONAL(IS_GDC_12, test "x$GDC_VERSION_MAJOR" = "x12")
 
 AC_CONFIG_FILES([
 	Makefile

docs/pixiv_down-bookmarked.1

@@ -1,4 +1,4 @@
-.Dd April 04, 2024
+.Dd August 01, 2024
 .Dt PIXIV_DOWN-BOOKMARKED 1
 .Os
 .Sh NAME
@@ -57,6 +57,13 @@ Remove works that are no longer available while running this
 command.  The IDs of missing works will still be logged to the
 .Pa pixiv_down-missing.txt
 file.
+.It Fl \-remove-from-file Ar file
+Remove works that are listed on each line in
+.Ar file .
+This file is generally the one that is generated by running this
+command, without the 
+.Fl \-remove-from-file
+option.
 .It Fl s, \-skip Ar offset
 Skip downloading the first
 .Ar offset
@@ -99,4 +106,4 @@ Written by
 .Pp
 Please report any bugs you may find at the bug tracker, which can be
 found on the website at
-.Lk https://yume-neru.neocities.org/p/pixiv_down.html
+.Lk https://yume-neru.neocities.org/p/pixiv_down.html

source/Makefile.am

@@ -47,12 +47,22 @@ pixiv_down_LINK = @DC@ $(D_OFLAG)pixiv_down
 
 if IS_GDC
 pixiv_down_DFLAGS += -fversion=PD_USE_MAGICK
-else
+
+# Fix for https://forum.dlang.org/post/ftibightqjluvjrhjxtm@forum.dlang.org
+# 	GDC undefined reference when linking separately compiled files.
+#
+# This is only really needed for files that use std.format.format,
+# but determining those files is more effort than it's worth.
+if IS_GDC_12
+pixiv_down_DFLAGS += -fall-instantiations
+endif # IS_GDC_12
+
+endif # IS_GDC
 
 if IS_LDC
 pixiv_down_DFLAGS += --d-version=PD_USE_MAGICK
-else
-pixiv_down_DFLAGS += -version=PD_USE_MAGICK
-endif
+endif # IS_LDC
 
-endif
+if IS_DMD
+pixiv_down_DFLAGS += -version=PD_USE_MAGICK
+endif # IS_DMD

source/app.d

@@ -372,10 +372,6 @@ version(PD_USE_MAGICK)
    }
    infof("config outputDirectory: %s", config.outputDirectory);
 
-   if (args[1] != "help") {
-      config.csrfToken = fetchCSRFToken(config.sessionid);
-      infof("CSRF token: %s", config.csrfToken);
-   }
    config.locale = getLocale();
    infof("Current locale = %s", config.locale);
 

source/cmds/bookmarked.d

@@ -28,13 +28,18 @@ public void displayBookmarkedHelp()
       "\nThis command allows you to download all of your bookmarked works.\n" ~
       "A list containing the works that are no longer available (because\n" ~
       "they've been removed or made private) can be found in a file called\n" ~
-      "'pixiv_down-missing.txt' after running the `bookmarked' command." ~
+      "'pixiv_down-missing.txt' after running the `bookmarked' command.\n" ~
       "\nOptions:\n" ~
-      "   -h, --help       \tDisplay this help message and exit.\n" ~
-      "   -s, --skip OFFSET\tSkip downloading the first OFFSET creators.\n" ~
-      "   --novels         \tDownload bookmarked novels instead of artworks.\n" ~
-      "   --private        \tDownload your privately bookmarked works instead.\n" ~
-      "   --remove-invalid \tRemove bookmarks for works that are no longer available.\n");
+      "   -h, --help             \tDisplay this help message and exit.\n" ~
+      "   -s, --skip OFFSET      \tSkip downloading the first OFFSET creators.\n" ~
+      "   --novels               \tDownload bookmarked novels instead of artworks.\n" ~
+      "   --private              \tDownload your privately bookmarked works instead.\n" ~
+      "   --remove-invalid       \tRemove bookmarks for works that are no longer available.\n" ~
+      "   --remove-from-file FILE\tRemove bookmarks for works listed in FILE.\n" ~
+      "\nThe --remove-invalid and --remove-from-file options DO NOT remove any\n" ~
+      "files from your system, they only remove the \"bookmarked\" status on\n" ~
+      "pixiv.  The FILE for --remove-from-file is expected to be the generated\n" ~
+      "pixiv_down-missing.txt.");
 }
 
 /**
@@ -49,6 +54,7 @@ public void displayBookmarkedHelp()
  */
 public int bookmarkedHandle(string[] args, in Config config)
 {
+   import std.file : exists;
    import std.getopt : getopt, GetOptException, GetOptOption = config;
 
    Options options;
@@ -58,7 +64,8 @@ public int bookmarkedHandle(string[] args, in Config config)
          "private|p", &options.privateRequested,
          "skip|s", &options.offset,
          "remove-invalid", &options.removeInvalid,
-         "novels", &options.novelsRequested);
+         "novels", &options.novelsRequested,
+         "remove-from-file", &options.removalFilePath);
 
       if (helpInformation.helpWanted) {
          displayBookmarkedHelp();
@@ -70,6 +77,11 @@ public int bookmarkedHandle(string[] args, in Config config)
       return 1;
    }
 
+   if ((options.removalFilePath != string.init) && (exists(options.removalFilePath))) {
+      removeBookmarksFromFile(options, config);
+      return 0;
+   }
+
    fetchAndDownloadBookmarks(options, config);
    return 0;
 }
@@ -88,14 +100,17 @@ struct Options
    bool removeInvalid;
    bool novelsRequested;
    long offset;
+   string removalFilePath;
 }
 
 void fetchAndDownloadBookmarks(in Options options, in Config config)
 {
-   long processedIDs = options.offset;
    long totalIDs;
+   long processedIDs = options.offset;
    long numberOfMissingIDs = 0;
 
+   const csrfToken = fetchCSRFToken(config.sessionid);
+
    do {
       trace("fetching user bookmarks...");
       Bookmarks bookmarks = fetchUserBookmarks(
@@ -106,12 +121,12 @@ void fetchAndDownloadBookmarks(in Options options, in Config config)
       /* Break early incase someone has manually removed bookmarks
        * while we've been downloading. */
       if (totalIDs <= processedIDs) {
-         warningf("totalIDs (%d) has changed to be less than processedIDs (%d)", totalIDs,
+         warningf("totalIDs (≈%d) has changed to be less than processedIDs (%d)", totalIDs,
             processedIDs);
          break;
       }
 
-      string[] missingIDs = downloadBookmarks(bookmarks, options, config);
+      string[] missingIDs = downloadBookmarks(bookmarks, csrfToken, options, config);
 
       numberOfMissingIDs += missingIDs.length;
       if (missingIDs.length > 0) {
@@ -140,21 +155,24 @@ void fetchAndDownloadBookmarks(in Options options, in Config config)
    writeln("Finished downloading bookmarked works.");
 }
 
-/// Returns number of bookmarks processed.
-string[] downloadBookmarks(Bookmarks bookmarks, in Options options, in Config config)
+/// Returns an array containing all Missing IDs.
+string[] downloadBookmarks(Bookmarks bookmarks, in string csrfToken, in Options options, in Config config)
 {
+   import std.format : format;
+
    string[] missingIDs;
    const type = options.novelsRequested ? "novels" : "illusts";
 
    foreach(bookmark; bookmarks.works) {
       if (bookmark.isMasked) {
          warningf("masked bookmark ID:%s reason:%s", bookmark.id, bookmark.maskReason);
-         missingIDs ~= bookmark.id;
+         missingIDs ~= format("%s\t%s\t%s", bookmark.id, bookmark.bookmarkData.id, type);
          if (options.removeInvalid) {
-            bool success = postBookmarksDelete(bookmark.bookmarkData.id, type, config);
+            const success = postBookmarksDelete(bookmark.bookmarkData.id, csrfToken, type, config);
             if (success) {
                info("successfully removed masked bookmark");
             }
+            sleep(1, 3, false);
          }
          continue;
       }
@@ -166,7 +184,7 @@ string[] downloadBookmarks(Bookmarks bookmarks, in Options options, in Config co
          ArtworkInfo artworkInfo = fetchArtworkInfo(bookmark.id, config);
          downloadArtwork(artworkInfo, config);
       }
-      sleep(2, 7);
+      sleep(4, 9);
       Term.goUpAndClearLine(Yes.useStderr);
    }
 
@@ -201,3 +219,58 @@ void writeMissingIDs(string[] ids, bool removeInvalid)
       idFile.writeln(id);
    }
 }
+
+void removeBookmarksFromFile(in Options options, in Config config)
+{
+   import std.algorithm.searching : any;
+   import std.string : split, stripRight;
+   import pixiv : postBookmarksDelete;
+
+   const csrfToken = fetchCSRFToken(config.sessionid);
+
+   bool encounteredErrors = false;
+   auto missingIDs = File(options.removalFilePath, "r");
+   foreach(string line; missingIDs.byLineCopy(No.keepTerminator)) {
+      // Skip empty lines and comment lines
+      if (line.length == 0 || line[0] == '#') {
+         continue;
+      }
+      // <work_id>\t<bookmark_id>\t<work_type>
+      const segments = line.split("\t");
+      if (segments.length != 3) {
+         continue;
+      }
+
+      const workID = segments[0];
+      const bookmarkID = segments[1];
+      const workType = segments[2];
+      if (any!"a < '0' || a > '9'"(workID)) {
+         stderr.writefln("Failed to remove bookmark: Invalid work ID: ", workID);
+         continue;
+      }
+      if (any!"a < '0' || a > '9'"(bookmarkID)) {
+         stderr.writefln("Failed to remove bookmark: Invalid bookmark ID: ", bookmarkID);
+         errorf("Invalid bookmark ID: %s (workID = %s, workType = %s)", bookmarkID, workID,
+            workType);
+         continue;
+      }
+      if (workType != "novels" && workType != "illusts") {
+         stderr.writefln("Failed to remove bookmark: Invalid type: ", workType);
+         errorf("Invalid bookmark type: %s (workID = %s, bookmarkID = %s)", workType, workID,
+            bookmarkID);
+      }
+
+      const success = postBookmarksDelete(bookmarkID, csrfToken, workType, config);
+      if (!success) {
+         encounteredErrors = true;
+         stderr.writefln("Failed to remove bookmark for ID %s", workID);
+         errorf("Failed to unbookmark bookmarkID %s (workID = %s, workType = %s)", bookmarkID,
+            workID, workType);
+      }
+      sleep(2, 7);
+   }
+
+   if (encounteredErrors) {
+      stderr.writeln("There were some errors removing bookmarks.");
+   }
+}

source/configuration.d

@@ -32,7 +32,7 @@ struct Config
    string outputDirectory;
    /// The PHPSESSID of the current account.
    string sessionid;
-   /// Cross Site Request Forgery Token
+   /// Cross Site Request Forgery Token (unused)
    string csrfToken;
    /// Two character locale used for pixiv requests
    string locale;

source/pixiv.d

@@ -236,28 +236,51 @@ ArtworkPage[] fetchArtworkPages(string id, in Config config)
    return pages;
 }
 
+
+import std.regex : ctRegex;
+private enum TOKEN_Regex = ctRegex!(`token":"([^"]+)"`);
+
+/**
+ * Fetch the Cross-Site Request Forgery token that will be used to
+ * un-bookmark works.
+ *
+ * Params:
+ *  sessionID = The `PHPSESSID` cookie
+ *
+ * Returns: A string containing the CSRF token.
+ */
 string fetchCSRFToken(in string sessionID)
 {
    import std.net.curl;
-   import std.regex;
+   import std.regex : matchFirst;
+   import std.string : split;
+
+   const userID = split(sessionID, "_")[0];
 
    auto html = appender!string;
 
-   auto client = makeHTTPClient(sessionID);
-   client.url = "https://www.pixiv.net/";
+   // Make our own instead of using util.makeHTTPClient so that we can
+   // better control our headers.
+   auto client = HTTP();
+   client.addRequestHeader("Accept", "text/html,application/xhtml+xml");
+   client.addRequestHeader("Host", "www.pixiv.net");
+   client.addRequestHeader("Referer", "https://www.pixiv.net/users/"~userID);
+   client.setCookie("PHPSESSID=" ~ sessionID);
+   client.setUserAgent(UserAgent);
    client.onReceive = (ubyte[] data) {
       html ~= data;
       return data.length;
    };
+   client.url = "https://www.pixiv.net/users/"~userID~"/following";
    client.perform();
 
-   auto tokenRegex = ctRegex!(`token":?"([^"]+)"`);
-   auto tokenMatch = matchFirst(html[], tokenRegex);
-   if (tokenMatch.empty) {
-      error("Failed to retrieve CSRF token.");
-      return "";
+   auto tokenCapture = matchFirst(html[], TOKEN_Regex);
+   if (tokenCapture.empty || tokenCapture.length < 1) {
+      throw new Exception("Could not retrieve CSRF Token");
    }
-   return tokenMatch[1];
+   trace("successfully retrieved CSRF Token");
+
+   return tokenCapture[1];
 }
 
 
@@ -526,23 +549,29 @@ User fetchUser(string id, in Config config)
 }
 
 // /ajax/illusts/bookmarks/delete
-bool postBookmarksDelete(string workID, string type, in Config config)
+bool postBookmarksDelete(string bookmarkID, in string csrfToken, in string type, in Config config)
 {
    import std.net.curl;
-   import std.stdio;
+   import std.string : split;
 
+   const uid = config.sessionid.split("_")[0];
    auto response = appender!string;
 
-   auto client = makeHTTPClient(config.sessionid);
-   client.url = format("https://www.pixiv.net/ajax/%s/bookmarks/delete", type);
+   auto client = HTTP();
    client.method = HTTP.Method.post;
-
+   client.url = "https://www.pixiv.net/ajax/"~type~"/bookmarks/delete";
+   client.setUserAgent(UserAgent);
+   client.setCookie("PHPSESSID="~config.sessionid);
+   client.addRequestHeader("Origin", "https://www.pixiv.net");
+   client.addRequestHeader("X-Csrf-Token", csrfToken);
+   client.addRequestHeader("Referer", type == "illusts" ?
+         "https://www.pixiv.net/users/"~uid~"/bookmarks/artworks" :
+         "https://www.pixiv.net/users/"~uid~"/bookmarks/novels");
    if (type == "novels") {
-      client.setPostData(format("del=1&book_id=%s", workID), "application/x-www-form-urlencoded");
+      client.setPostData("del=1&book_id="~bookmarkID, "application/x-www-form-urlencoded");
    } else {
-      client.setPostData(format("bookmark_id=%s", workID), "application/x-www-form-urlencoded");
+      client.setPostData("bookmark_id="~bookmarkID, "application/x-www-form-urlencoded");
    }
-   client.addRequestHeader("x-csrf-token", config.csrfToken);
    client.onReceive = (ubyte[] data) {
       response ~= data;
       return data.length;
@@ -554,6 +583,7 @@ bool postBookmarksDelete(string workID, string type, in Config config)
       errorf("POST /%s/bookmarks/delete: %s", type, json["message"].str);
       return false;
    }
+
    return true;
 }