pixiv_down/source/pd/pixiv.d

/*
 * pixiv_down - CLI-based downloading tool for https://www.pixiv.net.
 * Copyright (C) 2024  Mio
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, version 3 of the License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
module pd.pixiv;

import std.array : appender;
import std.format : format;
import std.json : JSONException, JSONValue, parseJSON;

import std.experimental.logger;

import pd.configuration;
import pd.utils;

static if (__VERSION__ <= 2081L) {
   import std.json : JSON_TYPE;

   /*
      While we could just alias JSONType = JSON_TYPE and use the ALL CAPS
      version of the enum members in the code, they are technically deprecated.
   */
   enum JSONType : byte
   {
      null_ = JSON_TYPE.NULL,
      string = JSON_TYPE.STRING,
      integer = JSON_TYPE.INTEGER,
      uinteger = JSON_TYPE.UINTEGER,
      float_ = JSON_TYPE.FLOAT,
      array = JSON_TYPE.ARRAY,
      object = JSON_TYPE.OBJECT,
      true_ = JSON_TYPE.TRUE,
      false_ = JSON_TYPE.FALSE
   }
} else {
   import std.json : JSONType;
}

static if (__VERSION__ <= 2082L) {
   bool boolean(const ref JSONValue self) pure @safe
   {
      if (self.type == JSONType.true_) return true;
      if (self.type == JSONType.false_) return false;

      throw new JSONException("JSONValue is not a boolean type");
   }
}


immutable struct ArtworkInfo
{
   string id;
   string title;
   string type;
   string userId;
   string userName;
   long numberOfPages;
   string originalURL;
   string createDate;
   bool isR18;
}

immutable struct ArtworkPage
{
   string originalURL;
}

// https://www.pixiv.net/ajax/user/:id/:type/bookmarks
immutable struct Bookmarks
{
   Bookmark[] works;
   long total;
}

immutable struct Bookmark
{
   string id;
   bool isMasked;
   string maskReason;
   BookmarkData bookmarkData;
}

immutable struct BookmarkData
{
   string id;
   // 'private' in JSON
   bool isPrivate;
}

// https://www.pixiv.net/ajax/novel/$id
immutable struct NovelInfo
{
   string id;
   string title;
   string userId;
   string userName;
   string content;
   string description;
   string createDate;
   bool isR18;
}

immutable struct UgoiraFrame
{
   string filename;
   long delay;
}

immutable struct UgoiraInfo
{
   UgoiraFrame[] frames;
   string originalSource;
   string mimeType;
}

immutable struct User
{
   string id;
   string userName;
}

immutable struct UserProfile
{
   string[] illusts;
   string[] manga;
   string[] novels;
}

class PixivException : Exception
{
   this(string msg)
   {
      super(msg);
   }
}

class PixivJSONException : PixivException
{
   this(string msg)
   {
      super(msg);
   }
}

ArtworkInfo fetchArtworkInfo(string id, in Config config)
{
   auto json = makeJSONRequest(format("/illust/%s", id), config);

   try {
      auto jsonBody = json["body"];

      string type;
      switch (jsonBody["illustType"].integer)
      {
      case 0:
         type = "illustration";
         break;
      case 1:
         type = "manga";
         break;
      case 2:
         type = "ugoira";
         break;
      case 3:
         type = "novel";
         break;
      default:
         type = "unknown";
         errorf("Unknown artwork type: %d", jsonBody["illustType"].integer);
         break;
      }

      return ArtworkInfo(
         jsonBody["id"].str,
         jsonBody["title"].str,
         type,
         jsonBody["userId"].str,
         jsonBody["userName"].str,
         jsonBody["pageCount"].integer,
         jsonBody["urls"]["original"].str,
         jsonBody["createDate"].str,
         jsonBody["xRestrict"].integer == 1
      );
   } catch (JSONException e) {
      errorf("parsing JSON: %s", e.msg);
      if ("message" in json && json["message"].str != "") {
         throw new PixivJSONException(json["message"].str);
      }
      throw new PixivJSONException(e.msg);
   }
}

ArtworkPage[] fetchArtworkPages(string id, in Config config)
{
   auto response = appender!string;
   ArtworkPage[] pages;

   auto client = makeHTTPClient(config.sessionid);
   client.url = format("https://www.pixiv.net/ajax/illust/%s/pages", id);
   client.onReceive = (ubyte[] data) {
      response.put(data);
      return data.length;
   };
   client.perform();

   auto json = parseJSON(response[]);

   try {
      auto jsonBody = json["body"];
      pages.reserve(jsonBody.array.length);
      foreach(ref page; jsonBody.array) {
         pages ~= ArtworkPage(page["urls"]["original"].str);
      }
   } catch (JSONException e) {
      if ("message" in json) {
         auto msg = json["message"].str;
         if (msg.length != 0) {
            throw new PixivJSONException(msg);
         }
      }
      throw new PixivJSONException(e.msg);
   }

   return pages;
}


import std.regex : ctRegex;
private enum TOKEN_Regex = ctRegex!(`token":"([^"]+)"`);

/**
 * Fetch the Cross-Site Request Forgery token that will be used to
 * un-bookmark works.
 *
 * Params:
 *  sessionID = The `PHPSESSID` cookie
 *
 * Returns: A string containing the CSRF token.
 */
string fetchCSRFToken(in string sessionID)
{
   import std.net.curl: HTTP;
   import std.regex : matchFirst;
   import std.string : split;

   const userID = split(sessionID, "_")[0];

   auto html = appender!string;

   // Make our own instead of using util.makeHTTPClient so that we can
   // better control our headers.
   auto client = HTTP();
   client.addRequestHeader("Accept", "text/html,application/xhtml+xml");
   client.addRequestHeader("Host", "www.pixiv.net");
   client.addRequestHeader("Referer", "https://www.pixiv.net/users/"~userID);
   client.setCookie("PHPSESSID=" ~ sessionID);
   client.setUserAgent(UserAgent);
   client.onReceive = (ubyte[] data) {
      html ~= data;
      return data.length;
   };
   client.url = "https://www.pixiv.net/users/"~userID~"/following";
   client.perform();

   auto tokenCapture = matchFirst(html[], TOKEN_Regex);
   if (tokenCapture.empty || tokenCapture.length < 1) {
      throw new Exception("Could not retrieve CSRF Token");
   }
   trace("successfully retrieved CSRF Token");

   return tokenCapture[1];
}


// https://www.pixiv.net/ajax/novel/$id
NovelInfo fetchNovelInfo(string id, in Config config)
{
   auto json = makeJSONRequest(format("/novel/%s", id), config);

   try {
      auto jsonBody = json["body"];

      return NovelInfo(
         jsonBody["id"].str,
         jsonBody["title"].str,
         jsonBody["userId"].str,
         jsonBody["userName"].str,
         jsonBody["content"].str,
         jsonBody["description"].str,
         jsonBody["createDate"].str,
         jsonBody["xRestrict"].integer == 1
      );
   } catch (JSONException e) {
      if ("message" in json && json["message"].str != "") {
         throw new PixivJSONException(json["message"].str);
      }
      throw new PixivJSONException(e.msg);
   }
}

// https://www.pixiv.net/ajax/follow_latest/novel?p=1&mode=all
string[] fetchNovelLatest(int page, in Config config)
{
   import std.conv : to;

   string[string] params = [
      "p": to!string(page),
      "mode": "all"
   ];
   auto json = makeJSONRequest("/follow_latest/novel", config, params);

   try {
      string[] ret;
      auto jsonBody = json["body"];

      if (jsonBody["thumbnails"]["novel"].type != JSONType.array) {
         warning("Novel thumbnails was not an array");
         return [];
      }

      ret.reserve(jsonBody["thumbnails"]["novel"].array.length);

      foreach(novel; jsonBody["thumbnails"]["novel"].array) {
         ret ~= novel["id"].str;
      }
      return ret;
   } catch (JSONException e) {
      errorf("parsing JSON: %s", e.msg);
      if ("message" in json && json["message"].str != "") {
         throw new PixivJSONException(json["message"].str);
      }
      throw new PixivJSONException(e.msg);
   }
}

string[] fetchFollowLatest(int page, in Config config)
{
   import std.conv : to;

   string[string] params = [
      "p":  to!string(page),
      "mode": "all"
   ];
   auto json = makeJSONRequest("/follow_latest/illust", config, params);

   try {
      string[] ids;
      auto jsonBody = json["body"];

      ids.reserve(jsonBody["thumbnails"]["illust"].array.length);

      // TODO(mio): Novels exist in body.thumbnails.novel
      foreach(thumbnail; jsonBody["thumbnails"]["illust"].array) {
         ids ~= thumbnail["id"].str;
      }

      return ids;
   } catch (JSONException e) {
      if ("message" in json && json["message"].str != "") {
         throw new PixivJSONException(json["message"].str);
      }
      throw new PixivJSONException(e.msg);
   }
}

User[] fetchFollowing(bool private_, long skip, out long total, in Config config)
{
   import std.conv : to;
   import std.string : split;

   string sSkip = to!string(skip);

   const id = split(config.sessionid, '_')[0];

   string[string] params = [
      "offset": sSkip,
      "rest": private_ ? "hide" : "show",
      "limit": "24"
   ];

   auto json = makeJSONRequest(format("/user/%s/following", id), config, params);

   User[] users;

   try {
      auto jsonBody = json["body"];
      total = jsonBody["total"].integer;
      users.reserve(total);

      foreach(user; jsonBody["users"].array) {
         users ~= User(user["userId"].str, user["userName"].str);
      }

      return users;
   } catch (JSONException e) {
      if ("message" in json && json["message"].str != "") {
         throw new PixivJSONException(json["message"].str);
      }
      throw new PixivJSONException(e.msg);
   }
}

UgoiraInfo fetchUgoiraInfo(ArtworkInfo info, in Config config)
{
   auto json = makeJSONRequest(format("/illust/%s/ugoira_meta", info.id),
      config);

   try {
      UgoiraFrame[] frames;

      auto jsonBody = json["body"];

      foreach(frame; jsonBody["frames"].array) {
         frames ~= UgoiraFrame(frame["file"].str, frame["delay"].integer);
      }

      return UgoiraInfo(
         frames,
         jsonBody["originalSrc"].str,
         jsonBody["mime_type"].str
      );
   } catch (JSONException e) {
      errorf("parsing JSON: %s", e.msg);
      if ("message" in json && json["message"].str.length != 0) {
         errorf("pixiv error: %s", json["message"].str);
         throw new PixivJSONException(json["message"].str);
      }
      throw new PixivJSONException(e.msg);
   }
}

Bookmarks fetchUserBookmarks(string contentType, bool isPrivate, long offset, in Config config)
{
   import std.conv : to;
   import std.string : split;

   string[string] params = [
      "rest": isPrivate ? "hide" : "show",
      "offset": to!string(offset),
      "limit": "48",
      "tag": ""
   ];

   const userId = config.sessionid.split("_")[0];
   infof("current user ID: %s", userId);
   auto path = format("/user/%s/%s/bookmarks", userId, contentType);
   auto json = makeJSONRequest(path, config, params);

   try {
      Bookmark[] works;

      auto jsonBody = json["body"];

      foreach(bookmark; jsonBody["works"].array) {
         if (bookmark["isMasked"].boolean) {
            works ~= Bookmark(
               bookmark["id"].integer.to!string,
               bookmark["isMasked"].boolean,
               bookmark["maskReason"].str,
               BookmarkData(
                  bookmark["bookmarkData"]["id"].str,
                  bookmark["bookmarkData"]["private"].boolean
               )
            );
         } else {
            works ~= Bookmark(
               bookmark["id"].str,
               bookmark["isMasked"].boolean,
               "",
               BookmarkData(
                  bookmark["bookmarkData"]["id"].str,
                  bookmark["bookmarkData"]["private"].boolean
               )
            );
         }
      }

      return Bookmarks(works, jsonBody["total"].integer);
   } catch (JSONException e) {
      if ("message" in json && json["message"].str != "") {
         throw new PixivJSONException(json["message"].str);
      }
      throw new PixivJSONException(e.msg);
   }
}

UserProfile fetchUserProfie(string id, in Config config)
{
   auto path = format("/user/%s/profile/all", id);
   auto json = makeJSONRequest(path, config);

   try {
      import std.exception : assumeUnique;

      auto jsonBody = json["body"];

      // When an artist doesn't have any works of a certain type, then the
      // returned JSON type is an array. When they DO, it's an object.

      immutable illusts = (jsonBody["illusts"].type == JSONType.object) ?
         assumeUnique(jsonBody["illusts"].object.keys) :
         [];
      immutable manga = jsonBody["manga"].type == JSONType.object ?
         assumeUnique(jsonBody["manga"].object.keys) :
         [];
      immutable novels = (jsonBody["novels"].type == JSONType.object) ?
         assumeUnique(jsonBody["novels"].object.keys) :
         [];

      return UserProfile(illusts, manga, novels);
   } catch (JSONException e) {
      if ("message" in json && json["message"].str != "") {
         throw new PixivJSONException(json["message"].str);
      }
      throw new PixivJSONException(e.msg);
   }
}

User fetchUser(string id, in Config config)
{
   auto path = format("/user/%s", id);
   auto json = makeJSONRequest(path, config, ["full": "1"]);

   try {
      auto jsonBody = json["body"];

      return User(
         jsonBody["userId"].str,
         jsonBody["name"].str
      );
   } catch (JSONException e) {
      if ("message" in json && json["message"].str != "") {
         throw new PixivJSONException(json["message"].str);
      }
      throw new PixivJSONException(e.msg);
   }
}

// /ajax/illusts/bookmarks/delete
bool postBookmarksDelete(string bookmarkID, in string csrfToken, in string type, in Config config)
{
   import std.net.curl;
   import std.string : split;

   const uid = config.sessionid.split("_")[0];
   auto response = appender!string;

   auto client = HTTP();
   client.method = HTTP.Method.post;
   client.url = "https://www.pixiv.net/ajax/"~type~"/bookmarks/delete";
   client.setUserAgent(UserAgent);
   client.setCookie("PHPSESSID="~config.sessionid);
   client.addRequestHeader("Origin", "https://www.pixiv.net");
   client.addRequestHeader("X-Csrf-Token", csrfToken);
   client.addRequestHeader("Referer", type == "illusts" ?
         "https://www.pixiv.net/users/"~uid~"/bookmarks/artworks" :
         "https://www.pixiv.net/users/"~uid~"/bookmarks/novels");
   if (type == "novels") {
      client.setPostData("del=1&book_id="~bookmarkID, "application/x-www-form-urlencoded");
   } else {
      client.setPostData("bookmark_id="~bookmarkID, "application/x-www-form-urlencoded");
   }
   client.onReceive = (ubyte[] data) {
      response ~= data;
      return data.length;
   };
   client.perform();

   auto json = parseJSON(response.data);
   if (json["error"].boolean) {
      errorf("POST /%s/bookmarks/delete: %s", type, json["message"].str);
      return false;
   }

   return true;
}

private:

JSONValue makeJSONRequest(string path, in Config config, string[string] args = string[string].init)
{
   import mlib.search_params;

   scope params = new URLSearchParams();
   foreach(const ref key, const ref value; args) {
      params.append(key, value);
   }
   params.append("lang", config.locale);

   auto response = appender!string;
   auto client = makeHTTPClient(config.sessionid);
   client.url = format("https://www.pixiv.net/ajax%s?%s", path, params.toString());
   client.onReceive = (ubyte[] data) {
      response.put(data);
      return data.length;
   };
   tracef("Making request to %s?%s", path, params.toString());
   client.perform();

   return parseJSON(response[]);
}