pixiv_down/source/app/cmds/daily.d

/*
 * pixiv_down - CLI-based downloading tool for https://www.pixiv.net.
 * Copyright (C) 2024  Mio
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, version 3 of the License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
module app.cmds.daily;

import pd.configuration;
import pd.pixiv;
import pd.pixiv_downloader;
import std.datetime.systime;
import std.experimental.logger;
import mlib.term;
import app.util;

import std.typecons: nullable, Nullable;

public int dailyHandle(string[] args, in Config config)
{
   import core.time : TimeException;
   import std.algorithm.searching : canFind;
   import std.datetime.date : Date;
   import std.getopt : getopt, GetOptException, GetOptOption = config;
   import std.stdio : stderr;

   bool restrict = false;
   string rawBeginDate;
   string rawEndDate;
   SysTime beginDate;
   SysTime endDate;
   bool force;

   try {
      getopt(args,
         "begin|b", &rawBeginDate,
         "end|e", &rawEndDate,
         "force|f", &force,
         "sfw-only|s", &restrict);
   } catch (GetOptException e) {
      stderr.writefln("pixiv_down daily: %s", e.msg);
      stderr.writefln("Run 'pixiv_down help daily' for more information");
      return 1;
   }

   tracef("current date = %s", (cast(Date)Clock.currTime).toISOExtString);
   tracef("  begin date = %s", rawBeginDate);
   tracef("    end date = %s", rawEndDate);

   beginDate = Clock.currTime;

   if (string.init != rawBeginDate) {
      try {
         beginDate = SysTime.fromISOExtString(rawBeginDate ~ "T00:00:00+00:00");
      } catch (TimeException te) {
         stderr.writefln("pixiv_down daily: Failed to parse the begin DATE: %s", rawBeginDate);
         stderr.writeln("Make sure it's in the format YYYY-MM-DD, for example: 2023-06-20");
         return 1;
      }
   }

   /* Automatically determine the end date, based off of the last run
    * of this command, but allow people to specify a custom end date. */
   if (string.init != rawEndDate) {
      try {
         endDate = SysTime.fromISOExtString(rawEndDate ~ "T00:00:00+00:00");
      } catch (TimeException te) {
         stderr.writefln("pixiv_down daily: Failed to parse the end DATE: %s", rawEndDate);
         stderr.writeln("Make sure it's in the format YYYY-MM-DD, for example: 2023-06-20");
         return 1;
      }
   } else if (string.init != rawBeginDate) {
      stderr.writeln("pixiv_down daily: Cannot determine END date when BEGIN date has been provided.");
      stderr.writeln("                  Please manually provide and END date with the -e option.");
      return 1;
   } else {
      auto lastRunTime = fetchLastStartTime();
      if (lastRunTime.isNull()) {
         stderr.writeln("pixiv_down daily: No end date could be determined, please provide date.");
         stderr.writeln("                  For example `pixiv_down daily -e 2023-06-20`");
         return 1;
      }
      endDate = lastRunTime.get();
   }

   /*
    * If beginDate and endDate are the same, then we want to download
    * all images for that day.
    */
   if (endDate.stdTime == beginDate.stdTime) {
      beginDate.hour = 23;
      beginDate.minute = 59;
      beginDate.second = 59;
   }

   if (endDate.stdTime > beginDate.stdTime) {
      stderr.writeln("pixiv_down daily: Cannot download. Provided END date is after BEGIN date.");
      stderr.writeln("                  pixiv_down downloads from newer date to older date.");
      return 1;
   }
   stderr.writeln("Begin = ", (cast(Date)beginDate).toISOExtString());
   stderr.writeln("End   = ", (cast(Date)endDate).toISOExtString());

   downloadDaily(DailyOptions(beginDate, endDate, restrict, force), config);
   saveLastStartTime(beginDate);

   return 0;
}

void displayDailyHelp()
{
   import std.stdio : stderr;

   stderr.writefln(
      "pixiv_down daily - Download new content from followed artists.\n" ~
      "\nUsage:\tpixiv_down daily [options]\n" ~
      "\nThis command will, by default, download all the recently uploaded\n" ~
      "from the people you follow up to (and including) the end date.\n" ~
      "This will also download both the \"safe\" and \"r18\" works.  Use\n" ~
      "the --sfw-only flag to change this.\n" ~
      "\nBoth the BEGIN and END dates must follow the format YYYY-MM-DD,\n" ~
      "for example: 2023-06-20.  The END date is only required for the\n" ~
      "first run of the daily command.  Afterwards, the end date will be\n" ~
      "determined automatically based on the last run's BEGIN date,\n" ~
      "regardless of whether a BEGIN date was provided or not.\n" ~
      "\nOptions:\n" ~
      "   -b, --begin    BEGIN    \tThe date the begin downloading from.\n" ~
      "   -e, --end      END      \tThe date to finish downloading at.\n" ~
      "   -f, --force             \tForce download all artworks (overwrite).\n" ~
      "   -s, --sfw-only          \tOnly download SFW content (not R-18).\n" ~
      "   -h, --help              \tDisplay this help message and exit.\n" ~
      "\nExamples:\n" ~
      "\n  Download all content from the current time until 2023-06-19:\n" ~
      "       pixiv_down daily --end 2023-06-19\n" ~
      "\n  Download all content from the 20th June, 2023:\n" ~
      "       pixiv_down daily --begin 2021-06-20 --end 2021-06-20\n" ~
      "\nI wouldn't recommend downloading all content from a specific day\n" ~
      "which is over a month or two ago, since this command isn't really\n" ~
      "designed for that. It'll require a lot of requests to the pixiv\n" ~
      "server and will take a long while.  You'd be better using either\n" ~
      "the 'artist' or 'following' command.");
}


private:

struct DailyOptions
{
   const SysTime begin;
   const SysTime end;
   const bool sfwOnly;
   const bool force;
}


void downloadDaily(in DailyOptions options, in Config config)
{
   trace("downloadDaily -- begin");
   downloadLatestArtworks(options, config);
   trace("downloadDaily ---- finished artworks");
   sleep(5, 10);
   downloadLatestNovels(options, config);
   trace("downloadDaily ---- finished novels");

   trace("downloadDaily -- end");
}

void downloadLatestArtworks(in DailyOptions options, in Config config)
{
   import std.algorithm.iteration : map;
   import app.util : sleep;

   bool pastEndDate = false;
   int page = 1;

   do {
      page > 1 && sleep(10, 15);
      const ids = fetchFollowLatest(page, config);
      foreach(id; ids) {
         const artwork = fetchArtworkInfo(id, config);
         const createDate = SysTime.fromISOExtString(artwork.createDate);

         // Only download artworks uploaded on or after `begin`.
         if (createDate > options.begin) {
            continue;
         }

         // Only download posts uploaded on or before `end`.
         if (createDate < options.end) {
            pastEndDate = true;
            info("daily downloads has past `endDate`");
            break;
         }

         if (options.sfwOnly && artwork.isR18) {
            continue;
         }

         downloadArtwork(artwork, config, options.force);
         sleep(1, 5);
         Term.goUpAndClearLine(1, Yes.useStderr);
      }

      if (pastEndDate == false) {
         tracef("finished downloading page %d", page);
         page += 1;
         tracef("preparing to download page %d", page);
      }
   } while (pastEndDate == false);
}

void downloadLatestNovels(in DailyOptions options, in Config config)
{
   import std.stdio : stdout;

   int page = 1;
   bool pastEndDate = false;

   do {
      page > 1 && sleep(10, 15);

      const ids = fetchNovelLatest(page, config);
      foreach(id; ids) {
         const novel = fetchNovelInfo(id, config);
         const createDate = SysTime.fromISOExtString(novel.createDate);

         if (createDate > options.begin) {
            continue;
         }

         if (createDate < options.end) {
            pastEndDate = true;
            info("daily downloads has past `endDate`");
            break;
         }

         if (options.sfwOnly && novel.isR18) {
            continue;
         }

         downloadNovel(novel, config, options.force);
      }

      if (!pastEndDate) {
         tracef("finished downloading page %d", page);
         page += 1;
         tracef("preparing to download page %d", page);
      }
   } while (!pastEndDate);
}

Nullable!SysTime fetchLastStartTime() nothrow
{
   import mlib.directories: getProjectDirectories;
   import std.datetime.date: Date;
   import std.path: buildPath;
   import std.stdio: File;

   Nullable!SysTime sysTime;
   auto dirs = getProjectDirectories(null, "YumeNeru Software", "pixiv_down");

   try {
      auto file = File(buildPath(dirs.stateDir, "last_daily_run"), "r");
      auto date = Date.fromISOExtString(file.readln());
      sysTime = SysTime(date);
   } catch (Exception) {
      return sysTime;
   }

   return sysTime;
}

void saveLastStartTime(const ref SysTime startDate)
{
   import mlib.directories: getProjectDirectories;
   import std.datetime.date: Date;
   import std.path: buildPath;
   import std.stdio: File;

   auto dirs = getProjectDirectories(null, "YumeNeru Software", "pixiv_down");

   auto file = File(buildPath(dirs.stateDir, "last_daily_run"), "w+");
   file.writeln((cast(Date)startDate).toISOExtString());
}