luce/tools/crash_test.cr

# SPDX-FileCopyrightText: 2024 mio <stigma@disroot.org>
# SPDX-License-Identifier: BSD-3-Clause

# A rather slow (and not necessarily correct*) method of testing luce.
# It retrieves a list of shards from shardbox.org (by category), and
# attempts to parse any markdown file found in the shard's repository.
#
# After each test there is a delay (SHARD_TEST_DELAY_SECONDS), and
# once a category has been complete there is another delay
# (CATEGORY_TEST_DELAY_SECONDS).
#
# In its current incarnation, this script may take a few hours to
# complete.
#
# * This only checks whether luce raises an exception, not if the HTML
#   generated is correct.
require "file_utils"
require "http/client"
require "yaml"

require "../src/luce.cr"

SHARD_TEST_DELAY_SECONDS    =  8
CATEGORY_TEST_DELAY_SECONDS = 16

enum TestResultType
  Success
  Skipped
  Failed
end

record TestResult, type : TestResultType, message : String

# tip to update:
#    git clone https://github.com/shardbox/catalog --depth 1
#    cd catalog/catalog
#    find -printf "%f\n" | sort | cut -d'.' -f1
SHARDBOX_CATEGORIES = [
  "Algorithms_and_Data_structures",
  "Api_Builders",
  "Audio",
  "Blockchain",
  "Caching",
  "C_bindings",
  "CLI_Applications",
  "CLI_Builders",
  "CLI_Utils",
  "Code_Analysis_and_Metrics",
  "Compression",
  "Concurrency",
  "Configuration",
  "Converters",
  "Cryptography",
  "Database_Drivers_Clients",
  "Database_Tools",
  "Data_Formats",
  "Data_Generators",
  "Debugging",
  "Dependency_Injection",
  "Development_Tools",
  "Email",
  "Environment_Management",
  "Examples_and_funny_stuff",
  "Feature_Flipping",
  "Framework_Components",
  "Game_Development",
  "GUI_Development",
  "GUI_library",
  "HTML_Builders",
  "HTML_XML_Parsing",
  "HTTP",
  "Image_processing",
  "Implementations_Compilers",
  "Internationalization",
  "Logging_and_monitoring",
  "Machine_Learning",
  "Markdown_Text_Processors",
  "Misc",
  "Natural_Language",
  "Networking",
  "Network_Protocols",
  "ORM_ODM_Extensions",
  "Package_Management",
  "Parser_Generators",
  "Processes_and_Threads",
  "Project_Generators",
  "Queues_and_Messaging",
  "Routing",
  "Scheduling",
  "Science_and_Data_analysis",
  "Search",
  "Serverless_Computing",
  "System",
  "Task_management",
  "Template_Engine",
  "Testing",
  "Third-party_APIs",
  "Time_Date",
  "Uncategorized",
  "Validation",
  "Web_Applications",
  "Web_Frameworks",
  "Web_Servers",
]

def fetch_category_yaml(category : String) : YAML::Any?
  puts "Downloading #{category} YAML file..."
  yaml_file_res = HTTP::Client.get "https://raw.githubusercontent.com/shardbox/catalog/master/catalog/#{category}.yml"
  unless yaml_file_res.success?
    STDERR.puts "Failed to download YAML file for category: #{category}"
    STDERR.puts "#{yaml_file_res.status} (#{yaml_file_res.status_code}): #{yaml_file_res.status_message}"
    return nil
  end
  YAML.parse(yaml_file_res.body)
end

def test_shard(shard : YAML::Any) : TestResult
  url = if shard["github"]?
          "https://github.com/#{shard["github"]}"
        elsif shard["gitlab"]?
          "https://gitlab.com/#{shard["gitlab"]}"
        elsif shard["bitbucket"]?
          "https://bitbucket.org/#{shard["bitbucket"]}"
        elsif shard["git"]?
          "#{shard["git"]}"
        else
          raise "Unsupported shard host: #{shard}"
        end
  HTTP::Client.head(url) do |response|
    return TestResult.new(TestResultType::Skipped, "404: #{url}") unless response.success?
  end
  dir = File.basename(url)
  # TODO: Get terminal width and truncate url
  puts "    Cloning git repository: #{url}"
  Process.run("git", ["clone", url, "--depth=1", dir])

  done = Channel(Nil).new

  Dir.glob("#{dir}/**/*.md").each do |md|
    spawn do
      contents = File.read(md)
      Luce.to_html(contents, extension_set: Luce::ExtensionSet::GITHUB_WEB)
      done.send(nil)
    end

    select
    when res = done.receive
      break
    when timeout(2.minutes)
      return TestResult.new(TestResultType::Failed, "#{shard}\n      Reason: Timed out")
    end
  rescue ex
    return TestResult.new(TestResultType::Failed, "#{shard}\n      Reason:    #{ex}")
  end
  sleep SHARD_TEST_DELAY_SECONDS
  TestResult.new(TestResultType::Success, "")
ensure
  FileUtils.rm_rf(dir) unless dir.nil?
end

def main(args = ARGV)
  errors = [] of String
  skipped = [] of String
  scanned_shards = 0
  start = Time.monotonic

  Process.on_interrupt do
    print "\r"

    puts "## Finished scanning"
    puts "Scanned #{scanned_shards} shards in #{Time.monotonic - start}"
    unless skipped.empty?
      puts "Skipped shards:"
      skipped.each { |skip| puts "    #{skip}" }
      puts "Found #{skipped.size} case(s) where we couldn't test Luce.to_html"
    end
    unless errors.empty?
      puts "Found issues:"
      errors.each { |error| puts "    #{error}" }
      puts "Found #{errors.size} case(s) where Luce.to_html raised an error!"
    end
    abort
  end

  SHARDBOX_CATEGORIES.each do |category|
    yaml = fetch_category_yaml(category)
    next if yaml.nil?
    puts "\rTesting all shards from #{yaml["name"]}..."
    yaml["shards"].as_a.each do |shard|
      result = test_shard shard
      case result.type
      when TestResultType::Failed
        errors << result.message
      when TestResultType::Skipped
        skipped << result.message
      end
      scanned_shards += 1
    end
    sleep CATEGORY_TEST_DELAY_SECONDS
  end

  puts "## Finished scanning"
  puts "Scanned #{scanned_shards} shards in #{Time.monotonic - start}"
  unless skipped.empty?
    puts "Skipped shards:"
    skipped.each { |skip| puts "    #{skip}" }
    puts "Found #{skipped.size} case(s) where we couldn't test Luce.to_html"
  end
  unless errors.empty?
    puts "Found issues:"
    errors.each { |error| puts "    #{error}" }
    puts "Found #{errors.size} case(s) where Luce.to_html raised an error!"
  end
end

begin
  Process.run("git")
rescue
  STDERR.puts "`git` executable not available. See https://git-scm.com"
  exit 1
end

main