blacklist/post_fsd_wiki.phantomjs

/*\ post_fsd_wiki.phantomjs
|*|
|*|   this script is used to create/modify mediawiki pages on the FSD
|*|
|*|   one such wiki page for each blacklisted package
|*|
|*|   it must be executed by the phantomjs program
|*|   and have access to a JSON file as created by the report.rb script
\*/


var BLACKLIST_DATA_FILE   = './blacklist-testdata.json' ;
var WIKI_TEMPLATE_FILE    = './wiki-template.json' ;
var WIKI_BASE_URL         = 'http://localhost/mediawiki/index.php' ;
// var WIKI_BASE_URL  = 'https://directory.fsf.org/wiki?title=' ;
var LOGIN_URL             = WIKI_BASE_URL + '?title=Special:UserLogin' ;
var EDIT_URL              = WIKI_BASE_URL + '?title=_TITLE_&action=edit' ;
var USAGE_MSG             = "USAGE: phantomjs ./post_fsd_wiki.phantomjs <WIKI_LOGIN> <WIKI_PASS>" ;
var TITLE_URL_REGEX       = /_TITLE_/ ;
var PARABOLA_ENTRY_HEADER = '== Parabola Blacklist Description ==\n' ;
var WIKI_TEXT_BEGIN       = '<!-- PARABOLA BLACKLIST BEGIN (DO NOT EDIT) -->' + PARABOLA_ENTRY_HEADER ;
var WIKI_TEXT_END         = '\n<!-- PARABOLA BLACKLIST END -->\n' ;
var WIKI_TEXT_REGEX       = RegExp(WIKI_TEXT_BEGIN.replace(/\((.*)\)/ , '\\($1\\)') + '(.|\n)*' + WIKI_TEXT_END) ;
var LOGIN_IMPUT_ID        = 'wpName1' ;
var PASS_IMPUT_ID         = 'wpPassword1' ;
var LOGIN_SUBMIT_IMPUT_ID = 'wpLoginAttempt' ;
var USERNAME_LI_ID        = 'pt-userpage' ;
var EDIT_TEXT_INPUT_ID    = 'wpTextbox1' ;
var EDIT_SUBMIT_INPUT_ID  = 'wpSave' ;
var WIKI_TITLE_H1_ID      = 'firstHeading' ;
var WIKI_TEXT_DIV_ID      = 'mw-content-text' ;
var CATEGORIES_DIV_ID     = 'catlinks' ;
var PACKAGE_NAME_KEY      = 'original_package' ;  // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[0]
var REPLACEMENT_KEY       = 'libre_replacement' ; // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[1]
var REFERENCE_KEY         = 'ref' ;               // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[2]
var ENTRY_ID_KEY          = 'id' ;                // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[3]
var DESCRIPTION_KEY       = 'short_description' ; // BLACKLIST_DATA_FILE JSON per ./SYNTAX CSV[4]
var BLACKLIST_TAGS_KEY    = 'blacklist_tags'      // BLACKLIST_DATA_FILE JSON
var NONFREE_TAG           = 'nonfree' ;
var SEMIFREE_TAG          = 'semifree' ;
var USES_NONFREE_TAG      = 'uses-nonfree' ;
var BRANDING_TAG          = 'branding' ;
var TECHNICAL_TAG         = 'technical' ;
var HAS_REPLACEMENT_TAG   = 'FIXME:package' ;
var NEEDS_DESC_TAG        = 'FIXME:description' ;
var ACCEPTED_TAGS         = [ NONFREE_TAG         , SEMIFREE_TAG   , USES_NONFREE_TAG ,
                              HAS_REPLACEMENT_TAG , NEEDS_DESC_TAG                    ] ;
var STEP_FUNCTION_KEY     = 'step-function' ;
var PAGELOAD_WAIT_KEY     = 'wait-for-pageload' ;
var STEP_TIMEOUT          = 3000 ; var StepTimeout ;

var System     = require('system') ;
var Args       = System.args ;
var Page       = require('webpage').create() ;
var WIKI_LOGIN = Args[1] || '' ;
var WIKI_PASS  = Args[2] || '' ;
var WikiPages  = [] ; var WikiPage ;
var Steps      = [] ; var Step ;
var StepN      = 0 ;
var IsReady    = true ;
var ShouldWait = false ;
var ShouldQuit = false ;


if (WIKI_LOGIN == '' || WIKI_PASS == '') { LOG(USAGE_MSG) ; phantom.exit() ; } LOG('') ;


/* steps */

function Prepare()
{
  LOG("loading BLACKLIST_DATA_FILE") ; var blacklist_data = require(BLACKLIST_DATA_FILE) ;

  if (!IsA(blacklist_data , Array )) { ForceQuit("failed to load package data") ;  return ; }

  LOG("found " + blacklist_data.length + " packages") ;

  blacklist_data.forEach(function(package_data)
  {
    var package_name   = package_data[PACKAGE_NAME_KEY  ] && package_data[PACKAGE_NAME_KEY  ].trim() ;
    var replacement    = package_data[REPLACEMENT_KEY   ] && package_data[REPLACEMENT_KEY   ].trim() ;
    var description    = package_data[DESCRIPTION_KEY   ] && package_data[DESCRIPTION_KEY   ].trim() ;
    var blacklist_tags = package_data[BLACKLIST_TAGS_KEY] ;

    if ((!IsA(package_name   , String)                     ) ||
        (!IsA(replacement    , String) && replacement != '') ||
        (!IsA(description    , String)                     ) ||
        (!IsA(blacklist_tags , Array )                     )  )
    { LOG_ERR("invalid package data - ignoring:\n" + JSON.stringify(package_data)) ; return ; }

    package_name         = package_name.toLowerCase().replace(/ /g , '_') ;
    var replacement_text = (!!replacement) ? '*Parabola Replacement: ' + replacement + '\n\n' : '' ;
    blacklist_tags       = blacklist_tags.filter(function(tag) { return ~ACCEPTED_TAGS.indexOf(tag) ; })
                                         .map   (function(tag) { return 'FSDG_' + tag                 }) ;
    if (blacklist_tags.length == 0) { LOG(package_name + " tags are uninteresting - ignoring") ; return ; }

    LOG("loading package data: " + package_name) ;

    var wiki_page                 = {} ;
    wiki_page[PACKAGE_NAME_KEY  ] = package_name ;
    wiki_page[REPLACEMENT_KEY   ] = replacement_text
    wiki_page[DESCRIPTION_KEY   ] = description ;
    wiki_page[BLACKLIST_TAGS_KEY] = blacklist_tags ;
    WikiPages.push(wiki_page) ;
  }) ;

  IsReady = true ;
}

function LoadLoginPage()
{
  if (WikiPages.length > 0) { LOG(WikiPages.length + " WikiPages to consider") ; }
  else                      { ForceQuit("no WikiPages to consider") ; return ;   }

  GetUrl(LOGIN_URL) ;
}

function SubmitLogin()
{
  EvalQuitOnErr(Page.evaluate(function(login_input_id , pass_input_id , submit_input_id ,
                                       wiki_login     , wiki_pass     , IsA             )
  {
    var login_input   = document.getElementById(login_input_id ) ;
    var pass_input    = document.getElementById(pass_input_id  ) ;
    var submit_button = document.getElementById(submit_input_id) ;
    var login_form    = submit_button.form ;

    if (!IsA(login_input   , 'INPUT' ) || !IsA(pass_input , 'INPUT') ||
        !IsA(submit_button , 'BUTTON') || !IsA(login_form , 'FORM' )  )
      return "ERROR: invalid login page" ;

    login_input.value = wiki_login ;
    pass_input.value  = wiki_pass ;

    login_form.submit() ;

    return '' ;
  } , LOGIN_IMPUT_ID , PASS_IMPUT_ID , LOGIN_SUBMIT_IMPUT_ID , WIKI_LOGIN , WIKI_PASS , IsA)) ;

  IsReady = true ;
}

function VerifyLogin()
{
  EvalQuitOnErr(Page.evaluate(function(username_li_id , wiki_login , IsA , LOG)
  {
    var username_li = document.getElementById(username_li_id) ;
    var username    = !!username_li && username_li.textContent.toLowerCase() ;

    if (!IsA(username_li , 'LI')           ||
        username != wiki_login.toLowerCase())
      return "ERROR: login failed" ;

    LOG("signed in as: " + username) ; return '' ;
  } , USERNAME_LI_ID , WIKI_LOGIN , IsA , LOG)) ;

  IsReady = true ;
}

function LoadWikiData()
{
  LOG(WikiPages.length + " WikiPages remaining") ;

  if (!!(WikiPage = WikiPages.shift())) IsReady = true ; else Done() ;
}

function LoadEditPage()
{
  var page_title = WikiPage[PACKAGE_NAME_KEY] ;

  GetUrl(EditPageUrl(page_title)) ;
}

function SubmitEdit()
{
  var page_title       = WikiPage[PACKAGE_NAME_KEY  ] ;
  var replacement_text = WikiPage[REPLACEMENT_KEY   ] ;
  var wiki_text        = WikiPage[DESCRIPTION_KEY   ] ;
  var wiki_categories  = WikiPage[BLACKLIST_TAGS_KEY] ;
  wiki_text            = WIKI_TEXT_BEGIN + replacement_text + wiki_text + '\n\n' ;
  wiki_categories.forEach(function(category) { wiki_text += '[[Category:' + category + ']]\n' }) ;
  wiki_text           += WIKI_TEXT_END

  if (Page.url != EditPageUrl(page_title)) { ForceQuit("edit page load failed") ; return ; }

  var post_wiki_text = EvalQuitOnErr(Page.evaluate(function(wiki_text_input_id , wiki_submit_button_id ,
                                                            wiki_text_regex    , new_wiki_text         ,
                                                            IsA                , DBG                   )
  {
    var wiki_text_input    = document.getElementById(wiki_text_input_id   ) ;
    var wiki_submit_button = document.getElementById(wiki_submit_button_id) ;
    var existing_wiki_text = !!wiki_text_input && wiki_text_input.value ;

DBG('SubmitEditEvalIn')(wiki_text_input , wiki_submit_button) ;

    if (!IsA(wiki_text_input    , 'TEXTAREA') ||
        !IsA(wiki_submit_button , 'INPUT'   )  )
      return "ERROR: invalid edit page" ;

    var modified_wiki_text = existing_wiki_text.replace(wiki_text_regex , new_wiki_text) ;
    var concat_wiki_text   = existing_wiki_text + new_wiki_text ;
    var post_wiki_text     = (!wiki_text_regex.test(existing_wiki_text)) ? concat_wiki_text   :
                             (modified_wiki_text != existing_wiki_text ) ? modified_wiki_text : '' ;

DBG('SubmitEditEvalOut')(new_wiki_text , existing_wiki_text , modified_wiki_text , post_wiki_text , wiki_text_regex) ;

    if (!!post_wiki_text) { wiki_text_input.value = post_wiki_text ; wiki_submit_button .click() ; }

    return post_wiki_text ;
  } , EDIT_TEXT_INPUT_ID , EDIT_SUBMIT_INPUT_ID , WIKI_TEXT_REGEX , wiki_text , IsA , DBG)) ;

DBG('SubmitEditOut')(post_wiki_text) ;

  if (post_wiki_text == '') { LOG("wiki text has not changed - skipping") ; NextPage() ; }
}

function VerifyEdit()
{
  var page_title       = WikiPage[PACKAGE_NAME_KEY  ] ;
  var replacement_text = WikiPage[REPLACEMENT_KEY   ] ;
  var wiki_text        = WikiPage[DESCRIPTION_KEY   ] ;
  var wiki_categories  = WikiPage[BLACKLIST_TAGS_KEY] ;
  var expected_url     = WIKI_BASE_URL + '/' + page_title ;
  page_title           = page_title              .replace(/_/g , ' ') ;
  wiki_text            = PARABOLA_ENTRY_HEADER   .replace(/=/g  , '').trim() + '\n'             +
                         replacement_text        .replace(/\*/g , '').trim() + '\n' + wiki_text ;
  var categories       = wiki_categories.join('').replace(/_/g , ' ') ;
  var categories_text  = ((wiki_categories.length > 1) ? 'Categories: ' : 'Category: ') + categories ;

if (DEBUG_VB) DUMPFILE() ;

  if (Page.url.toLowerCase() != expected_url) { ForceQuit("edit post expected_url failed") ; return ; }

  var err = EvalQuitOnErr(Page.evaluate(function(wiki_title_h1_id , wiki_text_div_id , categories_div_id ,
                                                 page_title       , new_wiki_text    , new_categories    ,
                                                 IsA              , DBG                                  )
  {
    var wiki_title_h1       = document.getElementById(wiki_title_h1_id ) ;
    var wiki_text_div       = document.getElementById(wiki_text_div_id ) ;
    var categories_div      = document.getElementById(categories_div_id) ;
    var existing_wiki_title = !!wiki_title_h1  && wiki_title_h1 .textContent.toLowerCase() ;
    var existing_wiki_text  = !!wiki_text_div  && wiki_text_div .textContent.trim() ;
    var existing_categories = !!categories_div && categories_div.textContent.trim() ;
    var wiki_text_regex     = RegExp(new_wiki_text ) ;
    var categories_regex    = RegExp(new_categories) ;

DBG('VerifyEdit')(wiki_title_h1 , wiki_text_div , categories_div , existing_wiki_title , page_title , existing_wiki_text , new_wiki_text , existing_categories , new_categories , wiki_text_regex , categories_regex) ;

    if (!IsA(wiki_title_h1  , 'H1' ) || existing_wiki_title != page_title           ||
        !IsA(wiki_text_div  , 'DIV') || !wiki_text_regex .test(existing_wiki_text ) ||
        !IsA(categories_div , 'DIV') || !categories_regex.test(existing_categories)  )
      return "ERROR: edit post data failed" ;

    return '' ;
  } , WIKI_TITLE_H1_ID , WIKI_TEXT_DIV_ID , CATEGORIES_DIV_ID            ,
      page_title       , wiki_text        , categories_text   , IsA , DBG)) ;

  if (!err) LOG("successfully updated: " + page_title) ; NextPage() ;
}


/* main loop */

function DefineStep(step_function , should_wait_for_pageload)
{
  var step_data                = {} ;
  step_data[STEP_FUNCTION_KEY] = step_function ;
  step_data[PAGELOAD_WAIT_KEY] = should_wait_for_pageload ;
  Steps.push(step_data) ;
}

function MainLoop()
{
DBG('MainLoop')() ;

  if      (!!ShouldQuit) { LOG_ERR(ShouldQuit) ; Done() ; return ; }
  else if (!IsReady    ) { PumpMainLoop() ;               return ; }

  clearTimeout(StepTimeout) ;

  StepTimeout   = setTimeout(function() { ForceQuit("timeout executing step") ; } , STEP_TIMEOUT) ;
  var step_data = Steps[StepN++] ;
  var step      = step_data && step_data[STEP_FUNCTION_KEY] ;
  ShouldWait    = step_data && step_data[PAGELOAD_WAIT_KEY] ;
  IsReady       = false ;

  if (typeof step !== 'function') Done() ;
  else                            { LOG_STEP(step.name) ; step() ; PumpMainLoop() ; }
}

function PumpMainLoop() { setTimeout(MainLoop , 250) ; }

function ForceQuit(quit_msg) { ShouldQuit = quit_msg || 'script error' ; PumpMainLoop() ; }

function Done() { LOG("done") ; phantom.exit() ; LOG('') ; }


/* helpers */

function GetUrl(get_url)
{
DBG('GetUrl')(get_url) ;

  Page.open(get_url , function(status) { if (status != 'success') ForceQuit("status: " + status) ; }) ;
}

function EvalQuitOnErr(result)
{
//   CAVEATS: this function is intended to handle the return value of 'Page.evaluate' callbacks
//            on success, the 'Page.evaluate' callback must return a string not matching /^ERROR:.*/
//            the Step function must set IsReady to true sometime after this function returns
//            unless it is waiting for a page load, in which case, 'Page.onLoadFinished' will flag it

  if (typeof result != 'string' || !result.indexOf('ERROR:')) ForceQuit(result) ;
  else                                                        IsReady = !ShouldWait ;

  return result ;
}

function NextPage() { StepN = NEXT_PAGE_STEP ; IsReady = true ; }

function IsA(an_object , expected_type)
{
  if (!an_object || an_object == [] || an_object == '') return false ;

  switch(expected_type)
  {
    case Array      : return Object.prototype.toString.call(an_object) == '[object Array]' ;  break ;
    case String     : return Object.prototype.toString.call(an_object) == '[object String]' ; break ;
    case 'BUTTON'   :
    case 'DIV'      :
    case 'FORM'     :
    case 'H1'       :
    case 'INPUT'    :
    case 'LI'       :
    case 'TEXTAREA' : return an_object.nodeName == expected_type ;                            break ;
    default         : return false ;                                                          break ;
  } ;
}

function EditPageUrl(page_title) { return EDIT_URL.replace(TITLE_URL_REGEX , page_title) ; }


/* event hendlers */

Page.onLoadStarted  = function() { IsReady = false ; LOG_ARGS.apply("Page.onLoadStarted" , arguments) ;              } ;

Page.onLoadFinished = function() { IsReady = true ;  LOG_ARGS.apply("Page.onLoadFinished " + Page.url , arguments) ; } ;

Page.onUrlChanged   = function() {                   LOG_ARGS.apply("Page.onUrlChanged"  , arguments) ;              } ;


/* logging */

Page.onConsoleMessage = function(msg) { DBG('')(msg) ; } ;

var DEBUG = true ; var DEBUG_EVS = DEBUG && false ; var DEBUG_VB = DEBUG && false ;
function LOG     (log , color) { console.log((color || '\033[00;32m') + log  + '\033[00m'   ) ; }
function LOG_STEP(name)        { LOG("Step: "                         + name , '\033[01;32m') ; }
function LOG_ERR (err)         { LOG("ERROR: "                        + err  , '\033[00;31m') ; }
function LOG_DBG (dbg)         { if (!DEBUG    ) return ;
                                 LOG("DEBUG: "                        + dbg  , '\033[00;34m') ; }
function LOG_ARGS()            { if (!DEBUG_EVS) return ;
                                 LOG("EVENT: " + this) ; var args = arguments ;
                                 for (arg in args) LOG("  arg: " + JSON.stringify(args[arg])) ; }
function DUMPFILE()            { DBG('')("dumping to file: " + Page.url.toLowerCase()) ;
                                 require('fs').write('dump.txt' , Page.content , 'w') ;         }
function DBG(context)
{
  function Dbg(dbg) { console.log('\033[00;34m' + dbg  + '\033[00m') ; }
  function DbgSubmitEditEvalIn(wiki_text_input , wiki_submit_button)
  {
    Dbg("SubmitEdit() wiki_text_input="            + wiki_text_input            ) ;
    Dbg("SubmitEdit() wiki_text_input   .nodeName" + wiki_text_input   .nodeName) ;
    Dbg("SubmitEdit() wiki_submit_button"          + wiki_submit_button         ) ;
    Dbg("SubmitEdit() wiki_submit_button.nodeName" + wiki_submit_button.nodeName) ; }
  function DbgSubmitEditEvalOut(new_wiki_text , existing_wiki_text , modified_wiki_text , post_wiki_text , wiki_text_regex)
  { Dbg("SubmitEdit() new_wiki_text="                              + new_wiki_text                             ) ;
    Dbg("SubmitEdit() existing_wiki_text="                         + existing_wiki_text                        ) ;
    Dbg("SubmitEdit() modified_wiki_text="                         + modified_wiki_text                        ) ;
    Dbg("SubmitEdit() post_wiki_text="                             + post_wiki_text                            ) ;
    Dbg("SubmitEdit() (modified_wiki_text == existing_wiki_text)=" + (modified_wiki_text == existing_wiki_text)) ;
    Dbg("SubmitEdit() wiki_text_regex.test(existing_wiki_text  )=" + wiki_text_regex.test(existing_wiki_text)  ) ; }
  function DbgSubmitEditOut(post_wiki_text) { if (DEBUG_VB) Dbg("SubmitEdit() returned post_wiki_text=" + post_wiki_text) ; }
  function DbgVerifyEdit(wiki_title_h1 , wiki_text_div , categories_div , existing_wiki_title , page_title , existing_wiki_text , new_wiki_text , existing_categories , new_categories , wiki_text_regex , categories_regex)
  { Dbg("VerifyEdit() wiki_title_h1="                              + wiki_title_h1  + " wiki_title_h1 .nodeName=" + wiki_title_h1.nodeName ) ;
    Dbg("VerifyEdit() wiki_text_div="                              + wiki_text_div  + " wiki_text_div .nodeName=" + wiki_text_div.nodeName ) ;
    Dbg("VerifyEdit() categories_div="                             + categories_div + " categories_div.nodeName=" + categories_div.nodeName) ;
    Dbg("VerifyEdit() existing_wiki_title="                        + existing_wiki_title                                                   ) ;
    Dbg("VerifyEdit() page_title="                                 + page_title                                                            ) ;
    Dbg("VerifyEdit() existing_wiki_text="                         + existing_wiki_text                                                    ) ;
    Dbg("VerifyEdit() new_wiki_text="                              + new_wiki_text                                                         ) ;
    Dbg("VerifyEdit() existing_categories="                        + existing_categories                                                   ) ;
    Dbg("VerifyEdit() new_categories="                             + new_categories                                                        ) ;
    Dbg("VerifyEdit() (existing_wiki_title == page_title       )=" + (existing_wiki_title == page_title)                                   ) ;
    Dbg("VerifyEdit() wiki_text_regex .test(existing_wiki_text )=" + wiki_text_regex .test(existing_wiki_text)                             ) ;
    Dbg("VerifyEdit() categories_regex.test(existing_categories)=" + categories_regex.test(existing_categories)                            ) ; }
  function DbgMainLoop() { if (DEBUG_VB) Dbg("MainLoop() ShouldQuit=" + ShouldQuit + " IsReady=" + IsReady) ; }
  function DbgGetUrl(get_url) { Dbg("GetUrl() " + get_url) ; }

  return eval('Dbg' + context) ;
} if (!DEBUG) DBG = function(context) { return function(){} } ;


/* main entry */

DefineStep(Prepare       , false) ;
DefineStep(LoadLoginPage , true ) ;
DefineStep(SubmitLogin   , true ) ;
DefineStep(VerifyLogin   , false) ;
DefineStep(LoadWikiData  , false) ;
DefineStep(LoadEditPage  , true ) ;
DefineStep(SubmitEdit    , true ) ;
DefineStep(VerifyEdit    , false) ;
for (step_n in Steps) if (Steps[step_n][STEP_FUNCTION_KEY].name == 'LoadWikiData') NEXT_PAGE_STEP = step_n ;

MainLoop() ;