Lexer.php 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383
  1. <?php
  2. /**
  3. * Forgivingly lexes HTML (SGML-style) markup into tokens.
  4. *
  5. * A lexer parses a string of SGML-style markup and converts them into
  6. * corresponding tokens. It doesn't check for well-formedness, although its
  7. * internal mechanism may make this automatic (such as the case of
  8. * HTMLPurifier_Lexer_DOMLex). There are several implementations to choose
  9. * from.
  10. *
  11. * A lexer is HTML-oriented: it might work with XML, but it's not
  12. * recommended, as we adhere to a subset of the specification for optimization
  13. * reasons. This might change in the future. Also, most tokenizers are not
  14. * expected to handle DTDs or PIs.
  15. *
  16. * This class should not be directly instantiated, but you may use create() to
  17. * retrieve a default copy of the lexer. Being a supertype, this class
  18. * does not actually define any implementation, but offers commonly used
  19. * convenience functions for subclasses.
  20. *
  21. * @note The unit tests will instantiate this class for testing purposes, as
  22. * many of the utility functions require a class to be instantiated.
  23. * This means that, even though this class is not runnable, it will
  24. * not be declared abstract.
  25. *
  26. * @par
  27. *
  28. * @note
  29. * We use tokens rather than create a DOM representation because DOM would:
  30. *
  31. * @par
  32. * -# Require more processing and memory to create,
  33. * -# Is not streamable, and
  34. * -# Has the entire document structure (html and body not needed).
  35. *
  36. * @par
  37. * However, DOM is helpful in that it makes it easy to move around nodes
  38. * without a lot of lookaheads to see when a tag is closed. This is a
  39. * limitation of the token system and some workarounds would be nice.
  40. */
  41. class HTMLPurifier_Lexer
  42. {
  43. /**
  44. * Whether or not this lexer implements line-number/column-number tracking.
  45. * If it does, set to true.
  46. */
  47. public $tracksLineNumbers = false;
  48. // -- STATIC ----------------------------------------------------------
  49. /**
  50. * Retrieves or sets the default Lexer as a Prototype Factory.
  51. *
  52. * By default HTMLPurifier_Lexer_DOMLex will be returned. There are
  53. * a few exceptions involving special features that only DirectLex
  54. * implements.
  55. *
  56. * @note The behavior of this class has changed, rather than accepting
  57. * a prototype object, it now accepts a configuration object.
  58. * To specify your own prototype, set %Core.LexerImpl to it.
  59. * This change in behavior de-singletonizes the lexer object.
  60. *
  61. * @param HTMLPurifier_Config $config
  62. * @return HTMLPurifier_Lexer
  63. * @throws HTMLPurifier_Exception
  64. */
  65. public static function create($config)
  66. {
  67. if (!($config instanceof HTMLPurifier_Config)) {
  68. $lexer = $config;
  69. trigger_error(
  70. "Passing a prototype to
  71. HTMLPurifier_Lexer::create() is deprecated, please instead
  72. use %Core.LexerImpl",
  73. E_USER_WARNING
  74. );
  75. } else {
  76. $lexer = $config->get('Core.LexerImpl');
  77. }
  78. $needs_tracking =
  79. $config->get('Core.MaintainLineNumbers') ||
  80. $config->get('Core.CollectErrors');
  81. $inst = null;
  82. if (is_object($lexer)) {
  83. $inst = $lexer;
  84. } else {
  85. if (is_null($lexer)) {
  86. do {
  87. // auto-detection algorithm
  88. if ($needs_tracking) {
  89. $lexer = 'DirectLex';
  90. break;
  91. }
  92. if (class_exists('DOMDocument', false) &&
  93. method_exists('DOMDocument', 'loadHTML') &&
  94. !extension_loaded('domxml')
  95. ) {
  96. // check for DOM support, because while it's part of the
  97. // core, it can be disabled compile time. Also, the PECL
  98. // domxml extension overrides the default DOM, and is evil
  99. // and nasty and we shan't bother to support it
  100. $lexer = 'DOMLex';
  101. } else {
  102. $lexer = 'DirectLex';
  103. }
  104. } while (0);
  105. } // do..while so we can break
  106. // instantiate recognized string names
  107. switch ($lexer) {
  108. case 'DOMLex':
  109. $inst = new HTMLPurifier_Lexer_DOMLex();
  110. break;
  111. case 'DirectLex':
  112. $inst = new HTMLPurifier_Lexer_DirectLex();
  113. break;
  114. case 'PH5P':
  115. $inst = new HTMLPurifier_Lexer_PH5P();
  116. break;
  117. default:
  118. throw new HTMLPurifier_Exception(
  119. "Cannot instantiate unrecognized Lexer type " .
  120. htmlspecialchars($lexer)
  121. );
  122. }
  123. }
  124. if (!$inst) {
  125. throw new HTMLPurifier_Exception('No lexer was instantiated');
  126. }
  127. // once PHP DOM implements native line numbers, or we
  128. // hack out something using XSLT, remove this stipulation
  129. if ($needs_tracking && !$inst->tracksLineNumbers) {
  130. throw new HTMLPurifier_Exception(
  131. 'Cannot use lexer that does not support line numbers with ' .
  132. 'Core.MaintainLineNumbers or Core.CollectErrors (use DirectLex instead)'
  133. );
  134. }
  135. return $inst;
  136. }
  137. // -- CONVENIENCE MEMBERS ---------------------------------------------
  138. public function __construct()
  139. {
  140. $this->_entity_parser = new HTMLPurifier_EntityParser();
  141. }
  142. /**
  143. * Most common entity to raw value conversion table for special entities.
  144. * @type array
  145. */
  146. protected $_special_entity2str =
  147. array(
  148. '&quot;' => '"',
  149. '&amp;' => '&',
  150. '&lt;' => '<',
  151. '&gt;' => '>',
  152. '&#39;' => "'",
  153. '&#039;' => "'",
  154. '&#x27;' => "'"
  155. );
  156. public function parseText($string, $config) {
  157. return $this->parseData($string, false, $config);
  158. }
  159. public function parseAttr($string, $config) {
  160. return $this->parseData($string, true, $config);
  161. }
  162. /**
  163. * Parses special entities into the proper characters.
  164. *
  165. * This string will translate escaped versions of the special characters
  166. * into the correct ones.
  167. *
  168. * @param string $string String character data to be parsed.
  169. * @return string Parsed character data.
  170. */
  171. public function parseData($string, $is_attr, $config)
  172. {
  173. // following functions require at least one character
  174. if ($string === '') {
  175. return '';
  176. }
  177. // subtracts amps that cannot possibly be escaped
  178. $num_amp = substr_count($string, '&') - substr_count($string, '& ') -
  179. ($string[strlen($string) - 1] === '&' ? 1 : 0);
  180. if (!$num_amp) {
  181. return $string;
  182. } // abort if no entities
  183. $num_esc_amp = substr_count($string, '&amp;');
  184. $string = strtr($string, $this->_special_entity2str);
  185. // code duplication for sake of optimization, see above
  186. $num_amp_2 = substr_count($string, '&') - substr_count($string, '& ') -
  187. ($string[strlen($string) - 1] === '&' ? 1 : 0);
  188. if ($num_amp_2 <= $num_esc_amp) {
  189. return $string;
  190. }
  191. // hmm... now we have some uncommon entities. Use the callback.
  192. if ($config->get('Core.LegacyEntityDecoder')) {
  193. $string = $this->_entity_parser->substituteSpecialEntities($string);
  194. } else {
  195. if ($is_attr) {
  196. $string = $this->_entity_parser->substituteAttrEntities($string);
  197. } else {
  198. $string = $this->_entity_parser->substituteTextEntities($string);
  199. }
  200. }
  201. return $string;
  202. }
  203. /**
  204. * Lexes an HTML string into tokens.
  205. * @param $string String HTML.
  206. * @param HTMLPurifier_Config $config
  207. * @param HTMLPurifier_Context $context
  208. * @return HTMLPurifier_Token[] array representation of HTML.
  209. */
  210. public function tokenizeHTML($string, $config, $context)
  211. {
  212. trigger_error('Call to abstract class', E_USER_ERROR);
  213. }
  214. /**
  215. * Translates CDATA sections into regular sections (through escaping).
  216. * @param string $string HTML string to process.
  217. * @return string HTML with CDATA sections escaped.
  218. */
  219. protected static function escapeCDATA($string)
  220. {
  221. return preg_replace_callback(
  222. '/<!\[CDATA\[(.+?)\]\]>/s',
  223. array('HTMLPurifier_Lexer', 'CDATACallback'),
  224. $string
  225. );
  226. }
  227. /**
  228. * Special CDATA case that is especially convoluted for <script>
  229. * @param string $string HTML string to process.
  230. * @return string HTML with CDATA sections escaped.
  231. */
  232. protected static function escapeCommentedCDATA($string)
  233. {
  234. return preg_replace_callback(
  235. '#<!--//--><!\[CDATA\[//><!--(.+?)//--><!\]\]>#s',
  236. array('HTMLPurifier_Lexer', 'CDATACallback'),
  237. $string
  238. );
  239. }
  240. /**
  241. * Special Internet Explorer conditional comments should be removed.
  242. * @param string $string HTML string to process.
  243. * @return string HTML with conditional comments removed.
  244. */
  245. protected static function removeIEConditional($string)
  246. {
  247. return preg_replace(
  248. '#<!--\[if [^>]+\]>.*?<!\[endif\]-->#si', // probably should generalize for all strings
  249. '',
  250. $string
  251. );
  252. }
  253. /**
  254. * Callback function for escapeCDATA() that does the work.
  255. *
  256. * @warning Though this is public in order to let the callback happen,
  257. * calling it directly is not recommended.
  258. * @param array $matches PCRE matches array, with index 0 the entire match
  259. * and 1 the inside of the CDATA section.
  260. * @return string Escaped internals of the CDATA section.
  261. */
  262. protected static function CDATACallback($matches)
  263. {
  264. // not exactly sure why the character set is needed, but whatever
  265. return htmlspecialchars($matches[1], ENT_COMPAT, 'UTF-8');
  266. }
  267. /**
  268. * Takes a piece of HTML and normalizes it by converting entities, fixing
  269. * encoding, extracting bits, and other good stuff.
  270. * @param string $html HTML.
  271. * @param HTMLPurifier_Config $config
  272. * @param HTMLPurifier_Context $context
  273. * @return string
  274. * @todo Consider making protected
  275. */
  276. public function normalize($html, $config, $context)
  277. {
  278. // normalize newlines to \n
  279. if ($config->get('Core.NormalizeNewlines')) {
  280. $html = str_replace("\r\n", "\n", $html);
  281. $html = str_replace("\r", "\n", $html);
  282. }
  283. if ($config->get('HTML.Trusted')) {
  284. // escape convoluted CDATA
  285. $html = $this->escapeCommentedCDATA($html);
  286. }
  287. // escape CDATA
  288. $html = $this->escapeCDATA($html);
  289. $html = $this->removeIEConditional($html);
  290. // extract body from document if applicable
  291. if ($config->get('Core.ConvertDocumentToFragment')) {
  292. $e = false;
  293. if ($config->get('Core.CollectErrors')) {
  294. $e =& $context->get('ErrorCollector');
  295. }
  296. $new_html = $this->extractBody($html);
  297. if ($e && $new_html != $html) {
  298. $e->send(E_WARNING, 'Lexer: Extracted body');
  299. }
  300. $html = $new_html;
  301. }
  302. // expand entities that aren't the big five
  303. if ($config->get('Core.LegacyEntityDecoder')) {
  304. $html = $this->_entity_parser->substituteNonSpecialEntities($html);
  305. }
  306. // clean into wellformed UTF-8 string for an SGML context: this has
  307. // to be done after entity expansion because the entities sometimes
  308. // represent non-SGML characters (horror, horror!)
  309. $html = HTMLPurifier_Encoder::cleanUTF8($html);
  310. // if processing instructions are to removed, remove them now
  311. if ($config->get('Core.RemoveProcessingInstructions')) {
  312. $html = preg_replace('#<\?.+?\?>#s', '', $html);
  313. }
  314. $hidden_elements = $config->get('Core.HiddenElements');
  315. if ($config->get('Core.AggressivelyRemoveScript') &&
  316. !($config->get('HTML.Trusted') || !$config->get('Core.RemoveScriptContents')
  317. || empty($hidden_elements["script"]))) {
  318. $html = preg_replace('#<script[^>]*>.*?</script>#i', '', $html);
  319. }
  320. return $html;
  321. }
  322. /**
  323. * Takes a string of HTML (fragment or document) and returns the content
  324. * @todo Consider making protected
  325. */
  326. public function extractBody($html)
  327. {
  328. $matches = array();
  329. $result = preg_match('|(.*?)<body[^>]*>(.*)</body>|is', $html, $matches);
  330. if ($result) {
  331. // Make sure it's not in a comment
  332. $comment_start = strrpos($matches[1], '<!--');
  333. $comment_end = strrpos($matches[1], '-->');
  334. if ($comment_start === false ||
  335. ($comment_end !== false && $comment_end > $comment_start)) {
  336. return $matches[2];
  337. }
  338. }
  339. return $html;
  340. }
  341. }
  342. // vim: et sw=4 sts=4