httpclient.php 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421
  1. <?php
  2. /**
  3. * StatusNet, the distributed open-source microblogging tool
  4. *
  5. * Utility for doing HTTP-related things
  6. *
  7. * PHP version 5
  8. *
  9. * LICENCE: This program is free software: you can redistribute it and/or modify
  10. * it under the terms of the GNU Affero General Public License as published by
  11. * the Free Software Foundation, either version 3 of the License, or
  12. * (at your option) any later version.
  13. *
  14. * This program is distributed in the hope that it will be useful,
  15. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17. * GNU Affero General Public License for more details.
  18. *
  19. * You should have received a copy of the GNU Affero General Public License
  20. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  21. *
  22. * @category Action
  23. * @package StatusNet
  24. * @author Evan Prodromou <evan@status.net>
  25. * @copyright 2009 StatusNet, Inc.
  26. * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
  27. * @link http://status.net/
  28. */
  29. if (!defined('GNUSOCIAL')) { exit(1); }
  30. /**
  31. * Useful structure for HTTP responses
  32. *
  33. * We make HTTP calls in several places, and we have several different
  34. * ways of doing them. This class hides the specifics of what underlying
  35. * library (curl or PHP-HTTP or whatever) that's used.
  36. *
  37. * This extends the HTTP_Request2_Response class with methods to get info
  38. * about any followed redirects.
  39. *
  40. * Originally used the name 'HTTPResponse' to match earlier code, but
  41. * this conflicts with a class in in the PECL HTTP extension.
  42. *
  43. * @category HTTP
  44. * @package StatusNet
  45. * @author Evan Prodromou <evan@status.net>
  46. * @author Brion Vibber <brion@status.net>
  47. * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
  48. * @link http://status.net/
  49. */
  50. class GNUsocial_HTTPResponse extends HTTP_Request2_Response
  51. {
  52. function __construct(HTTP_Request2_Response $response, $url, $redirects=0)
  53. {
  54. foreach (get_object_vars($response) as $key => $val) {
  55. $this->$key = $val;
  56. }
  57. $this->url = strval($url);
  58. $this->redirectCount = intval($redirects);
  59. }
  60. /**
  61. * Get the count of redirects that have been followed, if any.
  62. * @return int
  63. */
  64. function getRedirectCount()
  65. {
  66. return $this->redirectCount;
  67. }
  68. /**
  69. * Gets the target URL, before any redirects. Use getEffectiveUrl() for final target.
  70. * @return string URL
  71. */
  72. function getUrl()
  73. {
  74. return $this->url;
  75. }
  76. /**
  77. * Check if the response is OK, generally a 200 or other 2xx status code.
  78. * @return bool
  79. */
  80. function isOk()
  81. {
  82. $status = $this->getStatus();
  83. return ($status >= 200 && $status < 300);
  84. }
  85. }
  86. /**
  87. * Utility class for doing HTTP client stuff
  88. *
  89. * We make HTTP calls in several places, and we have several different
  90. * ways of doing them. This class hides the specifics of what underlying
  91. * library (curl or PHP-HTTP or whatever) that's used.
  92. *
  93. * This extends the PEAR HTTP_Request2 package:
  94. * - sends StatusNet-specific User-Agent header
  95. * - 'follow_redirects' config option, defaulting on
  96. * - 'max_redirs' config option, defaulting to 10
  97. * - extended response class adds getRedirectCount() and getUrl() methods
  98. * - get() and post() convenience functions return body content directly
  99. *
  100. * @category HTTP
  101. * @package StatusNet
  102. * @author Evan Prodromou <evan@status.net>
  103. * @author Brion Vibber <brion@status.net>
  104. * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
  105. * @link http://status.net/
  106. */
  107. class HTTPClient extends HTTP_Request2
  108. {
  109. function __construct($url=null, $method=self::METHOD_GET, $config=array())
  110. {
  111. if (is_int(common_config('http', 'timeout'))) {
  112. // Reasonably you shouldn't set http/timeout to 0 because of
  113. // malicious remote servers that can cause infinitely long
  114. // responses... But the default in HTTP_Request2 is 0 for
  115. // some reason and should probably be considered a valid value.
  116. $this->config['timeout'] = common_config('http', 'timeout');
  117. } else {
  118. common_log(LOG_ERR, 'config option http/timeout is not an integer value: '._ve(common_config('http', 'timeout')));
  119. }
  120. if (!empty(common_config('http', 'connect_timeout'))) {
  121. $this->config['connect_timeout'] = common_config('http', 'connect_timeout');
  122. }
  123. $this->config['max_redirs'] = 10;
  124. $this->config['follow_redirects'] = true;
  125. // We've had some issues with keepalive breaking with
  126. // HEAD requests, such as to youtube which seems to be
  127. // emitting chunked encoding info for an empty body
  128. // instead of not emitting anything. This may be a
  129. // bug on YouTube's end, but the upstream libray
  130. // ought to be investigated to see if we can handle
  131. // it gracefully in that case as well.
  132. $this->config['protocol_version'] = '1.0';
  133. // Default state of OpenSSL seems to have no trusted
  134. // SSL certificate authorities, which breaks hostname
  135. // verification and means we have a hard time communicating
  136. // with other sites' HTTPS interfaces.
  137. //
  138. // Turn off verification unless we've configured a CA bundle.
  139. if (common_config('http', 'ssl_cafile')) {
  140. $this->config['ssl_cafile'] = common_config('http', 'ssl_cafile');
  141. } else {
  142. $this->config['ssl_verify_peer'] = false;
  143. }
  144. // This means "verify the cert hostname against what we connect to", it does not
  145. // imply CA trust or anything like that. Just the hostname.
  146. $this->config['ssl_verify_host'] = common_config('http', 'ssl_verify_host');
  147. if (common_config('http', 'curl') && extension_loaded('curl')) {
  148. $this->config['adapter'] = 'HTTP_Request2_Adapter_Curl';
  149. }
  150. foreach (array('host', 'port', 'user', 'password', 'auth_scheme') as $cf) {
  151. $k = 'proxy_'.$cf;
  152. $v = common_config('http', $k);
  153. if (!empty($v)) {
  154. $this->config[$k] = $v;
  155. }
  156. }
  157. parent::__construct($url, $method, $config);
  158. $this->setHeader('User-Agent', self::userAgent());
  159. }
  160. /**
  161. * Convenience/back-compat instantiator
  162. * @return HTTPClient
  163. */
  164. public static function start()
  165. {
  166. return new HTTPClient();
  167. }
  168. /**
  169. * Quick static function to GET a URL
  170. */
  171. public static function quickGet($url, $accept=null, array $params=array(), array $headers=array())
  172. {
  173. if (!empty($params)) {
  174. $params = http_build_query($params, null, '&');
  175. if (strpos($url, '?') === false) {
  176. $url .= '?' . $params;
  177. } else {
  178. $url .= '&' . $params;
  179. }
  180. }
  181. $client = new HTTPClient();
  182. if (!is_null($accept)) {
  183. $client->setHeader('Accept', $accept);
  184. }
  185. $response = $client->get($url, $headers);
  186. if (!$response->isOk()) {
  187. // TRANS: Exception. %s is the URL we tried to GET.
  188. throw new Exception(sprintf(_m('Could not GET URL %s.'), $url), $response->getStatus());
  189. }
  190. return $response->getBody();
  191. }
  192. public static function quickGetJson($url, $params=array())
  193. {
  194. $data = json_decode(self::quickGet($url, null, $params));
  195. if (is_null($data)) {
  196. common_debug('Could not decode JSON data from URL: '.$url);
  197. throw new ServerException('Could not decode JSON data from URL');
  198. }
  199. return $data;
  200. }
  201. /**
  202. * If you want an Accept header, put it in $headers
  203. */
  204. public static function quickHead($url, array $params=array(), array $headers=array())
  205. {
  206. if (!empty($params)) {
  207. $params = http_build_query($params, null, '&');
  208. if (strpos($url, '?') === false) {
  209. $url .= '?' . $params;
  210. } else {
  211. $url .= '&' . $params;
  212. }
  213. }
  214. $client = new HTTPClient();
  215. $response = $client->head($url, $headers);
  216. if (!$response->isOk()) {
  217. // TRANS: Exception. %s is the URL we tried to GET.
  218. throw new Exception(sprintf(_m('Could not GET URL %s.'), $url), $response->getStatus());
  219. }
  220. return $response->getHeader();
  221. }
  222. /**
  223. * Convenience function to run a GET request.
  224. *
  225. * @return GNUsocial_HTTPResponse
  226. * @throws HTTP_Request2_Exception
  227. */
  228. public function get($url, $headers=array())
  229. {
  230. return $this->doRequest($url, self::METHOD_GET, $headers);
  231. }
  232. /**
  233. * Convenience function to run a HEAD request.
  234. *
  235. * NOTE: Will probably turn into a GET request if you let it follow redirects!
  236. * That option is only there to be flexible and may be removed in the future!
  237. *
  238. * @return GNUsocial_HTTPResponse
  239. * @throws HTTP_Request2_Exception
  240. */
  241. public function head($url, $headers=array(), $follow_redirects=false)
  242. {
  243. // Save the configured value for follow_redirects
  244. $old_follow = $this->config['follow_redirects'];
  245. try {
  246. // Temporarily (possibly) override the follow_redirects setting
  247. $this->config['follow_redirects'] = $follow_redirects;
  248. return $this->doRequest($url, self::METHOD_HEAD, $headers);
  249. } catch (Exception $e) {
  250. // Let the exception go on its merry way.
  251. throw $e;
  252. } finally {
  253. // reset to the old value
  254. $this->config['follow_redirects'] = $old_follow;
  255. }
  256. //we've either returned or thrown exception here
  257. }
  258. /**
  259. * Convenience function to POST form data.
  260. *
  261. * @param string $url
  262. * @param array $headers optional associative array of HTTP headers
  263. * @param array $data optional associative array or blob of form data to submit
  264. * @return GNUsocial_HTTPResponse
  265. * @throws HTTP_Request2_Exception
  266. */
  267. public function post($url, $headers=array(), $data=array())
  268. {
  269. if ($data) {
  270. $this->addPostParameter($data);
  271. }
  272. return $this->doRequest($url, self::METHOD_POST, $headers);
  273. }
  274. /**
  275. * @param string $url The URL including possible querystring
  276. * @param string $method The HTTP method to use
  277. * @param array $headers List of already formatted strings
  278. * (not an associative array, to allow
  279. * multiple same-named headers)
  280. *
  281. * @return GNUsocial_HTTPResponse
  282. * @throws HTTP_Request2_Exception
  283. */
  284. protected function doRequest($url, $method, array $headers=array())
  285. {
  286. $this->setUrl($url);
  287. // Workaround for HTTP_Request2 not setting up SNI in socket contexts;
  288. // This fixes cert validation for SSL virtual hosts using SNI.
  289. // Requires PHP 5.3.2 or later and OpenSSL with SNI support.
  290. if ($this->url->getScheme() == 'https' && defined('OPENSSL_TLSEXT_SERVER_NAME')) {
  291. $this->config['ssl_SNI_enabled'] = true;
  292. $this->config['ssl_SNI_server_name'] = $this->url->getHost();
  293. }
  294. $this->setMethod($method);
  295. foreach ($headers as $header) {
  296. $this->setHeader($header);
  297. }
  298. $response = $this->send();
  299. if (is_null($response)) {
  300. // TRANS: Failed to retrieve a remote web resource, %s is the target URL.
  301. throw new NoHttpResponseException($url);
  302. }
  303. return $response;
  304. }
  305. protected function log($level, $detail) {
  306. $method = $this->getMethod();
  307. $url = $this->getUrl();
  308. common_log($level, __CLASS__ . ": HTTP $method $url - $detail");
  309. }
  310. /**
  311. * Pulls up GNU Social's customized user-agent string, so services
  312. * we hit can track down the responsible software.
  313. *
  314. * @return string
  315. */
  316. static public function userAgent()
  317. {
  318. return GNUSOCIAL_ENGINE . '/' . GNUSOCIAL_VERSION
  319. . ' (' . GNUSOCIAL_CODENAME . ')';
  320. }
  321. /**
  322. * Actually performs the HTTP request and returns a
  323. * GNUsocial_HTTPResponse object with response body and header info.
  324. *
  325. * Wraps around parent send() to add logging and redirection processing.
  326. *
  327. * @return GNUsocial_HTTPResponse
  328. * @throw HTTP_Request2_Exception
  329. */
  330. public function send()
  331. {
  332. $maxRedirs = intval($this->config['max_redirs']);
  333. if (empty($this->config['max_redirs'])) {
  334. $maxRedirs = 0;
  335. }
  336. $redirs = 0;
  337. $redirUrls = array();
  338. do {
  339. try {
  340. $response = parent::send();
  341. } catch (Exception $e) {
  342. $this->log(LOG_ERR, $e->getMessage());
  343. throw $e;
  344. }
  345. $code = $response->getStatus();
  346. $effectiveUrl = $response->getEffectiveUrl();
  347. $redirUrls[] = $effectiveUrl;
  348. $response->redirUrls = $redirUrls;
  349. if ($code >= 200 && $code < 300) {
  350. $reason = $response->getReasonPhrase();
  351. $this->log(LOG_INFO, "$code $reason");
  352. } elseif ($code >= 300 && $code < 400) {
  353. $url = $this->getUrl();
  354. $target = $response->getHeader('Location');
  355. if (++$redirs >= $maxRedirs) {
  356. common_log(LOG_ERR, __CLASS__ . ": Too many redirects: skipping $code redirect from $url to $target");
  357. break;
  358. }
  359. try {
  360. $this->setUrl($target);
  361. $this->setHeader('Referer', $url);
  362. common_log(LOG_INFO, __CLASS__ . ": Following $code redirect from $url to $target");
  363. continue;
  364. } catch (HTTP_Request2_Exception $e) {
  365. common_log(LOG_ERR, __CLASS__ . ": Invalid $code redirect from $url to $target");
  366. }
  367. } else {
  368. $reason = $response->getReasonPhrase();
  369. $this->log(LOG_ERR, "$code $reason");
  370. }
  371. break;
  372. } while ($maxRedirs);
  373. return new GNUsocial_HTTPResponse($response, $this->getUrl(), $redirs);
  374. }
  375. public static function get_filename(string $url, array $headers = null) : ?string {
  376. if ($headers === null) {
  377. $head = (new HTTPClient())->head($url);
  378. $headers = $head->getHeader();
  379. $headers = array_change_key_case($headers, CASE_LOWER);
  380. }
  381. if (array_key_exists('content-disposition', $headers) &&
  382. preg_match('/^.+; filename="(.+?)"$/', $headers['content-disposition'], $matches) === 1) {
  383. return $matches[1];
  384. } else {
  385. common_log(LOG_INFO, "Couldn't determine filename for url: {$url}");
  386. return null;
  387. }
  388. }
  389. }