File_redirection.php 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493
  1. <?php
  2. // This file is part of GNU social - https://www.gnu.org/software/social
  3. //
  4. // GNU social is free software: you can redistribute it and/or modify
  5. // it under the terms of the GNU Affero General Public License as published by
  6. // the Free Software Foundation, either version 3 of the License, or
  7. // (at your option) any later version.
  8. //
  9. // GNU social is distributed in the hope that it will be useful,
  10. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. // GNU Affero General Public License for more details.
  13. //
  14. // You should have received a copy of the GNU Affero General Public License
  15. // along with GNU social. If not, see <http://www.gnu.org/licenses/>.
  16. defined('GNUSOCIAL') || die();
  17. /**
  18. * Table Definition for file_redirection
  19. */
  20. class File_redirection extends Managed_DataObject
  21. {
  22. ###START_AUTOCODE
  23. /* the code below is auto generated do not remove the above tag */
  24. public $__table = 'file_redirection'; // table name
  25. public $urlhash; // varchar(64) primary_key not_null
  26. public $url; // text
  27. public $file_id; // int(4)
  28. public $redirections; // int(4)
  29. public $httpcode; // int(4)
  30. public $modified; // datetime() not_null default_CURRENT_TIMESTAMP
  31. /* the code above is auto generated do not remove the tag below */
  32. ###END_AUTOCODE
  33. protected $file; /* Cache the associated file sometimes */
  34. public static function schemaDef()
  35. {
  36. return [
  37. 'fields' => [
  38. 'urlhash' => ['type' => 'varchar', 'length' => 64, 'not null' => true, 'description' => 'sha256 hash of the URL'],
  39. 'url' => ['type' => 'text', 'description' => 'short URL (or any other kind of redirect) for file (id)'],
  40. 'file_id' => ['type' => 'int', 'description' => 'short URL for what URL/file'],
  41. 'redirections' => ['type' => 'int', 'description' => 'redirect count'],
  42. 'httpcode' => ['type' => 'int', 'description' => 'HTTP status code (20x, 30x, etc.)'],
  43. 'modified' => ['type' => 'datetime', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'],
  44. ],
  45. 'primary key' => ['urlhash'],
  46. 'foreign keys' => [
  47. 'file_redirection_file_id_fkey' => ['file', ['file_id' => 'id']],
  48. ],
  49. ];
  50. }
  51. public static function getByUrl($url)
  52. {
  53. return self::getByPK(['urlhash' => File::hashurl($url)]);
  54. }
  55. public static function _commonHttp($url, $redirs)
  56. {
  57. $request = new HTTPClient($url);
  58. $request->setConfig([
  59. 'connect_timeout' => 10, // # seconds to wait
  60. 'max_redirs' => $redirs, // # max number of http redirections to follow
  61. 'follow_redirects' => false, // We follow redirects ourselves in lib/httpclient.php
  62. 'store_body' => false, // We won't need body content here.
  63. ]);
  64. return $request;
  65. }
  66. /**
  67. * Check if this URL is a redirect and return redir info.
  68. *
  69. * Most code should call File_redirection::where instead, to check if we
  70. * already know that redirection and avoid extra hits to the web.
  71. *
  72. * The URL is hit and any redirects are followed, up to 10 levels or until
  73. * a protected URL is reached.
  74. *
  75. * @param $short_url
  76. * @param int $redirs
  77. * @param bool $protected
  78. * @return mixed one of:
  79. * string - target URL, if this is a direct link or can't be followed
  80. * array - redirect info if this is an *unknown* redirect:
  81. * associative array with the following elements:
  82. * code: HTTP status code
  83. * redirects: count of redirects followed
  84. * url: URL string of final target
  85. * type (optional): MIME type from Content-Type header
  86. * size (optional): byte size from Content-Length header
  87. * time (optional): timestamp from Last-Modified header
  88. */
  89. public static function lookupWhere($short_url, $redirs = 10, $protected = false)
  90. {
  91. if ($redirs < 0) {
  92. return false;
  93. }
  94. if (strpos($short_url, '://') === false) {
  95. return $short_url;
  96. }
  97. try {
  98. $request = self::_commonHttp($short_url, $redirs);
  99. // Don't include body in output
  100. $request->setMethod(HTTP_Request2::METHOD_HEAD);
  101. $response = $request->send();
  102. if (405 == $response->getStatus() || 204 == $response->getStatus()) {
  103. // HTTP 405 Unsupported Method
  104. // Server doesn't support HEAD method? Can this really happen?
  105. // We'll try again as a GET and ignore the response data.
  106. //
  107. // HTTP 204 No Content
  108. // YFrog sends 204 responses back for our HEAD checks, which
  109. // seems like it may be a logic error in their servers. If
  110. // we get a 204 back, re-run it as a GET... if there's really
  111. // no content it'll be cheap. :)
  112. $request = self::_commonHttp($short_url, $redirs);
  113. $response = $request->send();
  114. } elseif (400 == $response->getStatus()) {
  115. throw new Exception('Got error 400 on HEAD request, will not go further.');
  116. }
  117. } catch (Exception $e) {
  118. // Invalid URL or failure to reach server
  119. common_log(LOG_ERR, "Error while following redirects for $short_url: " . $e->getMessage());
  120. return $short_url;
  121. }
  122. // if last url after all redirections is protected,
  123. // use the url before it in the redirection chain
  124. if ($response->getRedirectCount() && File::isProtected($response->getEffectiveUrl())) {
  125. $return_url = $response->redirUrls[$response->getRedirectCount() - 1];
  126. } else {
  127. $return_url = $response->getEffectiveUrl();
  128. }
  129. $ret = array('code' => $response->getStatus()
  130. , 'redirects' => $response->getRedirectCount()
  131. , 'url' => $return_url);
  132. $type = $response->getHeader('Content-Type');
  133. if ($type) {
  134. $ret['type'] = $type;
  135. }
  136. if ($protected) {
  137. $ret['protected'] = true;
  138. }
  139. $size = $response->getHeader('Content-Length'); // @fixme bytes?
  140. if ($size) {
  141. $ret['size'] = $size;
  142. }
  143. $time = $response->getHeader('Last-Modified');
  144. if ($time) {
  145. $ret['time'] = strtotime($time);
  146. }
  147. return $ret;
  148. }
  149. /**
  150. * Check if this URL is a redirect and return redir info.
  151. * If a File record is present for this URL, it is not considered a redirect.
  152. * If a File_redirection record is present for this URL, the recorded target is returned.
  153. *
  154. * If no File or File_redirect record is present, the URL is hit and any
  155. * redirects are followed, up to 10 levels or until a protected URL is
  156. * reached.
  157. *
  158. * @param string $in_url
  159. * @param boolean $discover true to attempt dereferencing the redirect if we don't know it already
  160. * @return File_redirection
  161. * @throws EmptyPkeyValueException
  162. * @throws ServerException
  163. */
  164. public static function where($in_url, $discover = true)
  165. {
  166. $redir = new File_redirection();
  167. $redir->url = $in_url;
  168. $redir->urlhash = File::hashurl($redir->url);
  169. $redir->redirections = 0;
  170. try {
  171. $r = File_redirection::getByUrl($in_url);
  172. try {
  173. $f = File::getByID($r->file_id);
  174. $r->file = $f;
  175. $r->redir_url = $f->url;
  176. } catch (NoResultException $e) {
  177. // Invalid entry, delete and run again
  178. common_log(
  179. LOG_ERR,
  180. 'Could not find File with id=' . $r->file_id . ' referenced in File_redirection, deleting File redirection entry and and trying again...'
  181. );
  182. $r->delete();
  183. return self::where($in_url);
  184. }
  185. // File_redirecion and File record found, return both
  186. return $r;
  187. } catch (NoResultException $e) {
  188. // File_redirecion record not found, but this might be a direct link to a file
  189. try {
  190. $f = File::getByUrl($in_url);
  191. $redir->file_id = $f->id;
  192. $redir->file = $f;
  193. return $redir;
  194. } catch (NoResultException $e) {
  195. // nope, this was not a direct link to a file either, let's keep going
  196. }
  197. }
  198. if ($discover) {
  199. // try to follow redirects and get the final url
  200. $redir_info = File_redirection::lookupWhere($in_url);
  201. if (is_string($redir_info)) {
  202. $redir_info = array('url' => $redir_info);
  203. }
  204. // the last url in the redirection chain can actually be a redirect!
  205. // this is the case with local /attachment/{file_id} links
  206. // in that case we have the file id already
  207. try {
  208. $r = File_redirection::getByUrl($redir_info['url']);
  209. $f = File::getKV('id', $r->file_id);
  210. if ($f instanceof File) {
  211. $redir->file = $f;
  212. $redir->redir_url = $f->url;
  213. } else {
  214. // Invalid entry in File_redirection, delete and run again
  215. common_log(
  216. LOG_ERR,
  217. 'Could not find File with id=' . $r->file_id . ' referenced in File_redirection, deleting File_redirection entry and trying again...'
  218. );
  219. $r->delete();
  220. return self::where($in_url);
  221. }
  222. } catch (NoResultException $e) {
  223. // save the file now when we know that we don't have it in File_redirection
  224. try {
  225. $redir->file = File::saveNew($redir_info, $redir_info['url']);
  226. } catch (ServerException $e) {
  227. common_log(LOG_ERR, $e);
  228. }
  229. }
  230. // If this is a redirection and we have a file to redirect to, save it
  231. // (if it doesn't exist in File_redirection already)
  232. if ($redir->file instanceof File && $redir_info['url'] != $in_url) {
  233. try {
  234. $file_redir = File_redirection::getByUrl($in_url);
  235. } catch (NoResultException $e) {
  236. $file_redir = new File_redirection();
  237. $file_redir->urlhash = File::hashurl($in_url);
  238. $file_redir->url = $in_url;
  239. $file_redir->file_id = $redir->file->getID();
  240. $file_redir->insert();
  241. $file_redir->redir_url = $redir->file->url;
  242. }
  243. $file_redir->file = $redir->file;
  244. return $file_redir;
  245. }
  246. }
  247. return $redir;
  248. }
  249. /**
  250. * Shorten a URL with the current user's configured shortening
  251. * options, if applicable.
  252. *
  253. * If it cannot be shortened or the "short" URL is longer than the
  254. * original, the original is returned.
  255. *
  256. * If the referenced item has not been seen before, embedding data
  257. * may be saved.
  258. *
  259. * @param string $long_url
  260. * @param User $user whose shortening options to use; defaults to the current web session user
  261. * @return string
  262. */
  263. public static function makeShort($long_url, $user = null)
  264. {
  265. $canon = File_redirection::_canonUrl($long_url);
  266. $short_url = File_redirection::_userMakeShort($canon, $user);
  267. // Did we get one? Is it shorter?
  268. return !empty($short_url) ? $short_url : $long_url;
  269. }
  270. /**
  271. * Shorten a URL with the current user's configured shortening
  272. * options, if applicable.
  273. *
  274. * If it cannot be shortened or the "short" URL is longer than the
  275. * original, the original is returned.
  276. *
  277. * If the referenced item has not been seen before, embedding data
  278. * may be saved.
  279. *
  280. * @param string $long_url
  281. * @param $user
  282. * @return string
  283. */
  284. public static function forceShort($long_url, $user)
  285. {
  286. $canon = File_redirection::_canonUrl($long_url);
  287. $short_url = File_redirection::_userMakeShort($canon, $user, true);
  288. // Did we get one? Is it shorter?
  289. return !empty($short_url) ? $short_url : $long_url;
  290. }
  291. public static function _userMakeShort($long_url, User $user = null, $force = false)
  292. {
  293. $short_url = common_shorten_url($long_url, $user, $force);
  294. if (!empty($short_url) && $short_url != $long_url) {
  295. $short_url = (string)$short_url;
  296. // store it
  297. try {
  298. $file = File::getByUrl($long_url);
  299. } catch (NoResultException $e) {
  300. // Check if the target URL is itself a redirect...
  301. // This should already have happened in processNew in common_shorten_url()
  302. $redir = File_redirection::where($long_url);
  303. $file = $redir->file;
  304. }
  305. // Now we definitely have a File object in $file
  306. try {
  307. $file_redir = File_redirection::getByUrl($short_url);
  308. } catch (NoResultException $e) {
  309. $file_redir = new File_redirection();
  310. $file_redir->urlhash = File::hashurl($short_url);
  311. $file_redir->url = $short_url;
  312. $file_redir->file_id = $file->getID();
  313. $file_redir->insert();
  314. }
  315. return $short_url;
  316. }
  317. return null;
  318. }
  319. /**
  320. * Basic attempt to canonicalize a URL, cleaning up some standard variants
  321. * such as funny syntax or a missing path. Used internally when cleaning
  322. * up URLs for storage and following redirect chains.
  323. *
  324. * Note that despite being on File_redirect, this function DOES NOT perform
  325. * any dereferencing of redirects.
  326. *
  327. * @param string $in_url input URL
  328. * @param string $default_scheme if given a bare link; defaults to 'http://'
  329. * @return string
  330. */
  331. public static function _canonUrl($in_url, $default_scheme = 'http://')
  332. {
  333. if (empty($in_url)) {
  334. return false;
  335. }
  336. $out_url = $in_url;
  337. $p = parse_url($out_url);
  338. if (empty($p['host']) || empty($p['scheme'])) {
  339. list($scheme) = explode(':', $in_url, 2);
  340. switch (strtolower($scheme)) {
  341. case 'fax':
  342. case 'tel':
  343. $out_url = str_replace('.-()', '', $out_url);
  344. break;
  345. // non-HTTP schemes, so no redirects
  346. case 'bitcoin':
  347. case 'mailto':
  348. case 'aim':
  349. case 'jabber':
  350. case 'xmpp':
  351. // don't touch anything
  352. break;
  353. // URLs without domain name, so no redirects
  354. case 'magnet':
  355. // don't touch anything
  356. break;
  357. // URLs with coordinates, not browsable domain names
  358. case 'geo':
  359. // don't touch anything
  360. break;
  361. default:
  362. $out_url = $default_scheme . ltrim($out_url, '/');
  363. $p = parse_url($out_url);
  364. if (empty($p['scheme'])) {
  365. return false;
  366. }
  367. break;
  368. }
  369. }
  370. if (('ftp' == $p['scheme']) || ('ftps' == $p['scheme']) || ('http' == $p['scheme']) || ('https' == $p['scheme'])) {
  371. if (empty($p['host'])) {
  372. return false;
  373. }
  374. if (empty($p['path'])) {
  375. $out_url .= '/';
  376. }
  377. }
  378. return $out_url;
  379. }
  380. public static function saveNew($data, $file_id, $url)
  381. {
  382. $file_redir = new File_redirection;
  383. $file_redir->urlhash = File::hashurl($url);
  384. $file_redir->url = $url;
  385. $file_redir->file_id = $file_id;
  386. $file_redir->redirections = intval($data['redirects']);
  387. $file_redir->httpcode = intval($data['code']);
  388. $file_redir->insert();
  389. }
  390. public static function beforeSchemaUpdate()
  391. {
  392. $table = strtolower(get_called_class());
  393. $schema = Schema::get();
  394. $schemadef = $schema->getTableDef($table);
  395. // 2015-02-19 We have to upgrade our table definitions to have the urlhash field populated
  396. if (isset($schemadef['fields']['urlhash']) && in_array('urlhash', $schemadef['primary key'])) {
  397. // We already have the urlhash field, so no need to migrate it.
  398. return;
  399. }
  400. echo "\nFound old $table table, upgrading it to contain 'urlhash' field...";
  401. // We have to create a urlhash that is _not_ the primary key,
  402. // transfer data and THEN run checkSchema
  403. $schemadef['fields']['urlhash'] = [
  404. 'type' => 'varchar',
  405. 'length' => 64,
  406. 'not null' => true,
  407. 'description' => 'sha256 hash of the URL',
  408. ];
  409. $schemadef['fields']['url'] = [
  410. 'type' => 'text',
  411. 'description' => 'short URL (or any other kind of redirect) for file (id)',
  412. ];
  413. unset($schemadef['primary key']);
  414. $schema->ensureTable($table, $schemadef);
  415. echo "DONE.\n";
  416. $classname = ucfirst($table);
  417. $tablefix = new $classname;
  418. // urlhash is hash('sha256', $url) in the File table
  419. echo "Updating urlhash fields in $table table...";
  420. switch (common_config('db', 'type')) {
  421. case 'pgsql':
  422. $url_sha256 = 'encode(sha256(CAST("url" AS bytea)), \'hex\')';
  423. break;
  424. case 'mysql':
  425. $url_sha256 = 'sha2(`url`, 256)';
  426. break;
  427. default:
  428. throw new ServerException('Unknown DB type selected.');
  429. }
  430. $tablefix->query(sprintf(
  431. 'UPDATE %1$s SET urlhash = %2$s;',
  432. $tablefix->escapedTableName(),
  433. $url_sha256
  434. ));
  435. echo "DONE.\n";
  436. echo "Resuming core schema upgrade...";
  437. }
  438. public function getFile()
  439. {
  440. if (!$this->file instanceof File) {
  441. $this->file = File::getByID($this->file_id);
  442. }
  443. return $this->file;
  444. }
  445. }