123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608 |
- <?php
- defined('GNUSOCIAL') || die();
- require_once dirname(__DIR__) . '/twitter.php';
- class TwitterImport
- {
- public $avatarsizename = 'reasonably_small';
- public $avatarsize = 128;
- public function importStatus($status)
- {
-
- $source = mb_strtolower(common_config('integration', 'source'));
- if (preg_match("/$source/", mb_strtolower($status->source))) {
- common_debug(__METHOD__ . ' - Skipping import of status ' .
- twitter_id($status) . " with source {$source}");
- return null;
- }
-
-
- if ($status->user->protected) {
- return null;
- }
- $notice = $this->saveStatus($status);
- return $notice;
- }
- public function name()
- {
- return get_class($this);
- }
- public function saveStatus($status)
- {
- $profile = $this->ensureProfile($status->user);
- if (empty($profile)) {
- common_log(LOG_ERR, __METHOD__ . ' - Problem saving notice. No associated Profile.');
- return null;
- }
- $statusId = twitter_id($status);
- $statusUri = $this->makeStatusURI($status->user->screen_name, $statusId);
-
- $n2s = Notice_to_status::getKV('status_id', $statusId);
- if (!empty($n2s)) {
- common_log(
- LOG_INFO,
- __METHOD__ . " - Ignoring duplicate import: {$statusId}"
- );
- return Notice::getKV('id', $n2s->notice_id);
- }
- $dupe = Notice::getKV('uri', $statusUri);
- if ($dupe instanceof Notice) {
-
- Notice_to_status::saveNew($dupe->id, $statusId);
- common_log(
- LOG_INFO,
- __METHOD__ . " - Ignoring duplicate import: {$statusId}"
- );
- return $dupe;
- }
-
- if (!empty($status->retweeted_status)) {
- common_log(LOG_INFO, "Status {$statusId} is a retweet of " . twitter_id($status->retweeted_status) . ".");
- $original = $this->saveStatus($status->retweeted_status);
- if (empty($original)) {
- return null;
- } else {
- $author = $original->getProfile();
-
-
- $content = sprintf(
- _m('RT @%1$s %2$s'),
- $author->nickname,
- $original->content
- );
- if (Notice::contentTooLong($content)) {
- $contentlimit = Notice::maxContent();
- $content = mb_substr($content, 0, $contentlimit - 4) . ' ...';
- }
- $repeat = Notice::saveNew(
- $profile->id,
- $content,
- 'twitter',
- [
- 'repeat_of' => $original->id,
- 'uri' => $statusUri,
- 'is_local' => Notice::GATEWAY,
- 'object_type' => ActivityObject::NOTE,
- 'verb' => ActivityVerb::POST,
- ]
- );
- common_log(LOG_INFO, "Saved {$repeat->id} as a repeat of {$original->id}");
- Notice_to_status::saveNew($repeat->id, $statusId);
- return $repeat;
- }
- }
- $notice = new Notice();
- $notice->profile_id = $profile->id;
- $notice->uri = $statusUri;
- $notice->url = $statusUri;
- $notice->verb = ActivityVerb::POST;
- $notice->object_type = ActivityObject::NOTE;
- $notice->created = strftime(
- '%Y-%m-%d %H:%M:%S',
- strtotime($status->created_at)
- );
- $notice->source = 'twitter';
- $notice->reply_to = null;
- $replyTo = twitter_id($status, 'in_reply_to_status_id');
- if (!empty($replyTo)) {
- common_log(LOG_INFO, "Status {$statusId} is a reply to status {$replyTo}");
- $n2s = Notice_to_status::getKV('status_id', $replyTo);
- if (empty($n2s)) {
- common_log(LOG_INFO, "Couldn't find local notice for status {$replyTo}");
- } else {
- $reply = Notice::getKV('id', $n2s->notice_id);
- if (empty($reply)) {
- common_log(LOG_INFO, "Couldn't find local notice for status {$replyTo}");
- } else {
- common_log(LOG_INFO, "Found local notice {$reply->id} for status {$replyTo}");
- $notice->reply_to = $reply->id;
- $notice->conversation = $reply->conversation;
- }
- }
- }
- $notice->is_local = Notice::GATEWAY;
- $notice->content = html_entity_decode($this->linkify($status, false), ENT_QUOTES, 'UTF-8');
- $notice->rendered = $this->linkify($status, true);
- if (Event::handle('StartNoticeSave', array(&$notice))) {
- if (empty($notice->conversation)) {
- $conv = Conversation::create();
- common_log(LOG_INFO, "No known conversation for status {$statusId} so a new one ({$conv->getID()}) was created.");
- $notice->conversation = $conv->getID();
- }
- $id = $notice->insert();
- if ($id === false) {
- common_log_db_error($notice, 'INSERT', __FILE__);
- common_log(LOG_ERR, __METHOD__ . ' - Problem saving notice.');
- }
- Event::handle('EndNoticeSave', array($notice));
- }
- Notice_to_status::saveNew($notice->id, $statusId);
- $this->saveStatusMentions($notice, $status);
- $this->saveStatusAttachments($notice, $status);
- $notice->blowOnInsert();
- return $notice;
- }
-
- public function makeStatusURI($username, $id)
- {
- return 'https://twitter.com/'
- . $username
- . '/status/'
- . $id;
- }
-
- protected function getProfileByUrl($nickname, $profileurl)
- {
- $profile = new Profile();
- $profile->nickname = $nickname;
- $profile->profileurl = $profileurl;
- $profile->limit(1);
- if (!$profile->find(true)) {
- $profile->profileurl = str_replace('https://', 'http://', $profileurl);
- if (!$profile->find(true)) {
- throw new NoResultException($profile);
- }
- }
- return $profile;
- }
- protected function ensureProfile($twuser)
- {
-
- $profileurl = 'https://twitter.com/' . $twuser->screen_name;
- try {
- $profile = $this->getProfileByUrl($twuser->screen_name, $profileurl);
- $this->updateAvatar($twuser, $profile);
- return $profile;
- } catch (NoResultException $e) {
- common_debug(__METHOD__ . ' - Adding profile and remote profile ' .
- "for Twitter user: $profileurl.");
- }
- $profile = new Profile();
- $profile->query('START TRANSACTION');
- $profile->nickname = $twuser->screen_name;
- $profile->fullname = $twuser->name;
- $profile->homepage = $twuser->url;
- $profile->bio = $twuser->description;
- $profile->location = $twuser->location;
- $profile->profileurl = $profileurl;
- $profile->created = common_sql_now();
- try {
- $id = $profile->insert();
- if (empty($id)) {
- throw new Exception('Failed insert');
- }
- } catch (Exception $e) {
- common_log(LOG_WARNING, __METHOD__ . " Couldn't insert profile: " . $e->getMessage());
- common_log_db_error($profile, 'INSERT', __FILE__);
- $profile->query("ROLLBACK");
- return false;
- }
- $profile->query("COMMIT");
- $this->updateAvatar($twuser, $profile);
- return $profile;
- }
-
- protected function updateAvatar($twuser, Profile $profile)
- {
- $path_parts = pathinfo($twuser->profile_image_url);
- $ext = isset($path_parts['extension'])
- ? '.'.$path_parts['extension']
- : '';
- $img_root = basename($path_parts['basename'], '_normal'.$ext);
- $filename = "Twitter_{$twuser->id}_{$img_root}_{$this->avatarsizename}{$ext}";
- try {
- $avatar = Avatar::getUploaded($profile);
- if ($avatar->filename === $filename) {
- return null;
- }
- common_debug(sprintf(
- '%s - Updating profile avatar (profile_id=%d) from %s to %s',
- __METHOD__,
- $profile->id,
- $avatar->filename,
- $filename
- ));
-
- } catch (NoAvatarException $e) {
-
-
- common_debug(__METHOD__ . " - No avatar found for (profile_id={$profile->id})");
- }
-
- $url = "{$path_parts['dirname']}/{$img_root}_{$this->avatarsizename}{$ext}";
- $mediatype = $this->getMediatype(mb_substr($ext, 1));
- try {
- $this->newAvatar($profile, $url, $filename, $mediatype);
- } catch (Exception $e) {
- if (file_exists(Avatar::path($filename))) {
- unlink(Avatar::path($filename));
- }
- return false;
- }
- return true;
- }
- protected function getMediatype($ext)
- {
- $mediatype = null;
- switch (strtolower($ext)) {
- case 'jpeg':
- case 'jpg':
- $mediatype = 'image/jpeg';
- break;
- case 'gif':
- $mediatype = 'image/gif';
- break;
- default:
- $mediatype = 'image/png';
- }
- return $mediatype;
- }
- protected function newAvatar(Profile $profile, $url, $filename, $mediatype)
- {
-
- Avatar::deleteFromProfile($profile);
-
- $this->fetchRemoteUrl($url, Avatar::path($filename));
- $avatar = new Avatar();
- $avatar->profile_id = $profile->id;
- $avatar->original = true;
- $avatar->mediatype = $mediatype;
- $avatar->filename = $filename;
- $avatar->width = $this->avatarsize;
- $avatar->height = $this->avatarsize;
- $avatar->created = common_sql_now();
- $id = $avatar->insert();
- if (empty($id)) {
- common_log(LOG_WARNING, __METHOD__ . " Couldn't insert avatar - " . $e->getMessage());
- common_log_db_error($avatar, 'INSERT', __FILE__);
- throw new ServerException('Could not insert avatar');
- }
- common_debug(__METHOD__ . " - Saved new avatar for {$profile->id}.");
- return $avatar;
- }
-
- protected function fetchRemoteUrl($url, $filename)
- {
- common_debug(__METHOD__ . " - Fetching Twitter avatar: {$url} to {$filename}");
- $request = HTTPClient::start();
- $request->setConfig('connect_timeout', 3);
- $request->setConfig('timeout', 6);
- $response = $request->get($url);
- if ($response->isOk()) {
- if (!file_put_contents($filename, $response->getBody())) {
- throw new ServerException('Failed saving fetched file');
- }
- } else {
- throw new Exception('Unexpected HTTP status code');
- }
- return true;
- }
- const URL = 1;
- const HASHTAG = 2;
- const MENTION = 3;
- public function linkify($status, $html = false)
- {
- $text = $status->text;
- if (empty($status->entities)) {
- $statusId = twitter_id($status);
- common_log(LOG_WARNING, "No entities data for {$statusId}; trying to fake up links ourselves.");
- $text = common_replace_urls_callback($text, 'common_linkify');
- $text = preg_replace_callback(
- '/(^|\"\;|\'|\(|\[|\{|\s+)#([\pL\pN_\-\.]{1,64})/',
- function ($m) {
- return $m[1] . '#'.TwitterStatusFetcher::tagLink($m[2]);
- },
- $text
- );
- $text = preg_replace_callback(
- '/(^|\s+)@([a-z0-9A-Z_]{1,64})/',
- function ($m) {
- return $m[1] . '@'.TwitterStatusFetcher::atLink($m[2]);
- },
- $text
- );
- return $text;
- }
-
-
-
- $toReplace = array();
- if (!empty($status->entities->urls)) {
- foreach ($status->entities->urls as $url) {
- $toReplace[$url->indices[0]] = array(self::URL, $url);
- }
- }
- if (!empty($status->entities->hashtags)) {
- foreach ($status->entities->hashtags as $hashtag) {
- $toReplace[$hashtag->indices[0]] = array(self::HASHTAG, $hashtag);
- }
- }
- if (!empty($status->entities->user_mentions)) {
- foreach ($status->entities->user_mentions as $mention) {
- $toReplace[$mention->indices[0]] = array(self::MENTION, $mention);
- }
- }
-
- ksort($toReplace);
- $result = '';
- $cursor = 0;
- foreach ($toReplace as $part) {
- list($type, $object) = $part;
- $start = $object->indices[0];
- $end = $object->indices[1];
- if ($cursor < $start) {
-
- $result .= $this->twitEscape(mb_substr($text, $cursor, $start - $cursor));
- $cursor = $start;
- }
- $orig = $this->twitEscape(mb_substr($text, $start, $end - $start));
- switch ($type) {
- case self::URL:
- $linkText = $this->makeUrlLink($object, $orig, $html);
- break;
- case self::HASHTAG:
- if ($html) {
- $linkText = $this->makeHashtagLink($object, $orig);
- } else {
- $linkText = $orig;
- }
- break;
- case self::MENTION:
- if ($html) {
- $linkText = $this->makeMentionLink($object, $orig);
- } else {
- $linkText = $orig;
- }
- break;
- default:
- $linkText = $orig;
- continue;
- }
- $result .= $linkText;
- $cursor = $end;
- }
- $last = $this->twitEscape(mb_substr($text, $cursor));
- $result .= $last;
- return $result;
- }
- public function twitEscape($str)
- {
-
-
-
-
-
-
-
-
- return htmlspecialchars(html_entity_decode($str, ENT_COMPAT, 'UTF-8'));
- }
- public function makeUrlLink($object, $orig, $html)
- {
- if ($html) {
- return '<a href="'.htmlspecialchars($object->expanded_url).'" class="extlink">'.htmlspecialchars($object->display_url).'</a>';
- } else {
- return htmlspecialchars($object->expanded_url);
- }
- }
- public function makeHashtagLink($object, $orig)
- {
- return "#" . self::tagLink($object->text, substr($orig, 1));
- }
- public function makeMentionLink($object, $orig)
- {
- return "@".self::atLink($object->screen_name, $object->name, substr($orig, 1));
- }
- public static function tagLink($tag, $orig)
- {
- return "<a href='https://twitter.com/search?q=%23{$tag}' class='hashtag'>{$orig}</a>";
- }
- public static function atLink($screenName, $fullName, $orig)
- {
- if (!empty($fullName)) {
- return "<a href='https://twitter.com/{$screenName}' title='{$fullName}'>{$orig}</a>";
- } else {
- return "<a href='https://twitter.com/{$screenName}'>{$orig}</a>";
- }
- }
- public function saveStatusMentions($notice, $status)
- {
- $mentions = array();
- if (empty($status->entities) || empty($status->entities->user_mentions)) {
- return;
- }
- foreach ($status->entities->user_mentions as $mention) {
- try {
- $flink = Foreign_link::getByForeignID($mention->id, TWITTER_SERVICE);
- $user = $flink->getUser();
- $reply = new Reply();
- $reply->notice_id = $notice->id;
- $reply->profile_id = $user->id;
- $reply->modified = $notice->created;
- common_log(LOG_INFO, __METHOD__ . ": saving reply: notice {$notice->id} to profile {$user->id}");
- $id = $reply->insert();
- } catch (NoSuchUserException $e) {
- common_log(LOG_WARNING, 'No local user found for Foreign_link with id: '.$mention->id);
- } catch (NoResultException $e) {
- common_log(LOG_WARNING, 'No foreign link or profile found for Foreign_link with id: '.$mention->id);
- }
- }
- }
-
- public function saveStatusAttachments(Notice $notice, $status)
- {
- if (common_config('attachments', 'process_links')) {
- if (!empty($status->entities) && !empty($status->entities->urls)) {
- foreach ($status->entities->urls as $url) {
- try {
- File::processNew($url->url, $notice);
- } catch (ServerException $e) {
-
- }
- }
- }
- }
- }
- }
|