changeset 234:7a773720d81f legit-client

Totally legit client
author nanaya <me@nanaya.net>
date Fri, 14 Jul 2023 22:42:20 +0900
parents 0f0cc55ff11b
children 4222343d9433
files app/controllers/tweets_controller.rb app/helpers/application_helper.rb app/lib/cached_fetch.rb app/lib/legit_client.rb app/views/tweets/_tweet.atom.erb app/views/tweets/_tweet.html.erb app/views/tweets/show.atom.erb
diffstat 7 files changed, 226 insertions(+), 70 deletions(-) [+]
line wrap: on
line diff
--- a/app/controllers/tweets_controller.rb	Fri Jul 14 01:45:40 2023 +0900
+++ b/app/controllers/tweets_controller.rb	Fri Jul 14 22:42:20 2023 +0900
@@ -1,39 +1,38 @@
 class TweetsController < ApplicationController
   def index
-    return redirect if params[:id].present? || params[:name].present?
+    return redirect if params[:name].present?
   end
 
   def show
     return redirect if params[:id][/\D/].present?
 
-    client = Tweet.new(params[:id].to_i)
-    @user = client.user
+    @user = CachedFetch.user_by_id params[:id]
+
+    if @user.nil?
+      head :not_found
+      return
+    end
 
     return redirect if normalized_screen_name != params[:name]
 
-    @tweets = client.timeline
-  rescue Twitter::Error::Forbidden
-    head :forbidden
-  rescue Twitter::Error::NotFound
-    head :not_found
-  rescue Twitter::Error::Unauthorized
-    head :forbidden
+    @tweets = CachedFetch.timeline params[:id]
+
+    head :not_found if @tweets.nil?
   end
 
   def redirect
-    @user ||= Tweet.new(params[:id].presence || params[:name]).user
-    redirect_to tweet_path(@user.id, normalized_screen_name)
-  rescue Twitter::Error::Forbidden
-    head :forbidden
-  rescue Twitter::Error::NotFound
-    head :not_found
-  rescue Twitter::Error::Unauthorized
-    head :forbidden
+    @user ||= CachedFetch.user_by_username(params[:name])
+
+    if @user.nil?
+      head :not_found
+    else
+      redirect_to tweet_path(@user[:id], normalized_screen_name)
+    end
   end
 
   private
 
   def normalized_screen_name
-    @user.screen_name.presence || '_'
+    @user[:username].presence || '_'
   end
 end
--- a/app/helpers/application_helper.rb	Fri Jul 14 01:45:40 2023 +0900
+++ b/app/helpers/application_helper.rb	Fri Jul 14 22:42:20 2023 +0900
@@ -5,27 +5,17 @@
     "tag:rsstweet@nanaya.pro,2014:#{id}"
   end
 
-  def expand_url(text, *urls)
-    urls.flatten!
-
-    urls = urls.reduce({}) do |result, u|
-      if u.try(:[], :url)
-        result[u[:url]] = u[:expanded_url]
-      end
-
-      result
-    end
-
+  def expand_url(text, urls)
     text.gsub /https?:\/\/t\.co\/[A-Za-z0-9]+/ do |url|
       urls[url] || url
     end
   end
 
   def status_url(tweet)
-    status_url_base tweet.user.screen_name, tweet.id
+    status_url_base tweet[:user][:username], tweet[:id]
   end
 
-  def status_url_base(screen_name, tweet_id)
-    "https://twitter.com/#{screen_name.presence || '_'}/status/#{tweet_id}"
+  def status_url_base(username, id)
+    "https://twitter.com/#{username.presence || '_'}/status/#{id}"
   end
 end
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/app/lib/cached_fetch.rb	Fri Jul 14 22:42:20 2023 +0900
@@ -0,0 +1,17 @@
+module CachedFetch
+  def self.timeline(user_id)
+    cached("timeline:#{user_id}") { LegitClient.timeline(user_id)&.[](:timeline) }
+  end
+
+  def self.user_by_id(user_id)
+    cached("user_by_id:#{user_id}") { LegitClient.user_by_id(user_id)&.[](:user) }
+  end
+
+  def self.user_by_username(username)
+    cached("user_by_username:#{username}") { LegitClient.user_by_username(username)&.[](:user) }
+  end
+
+  def self.cached(key, &block)
+    Rails.cache.fetch(key, expires_in: (15 + rand(15)).minutes, &block)
+  end
+end
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/app/lib/legit_client.rb	Fri Jul 14 22:42:20 2023 +0900
@@ -0,0 +1,144 @@
+module LegitClient
+  def self.timeline(user_id)
+    resp = fetch("https://twitter.com/i/api/graphql/1-5o8Qhfc2kWlu_2rWNcug/UserTweetsAndReplies?variables=%7B%22userId%22%3A#{escape_param user_id}%2C%22count%22%3A50%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_lists_timeline_redesign_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%2C%22withArticleRichContentState%22%3Afalse%7D")
+
+    begin
+      json = JSON.parse(resp)
+      {
+        timeline: normalize_timeline(json['data']['user']['result']['timeline_v2']['timeline']['instructions']),
+        raw: resp,
+      }
+    rescue => e
+      return if (json || {}).dig('data').is_a? Hash
+      Rails.logger.error("timeline fail: #{user_id}: #{resp}")
+      nil
+    end
+  end
+
+  def self.user_by_id(user_id)
+    resp = fetch("https://twitter.com/i/api/graphql/i_0UQ54YrCyqLUvgGzXygA/UserByRestId?variables=%7B%22userId%22%3A#{escape_param user_id}%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22hidden_profile_subscriptions_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%7D")
+
+    begin
+      json = JSON.parse(resp)
+      {
+        user: normalize_user(json['data']['user']['result']),
+        raw: resp,
+      }
+    rescue
+      return if (json || {}).dig('data').is_a? Hash
+      Rails.logger.error("user_by_id fail: #{user_id}: #{resp}")
+      nil
+    end
+  end
+
+  def self.user_by_username(username)
+    resp = fetch("https://twitter.com/i/api/graphql/xc8f1g7BYqr6VTzTbvNlGw/UserByScreenName?variables=%7B%22screen_name%22%3A#{escape_param username}%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22hidden_profile_subscriptions_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22subscriptions_verification_info_verified_since_enabled%22%3Atrue%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%7D")
+
+    begin
+      json = JSON.parse(resp)
+      {
+        user: normalize_user(json['data']['user']['result']),
+        raw: resp,
+      }
+    rescue
+      return if (json || {}).dig('data').is_a? Hash
+      Rails.logger.error("user_by_username fail: #{username}: #{resp}")
+      nil
+    end
+  end
+
+  def self.escape_param(param)
+    CGI.escape JSON.dump(param)
+  end
+
+  def self.fetch(uri)
+    Net::HTTP.get(URI(uri), $cfg[:headers])
+  end
+
+  def self.normalize_entity_media(json)
+    ret = {}
+
+    json.each do |entity_media|
+      val = {}
+
+      if entity_media['type'] == 'photo'
+        val[:url] = entity_media['media_url_https']
+      elsif entity_media['type'] == 'video'
+        val[:url] = entity_media['expanded_url']
+        val[:variants] = entity_media['video_info']['variants']
+          .filter { |variant| variant['bitrate'].present? }
+          .map do |variant|
+            {
+              bitrate: variant['bitrate'],
+              url: variant['url'],
+            }
+          end
+      end
+
+      if !val.empty?
+        val[:type] = entity_media['type']
+        val[:id] = entity_media['media_key']
+      end
+
+      ret[entity_media['display_url']] = val
+    end
+
+    ret
+  end
+
+  def self.normalize_entity_urls(json)
+    ret = {}
+
+    json.each do |entity_url|
+      ret[entity_url['url']] = entity_url['expanded_url']
+    end
+
+    ret
+  end
+
+  def self.normalize_timeline(json)
+    json.find { |instruction| instruction['type'] == 'TimelineAddEntries' }['entries']
+      .filter { |entry| entry['entryId'] =~ /\A(profile-conversation|tweet)-/ }
+      .reduce([]) do |acc, entry|
+        if entry['content']['entryType'] == 'TimelineTimelineItem'
+          acc.push(entry['content'])
+        else
+          entry['content']['items'].each do |item|
+            acc.push(item['item'])
+          end
+        end
+        acc
+      end.map { |rawTweet| normalize_tweet(rawTweet['itemContent']['tweet_results']['result']) }
+  end
+
+  def self.normalize_tweet(json)
+    return nil if json.nil?
+
+    return normalize_tweet(json['tweet']) if json['__typename'] == 'TweetWithVisibilityResults'
+
+    {
+      id: json['rest_id'],
+      created_at: Time.parse(json['legacy']['created_at']),
+      user: normalize_user(json['core']['user_results']['result']),
+      message: json['legacy']['full_text'],
+      retweet: normalize_tweet(json.dig('legacy', 'retweeted_status_result', 'result')),
+      quote: normalize_tweet(json.dig('quoted_status_result', 'result')),
+      quote_id: json['legacy']['quoted_status_id_str'],
+      reply_to_id: json['legacy']['in_reply_to_status_id_str'],
+      reply_to_user_id: json['legacy']['in_reply_to_user_id_str'],
+      reply_to_username: json['legacy']['in_reply_to_screen_name'],
+      entity_urls: normalize_entity_urls(json['legacy']['entities']['urls']),
+      entity_media: normalize_entity_media(json.dig('legacy', 'extended_entities', 'media') || []),
+    }
+  end
+
+  def self.normalize_user(json)
+    {
+      avatar_url: json['legacy']['profile_image_url_https'],
+      id: json['rest_id'],
+      name: json['legacy']['name'],
+      protected: json['legacy']['protected'] == true,
+      username: json['legacy']['screen_name'],
+    }
+  end
+end
--- a/app/views/tweets/_tweet.atom.erb	Fri Jul 14 01:45:40 2023 +0900
+++ b/app/views/tweets/_tweet.atom.erb	Fri Jul 14 22:42:20 2023 +0900
@@ -1,13 +1,16 @@
 <entry>
-  <id><%= atom_id "#{tweet.user.id}/#{tweet.id}" %></id>
-  <published><%= tweet.created_at.xmlschema %></published>
-  <updated><%= tweet.created_at.xmlschema %></updated>
+  <%
+    created_at = tweet[:created_at].xmlschema
+  %>
+  <id><%= atom_id "#{tweet[:user][:id]}/#{tweet[:id]}" %></id>
+  <published><%= created_at %></published>
+  <updated><%= created_at %></updated>
   <link rel="alternate" type="text/html" href="<%= status_url(tweet) %>"/>
-  <title><%= truncate tweet.unescaped_text, :length => 30 %></title>
+  <title><%= truncate tweet[:message], :length => 30 %></title>
   <content type="html">
     <%= render(:partial => "tweet", :formats => :html, :locals => { :tweet => tweet }).to_str %>
   </content>
   <author>
-    <name><%= tweet.user.screen_name %></name>
+    <name><%= tweet[:user][:username] %></name>
   </author>
 </entry>
--- a/app/views/tweets/_tweet.html.erb	Fri Jul 14 01:45:40 2023 +0900
+++ b/app/views/tweets/_tweet.html.erb	Fri Jul 14 22:42:20 2023 +0900
@@ -1,34 +1,32 @@
-<% if tweet.retweeted_status.present? %>
+<% if tweet[:retweet].present? %>
   <p>
-    <%= link_to status_url(tweet.retweeted_status) do %>
+    <%= link_to status_url(tweet[:retweet]) do %>
       <em>Retweeted:</em>
     <% end %>
   </p>
 
-  <%= render "tweet", :tweet => tweet.retweeted_status, :with_time => true %>
-<% else%>
+  <%= render "tweet", :tweet => tweet[:retweet], :with_time => true %>
+<% else %>
   <% if defined?(with_time) && with_time %>
     <p>
-      <small>Originally tweeted at <%= tweet.created_at.rfc822 %></small>
+      <small>Originally tweeted at <%= tweet[:created_at].rfc822 %></small>
     </p>
   <% end %>
 
-  <% if tweet.in_reply_to_status_id.present? %>
+  <% if tweet[:reply_to_id].present? %>
     <p>
       <small>
         Replying to
-        <%= link_to 'tweet', status_url_base(tweet.in_reply_to_screen_name, tweet.in_reply_to_status_id) %>
-        by <%= link_to tweet.in_reply_to_screen_name, "https://twitter.com/#{tweet.in_reply_to_screen_name}" %>
+        <%= link_to 'tweet', status_url_base(tweet[:reply_to_username], tweet[:reply_to_id]) %>
+        by <%= link_to "@#{tweet[:reply_to_username]}", "https://twitter.com/#{tweet[:reply_to_username]}" %>
       </small>
     </p>
   <% end %>
 
   <p>
-    <%# FIXME: Twitter gem doesn't support extended mode when writing this %>
     <%= auto_link(expand_url(
-          tweet.full_text_extended,
-          tweet.attrs[:entities][:urls],
-          tweet.attrs[:entities][:media]
+          tweet[:message],
+          { **tweet[:entity_urls], **tweet[:entity_media] }
         ))
         .gsub("\n", "<br />")
         .html_safe
@@ -36,31 +34,36 @@
   </p>
 
   <p>
-    <%= link_to "https://twitter.com/#{tweet.user.screen_name}" do %>
-      <%= image_tag tweet.user.profile_image_url_https.to_s, :alt => "profile image for #{tweet.user.name.printable}" %>
-      <%= tweet.user.name.printable -%>
+    <%= link_to "https://twitter.com/#{tweet[:user][:username]}" do %>
+      <%= image_tag tweet[:user][:avatar_url].to_s, :alt => "profile image for #{tweet[:user][:name].printable}" %>
+      <%= tweet[:user][:name].printable -%>
     <% end %>
   </p>
 
   <p>
-    <% tweet.media.each_with_index do |media, i| %>
-      <% if media.is_a? Twitter::Media::Photo %>
-        <%= link_to "#{media.media_url_https}?name=orig" do %>
-          <%= image_tag "#{media.media_url_https}?name=small", :alt => "attachment #{i + 1}" -%>
+    <% tweet[:entity_media].each do |_short_url, media| %>
+      <% if media[:type] == 'photo' %>
+        <%= link_to "#{media[:url]}?name=orig" do %>
+          <%= image_tag "#{media[:url]}?name=small", :alt => "attachment #{media[:id]}" -%>
         <% end %>
-      <% elsif media.is_a? Twitter::Media::Video %>
-        <%= video_tag media.video_info.variants
-          .select { |i| i.bitrate.is_a? Integer }
-          .sort_by { |i| -i.bitrate }
-          .map(&:url), width: '100%', controls: true
+      <% elsif media[:type] == 'video' %>
+        <%= video_tag media[:variants]
+          .sort_by { |variant| -variant[:bitrate] }
+          .map { |variant| variant[:url] }, width: '100%', controls: true
         %>
       <% end %>
     <% end %>
   </p>
 
-  <% if tweet.quoted_status.present? %>
+  <% if tweet[:quote_id].present? %>
     <blockquote>
-      <%= render "tweet", :tweet => tweet.quoted_status, :with_time => true %>
+      <% if tweet[:quote].present? %>
+        <%= render "tweet", :tweet => tweet[:quote], :with_time => true %>
+      <% else %>
+        <%= link_to status_url_base(nil, tweet[:quote_id]) do %>
+          <em>Preview not available, view tweet directly.</em>
+        <% end %>
+      <% end %>
     </blockquote>
   <% end %>
 <% end %>
--- a/app/views/tweets/show.atom.erb	Fri Jul 14 01:45:40 2023 +0900
+++ b/app/views/tweets/show.atom.erb	Fri Jul 14 22:42:20 2023 +0900
@@ -1,15 +1,15 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <feed xml:lang="en-US" xmlns="http://www.w3.org/2005/Atom">
-  <id><%= atom_id(@user.id) %></id>
+  <id><%= atom_id(@user[:id]) %></id>
 
-  <link rel="alternate" type="text/html" href="https://twitter.com/<%= @user.screen_name %>" />
+  <link rel="alternate" type="text/html" href="https://twitter.com/<%= @user[:username] %>" />
   <link rel="self" type="application/atom+xml" href="<%= request.url %>" />
 
-  <title><%= "#{@user.name.printable} (#{@user.screen_name})" %></title>
-  <icon><%= @user.profile_image_url_https %></icon>
-  <logo><%= @user.profile_image_url_https %></logo>
+  <title><%= "#{@user[:name].printable} (#{@user[:username]})" %></title>
+  <icon><%= @user[:avatar_url] %></icon>
+  <logo><%= @user[:avatar_url] %></logo>
 
-  <updated><%= (@tweets.first.try(:created_at) || Time.at(0)).xmlschema %></updated>
+  <updated><%= (@tweets.first.try(:[], :created_at) || Time.at(0)).xmlschema %></updated>
 
   <%= render :partial => "tweet", :collection => @tweets, :cached => true %>
 </feed>