Mercurial > rsstweet

changeset 264:4f86037f6e6a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Identifiable rate limited client
author nanaya <me@nanaya.net>
date Sun, 09 Feb 2025 03:48:26 +0900
parents 2fc76c51e184
children a8fd79a8d170
files app/lib/cached_fetch.rb app/lib/legit_client.rb test/lib/legit_client_test.rb
diffstat 3 files changed, 58 insertions(+), 54 deletions(-) [+]
line wrap: on
line diff
--- a/app/lib/cached_fetch.rb	Mon Dec 16 01:41:44 2024 +0900
+++ b/app/lib/cached_fetch.rb	Sun Feb 09 03:48:26 2025 +0900
@@ -1,14 +1,14 @@
 module CachedFetch
   def self.timeline(user_id)
-    cached("timeline:#{user_id}") { LegitClient.timeline(user_id)&.[](:timeline) }
+    cached("timeline:#{user_id}") { LegitClient.new.timeline(user_id)&.[](:timeline) }
   end
 
   def self.user_by_id(user_id)
-    cached("user_by_id:#{user_id}") { LegitClient.user_by_id(user_id)&.[](:user) }
+    cached("user_by_id:#{user_id}") { LegitClient.new.user_by_id(user_id)&.[](:user) }
   end
 
   def self.user_by_username(username)
-    cached("user_by_username:#{username}") { LegitClient.user_by_username(username)&.[](:user) }
+    cached("user_by_username:#{username}") { LegitClient.new.user_by_username(username)&.[](:user) }
   end
 
   def self.cached(key, &block)
--- a/app/lib/legit_client.rb	Mon Dec 16 01:41:44 2024 +0900
+++ b/app/lib/legit_client.rb	Sun Feb 09 03:48:26 2025 +0900
@@ -1,57 +1,14 @@
 require "net/http"
 
-module LegitClient
-  def self.timeline(user_id)
-    resp = fetch("https://x.com/i/api/graphql/1-5o8Qhfc2kWlu_2rWNcug/UserTweetsAndReplies?variables=%7B%22userId%22%3A#{escape_param user_id}%2C%22count%22%3A50%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_lists_timeline_redesign_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%2C%22withArticleRichContentState%22%3Afalse%7D")
-
-    handle_response resp, :timeline, "timeline(#{user_id})", ->(json) do
-      normalize_timeline json["data"]["user"]["result"]["timeline_v2"]["timeline"]["instructions"], user_id
-    end
-  end
-
-  def self.user_by_id(user_id)
-    resp = fetch("https://x.com/i/api/graphql/i_0UQ54YrCyqLUvgGzXygA/UserByRestId?variables=%7B%22userId%22%3A#{escape_param user_id}%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22hidden_profile_subscriptions_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%7D")
-
-    handle_response resp, :user, "user_by_id(#{user_id})", ->(json) do
-      normalize_user json["data"]["user"]["result"]
-    end
-  end
-
-  def self.user_by_username(username)
-    resp = fetch("https://x.com/i/api/graphql/xc8f1g7BYqr6VTzTbvNlGw/UserByScreenName?variables=%7B%22screen_name%22%3A#{escape_param username}%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22hidden_profile_subscriptions_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22subscriptions_verification_info_verified_since_enabled%22%3Atrue%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%7D")
-
-    handle_response resp, :user, "user_by_username(#{username})", ->(json) do
-      normalize_user json["data"]["user"]["result"]
-    end
+class LegitClient
+  def initialize
+    @id, @headers = $cfg[:headers].sample
   end
 
   def self.escape_param(param)
     CGI.escape JSON.dump(param)
   end
 
-  def self.fetch(uri)
-    Net::HTTP.get(URI(uri), $cfg[:headers].sample)
-  end
-
-  def self.handle_response(resp, key, error_key, callback)
-    json = JSON.parse(resp)
-    {
-      key => callback.call(json),
-      raw: resp
-    }
-  rescue => e
-    if json.is_a? Hash
-      if json["errors"].is_a? Array
-        return rate_limit_check(json)
-      elsif json["data"].is_a? Hash
-        return
-      end
-    end
-    Rails.logger.error("#{error_key} fail: #{resp}")
-
-    raise e
-  end
-
   def self.normalize_entity_media(json)
     ret = {}
 
@@ -154,9 +111,56 @@
     }
   end
 
-  def self.rate_limit_check(json)
+  def timeline(user_id)
+    resp = fetch("https://x.com/i/api/graphql/1-5o8Qhfc2kWlu_2rWNcug/UserTweetsAndReplies?variables=%7B%22userId%22%3A#{self.class.escape_param user_id}%2C%22count%22%3A50%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_lists_timeline_redesign_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%2C%22withArticleRichContentState%22%3Afalse%7D")
+
+    handle_response resp, :timeline, "timeline(#{user_id})", ->(json) do
+      self.class.normalize_timeline json["data"]["user"]["result"]["timeline_v2"]["timeline"]["instructions"], user_id
+    end
+  end
+
+  def user_by_id(user_id)
+    resp = fetch("https://x.com/i/api/graphql/i_0UQ54YrCyqLUvgGzXygA/UserByRestId?variables=%7B%22userId%22%3A#{self.class.escape_param user_id}%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22hidden_profile_subscriptions_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%7D")
+
+    handle_response resp, :user, "user_by_id(#{user_id})", ->(json) do
+      self.class.normalize_user json["data"]["user"]["result"]
+    end
+  end
+
+  def user_by_username(username)
+    resp = fetch("https://x.com/i/api/graphql/xc8f1g7BYqr6VTzTbvNlGw/UserByScreenName?variables=%7B%22screen_name%22%3A#{self.class.escape_param username}%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22hidden_profile_subscriptions_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22subscriptions_verification_info_verified_since_enabled%22%3Atrue%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%7D")
+
+    handle_response resp, :user, "user_by_username(#{username})", ->(json) do
+      self.class.normalize_user json["data"]["user"]["result"]
+    end
+  end
+
+  private def fetch(uri)
+    Net::HTTP.get(URI(uri), @headers)
+  end
+
+  private def handle_response(resp, key, error_key, callback)
+    json = JSON.parse(resp)
+    {
+      key => callback.call(json),
+      raw: resp
+    }
+  rescue => e
+    if json.is_a? Hash
+      if json["errors"].is_a? Array
+        return rate_limit_check(json)
+      elsif json["data"].is_a? Hash
+        return
+      end
+    end
+    Rails.logger.error("#{error_key} fail: #{resp}")
+
+    raise e
+  end
+
+  private def rate_limit_check(json)
     return unless json["errors"].any? { |err| err["code"] == 88 }
 
-    raise "Rate limited!"
+    raise "Rate limited! Client: #{@id}"
   end
 end
--- a/test/lib/legit_client_test.rb	Mon Dec 16 01:41:44 2024 +0900
+++ b/test/lib/legit_client_test.rb	Sun Feb 09 03:48:26 2025 +0900
@@ -2,14 +2,14 @@
 
 class LegitClientTest < ActiveSupport::TestCase
   test "loads timeline" do
-    assert_not_nil LegitClient.timeline("2791517370")
+    assert_not_nil LegitClient.new.timeline("2791517370")
   end
 
   test "loads user_by_id" do
-    assert_not_nil LegitClient.user_by_id("2791517370")
+    assert_not_nil LegitClient.new.user_by_id("2791517370")
   end
 
   test "loads user_by_username" do
-    assert_not_nil LegitClient.user_by_username("edogawa_test")
+    assert_not_nil LegitClient.new.user_by_username("edogawa_test")
   end
 end