Mercurial > rsstweet
annotate app/lib/legit_client.rb @ 264:4f86037f6e6a
Identifiable rate limited client
| author | nanaya <me@nanaya.net> |
|---|---|
| date | Sun, 09 Feb 2025 03:48:26 +0900 |
| parents | e2150dce4e90 |
| children |
| rev | line source |
|---|---|
| 260 | 1 require "net/http" |
| 259 | 2 |
| 264 | 3 class LegitClient |
| 4 def initialize | |
| 5 @id, @headers = $cfg[:headers].sample | |
| 234 | 6 end |
| 7 | |
| 8 def self.escape_param(param) | |
| 9 CGI.escape JSON.dump(param) | |
| 10 end | |
| 11 | |
| 12 def self.normalize_entity_media(json) | |
| 13 ret = {} | |
| 14 | |
| 15 json.each do |entity_media| | |
| 16 val = {} | |
| 17 | |
| 260 | 18 case entity_media["type"] |
| 19 when "animated_gif", "video" | |
| 20 val[:variants] = entity_media["video_info"]["variants"] | |
| 21 .filter { |variant| variant["bitrate"].present? } | |
|
253
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
22 .map do |variant| |
|
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
23 { |
| 260 | 24 bitrate: variant["bitrate"], |
| 25 url: variant["url"] | |
|
253
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
26 } |
|
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
27 end |
| 260 | 28 when "photo" |
| 29 val[:image_url] = entity_media["media_url_https"].sub(/\.([^.]+)$/, '?format=\1') | |
| 234 | 30 end |
| 31 | |
| 32 if !val.empty? | |
| 260 | 33 val[:url] = entity_media["expanded_url"] |
| 34 val[:type] = entity_media["type"] | |
| 35 val[:id] = entity_media["media_key"] | |
| 234 | 36 end |
| 37 | |
| 260 | 38 key = if ret[entity_media["url"]].nil? |
| 39 entity_media["url"] | |
|
237
961d362e42c7
The url in entity media isn't unique as they all point to the same thing
nanaya <me@nanaya.net>
parents:
236
diff
changeset
|
40 else |
| 260 | 41 entity_media["media_key"] |
|
237
961d362e42c7
The url in entity media isn't unique as they all point to the same thing
nanaya <me@nanaya.net>
parents:
236
diff
changeset
|
42 end |
|
961d362e42c7
The url in entity media isn't unique as they all point to the same thing
nanaya <me@nanaya.net>
parents:
236
diff
changeset
|
43 |
|
961d362e42c7
The url in entity media isn't unique as they all point to the same thing
nanaya <me@nanaya.net>
parents:
236
diff
changeset
|
44 ret[key] = val |
| 234 | 45 end |
| 46 | |
| 47 ret | |
| 48 end | |
| 49 | |
| 50 def self.normalize_entity_urls(json) | |
| 51 ret = {} | |
| 52 | |
|
240
c454ea4f7b34
Add support for note tweets (with no formatting)
nanaya <me@nanaya.net>
parents:
238
diff
changeset
|
53 (json || {}).each do |entity_url| |
| 260 | 54 ret[entity_url["url"]] = entity_url["expanded_url"] |
| 234 | 55 end |
| 56 | |
| 57 ret | |
| 58 end | |
| 59 | |
|
238
a04b4830eef2
Filter out non-own tweets included for replies
nanaya <me@nanaya.net>
parents:
237
diff
changeset
|
60 def self.normalize_timeline(json, user_id) |
| 252 | 61 json |
| 62 .reduce([]) do |acc, instruction| | |
| 260 | 63 case instruction["type"] |
| 64 when "TimelineAddEntries" then acc += instruction["entries"] | |
| 65 when "TimelinePinEntry" then acc << instruction["entry"] | |
| 252 | 66 end |
| 67 | |
| 68 acc | |
| 260 | 69 end.filter { |entry| entry["entryId"] =~ /\A(profile-conversation|tweet)-/ } |
| 234 | 70 .reduce([]) do |acc, entry| |
| 260 | 71 if entry["content"]["entryType"] == "TimelineTimelineItem" |
| 72 acc.push(entry["content"]) | |
| 234 | 73 else |
| 260 | 74 entry["content"]["items"].each do |item| |
| 75 acc.push(item["item"]) | |
| 234 | 76 end |
| 77 end | |
| 78 acc | |
| 260 | 79 end.map { |raw_tweet| normalize_tweet(raw_tweet["itemContent"]["tweet_results"]["result"]) } |
|
238
a04b4830eef2
Filter out non-own tweets included for replies
nanaya <me@nanaya.net>
parents:
237
diff
changeset
|
80 .filter { |tweet| !tweet.nil? && tweet.dig(:user, :id) == user_id } |
| 234 | 81 end |
| 82 | |
| 83 def self.normalize_tweet(json) | |
| 84 return nil if json.nil? | |
| 85 | |
| 260 | 86 return normalize_tweet(json["tweet"]) if json["__typename"] == "TweetWithVisibilityResults" |
| 234 | 87 |
| 88 { | |
| 260 | 89 id: json["rest_id"], |
| 90 created_at: Time.parse(json["legacy"]["created_at"]), | |
| 91 user: normalize_user(json["core"]["user_results"]["result"]), | |
| 92 message: json.dig("note_tweet", "note_tweet_results", "result", "text") || json["legacy"]["full_text"], | |
| 93 retweet: normalize_tweet(json.dig("legacy", "retweeted_status_result", "result")), | |
| 94 quote: normalize_tweet(json.dig("quoted_status_result", "result")), | |
| 95 quote_id: json["legacy"]["quoted_status_id_str"], | |
| 96 reply_to_id: json["legacy"]["in_reply_to_status_id_str"], | |
| 97 reply_to_user_id: json["legacy"]["in_reply_to_user_id_str"], | |
| 98 reply_to_username: json["legacy"]["in_reply_to_screen_name"], | |
| 99 entity_urls: { **normalize_entity_urls(json["legacy"]["entities"]["urls"]), **normalize_entity_urls(json.dig("note_tweet", "note_tweet_results", "result", "entity_set", "urls")) }, | |
| 100 entity_media: normalize_entity_media(json.dig("legacy", "extended_entities", "media") || []) | |
| 234 | 101 } |
| 102 end | |
| 103 | |
| 104 def self.normalize_user(json) | |
| 105 { | |
| 260 | 106 avatar_url: json["legacy"]["profile_image_url_https"], |
| 107 id: json["rest_id"], | |
| 108 name: json["legacy"]["name"], | |
| 109 protected: json["legacy"]["protected"] == true, | |
| 110 username: json["legacy"]["screen_name"] | |
| 234 | 111 } |
| 112 end | |
|
243
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
113 |
| 264 | 114 def timeline(user_id) |
| 115 resp = fetch("https://x.com/i/api/graphql/1-5o8Qhfc2kWlu_2rWNcug/UserTweetsAndReplies?variables=%7B%22userId%22%3A#{self.class.escape_param user_id}%2C%22count%22%3A50%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_lists_timeline_redesign_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%2C%22withArticleRichContentState%22%3Afalse%7D") | |
| 116 | |
| 117 handle_response resp, :timeline, "timeline(#{user_id})", ->(json) do | |
| 118 self.class.normalize_timeline json["data"]["user"]["result"]["timeline_v2"]["timeline"]["instructions"], user_id | |
| 119 end | |
| 120 end | |
| 121 | |
| 122 def user_by_id(user_id) | |
| 123 resp = fetch("https://x.com/i/api/graphql/i_0UQ54YrCyqLUvgGzXygA/UserByRestId?variables=%7B%22userId%22%3A#{self.class.escape_param user_id}%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22hidden_profile_subscriptions_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%7D") | |
| 124 | |
| 125 handle_response resp, :user, "user_by_id(#{user_id})", ->(json) do | |
| 126 self.class.normalize_user json["data"]["user"]["result"] | |
| 127 end | |
| 128 end | |
| 129 | |
| 130 def user_by_username(username) | |
| 131 resp = fetch("https://x.com/i/api/graphql/xc8f1g7BYqr6VTzTbvNlGw/UserByScreenName?variables=%7B%22screen_name%22%3A#{self.class.escape_param username}%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22hidden_profile_subscriptions_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22subscriptions_verification_info_verified_since_enabled%22%3Atrue%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%7D") | |
| 132 | |
| 133 handle_response resp, :user, "user_by_username(#{username})", ->(json) do | |
| 134 self.class.normalize_user json["data"]["user"]["result"] | |
| 135 end | |
| 136 end | |
| 137 | |
| 138 private def fetch(uri) | |
| 139 Net::HTTP.get(URI(uri), @headers) | |
| 140 end | |
| 141 | |
| 142 private def handle_response(resp, key, error_key, callback) | |
| 143 json = JSON.parse(resp) | |
| 144 { | |
| 145 key => callback.call(json), | |
| 146 raw: resp | |
| 147 } | |
| 148 rescue => e | |
| 149 if json.is_a? Hash | |
| 150 if json["errors"].is_a? Array | |
| 151 return rate_limit_check(json) | |
| 152 elsif json["data"].is_a? Hash | |
| 153 return | |
| 154 end | |
| 155 end | |
| 156 Rails.logger.error("#{error_key} fail: #{resp}") | |
| 157 | |
| 158 raise e | |
| 159 end | |
| 160 | |
| 161 private def rate_limit_check(json) | |
| 260 | 162 return unless json["errors"].any? { |err| err["code"] == 88 } |
|
243
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
163 |
| 264 | 164 raise "Rate limited! Client: #{@id}" |
|
243
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
165 end |
| 234 | 166 end |