Mercurial > rsstweet
annotate app/lib/legit_client.rb @ 258:20eb1429af2e
Remove newrelic
| author | nanaya <me@nanaya.net> |
|---|---|
| date | Sun, 15 Dec 2024 22:49:56 +0900 |
| parents | c6a50441a58d |
| children | 8b75d00c77ba |
| rev | line source |
|---|---|
| 234 | 1 module LegitClient |
| 2 def self.timeline(user_id) | |
| 257 | 3 resp = fetch("https://x.com/i/api/graphql/1-5o8Qhfc2kWlu_2rWNcug/UserTweetsAndReplies?variables=%7B%22userId%22%3A#{escape_param user_id}%2C%22count%22%3A50%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_lists_timeline_redesign_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%2C%22withArticleRichContentState%22%3Afalse%7D") |
| 234 | 4 |
|
243
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
5 handle_response resp, :timeline, "timeline(#{user_id})", ->(json) do |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
6 normalize_timeline json['data']['user']['result']['timeline_v2']['timeline']['instructions'], user_id |
| 234 | 7 end |
| 8 end | |
| 9 | |
| 10 def self.user_by_id(user_id) | |
| 257 | 11 resp = fetch("https://x.com/i/api/graphql/i_0UQ54YrCyqLUvgGzXygA/UserByRestId?variables=%7B%22userId%22%3A#{escape_param user_id}%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22hidden_profile_subscriptions_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%7D") |
| 234 | 12 |
|
243
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
13 handle_response resp, :user, "user_by_id(#{user_id})", ->(json) do |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
14 normalize_user json['data']['user']['result'] |
| 234 | 15 end |
| 16 end | |
| 17 | |
| 18 def self.user_by_username(username) | |
| 257 | 19 resp = fetch("https://x.com/i/api/graphql/xc8f1g7BYqr6VTzTbvNlGw/UserByScreenName?variables=%7B%22screen_name%22%3A#{escape_param username}%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22hidden_profile_subscriptions_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22subscriptions_verification_info_verified_since_enabled%22%3Atrue%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%7D") |
| 234 | 20 |
|
243
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
21 handle_response resp, :user, "user_by_username(#{username})", ->(json) do |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
22 normalize_user json['data']['user']['result'] |
| 234 | 23 end |
| 24 end | |
| 25 | |
| 26 def self.escape_param(param) | |
| 27 CGI.escape JSON.dump(param) | |
| 28 end | |
| 29 | |
| 30 def self.fetch(uri) | |
| 236 | 31 Net::HTTP.get(URI(uri), $cfg[:headers].sample) |
| 234 | 32 end |
| 33 | |
|
243
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
34 def self.handle_response(resp, key, error_key, callback) |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
35 json = JSON.parse(resp) |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
36 { |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
37 key => callback.call(json), |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
38 raw: resp, |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
39 } |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
40 rescue => e |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
41 if json.is_a? Hash |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
42 if json['errors'].is_a? Array |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
43 return rate_limit_check(json) |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
44 elsif json['data'].is_a? Hash |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
45 return |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
46 end |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
47 end |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
48 Rails.logger.error("#{error_key} fail: #{resp}") |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
49 |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
50 raise e |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
51 end |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
52 |
| 234 | 53 def self.normalize_entity_media(json) |
| 54 ret = {} | |
| 55 | |
| 56 json.each do |entity_media| | |
| 57 val = {} | |
| 58 | |
|
253
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
59 case entity_media['type'] |
|
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
60 when 'animated_gif', 'video' |
|
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
61 val[:variants] = entity_media['video_info']['variants'] |
|
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
62 .filter { |variant| variant['bitrate'].present? } |
|
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
63 .map do |variant| |
|
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
64 { |
|
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
65 bitrate: variant['bitrate'], |
|
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
66 url: variant['url'], |
|
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
67 } |
|
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
68 end |
|
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
69 when 'photo' |
|
d726e8b92dd1
Support animated gif (same as video)
nanaya <me@nanaya.net>
parents:
252
diff
changeset
|
70 val[:image_url] = entity_media['media_url_https'].sub(/\.([^.]+)$/, '?format=\1') |
| 234 | 71 end |
| 72 | |
| 73 if !val.empty? | |
|
237
961d362e42c7
The url in entity media isn't unique as they all point to the same thing
nanaya <me@nanaya.net>
parents:
236
diff
changeset
|
74 val[:url] = entity_media['expanded_url'] |
| 234 | 75 val[:type] = entity_media['type'] |
| 76 val[:id] = entity_media['media_key'] | |
| 77 end | |
| 78 | |
| 241 | 79 key = if ret[entity_media['url']].nil? |
| 80 entity_media['url'] | |
|
237
961d362e42c7
The url in entity media isn't unique as they all point to the same thing
nanaya <me@nanaya.net>
parents:
236
diff
changeset
|
81 else |
|
961d362e42c7
The url in entity media isn't unique as they all point to the same thing
nanaya <me@nanaya.net>
parents:
236
diff
changeset
|
82 entity_media['media_key'] |
|
961d362e42c7
The url in entity media isn't unique as they all point to the same thing
nanaya <me@nanaya.net>
parents:
236
diff
changeset
|
83 end |
|
961d362e42c7
The url in entity media isn't unique as they all point to the same thing
nanaya <me@nanaya.net>
parents:
236
diff
changeset
|
84 |
|
961d362e42c7
The url in entity media isn't unique as they all point to the same thing
nanaya <me@nanaya.net>
parents:
236
diff
changeset
|
85 ret[key] = val |
| 234 | 86 end |
| 87 | |
| 88 ret | |
| 89 end | |
| 90 | |
| 91 def self.normalize_entity_urls(json) | |
| 92 ret = {} | |
| 93 | |
|
240
c454ea4f7b34
Add support for note tweets (with no formatting)
nanaya <me@nanaya.net>
parents:
238
diff
changeset
|
94 (json || {}).each do |entity_url| |
| 234 | 95 ret[entity_url['url']] = entity_url['expanded_url'] |
| 96 end | |
| 97 | |
| 98 ret | |
| 99 end | |
| 100 | |
|
238
a04b4830eef2
Filter out non-own tweets included for replies
nanaya <me@nanaya.net>
parents:
237
diff
changeset
|
101 def self.normalize_timeline(json, user_id) |
| 252 | 102 json |
| 103 .reduce([]) do |acc, instruction| | |
| 104 case instruction['type'] | |
| 105 when 'TimelineAddEntries' then acc += instruction['entries'] | |
| 106 when 'TimelinePinEntry' then acc << instruction['entry'] | |
| 107 end | |
| 108 | |
| 109 acc | |
| 110 end.filter { |entry| entry['entryId'] =~ /\A(profile-conversation|tweet)-/ } | |
| 234 | 111 .reduce([]) do |acc, entry| |
| 112 if entry['content']['entryType'] == 'TimelineTimelineItem' | |
| 113 acc.push(entry['content']) | |
| 114 else | |
| 115 entry['content']['items'].each do |item| | |
| 116 acc.push(item['item']) | |
| 117 end | |
| 118 end | |
| 119 acc | |
| 249 | 120 end.map { |raw_tweet| normalize_tweet(raw_tweet['itemContent']['tweet_results']['result']) } |
|
238
a04b4830eef2
Filter out non-own tweets included for replies
nanaya <me@nanaya.net>
parents:
237
diff
changeset
|
121 .filter { |tweet| !tweet.nil? && tweet.dig(:user, :id) == user_id } |
| 234 | 122 end |
| 123 | |
| 124 def self.normalize_tweet(json) | |
| 125 return nil if json.nil? | |
| 126 | |
| 127 return normalize_tweet(json['tweet']) if json['__typename'] == 'TweetWithVisibilityResults' | |
| 128 | |
| 129 { | |
| 130 id: json['rest_id'], | |
| 131 created_at: Time.parse(json['legacy']['created_at']), | |
| 132 user: normalize_user(json['core']['user_results']['result']), | |
|
240
c454ea4f7b34
Add support for note tweets (with no formatting)
nanaya <me@nanaya.net>
parents:
238
diff
changeset
|
133 message: json.dig('note_tweet', 'note_tweet_results', 'result', 'text') || json['legacy']['full_text'], |
| 234 | 134 retweet: normalize_tweet(json.dig('legacy', 'retweeted_status_result', 'result')), |
| 135 quote: normalize_tweet(json.dig('quoted_status_result', 'result')), | |
| 136 quote_id: json['legacy']['quoted_status_id_str'], | |
| 137 reply_to_id: json['legacy']['in_reply_to_status_id_str'], | |
| 138 reply_to_user_id: json['legacy']['in_reply_to_user_id_str'], | |
| 139 reply_to_username: json['legacy']['in_reply_to_screen_name'], | |
|
240
c454ea4f7b34
Add support for note tweets (with no formatting)
nanaya <me@nanaya.net>
parents:
238
diff
changeset
|
140 entity_urls: { **normalize_entity_urls(json['legacy']['entities']['urls']), **normalize_entity_urls(json.dig('note_tweet', 'note_tweet_results', 'result', 'entity_set', 'urls')) }, |
| 234 | 141 entity_media: normalize_entity_media(json.dig('legacy', 'extended_entities', 'media') || []), |
| 142 } | |
| 143 end | |
| 144 | |
| 145 def self.normalize_user(json) | |
| 146 { | |
| 147 avatar_url: json['legacy']['profile_image_url_https'], | |
| 148 id: json['rest_id'], | |
| 149 name: json['legacy']['name'], | |
| 150 protected: json['legacy']['protected'] == true, | |
| 151 username: json['legacy']['screen_name'], | |
| 152 } | |
| 153 end | |
|
243
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
154 |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
155 def self.rate_limit_check(json) |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
156 return unless json['errors'].any? { |err| err['code'] == 88 } |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
157 |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
158 raise 'Rate limited!' |
|
bc2f45058c9e
Prevent caching of rate limited error and combine response handling
nanaya <me@nanaya.net>
parents:
241
diff
changeset
|
159 end |
| 234 | 160 end |