annotate app/lib/legit_client.rb @ 236:498043313523 legit-client

Support multiple headers
author nanaya <me@nanaya.net>
date Sat, 15 Jul 2023 01:40:31 +0900
parents 7a773720d81f
children 961d362e42c7
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
234
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
1 module LegitClient
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
2 def self.timeline(user_id)
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
3 resp = fetch("https://twitter.com/i/api/graphql/1-5o8Qhfc2kWlu_2rWNcug/UserTweetsAndReplies?variables=%7B%22userId%22%3A#{escape_param user_id}%2C%22count%22%3A50%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_lists_timeline_redesign_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%2C%22withArticleRichContentState%22%3Afalse%7D")
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
4
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
5 begin
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
6 json = JSON.parse(resp)
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
7 {
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
8 timeline: normalize_timeline(json['data']['user']['result']['timeline_v2']['timeline']['instructions']),
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
9 raw: resp,
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
10 }
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
11 rescue => e
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
12 return if (json || {}).dig('data').is_a? Hash
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
13 Rails.logger.error("timeline fail: #{user_id}: #{resp}")
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
14 nil
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
15 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
16 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
17
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
18 def self.user_by_id(user_id)
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
19 resp = fetch("https://twitter.com/i/api/graphql/i_0UQ54YrCyqLUvgGzXygA/UserByRestId?variables=%7B%22userId%22%3A#{escape_param user_id}%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22hidden_profile_subscriptions_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%7D")
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
20
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
21 begin
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
22 json = JSON.parse(resp)
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
23 {
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
24 user: normalize_user(json['data']['user']['result']),
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
25 raw: resp,
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
26 }
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
27 rescue
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
28 return if (json || {}).dig('data').is_a? Hash
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
29 Rails.logger.error("user_by_id fail: #{user_id}: #{resp}")
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
30 nil
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
31 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
32 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
33
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
34 def self.user_by_username(username)
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
35 resp = fetch("https://twitter.com/i/api/graphql/xc8f1g7BYqr6VTzTbvNlGw/UserByScreenName?variables=%7B%22screen_name%22%3A#{escape_param username}%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22hidden_profile_subscriptions_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22subscriptions_verification_info_verified_since_enabled%22%3Atrue%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Afalse%7D")
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
36
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
37 begin
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
38 json = JSON.parse(resp)
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
39 {
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
40 user: normalize_user(json['data']['user']['result']),
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
41 raw: resp,
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
42 }
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
43 rescue
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
44 return if (json || {}).dig('data').is_a? Hash
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
45 Rails.logger.error("user_by_username fail: #{username}: #{resp}")
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
46 nil
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
47 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
48 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
49
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
50 def self.escape_param(param)
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
51 CGI.escape JSON.dump(param)
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
52 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
53
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
54 def self.fetch(uri)
236
498043313523 Support multiple headers
nanaya <me@nanaya.net>
parents: 234
diff changeset
55 Net::HTTP.get(URI(uri), $cfg[:headers].sample)
234
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
56 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
57
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
58 def self.normalize_entity_media(json)
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
59 ret = {}
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
60
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
61 json.each do |entity_media|
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
62 val = {}
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
63
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
64 if entity_media['type'] == 'photo'
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
65 val[:url] = entity_media['media_url_https']
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
66 elsif entity_media['type'] == 'video'
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
67 val[:url] = entity_media['expanded_url']
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
68 val[:variants] = entity_media['video_info']['variants']
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
69 .filter { |variant| variant['bitrate'].present? }
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
70 .map do |variant|
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
71 {
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
72 bitrate: variant['bitrate'],
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
73 url: variant['url'],
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
74 }
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
75 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
76 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
77
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
78 if !val.empty?
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
79 val[:type] = entity_media['type']
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
80 val[:id] = entity_media['media_key']
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
81 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
82
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
83 ret[entity_media['display_url']] = val
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
84 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
85
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
86 ret
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
87 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
88
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
89 def self.normalize_entity_urls(json)
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
90 ret = {}
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
91
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
92 json.each do |entity_url|
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
93 ret[entity_url['url']] = entity_url['expanded_url']
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
94 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
95
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
96 ret
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
97 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
98
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
99 def self.normalize_timeline(json)
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
100 json.find { |instruction| instruction['type'] == 'TimelineAddEntries' }['entries']
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
101 .filter { |entry| entry['entryId'] =~ /\A(profile-conversation|tweet)-/ }
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
102 .reduce([]) do |acc, entry|
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
103 if entry['content']['entryType'] == 'TimelineTimelineItem'
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
104 acc.push(entry['content'])
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
105 else
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
106 entry['content']['items'].each do |item|
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
107 acc.push(item['item'])
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
108 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
109 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
110 acc
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
111 end.map { |rawTweet| normalize_tweet(rawTweet['itemContent']['tweet_results']['result']) }
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
112 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
113
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
114 def self.normalize_tweet(json)
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
115 return nil if json.nil?
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
116
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
117 return normalize_tweet(json['tweet']) if json['__typename'] == 'TweetWithVisibilityResults'
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
118
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
119 {
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
120 id: json['rest_id'],
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
121 created_at: Time.parse(json['legacy']['created_at']),
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
122 user: normalize_user(json['core']['user_results']['result']),
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
123 message: json['legacy']['full_text'],
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
124 retweet: normalize_tweet(json.dig('legacy', 'retweeted_status_result', 'result')),
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
125 quote: normalize_tweet(json.dig('quoted_status_result', 'result')),
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
126 quote_id: json['legacy']['quoted_status_id_str'],
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
127 reply_to_id: json['legacy']['in_reply_to_status_id_str'],
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
128 reply_to_user_id: json['legacy']['in_reply_to_user_id_str'],
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
129 reply_to_username: json['legacy']['in_reply_to_screen_name'],
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
130 entity_urls: normalize_entity_urls(json['legacy']['entities']['urls']),
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
131 entity_media: normalize_entity_media(json.dig('legacy', 'extended_entities', 'media') || []),
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
132 }
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
133 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
134
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
135 def self.normalize_user(json)
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
136 {
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
137 avatar_url: json['legacy']['profile_image_url_https'],
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
138 id: json['rest_id'],
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
139 name: json['legacy']['name'],
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
140 protected: json['legacy']['protected'] == true,
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
141 username: json['legacy']['screen_name'],
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
142 }
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
143 end
7a773720d81f Totally legit client
nanaya <me@nanaya.net>
parents:
diff changeset
144 end