erjemin/instaloader
erjemin/instaloader
1
0
Fork 0
Code Issues Wiki Activity

Rate-control non-graphql queries

Browse Source 
Non-GraphQL queries seem to be rate limited too, and this commit adds
a safe-seeming rate-controlling for this kind of queries.

This fixes #464.
This commit is contained in:
Alexander Graf 2020-02-03 17:19:49 +01:00
parent f966ea5bb9
commit de585becdb
1 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
instaloader/instaloadercontext.py
View File

@ -298,9 +298,14 @@ class InstaloaderContext:
def _graphql_request_count_per_sliding_window(self, query_hash: str) -> int: def _graphql_request_count_per_sliding_window(self, query_hash: str) -> int:
"""Return how many GraphQL requests can be done within the sliding window.""" """Return how many GraphQL requests can be done within the sliding window."""
if self.is_logged_in: if self.is_logged_in:
max_reqs = {'1cb6ec562846122743b61e492c85999f': 20, '33ba35852cb50da46f5b5e889df7d159': 20, 'iphone': 100} max_reqs = {'1cb6ec562846122743b61e492c85999f': 20,
'33ba35852cb50da46f5b5e889df7d159': 20,
'iphone': 100,
'other': 100}
else: else:
max_reqs = {'1cb6ec562846122743b61e492c85999f': 200, '33ba35852cb50da46f5b5e889df7d159': 200} max_reqs = {'1cb6ec562846122743b61e492c85999f': 200,
'33ba35852cb50da46f5b5e889df7d159': 200,
'other': 200}
return max_reqs.get(query_hash) or min(max_reqs.values()) return max_reqs.get(query_hash) or min(max_reqs.values())
def _graphql_query_waittime(self, query_hash: str, current_time: float, untracked_queries: bool = False) -> float: def _graphql_query_waittime(self, query_hash: str, current_time: float, untracked_queries: bool = False) -> float:
@ -366,6 +371,7 @@ class InstaloaderContext:
""" """
is_graphql_query = 'query_hash' in params and 'graphql/query' in path is_graphql_query = 'query_hash' in params and 'graphql/query' in path
is_iphone_query = host == 'i.instagram.com' is_iphone_query = host == 'i.instagram.com'
is_other_query = not is_graphql_query and host == "www.instagram.com"
sess = session if session else self._session sess = session if session else self._session
try: try:
self.do_sleep() self.do_sleep()
@ -373,6 +379,8 @@ class InstaloaderContext:
self._ratecontrol_graphql_query(params['query_hash']) self._ratecontrol_graphql_query(params['query_hash'])
if is_iphone_query: if is_iphone_query:
self._ratecontrol_graphql_query('iphone') self._ratecontrol_graphql_query('iphone')
if is_other_query:
self._ratecontrol_graphql_query('other')
resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False) resp = sess.get('https://{0}/{1}'.format(host, path), params=params, allow_redirects=False)
while resp.is_redirect: while resp.is_redirect:
redirect_url = resp.headers['location'] redirect_url = resp.headers['location']
@ -432,6 +440,8 @@ class InstaloaderContext:
self._ratecontrol_graphql_query(params['query_hash'], untracked_queries=True) self._ratecontrol_graphql_query(params['query_hash'], untracked_queries=True)
if is_iphone_query and isinstance(err, TooManyRequestsException): if is_iphone_query and isinstance(err, TooManyRequestsException):
self._ratecontrol_graphql_query('iphone', untracked_queries=True) self._ratecontrol_graphql_query('iphone', untracked_queries=True)
if is_other_query and isinstance(err, TooManyRequestsException):
self._ratecontrol_graphql_query('other', untracked_queries=True)
return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1) return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)
except KeyboardInterrupt: except KeyboardInterrupt:
self.error("[skipped by user]", repeat_at_end=False) self.error("[skipped by user]", repeat_at_end=False)