erjemin/instaloader
erjemin/instaloader
1
0
Fork 0
Code Issues Wiki Activity

Set graphql_count_per_slidingwindow to 20

Browse Source 
Instagram tightened their rate limits again.
Implemented hidden switch --graphql-rate-limit for easier adjustment of
graphql_count_per_slidingwindow.
Fixes #118
This commit is contained in:
André Koch-Kramer 2018-05-13 19:09:40 +02:00
parent b8797cf337
commit e1228501d2
3 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
instaloader/__main__.py
View File

@ -271,6 +271,7 @@ def main():
g_how.add_argument('--user-agent',
help='User Agent to use for HTTP requests. Defaults to \'{}\'.'.format(default_user_agent()))
g_how.add_argument('-S', '--no-sleep', action='store_true', help=SUPPRESS)
g_how.add_argument('--graphql-rate-limit', type=int, default=20, help=SUPPRESS)
g_how.add_argument('--max-connection-attempts', metavar='N', type=int, default=3,
help='Maximum number of connection attempts until a request is aborted. Defaults to 3. If a '
'connection fails, it can be manually skipped by hitting CTRL+C. Set this to 0 to retry '
@ -316,6 +317,7 @@ def main():
compress_json=not args.no_compress_json,
post_metadata_txt_pattern=post_metadata_txt_pattern,
storyitem_metadata_txt_pattern=storyitem_metadata_txt_pattern,
graphql_rate_limit=args.graphql_rate_limit,
max_connection_attempts=args.max_connection_attempts)
_main(loader,
args.profile,

6
instaloader/instaloader.py
View File

@ -106,9 +106,10 @@ class Instaloader:
compress_json: bool = True,
post_metadata_txt_pattern: str = None,
storyitem_metadata_txt_pattern: str = None,
graphql_rate_limit: int = 20,
max_connection_attempts: int = 3):
self.context = InstaloaderContext(sleep, quiet, user_agent, max_connection_attempts)
self.context = InstaloaderContext(sleep, quiet, user_agent, graphql_rate_limit, max_connection_attempts)
# configuration parameters
self.dirname_pattern = dirname_pattern or "{target}"
@ -131,7 +132,8 @@ class Instaloader:
self.filename_pattern, self.download_videos, self.download_video_thumbnails,
self.download_geotags, self.download_comments, self.save_metadata,
self.compress_json, self.post_metadata_txt_pattern,
self.storyitem_metadata_txt_pattern, self.context.max_connection_attempts)
self.storyitem_metadata_txt_pattern, self.context.graphql_count_per_slidingwindow,
self.context.max_connection_attempts)
new_loader.context.query_timestamps = self.context.query_timestamps
yield new_loader
self.context.error_log.extend(new_loader.context.error_log)

9
instaloader/instaloadercontext.py
View File

@ -46,8 +46,8 @@ class InstaloaderContext:
class :class:`Instaloader`.
"""
def __init__(self, sleep: bool = True, quiet: bool = False,
user_agent: Optional[str] = None, max_connection_attempts: int = 3):
def __init__(self, sleep: bool = True, quiet: bool = False, user_agent: Optional[str] = None,
graphql_count_per_slidingwindow: int = 20, max_connection_attempts: int = 3):
self.user_agent = user_agent if user_agent is not None else default_user_agent()
self._session = self.get_anonymous_session()
@ -56,7 +56,8 @@ class InstaloaderContext:
self.quiet = quiet
self.max_connection_attempts = max_connection_attempts
self._graphql_page_length = 50
self._graphql_count_per_slidingwindow = 25
self.graphql_count_per_slidingwindow = graphql_count_per_slidingwindow \
if graphql_count_per_slidingwindow else 20
self._root_rhx_gis = None
# error log, filled with error() and printed at the end of Instaloader.main()
@ -207,7 +208,7 @@ class InstaloaderContext:
return sliding_window if untracked_queries else 0
current_time = time.monotonic()
self.query_timestamps = list(filter(lambda t: t > current_time - sliding_window, self.query_timestamps))
if len(self.query_timestamps) < self._graphql_count_per_slidingwindow and not untracked_queries:
if len(self.query_timestamps) < self.graphql_count_per_slidingwindow and not untracked_queries:
return 0
return round(min(self.query_timestamps) + sliding_window - current_time) + 6
is_graphql_query = 'query_hash' in params and 'graphql/query' in path