From e00d77c234dfece7a61f4029e6661543a518f358 Mon Sep 17 00:00:00 2001 From: Lars Lindqvist Date: Mon, 20 Aug 2018 08:43:53 +0200 Subject: [PATCH 01/27] Basic CLI support for tagged posts. Squashed commit of the following (pr #154): commit 8fd56c379ff89ff634b510df8abde5a9c50218f0 Merge: 08f0ee7 a3777ca Author: Lars Lindqvist Date: Wed Aug 15 20:23:23 2018 +0200 Merge branch 'master' into master commit 08f0ee795c2273b09056031253781776c5c93bf4 Merge: 700b3a8 dcea0e9 Author: Lars Lindqvist Date: Sun Aug 5 15:25:55 2018 +0200 Merge branch 'master' into master commit 700b3a8d094f552caa638a9d91f9221392a8e3f0 Author: Lars Lindqvist Date: Sat Aug 4 16:26:59 2018 +0200 Basic CLI support for tagged posts. commit 5e3cd10cbcbec6d29abd6a56ab9c39294a8d44b3 Merge: af564f5 92653dc Author: Lars Lindqvist Date: Fri Aug 3 19:33:24 2018 +0200 Merge branch 'master' into master commit af564f5174d1ffd3af3f7e635b650651e1f7411a Author: Lars Lindqvist Date: Fri Aug 3 19:25:57 2018 +0200 Fix owner_profile for Profile.get_tagged_posts() commit 3cde1f7db4860edaca970c70543ed5bca0f97853 Author: Lars Lindqvist Date: Thu Jul 26 19:51:33 2018 +0200 Add meth:get_tagged_posts to Profile --- instaloader/__main__.py | 13 +++++++++++-- instaloader/instaloader.py | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/instaloader/__main__.py b/instaloader/__main__.py index 9f9c54e..a8f5aae 100644 --- a/instaloader/__main__.py +++ b/instaloader/__main__.py @@ -62,6 +62,7 @@ def _main(instaloader: Instaloader, targetlist: List[str], profile_pic: bool = True, profile_pic_only: bool = False, fast_update: bool = False, stories: bool = False, stories_only: bool = False, + tagged: bool = False, tagged_only: bool = False, post_filter_str: Optional[str] = None, storyitem_filter_str: Optional[str] = None) -> None: """Download set of profiles, hashtags etc. and handle logging in and session files if desired.""" @@ -171,7 +172,8 @@ def _main(instaloader: Instaloader, targetlist: List[str], for target in profiles: with instaloader.context.error_catcher(target): instaloader.download_profile(target, download_profile_pic, not download_profile_posts, - fast_update, post_filter=post_filter) + fast_update, download_tagged=tagged, + download_tagged_only=tagged_only, post_filter=post_filter) if anonymous_retry_profiles: instaloader.context.log("Downloading anonymously: {}" .format(' '.join([p.username for p in anonymous_retry_profiles]))) @@ -179,7 +181,8 @@ def _main(instaloader: Instaloader, targetlist: List[str], for target in anonymous_retry_profiles: with instaloader.context.error_catcher(target): anonymous_loader.download_profile(target, download_profile_pic, not download_profile_posts, - fast_update, post_filter=post_filter) + fast_update, download_tagged=tagged, + download_tagged_only=tagged_only, post_filter=post_filter) if download_profile_stories and profiles: with instaloader.context.error_catcher("Download stories"): instaloader.context.log("Downloading stories") @@ -254,6 +257,10 @@ def main(): g_what.add_argument('--stories-only', action='store_true', help='Rather than downloading regular posts of each specified profile, only download ' 'stories. Requires --login. Does not imply --no-profile-pic.') + g_what.add_argument('--tagged', action='store_true', + help='Also download posts where each profile is tagged.') + g_what.add_argument('--tagged-only', action='store_true', + help='Download only post where each profile is tagged, not their regular posts.') g_what.add_argument('--post-filter', '--only-if', metavar='filter', help='Expression that, if given, must evaluate to True for each post to be downloaded. Must be ' 'a syntactically valid python expression. Variables are evaluated to ' @@ -360,6 +367,8 @@ def main(): fast_update=args.fast_update, stories=args.stories, stories_only=args.stories_only, + tagged=args.tagged, + tagged_only=args.tagged_only, post_filter_str=args.post_filter, storyitem_filter_str=args.storyitem_filter) loader.close() diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index d8e5ec6..8d171ed 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -619,6 +619,23 @@ class Instaloader: if fast_update and not downloaded: break + def download_tagged(self, profile: Profile, fast_update: bool = False, + target: Optional[str] = None, + post_filter: Optional[Callable[[Post], bool]] = None) -> None: + if target is None: + target = profile.username + ':tagged' + self.context.log("Retrieving tagged posts for profile {}.".format(profile.username)) + count = 1 + for post in profile.get_tagged_posts(): + self.context.log("[%3i/???] " % (count), end="", flush=True) + count += 1 + if post_filter is not None and not post_filter(post): + self.context.log('<{} skipped>'.format(post)) + with self.context.error_catcher('Download tagged {}'.format(profile.username)): + downloaded = self.download_post(post, target) + if fast_update and not downloaded: + break + def _get_id_filename(self, profile_name: str) -> str: if ((format_string_contains_key(self.dirname_pattern, 'profile') or format_string_contains_key(self.dirname_pattern, 'target'))): @@ -686,6 +703,7 @@ class Instaloader: profile_pic: bool = True, profile_pic_only: bool = False, fast_update: bool = False, download_stories: bool = False, download_stories_only: bool = False, + download_tagged: bool = False, download_tagged_only: bool = False, post_filter: Optional[Callable[[Post], bool]] = None, storyitem_filter: Optional[Callable[[StoryItem], bool]] = None) -> None: """Download one profile""" @@ -739,6 +757,13 @@ class Instaloader: if download_stories_only: return + # Download tagged, if requested + if download_tagged or download_tagged_only: + with self.context.error_catcher('Download tagged of {}'.format(profile_name)): + self.download_tagged(profile, fast_update=fast_update, post_filter=post_filter) + if download_tagged_only: + return + # Iterate over pictures and download them self.context.log("Retrieving posts from profile {}.".format(profile_name)) totalcount = profile.mediacount From 9258b84695732ae13cf3ce07eb5befea2fe066dd Mon Sep 17 00:00:00 2001 From: AndCycle Date: Mon, 20 Aug 2018 09:00:51 +0200 Subject: [PATCH 02/27] Allow reading arguments from file Squashed commit of the following (pr #161): commit 58235a99b04a975d825f4f8f8431a6dcb7acccdc Author: AndCycle Date: Thu Aug 16 17:57:26 2018 +0800 doc: proper fix rst syntax commit 6620e3583c0b59e9447bcc44e5f573c61d02e6de Author: AndCycle Date: Thu Aug 16 17:29:25 2018 +0800 doc: fix syntax commit 7a048a3c0440ad900917eb865b72dcfb348da879 Author: AndCycle Date: Thu Aug 16 17:09:26 2018 +0800 doc: add info about read args from file. commit c4ceaf96365422f25357c54af03aadfa7222ce44 Author: AndCycle Date: Thu Aug 16 17:05:07 2018 +0800 alter `fromfile_prefix_chars` to plus sign to avoid conflict. commit d469b52b95de6bf86761b9315a78ecda2b033ac2 Author: AndCycle Date: Mon Aug 6 19:10:23 2018 +0800 * Allow reading arguments from file function referenced from instagram-scraper, which provide a convenient way to hide sensitive information. --- docs/cli-options.rst | 17 +++++++++++++++++ instaloader/__main__.py | 3 ++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/cli-options.rst b/docs/cli-options.rst index 2b6e6fc..1b585e9 100644 --- a/docs/cli-options.rst +++ b/docs/cli-options.rst @@ -192,3 +192,20 @@ Miscellaneous Options Disable user interaction, i.e. do not print messages (except errors) and fail if login credentials are needed but not given. This is handy for running :ref:`instaloader-as-cronjob`. + +.. option:: +args.txt + + Read arguments from file `args.txt`, a shortcut to provide argument from + file rather than command-line. This provide a convient way to hide login + info from CLI. and also can use for simplify managment of long arguments. + + .. note:: + + text file should separate arg with line break. + + args.txt example:: + + --login MYUSENAME + --password MYPASSWORD + --fast-update + diff --git a/instaloader/__main__.py b/instaloader/__main__.py index a8f5aae..f340c6d 100644 --- a/instaloader/__main__.py +++ b/instaloader/__main__.py @@ -207,7 +207,8 @@ def main(): parser = ArgumentParser(description=__doc__, add_help=False, usage=usage_string(), epilog="Report issues at https://github.com/instaloader/instaloader/issues. " "The complete documentation can be found at " - "https://instaloader.github.io/.") + "https://instaloader.github.io/.", + fromfile_prefix_chars='+') g_what = parser.add_argument_group('What to Download', 'Specify a list of profiles or #hashtags. For each of these, Instaloader ' From f7e6632f68b0474535aadb047400335fce82c461 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 20 Aug 2018 10:44:20 +0200 Subject: [PATCH 03/27] Document --tagged{,-only} in docs/cli-options.rst --- docs/cli-options.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/cli-options.rst b/docs/cli-options.rst index 1b585e9..6694676 100644 --- a/docs/cli-options.rst +++ b/docs/cli-options.rst @@ -74,6 +74,10 @@ automatically **finds it by its unique ID** and renames the folder likewise. Also **download stories** of each profile that is downloaded. Requires :option:`--login`. +.. option:: --tagged + + Also download posts where each profile is tagged. + .. option:: --no-metadata-json Do not create a JSON file containing the metadata of each post. @@ -93,6 +97,10 @@ automatically **finds it by its unique ID** and renames the folder likewise. If possible, use ``:stories`` target rather than :option:`--stories-only` with all your followees. ``:stories`` uses fewer API requests. +.. option:: --tagged-only + + Download only post where each profile is tagged, not their regular posts. + .. option:: --post-filter filter, --only-if filter Expression that, if given, must evaluate to True for each post to be From 5f57345f1bd2a70922bcc8df909c23f9ad8737b1 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 20 Aug 2018 09:31:30 +0200 Subject: [PATCH 04/27] "instaloader -- -SHORTCODE" to fetch single post Closes #129. --- docs/basic-usage.rst | 4 ++++ instaloader/__main__.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/docs/basic-usage.rst b/docs/basic-usage.rst index e45fd68..18d97f3 100644 --- a/docs/basic-usage.rst +++ b/docs/basic-usage.rst @@ -84,6 +84,10 @@ Instaloader supports the following targets: All profiles that are followed by ``profile``, i.e. the *followees* of ``profile`` (requires :option:`--login`). +- ``-post`` + The single **post** with the given shortcode. Must be preceeded by ``--`` in + the argument list to not be mistaken as an option flag. + Instaloader goes through all media matching the specified targets and downloads the pictures and videos and their captions. You can specify diff --git a/instaloader/__main__.py b/instaloader/__main__.py index f340c6d..ee4ed70 100644 --- a/instaloader/__main__.py +++ b/instaloader/__main__.py @@ -133,6 +133,8 @@ def _main(instaloader: Instaloader, targetlist: List[str], elif target[0] == '#': instaloader.download_hashtag(hashtag=target[1:], max_count=max_count, fast_update=fast_update, post_filter=post_filter) + elif target[0] == '-': + instaloader.download_post(Post.from_shortcode(instaloader.context, target[1:]), target) elif target == ":feed": instaloader.download_feed_posts(fast_update=fast_update, max_count=max_count, post_filter=post_filter) From 06b7edd6d51c459c1cf0f848c5fd6e34ecf43ffe Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 20 Aug 2018 10:37:51 +0200 Subject: [PATCH 05/27] --no-pictures flag to not download post pictures Closes #131. --- docs/cli-options.rst | 6 +++++ instaloader/__main__.py | 7 +++++ instaloader/instaloader.py | 54 ++++++++++++++++++++++---------------- 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/docs/cli-options.rst b/docs/cli-options.rst index 6694676..537bd18 100644 --- a/docs/cli-options.rst +++ b/docs/cli-options.rst @@ -35,6 +35,12 @@ automatically **finds it by its unique ID** and renames the folder likewise. Do not download profile picture. +.. option:: --no-pictures + + Do not download post pictures. Cannot be used together with + :option:`--fast-update`. Implies :option:`--no-video-thumbnails`, does not + imply :option:`--no-videos`. + .. option:: --no-videos, -V Do not download videos. diff --git a/instaloader/__main__.py b/instaloader/__main__.py index ee4ed70..ffd6d3d 100644 --- a/instaloader/__main__.py +++ b/instaloader/__main__.py @@ -230,6 +230,9 @@ def main(): help='Only download profile picture.') g_what.add_argument('--no-profile-pic', action='store_true', help='Do not download profile picture.') + g_what.add_argument('--no-pictures', action='store_true', + help='Do not download post pictures. Cannot be used together with --fast-update. ' + 'Implies --no-video-thumbnails, does not imply --no-videos.') g_what.add_argument('-V', '--no-videos', action='store_true', help='Do not download videos.') g_what.add_argument('--no-video-thumbnails', action='store_true', @@ -349,8 +352,12 @@ def main(): raise SystemExit("--no-captions and --post-metadata-txt or --storyitem-metadata-txt given; " "That contradicts.") + if args.no_pictures and args.fast_update: + raise SystemExit('--no-pictures and --fast-update cannot be used together.') + loader = Instaloader(sleep=not args.no_sleep, quiet=args.quiet, user_agent=args.user_agent, dirname_pattern=args.dirname_pattern, filename_pattern=args.filename_pattern, + download_pictures=not args.no_pictures, download_videos=not args.no_videos, download_video_thumbnails=not args.no_video_thumbnails, download_geotags=args.geotags, download_comments=args.comments, save_metadata=not args.no_metadata_json, diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index 8d171ed..fa5b4ed 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -80,6 +80,7 @@ class Instaloader: :param user_agent: :option:`--user-agent` :param dirname_pattern: :option:`--dirname-pattern`, default is ``{target}`` :param filename_pattern: :option:`--filename-pattern`, default is ``{date_utc}_UTC`` + :param download_pictures: not :option:`--no-pictures` :param download_videos: not :option:`--no-videos` :param download_video_thumbnails: not :option:`--no-video-thumbnails` :param download_geotags: :option:`--geotags` @@ -102,6 +103,7 @@ class Instaloader: user_agent: Optional[str] = None, dirname_pattern: Optional[str] = None, filename_pattern: Optional[str] = None, + download_pictures=True, download_videos: bool = True, download_video_thumbnails: bool = True, download_geotags: bool = True, @@ -118,6 +120,7 @@ class Instaloader: # configuration parameters self.dirname_pattern = dirname_pattern or "{target}" self.filename_pattern = filename_pattern or "{date_utc}_UTC" + self.download_pictures = download_pictures self.download_videos = download_videos self.download_video_thumbnails = download_video_thumbnails self.download_geotags = download_geotags @@ -133,11 +136,15 @@ class Instaloader: def anonymous_copy(self): """Yield an anonymous, otherwise equally-configured copy of an Instaloader instance; Then copy its error log.""" new_loader = Instaloader(self.context.sleep, self.context.quiet, self.context.user_agent, self.dirname_pattern, - self.filename_pattern, self.download_videos, self.download_video_thumbnails, - self.download_geotags, self.download_comments, self.save_metadata, - self.compress_json, self.post_metadata_txt_pattern, - self.storyitem_metadata_txt_pattern, self.context.graphql_count_per_slidingwindow, - self.context.max_connection_attempts) + self.filename_pattern, download_pictures=self.download_pictures, + download_videos=self.download_videos, + download_video_thumbnails=self.download_video_thumbnails, + download_geotags=self.download_geotags, download_comments=self.download_comments, + save_metadata=self.save_metadata, compress_json=self.compress_json, + post_metadata_txt_pattern=self.post_metadata_txt_pattern, + storyitem_metadata_txt_pattern=self.storyitem_metadata_txt_pattern, + graphql_rate_limit=self.context.graphql_count_per_slidingwindow, + max_connection_attempts=self.context.max_connection_attempts) new_loader.context.query_timestamps = self.context.query_timestamps yield new_loader self.context.error_log.extend(new_loader.context.error_log) @@ -343,25 +350,26 @@ class Instaloader: # Download the image(s) / video thumbnail and videos within sidecars if desired downloaded = False - if post.typename == 'GraphSidecar': - edge_number = 1 - for sidecar_node in post.get_sidecar_nodes(): - # Download picture or video thumbnail - if not sidecar_node.is_video or self.download_video_thumbnails is True: - downloaded |= self.download_pic(filename=filename, url=sidecar_node.display_url, - mtime=post.date_local, filename_suffix=str(edge_number)) - # Additionally download video if available and desired - if sidecar_node.is_video and self.download_videos is True: - downloaded |= self.download_pic(filename=filename, url=sidecar_node.video_url, - mtime=post.date_local, filename_suffix=str(edge_number)) - edge_number += 1 - elif post.typename == 'GraphImage': - downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local) - elif post.typename == 'GraphVideo': - if self.download_video_thumbnails is True: + if self.download_pictures: + if post.typename == 'GraphSidecar': + edge_number = 1 + for sidecar_node in post.get_sidecar_nodes(): + # Download picture or video thumbnail + if not sidecar_node.is_video or self.download_video_thumbnails is True: + downloaded |= self.download_pic(filename=filename, url=sidecar_node.display_url, + mtime=post.date_local, filename_suffix=str(edge_number)) + # Additionally download video if available and desired + if sidecar_node.is_video and self.download_videos is True: + downloaded |= self.download_pic(filename=filename, url=sidecar_node.video_url, + mtime=post.date_local, filename_suffix=str(edge_number)) + edge_number += 1 + elif post.typename == 'GraphImage': downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local) - else: - self.context.error("Warning: {0} has unknown typename: {1}".format(post, post.typename)) + elif post.typename == 'GraphVideo': + if self.download_video_thumbnails is True: + downloaded = self.download_pic(filename=filename, url=post.url, mtime=post.date_local) + else: + self.context.error("Warning: {0} has unknown typename: {1}".format(post, post.typename)) # Save caption if desired metadata_string = _ArbitraryItemFormatter(post).format(self.post_metadata_txt_pattern).strip() From 0f0ac13d7204584e839f6ef30176559d6fd3c5cd Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 20 Aug 2018 12:32:57 +0200 Subject: [PATCH 06/27] reorder --help output --- README.rst | 2 +- docs/basic-usage.rst | 16 +++++-- docs/cli-options.rst | 83 ++++++++++++++++++++--------------- docs/index.rst | 2 +- instaloader/__main__.py | 96 +++++++++++++++++++++++------------------ 5 files changed, 118 insertions(+), 81 deletions(-) diff --git a/README.rst b/README.rst index 3e03d4c..cdebeb5 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ :: - instaloader [--comments] [--geotags] [--stories] + instaloader [--comments] [--geotags] [--stories] [--tagged] [--login YOUR-USERNAME] [--fast-update] profile | "#hashtag" | :stories | :feed | :saved diff --git a/docs/basic-usage.rst b/docs/basic-usage.rst index 18d97f3..f60cd09 100644 --- a/docs/basic-usage.rst +++ b/docs/basic-usage.rst @@ -60,12 +60,20 @@ already have a valid session cookie file. What to Download ^^^^^^^^^^^^^^^^ +.. targets-start + Instaloader supports the following targets: - ``profile`` - Public profile, or private profile with :option:`--login`. For each profile - you download, :option:`--stories` instructs Instaloader to also - **download the user's stories**. + Public profile, or private profile with :option:`--login`. + + If an already-downloaded profile has been renamed, Instaloader automatically + finds it by its unique ID and renames the folder accordingly. + + Besides the profile's posts, its current profile picture is downloaded. For + each profile you download, :option:`--stories` instructs Instaloader to also + **download the user's stories**, and :option:`--tagged` to **download posts + where the user is tagged**. - ``"#hashtag"`` Posts with a certain **hashtag** (the quotes are usually necessary), @@ -88,6 +96,8 @@ Instaloader supports the following targets: The single **post** with the given shortcode. Must be preceeded by ``--`` in the argument list to not be mistaken as an option flag. +.. targets-end + Instaloader goes through all media matching the specified targets and downloads the pictures and videos and their captions. You can specify diff --git a/docs/cli-options.rst b/docs/cli-options.rst index 537bd18..2a090d2 100644 --- a/docs/cli-options.rst +++ b/docs/cli-options.rst @@ -13,27 +13,33 @@ feed), ``:stories`` (stories of your followees) or ``:saved`` (collection of posts marked as saved). Here we explain the additional options that can be given to Instaloader to -customize its behavior. To get a list of all flags, their abbreviations and -their descriptions, you may also run ``instaloader --help``. For an +customize its behavior. For an introduction on how to use Instaloader, see :ref:`download-pictures-from-instagram`. -What to Download -^^^^^^^^^^^^^^^^ +To get a list of all flags, their abbreviations and +their descriptions, you may also run:: -Specify a list of targets (profiles, #hashtags, ``:feed``, ``:stories`` or -``:saved``). For each of these, Instaloader creates a folder and stores all -posts along with the pictures's captions and the current **profile picture** -there. If an already-downloaded profile has been renamed, Instaloader -automatically **finds it by its unique ID** and renames the folder likewise. + instaloader --help -.. option:: --profile-pic-only, -P +Targets +^^^^^^^ - Only download profile picture. +Specify a list of targets. For each of these, Instaloader creates a folder and +stores all posts along with the pictures's captions there. -.. option:: --no-profile-pic +.. include:: basic-usage.rst + :start-after: targets-start + :end-before: targets-end - Do not download profile picture. +- ``filename.json[.xz]`` + Re-Download the given object + +- ``+args.txt`` + Read targets (and options) from given textfile. See :option:`+args.txt`. + +What to Download of each Post +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. option:: --no-pictures @@ -75,6 +81,26 @@ automatically **finds it by its unique ID** and renames the folder likewise. Template to write in txt file for each StoryItem. See :ref:`metadata-text-files`. +.. option:: --no-metadata-json + + Do not create a JSON file containing the metadata of each post. + +.. option:: --no-compress-json + + Do not xz compress JSON files, rather create pretty formatted JSONs. + + +What to Download of each Profile +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. option:: --profile-pic-only, -P + + Only download profile picture. + +.. option:: --no-profile-pic + + Do not download profile picture. + .. option:: --stories, -s Also **download stories** of each profile that is downloaded. Requires @@ -84,14 +110,6 @@ automatically **finds it by its unique ID** and renames the folder likewise. Also download posts where each profile is tagged. -.. option:: --no-metadata-json - - Do not create a JSON file containing the metadata of each post. - -.. option:: --no-compress-json - - Do not xz compress JSON files, rather create pretty formatted JSONs. - .. option:: --stories-only Rather than downloading regular posts of each specified profile, only @@ -107,6 +125,15 @@ automatically **finds it by its unique ID** and renames the folder likewise. Download only post where each profile is tagged, not their regular posts. +Which Posts to Download +^^^^^^^^^^^^^^^^^^^^^^^ + +.. option:: --fast-update, -F + + For each target, stop when encountering the first already-downloaded picture. + This flag is recommended when you use Instaloader to update your personal + Instagram archive. + .. option:: --post-filter filter, --only-if filter Expression that, if given, must evaluate to True for each post to be @@ -122,20 +149,6 @@ automatically **finds it by its unique ID** and renames the folder likewise. evaluated to :class:`instaloader.StoryItem` attributes. See :ref:`filter-posts` for more examples. - - -When to Stop Downloading -^^^^^^^^^^^^^^^^^^^^^^^^ - -If none of these options are given, Instaloader goes through all pictures -matching the specified targets. - -.. option:: --fast-update, -F - - For each target, stop when encountering the first already-downloaded picture. - This flag is recommended when you use Instaloader to update your personal - Instagram archive. - .. option:: --count COUNT, -c Do not attempt to download more than COUNT posts. Applies only to diff --git a/docs/index.rst b/docs/index.rst index 5f0987f..d55e7c8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -41,7 +41,7 @@ See :ref:`install` for more options on how to install Instaloader. :: - instaloader [--comments] [--geotags] [--stories] + instaloader [--comments] [--geotags] [--stories] [--tagged] [--login YOUR-USERNAME] [--fast-update] profile | "#hashtag" | :stories | :feed | :saved diff --git a/instaloader/__main__.py b/instaloader/__main__.py index ffd6d3d..85c1cde 100644 --- a/instaloader/__main__.py +++ b/instaloader/__main__.py @@ -18,7 +18,7 @@ def usage_string(): argv0 = os.path.basename(sys.argv[0]) argv0 = "instaloader" if argv0 == "__main__.py" else argv0 return """ -{0} [--comments] [--geotags] [--stories] +{0} [--comments] [--geotags] [--stories] [--tagged] {2:{1}} [--login YOUR-USERNAME] [--fast-update] {2:{1}} profile | "#hashtag" | :stories | :feed | :saved {0} --help""".format(argv0, len(argv0), '') @@ -212,77 +212,91 @@ def main(): "https://instaloader.github.io/.", fromfile_prefix_chars='+') - g_what = parser.add_argument_group('What to Download', - 'Specify a list of profiles or #hashtags. For each of these, Instaloader ' - 'creates a folder and ' - 'downloads all posts along with the pictures\'s ' - 'captions and the current profile picture. ' - 'If an already-downloaded profile has been renamed, Instaloader automatically ' - 'finds it by its unique ID and renames the folder likewise.') - g_what.add_argument('profile', nargs='*', metavar='profile|#hashtag', - help='Name of profile or #hashtag to download. ' - 'Alternatively, if --login is given: @ to download all followees of ' - '; the special targets ' - ':feed to download pictures from your feed; ' - ':stories to download the stories of your followees; or ' - ':saved to download the posts marked as saved.') - g_what.add_argument('-P', '--profile-pic-only', action='store_true', + g_targets = parser.add_argument_group("What to Download", + "Specify a list of targets. For each of these, Instaloader creates a folder " + "and downloads all posts. The following targets are supported:") + g_targets.add_argument('profile', nargs='*', + help="Download profile. If an already-downloaded profile has been renamed, Instaloader " + "automatically finds it by its unique ID and renames the folder likewise.") + g_targets.add_argument('_at_profile', nargs='*', metavar="@profile", + help="Download all followees of profile. Requires --login. " + "Consider using :feed rather than @yourself.") + g_targets.add_argument('_hashtag', nargs='*', metavar='"#hashtag"', help="Download #hashtag.") + g_targets.add_argument('_feed', nargs='*', metavar=":feed", + help="Download pictures from your feed. Requires --login.") + g_targets.add_argument('_stories', nargs='*', metavar=":stories", + help="Download the stories of your followees. Requires --login.") + g_targets.add_argument('_saved', nargs='*', metavar=":saved", + help="Download the posts that you marked as saved. Requires --login.") + g_targets.add_argument('_singlepost', nargs='*', metavar="-- -shortcode", + help="Download the post with the given shortcode") + g_targets.add_argument('_json', nargs='*', metavar="filename.json[.xz]", + help="Re-Download the given object.") + g_targets.add_argument('_fromfile', nargs='*', metavar="+args.txt", + help="Read targets (and options) from given textfile.") + + g_post = parser.add_argument_group("What to Download of each Post") + + g_prof = parser.add_argument_group("What to Download of each Profile") + + g_prof.add_argument('-P', '--profile-pic-only', action='store_true', help='Only download profile picture.') - g_what.add_argument('--no-profile-pic', action='store_true', + g_prof.add_argument('--no-profile-pic', action='store_true', help='Do not download profile picture.') - g_what.add_argument('--no-pictures', action='store_true', + g_post.add_argument('--no-pictures', action='store_true', help='Do not download post pictures. Cannot be used together with --fast-update. ' 'Implies --no-video-thumbnails, does not imply --no-videos.') - g_what.add_argument('-V', '--no-videos', action='store_true', + g_post.add_argument('-V', '--no-videos', action='store_true', help='Do not download videos.') - g_what.add_argument('--no-video-thumbnails', action='store_true', + g_post.add_argument('--no-video-thumbnails', action='store_true', help='Do not download thumbnails of videos.') - g_what.add_argument('-G', '--geotags', action='store_true', + g_post.add_argument('-G', '--geotags', action='store_true', help='Download geotags when available. Geotags are stored as a ' 'text file with the location\'s name and a Google Maps link. ' 'This requires an additional request to the Instagram ' 'server for each picture, which is why it is disabled by default.') - g_what.add_argument('-C', '--comments', action='store_true', + g_post.add_argument('-C', '--comments', action='store_true', help='Download and update comments for each post. ' 'This requires an additional request to the Instagram ' 'server for each post, which is why it is disabled by default.') - g_what.add_argument('--no-captions', action='store_true', + g_post.add_argument('--no-captions', action='store_true', help='Do not create txt files.') - g_what.add_argument('--post-metadata-txt', action='append', + g_post.add_argument('--post-metadata-txt', action='append', help='Template to write in txt file for each Post.') - g_what.add_argument('--storyitem-metadata-txt', action='append', + g_post.add_argument('--storyitem-metadata-txt', action='append', help='Template to write in txt file for each StoryItem.') - g_what.add_argument('--no-metadata-json', action='store_true', + g_post.add_argument('--no-metadata-json', action='store_true', help='Do not create a JSON file containing the metadata of each post.') - g_what.add_argument('--metadata-json', action='store_true', + g_post.add_argument('--metadata-json', action='store_true', help=SUPPRESS) - g_what.add_argument('--no-compress-json', action='store_true', + g_post.add_argument('--no-compress-json', action='store_true', help='Do not xz compress JSON files, rather create pretty formatted JSONs.') - g_what.add_argument('-s', '--stories', action='store_true', + g_prof.add_argument('-s', '--stories', action='store_true', help='Also download stories of each profile that is downloaded. Requires --login.') - g_what.add_argument('--stories-only', action='store_true', + g_prof.add_argument('--stories-only', action='store_true', help='Rather than downloading regular posts of each specified profile, only download ' 'stories. Requires --login. Does not imply --no-profile-pic.') - g_what.add_argument('--tagged', action='store_true', + g_prof.add_argument('--tagged', action='store_true', help='Also download posts where each profile is tagged.') - g_what.add_argument('--tagged-only', action='store_true', + g_prof.add_argument('--tagged-only', action='store_true', help='Download only post where each profile is tagged, not their regular posts.') - g_what.add_argument('--post-filter', '--only-if', metavar='filter', + + g_cond = parser.add_argument_group("Which Posts to Download") + + g_cond.add_argument('-F', '--fast-update', action='store_true', + help='For each target, stop when encountering the first already-downloaded picture. This ' + 'flag is recommended when you use Instaloader to update your personal Instagram archive.') + + g_cond.add_argument('--post-filter', '--only-if', metavar='filter', help='Expression that, if given, must evaluate to True for each post to be downloaded. Must be ' 'a syntactically valid python expression. Variables are evaluated to ' 'instaloader.Post attributes. Example: --post-filter=viewer_has_liked.') - g_what.add_argument('--storyitem-filter', metavar='filter', + g_cond.add_argument('--storyitem-filter', metavar='filter', help='Expression that, if given, must evaluate to True for each storyitem to be downloaded. ' 'Must be a syntactically valid python expression. Variables are evaluated to ' 'instaloader.StoryItem attributes.') - g_stop = parser.add_argument_group('When to Stop Downloading', - 'If none of these options are given, Instaloader goes through all pictures ' - 'matching the specified targets.') - g_stop.add_argument('-F', '--fast-update', action='store_true', - help='For each target, stop when encountering the first already-downloaded picture. This ' - 'flag is recommended when you use Instaloader to update your personal Instagram archive.') - g_stop.add_argument('-c', '--count', + g_cond.add_argument('-c', '--count', help='Do not attempt to download more than COUNT posts. ' 'Applies only to #hashtag and :feed.') From e388a1c966af61e88619e7f583f8e8c867b2b31f Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 20 Aug 2018 15:23:11 +0200 Subject: [PATCH 07/27] --no-posts; Deprecate --{profile-pic,stories}-only --- docs/cli-options.rst | 14 +++++-- instaloader/__main__.py | 78 ++++++++++++++++---------------------- instaloader/instaloader.py | 74 +++++++++++++++++++++++++++++++++++- 3 files changed, 115 insertions(+), 51 deletions(-) diff --git a/docs/cli-options.rst b/docs/cli-options.rst index 2a090d2..ac51d53 100644 --- a/docs/cli-options.rst +++ b/docs/cli-options.rst @@ -95,8 +95,15 @@ What to Download of each Profile .. option:: --profile-pic-only, -P + .. deprecated:: 4.1 + Use :option:`--no-posts`. + Only download profile picture. +.. option:: --no-posts + + Do not download regular posts. + .. option:: --no-profile-pic Do not download profile picture. @@ -112,6 +119,9 @@ What to Download of each Profile .. option:: --stories-only + .. deprecated:: 4.1 + Use :option:`--stories` :option:`--no-posts`. + Rather than downloading regular posts of each specified profile, only download stories. Requires :option:`--login`. Does not imply :option:`--no-profile-pic`. @@ -121,10 +131,6 @@ What to Download of each Profile If possible, use ``:stories`` target rather than :option:`--stories-only` with all your followees. ``:stories`` uses fewer API requests. -.. option:: --tagged-only - - Download only post where each profile is tagged, not their regular posts. - Which Posts to Download ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/instaloader/__main__.py b/instaloader/__main__.py index 85c1cde..baf50ca 100644 --- a/instaloader/__main__.py +++ b/instaloader/__main__.py @@ -58,12 +58,11 @@ def filterstr_to_filterfunc(filter_str: str, item_type: type): def _main(instaloader: Instaloader, targetlist: List[str], username: Optional[str] = None, password: Optional[str] = None, - sessionfile: Optional[str] = None, max_count: Optional[int] = None, - profile_pic: bool = True, profile_pic_only: bool = False, + sessionfile: Optional[str] = None, + download_profile_pic: bool = True, + download_posts=True, download_stories: bool = False, download_tagged: bool = False, fast_update: bool = False, - stories: bool = False, stories_only: bool = False, - tagged: bool = False, tagged_only: bool = False, - post_filter_str: Optional[str] = None, + max_count: Optional[int] = None, post_filter_str: Optional[str] = None, storyitem_filter_str: Optional[str] = None) -> None: """Download set of profiles, hashtags etc. and handle logging in and session files if desired.""" # Parse and generate filter function @@ -89,10 +88,6 @@ def _main(instaloader: Instaloader, targetlist: List[str], else: instaloader.interactive_login(username) instaloader.context.log("Logged in as %s." % username) - # Determine what to download - download_profile_pic = profile_pic or profile_pic_only - download_profile_posts = not (stories_only or profile_pic_only) - download_profile_stories = stories or stories_only # Try block for KeyboardInterrupt (save session on ^C) profiles = set() anonymous_retry_profiles = set() @@ -147,7 +142,7 @@ def _main(instaloader: Instaloader, targetlist: List[str], try: profile = instaloader.check_profile_id(target) if instaloader.context.is_logged_in and profile.has_blocked_viewer: - if download_profile_pic or (download_profile_posts and not profile.is_private): + if download_profile_pic or ((download_posts or download_tagged) and not profile.is_private): raise ProfileNotExistsException("{} blocked you; But we download her anonymously." .format(target)) else: @@ -157,39 +152,30 @@ def _main(instaloader: Instaloader, targetlist: List[str], except ProfileNotExistsException as err: # Not only our profile.has_blocked_viewer condition raises ProfileNotExistsException, # check_profile_id() also does, since access to blocked profile may be responded with 404. - if instaloader.context.is_logged_in and (download_profile_pic or download_profile_posts): + if instaloader.context.is_logged_in and (download_profile_pic or download_posts or + download_tagged): instaloader.context.log(err) instaloader.context.log("Trying again anonymously, helps in case you are just blocked.") with instaloader.anonymous_copy() as anonymous_loader: with instaloader.context.error_catcher(): anonymous_retry_profiles.add(anonymous_loader.check_profile_id(target)) - instaloader.context.log("Looks good.") + instaloader.context.error("Warning: {} will be downloaded anonymously (\"{}\")." + .format(target, err)) else: raise if len(profiles) > 1: instaloader.context.log("Downloading {} profiles: {}".format(len(profiles), ' '.join([p.username for p in profiles]))) - if download_profile_pic or download_profile_posts: - # Iterate through profiles list and download them - for target in profiles: - with instaloader.context.error_catcher(target): - instaloader.download_profile(target, download_profile_pic, not download_profile_posts, - fast_update, download_tagged=tagged, - download_tagged_only=tagged_only, post_filter=post_filter) - if anonymous_retry_profiles: - instaloader.context.log("Downloading anonymously: {}" - .format(' '.join([p.username for p in anonymous_retry_profiles]))) - with instaloader.anonymous_copy() as anonymous_loader: - for target in anonymous_retry_profiles: - with instaloader.context.error_catcher(target): - anonymous_loader.download_profile(target, download_profile_pic, not download_profile_posts, - fast_update, download_tagged=tagged, - download_tagged_only=tagged_only, post_filter=post_filter) - if download_profile_stories and profiles: - with instaloader.context.error_catcher("Download stories"): - instaloader.context.log("Downloading stories") - instaloader.download_stories(userids=list(profiles), fast_update=fast_update, - filename_target=None, storyitem_filter=storyitem_filter) + instaloader.download_profiles(profiles, + download_profile_pic, download_posts, download_tagged, download_stories, + fast_update, post_filter, storyitem_filter) + if anonymous_retry_profiles: + instaloader.context.log("Downloading anonymously: {}" + .format(' '.join([p.username for p in anonymous_retry_profiles]))) + with instaloader.anonymous_copy() as anonymous_loader: + anonymous_loader.download_profiles(anonymous_retry_profiles, + download_profile_pic, download_posts, download_tagged, + fast_update=fast_update, post_filter=post_filter) except KeyboardInterrupt: print("\nInterrupted by user.", file=sys.stderr) # Save session if it is useful @@ -240,7 +226,9 @@ def main(): g_prof = parser.add_argument_group("What to Download of each Profile") g_prof.add_argument('-P', '--profile-pic-only', action='store_true', - help='Only download profile picture.') + help=SUPPRESS) + g_prof.add_argument('--no-posts', action='store_true', + help="Do not download regular posts.") g_prof.add_argument('--no-profile-pic', action='store_true', help='Do not download profile picture.') g_post.add_argument('--no-pictures', action='store_true', @@ -274,12 +262,9 @@ def main(): g_prof.add_argument('-s', '--stories', action='store_true', help='Also download stories of each profile that is downloaded. Requires --login.') g_prof.add_argument('--stories-only', action='store_true', - help='Rather than downloading regular posts of each specified profile, only download ' - 'stories. Requires --login. Does not imply --no-profile-pic.') + help=SUPPRESS) g_prof.add_argument('--tagged', action='store_true', help='Also download posts where each profile is tagged.') - g_prof.add_argument('--tagged-only', action='store_true', - help='Download only post where each profile is tagged, not their regular posts.') g_cond = parser.add_argument_group("Which Posts to Download") @@ -369,6 +354,11 @@ def main(): if args.no_pictures and args.fast_update: raise SystemExit('--no-pictures and --fast-update cannot be used together.') + # Determine what to download + download_profile_pic = not args.no_profile_pic or args.profile_pic_only + download_posts = not (args.no_posts or args.stories_only or args.profile_pic_only) + download_stories = args.stories or args.stories_only + loader = Instaloader(sleep=not args.no_sleep, quiet=args.quiet, user_agent=args.user_agent, dirname_pattern=args.dirname_pattern, filename_pattern=args.filename_pattern, download_pictures=not args.no_pictures, @@ -385,14 +375,12 @@ def main(): username=args.login.lower() if args.login is not None else None, password=args.password, sessionfile=args.sessionfile, - max_count=int(args.count) if args.count is not None else None, - profile_pic=not args.no_profile_pic, - profile_pic_only=args.profile_pic_only, + download_profile_pic=download_profile_pic, + download_posts=download_posts, + download_stories=download_stories, + download_tagged=args.tagged, fast_update=args.fast_update, - stories=args.stories, - stories_only=args.stories_only, - tagged=args.tagged, - tagged_only=args.tagged_only, + max_count=int(args.count) if args.count is not None else None, post_filter_str=args.post_filter, storyitem_filter_str=args.storyitem_filter) loader.close() diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index fa5b4ed..fc36eaf 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -11,7 +11,7 @@ from contextlib import contextmanager, suppress from datetime import datetime, timezone from functools import wraps from io import BytesIO -from typing import Any, Callable, Iterator, List, Optional, Union +from typing import Any, Callable, Iterator, List, Optional, Set, Union from .exceptions import * from .instaloadercontext import InstaloaderContext @@ -707,6 +707,72 @@ class Instaloader: return profile raise ProfileNotExistsException("Profile {0} does not exist.".format(profile_name)) + def download_profiles(self, profiles: Set[Profile], + profile_pic: bool = True, posts: bool = True, tagged: bool = False, stories: bool = False, + fast_update: bool = False, + post_filter: Optional[Callable[[Post], bool]] = None, + storyitem_filter: Optional[Callable[[Post], bool]] = None): + """High-level method to download set of profiles. + + :param profiles: Set of profiles to download. + :param profile_pic: not :option:`--no-profile-pic`. + :param posts: not :option:`--no-posts`. + :param tagged: :option:`--tagged`. + :param stories: :option:`--stories`. + :param fast_update: :option:`--fast-update`. + :param post_filter: :option:`--post-filter`. + :param storyitem_filter: :option:`--post-filter`.""" + + for profile in profiles: + with self.context.error_catcher(profile.username): + profile_name = profile.username + + # Save metadata as JSON if desired. + if self.save_metadata: + json_filename = '{0}/{1}_{2}'.format(self.dirname_pattern.format(profile=profile_name, + target=profile_name), + profile_name, profile.userid) + self.save_metadata_json(json_filename, profile) + + # Download profile picture + if profile_pic: + with self.context.error_catcher('Download profile picture of {}'.format(profile_name)): + self.download_profilepic(profile) + + # Catch some errors + if profile.is_private: + if not self.context.is_logged_in: + raise LoginRequiredException("--login=USERNAME required.") + if not profile.followed_by_viewer and self.context.username != profile.username: + raise PrivateProfileNotFollowedException("Private but not followed.") + + # Download tagged, if requested + if tagged: + with self.context.error_catcher('Download tagged of {}'.format(profile_name)): + self.download_tagged(profile, fast_update=fast_update, post_filter=post_filter) + + # Iterate over pictures and download them + if posts: + self.context.log("Retrieving posts from profile {}.".format(profile_name)) + totalcount = profile.mediacount + count = 1 + for post in profile.get_posts(): + self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True) + count += 1 + if post_filter is not None and not post_filter(post): + self.context.log('') + continue + with self.context.error_catcher("Download {} of {}".format(post, profile_name)): + downloaded = self.download_post(post, target=profile_name) + if fast_update and not downloaded: + break + + if stories and profiles: + with self.context.error_catcher("Download stories"): + self.context.log("Downloading stories") + self.download_stories(userids=list(profiles), fast_update=fast_update, filename_target=None, + storyitem_filter=storyitem_filter) + def download_profile(self, profile_name: Union[str, Profile], profile_pic: bool = True, profile_pic_only: bool = False, fast_update: bool = False, @@ -714,7 +780,11 @@ class Instaloader: download_tagged: bool = False, download_tagged_only: bool = False, post_filter: Optional[Callable[[Post], bool]] = None, storyitem_filter: Optional[Callable[[StoryItem], bool]] = None) -> None: - """Download one profile""" + """Download one profile + + .. deprecated:: 4.1 + Use :meth:`Instaloader.download_profiles`. + """ # Get profile main page json # check if profile does exist or name has changed since last download From b91e73b090ba2b12571c1a531d592eceba6b5120 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 23 Aug 2018 14:44:37 +0200 Subject: [PATCH 08/27] set docs font size to 16px --- docs/_static/style.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/_static/style.css b/docs/_static/style.css index 80010e1..971da77 100644 --- a/docs/_static/style.css +++ b/docs/_static/style.css @@ -18,3 +18,7 @@ a { a:hover { color: #f48400; } + +body { + font-size: 16px; +} From 0dcc9129877aa8f64fd444f6c6e934d581780a3b Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 23 Aug 2018 17:25:28 +0200 Subject: [PATCH 09/27] doc: Note new features with versionadded --- docs/basic-usage.rst | 2 ++ docs/cli-options.rst | 23 ++++++++++++++--------- instaloader/instaloader.py | 7 ++++++- instaloader/structures.py | 4 +++- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/docs/basic-usage.rst b/docs/basic-usage.rst index f60cd09..ef7d6d1 100644 --- a/docs/basic-usage.rst +++ b/docs/basic-usage.rst @@ -96,6 +96,8 @@ Instaloader supports the following targets: The single **post** with the given shortcode. Must be preceeded by ``--`` in the argument list to not be mistaken as an option flag. + .. versionadded:: 4.1 + .. targets-end Instaloader goes through all media matching the specified targets and diff --git a/docs/cli-options.rst b/docs/cli-options.rst index ac51d53..34d3511 100644 --- a/docs/cli-options.rst +++ b/docs/cli-options.rst @@ -47,6 +47,8 @@ What to Download of each Post :option:`--fast-update`. Implies :option:`--no-video-thumbnails`, does not imply :option:`--no-videos`. + .. versionadded:: 4.1 + .. option:: --no-videos, -V Do not download videos. @@ -117,6 +119,8 @@ What to Download of each Profile Also download posts where each profile is tagged. + .. versionadded:: 4.1 + .. option:: --stories-only .. deprecated:: 4.1 @@ -228,17 +232,18 @@ Miscellaneous Options .. option:: +args.txt - Read arguments from file `args.txt`, a shortcut to provide argument from - file rather than command-line. This provide a convient way to hide login - info from CLI. and also can use for simplify managment of long arguments. + Read arguments from file `args.txt`, a shortcut to provide argument from + file rather than command-line. This provide a convient way to hide login + info from CLI. and also can use for simplify managment of long arguments. - .. note:: + .. note:: - text file should separate arg with line break. + text file should separate arg with line break. - args.txt example:: + args.txt example:: - --login MYUSENAME - --password MYPASSWORD - --fast-update + --login MYUSENAME + --password MYPASSWORD + --fast-update + .. versionadded:: 4.1 diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index fc36eaf..551c447 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -630,6 +630,9 @@ class Instaloader: def download_tagged(self, profile: Profile, fast_update: bool = False, target: Optional[str] = None, post_filter: Optional[Callable[[Post], bool]] = None) -> None: + """Download all posts where a profile is tagged. + + .. versionadded:: 4.1""" if target is None: target = profile.username + ':tagged' self.context.log("Retrieving tagged posts for profile {}.".format(profile.username)) @@ -721,7 +724,9 @@ class Instaloader: :param stories: :option:`--stories`. :param fast_update: :option:`--fast-update`. :param post_filter: :option:`--post-filter`. - :param storyitem_filter: :option:`--post-filter`.""" + :param storyitem_filter: :option:`--post-filter`. + + .. versionadded:: 4.1""" for profile in profiles: with self.context.error_catcher(profile.username): diff --git a/instaloader/structures.py b/instaloader/structures.py index 208a67d..6336465 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -614,7 +614,9 @@ class Profile: self._metadata('edge_saved_media'))) def get_tagged_posts(self) -> Iterator[Post]: - """Retrieve all posts where a profile is tagged.""" + """Retrieve all posts where a profile is tagged. + + .. versionadded:: 4.0.7""" self._obtain_metadata() yield from (Post(self._context, node, self if int(node['owner']['id']) == self.userid else None) for node in self._context.graphql_node_list("e31a871f7301132ceaab56507a66bbb7", From 91d5d5f867a6d82de7460482f81c47f572fb9ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Sat, 11 Aug 2018 07:25:33 +0200 Subject: [PATCH 10/27] Add class and functions for downloading highlights Requested in #162. --- docs/as-module.rst | 5 ++++ instaloader/__init__.py | 2 +- instaloader/instaloader.py | 54 ++++++++++++++++++++++++++++++++-- instaloader/structures.py | 59 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 115 insertions(+), 5 deletions(-) diff --git a/docs/as-module.rst b/docs/as-module.rst index f912dcc..15a3faf 100644 --- a/docs/as-module.rst +++ b/docs/as-module.rst @@ -135,6 +135,11 @@ User Stories .. autoclass:: StoryItem :no-show-inheritance: +Highlights +"""""""""" + +.. autoclass:: Highlight + Profiles """""""" diff --git a/instaloader/__init__.py b/instaloader/__init__.py index df41aeb..d39eae5 100644 --- a/instaloader/__init__.py +++ b/instaloader/__init__.py @@ -15,5 +15,5 @@ else: from .exceptions import * from .instaloader import Instaloader from .instaloadercontext import InstaloaderContext -from .structures import (Post, PostSidecarNode, PostComment, PostLocation, Profile, Story, StoryItem, +from .structures import (Highlight, Post, PostSidecarNode, PostComment, PostLocation, Profile, Story, StoryItem, load_structure_from_file, save_structure_to_file) diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index 551c447..9c4c614 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -15,7 +15,7 @@ from typing import Any, Callable, Iterator, List, Optional, Set, Union from .exceptions import * from .instaloadercontext import InstaloaderContext -from .structures import JsonExportable, Post, PostLocation, Profile, Story, StoryItem, save_structure_to_file +from .structures import Highlight, JsonExportable, Post, PostLocation, Profile, Story, StoryItem, save_structure_to_file def get_default_session_filename(username: str) -> str: @@ -190,7 +190,7 @@ class Instaloader: def update_comments(self, filename: str, post: Post) -> None: def _postcomment_asdict(comment): - return {'id': comment.id, + return {'id': comment.unique_id, 'created_at': int(comment.created_at_utc.replace(tzinfo=timezone.utc).timestamp()), 'text': comment.text, 'owner': comment.owner._asdict()} @@ -488,6 +488,56 @@ class Instaloader: self.context.log() return downloaded + @_requires_login + def get_highlights(self, userid: int) -> Iterator[Highlight]: + """Get all highlights from a user. + To use this, one needs to be logged in + + :param userid: ID of the profile whose highlights should get fetched. + """ + + data = self.context.graphql_query("7c16654f22c819fb63d1183034a5162f", + {"user_id": userid, "include_chaining": False, "include_reel": False, + "include_suggested_users": False, "include_logged_out_extras": False, + "include_highlight_reels": True})["data"]["user"]['edge_highlight_reels'] + if data is None: + raise BadResponseException('Bad highlights reel JSON.') + yield from (Highlight(self.context, edge['node']) for edge in data['edges']) + + @_requires_login + def download_highlights(self, + userid: int, + fast_update: bool = False, + filename_target: Optional[str] = None, + storyitem_filter: Optional[Callable[[StoryItem], bool]] = None) -> None: + """ + Download available highlights from a user whose ID is given. + To use this, one needs to be logged in + + :param userid: ID of the profile whose highlights should get downloaded. + :param fast_update: If true, abort when first already-downloaded picture is encountered + :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern + or None if profile name and the highlights' titles should be used instead + :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded + """ + for user_highlight in self.get_highlights(userid): + name = user_highlight.owner_username + self.context.log("Retrieving highlights \"{}\" from profile {}".format(user_highlight.title, name)) + totalcount = user_highlight.itemcount + count = 1 + for item in user_highlight.get_items(): + if storyitem_filter is not None and not storyitem_filter(item): + self.context.log("<{} skipped>".format(item), flush=True) + continue + self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True) + count += 1 + with self.context.error_catcher('Download highlights \"{}\" from user {}'.format(user_highlight.title, name)): + downloaded = self.download_storyitem(item, filename_target + if filename_target + else '{}/{}'.format(name, user_highlight.title)) + if fast_update and not downloaded: + break + @_requires_login def get_feed_posts(self) -> Iterator[Post]: """Get Posts of the user's feed. diff --git a/instaloader/structures.py b/instaloader/structures.py index 6336465..eee5472 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -782,7 +782,7 @@ class Story: # story is a Story object for item in story.get_items(): # item is a StoryItem object - L.download_storyitem(item, ':stores') + L.download_storyitem(item, ':stories') This class implements == and is hashable. @@ -805,7 +805,7 @@ class Story: return NotImplemented def __hash__(self) -> int: - return hash(self._unique_id) + return hash(self.unique_id) @property def unique_id(self) -> str: @@ -868,6 +868,61 @@ class Story: yield from (StoryItem(self._context, item, self.owner_profile) for item in reversed(self._node['items'])) +class Highlight(Story): + + def __init__(self, context: InstaloaderContext, node: Dict[str, Any]): + super().__init__(context, node) + self._items = None + + def __repr__(self): + return ''.format(self.owner_username, self.title) + + @property + def unique_id(self) -> int: + """A unique ID identifying this set of highlights.""" + return int(self._node['id']) + + @property + def owner_profile(self) -> Profile: + """:class:`Profile` instance of the highlights' owner.""" + if not self._owner_profile: + self._owner_profile = Profile(self._context, self._node['owner']) + return self._owner_profile + + @property + def title(self) -> str: + """The title of these highlights.""" + return self._node['title'] + + @property + def cover_url(self) -> str: + """URL of the highlights' cover.""" + return self._node['cover_media']['thumbnail_src'] + + @property + def cover_cropped_url(self) -> str: + """URL of the cropped version of the cover.""" + return self._node['cover_media_cropped_thumbnail']['url'] + + def _fetch_items(self): + if not self._items: + self._items = self._context.graphql_query("45246d3fe16ccc6577e0bd297a5db1ab", + {"reel_ids": [], "tag_names": [], "location_ids": [], + "highlight_reel_ids": [str(self.unique_id)], + "precomposed_overlay": False})['data']['reels_media'][0]['items'] + + @property + def itemcount(self) -> int: + """Count of items associated with the :class:`Highlight` instance.""" + self._fetch_items() + return len(self._items) + + def get_items(self) -> Iterator[StoryItem]: + """Retrieve all associated highlight items.""" + self._fetch_items() + yield from (StoryItem(self._context, item, self.owner_profile) for item in self._items) + + JsonExportable = Union[Post, Profile, StoryItem] From 43f52198ff6320838cc9421349cfbdf92a1342d4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 15 Aug 2018 20:06:00 +0200 Subject: [PATCH 11/27] Fix docs building and show Highlights inheritance --- docs/as-module.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/as-module.rst b/docs/as-module.rst index 15a3faf..b4d156c 100644 --- a/docs/as-module.rst +++ b/docs/as-module.rst @@ -139,6 +139,10 @@ Highlights """""""""" .. autoclass:: Highlight + :no-show-inheritance: + :inherited-members: + + Bases: :class:`Story` Profiles """""""" From 158c1433bbfdff9ce94ddedb856e8e363bc3d98d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Thu, 23 Aug 2018 16:20:42 +0200 Subject: [PATCH 12/27] Revert accidental change of comment.id to comment.unique_id --- instaloader/instaloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index 9c4c614..9c7e976 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -190,7 +190,7 @@ class Instaloader: def update_comments(self, filename: str, post: Post) -> None: def _postcomment_asdict(comment): - return {'id': comment.unique_id, + return {'id': comment.id, 'created_at': int(comment.created_at_utc.replace(tzinfo=timezone.utc).timestamp()), 'text': comment.text, 'owner': comment.owner._asdict()} From 54572fb1fcc2d022c5105b8eba0af69fc1a68651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Thu, 23 Aug 2018 16:26:02 +0200 Subject: [PATCH 13/27] Make the Highlight stuff accept Profile objects download_highlights(), get_highlights() and the Highlight class now accept and use the owner's Profile rather than creating it themselves. --- instaloader/instaloader.py | 14 ++++++++------ instaloader/structures.py | 3 ++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index 9c7e976..87dab74 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -489,24 +489,26 @@ class Instaloader: return downloaded @_requires_login - def get_highlights(self, userid: int) -> Iterator[Highlight]: + def get_highlights(self, user: Union[int, Profile]) -> Iterator[Highlight]: """Get all highlights from a user. To use this, one needs to be logged in - :param userid: ID of the profile whose highlights should get fetched. + :param user: ID or Profile of the user whose highlights should get fetched. """ + userid = user if isinstance(user, int) else user.userid data = self.context.graphql_query("7c16654f22c819fb63d1183034a5162f", {"user_id": userid, "include_chaining": False, "include_reel": False, "include_suggested_users": False, "include_logged_out_extras": False, "include_highlight_reels": True})["data"]["user"]['edge_highlight_reels'] if data is None: raise BadResponseException('Bad highlights reel JSON.') - yield from (Highlight(self.context, edge['node']) for edge in data['edges']) + yield from (Highlight(self.context, edge['node'], user if isinstance(user, Profile) else None) + for edge in data['edges']) @_requires_login def download_highlights(self, - userid: int, + user: Union[int, Profile], fast_update: bool = False, filename_target: Optional[str] = None, storyitem_filter: Optional[Callable[[StoryItem], bool]] = None) -> None: @@ -514,13 +516,13 @@ class Instaloader: Download available highlights from a user whose ID is given. To use this, one needs to be logged in - :param userid: ID of the profile whose highlights should get downloaded. + :param user: ID or Profile of the user whose highlights should get downloaded. :param fast_update: If true, abort when first already-downloaded picture is encountered :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern or None if profile name and the highlights' titles should be used instead :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded """ - for user_highlight in self.get_highlights(userid): + for user_highlight in self.get_highlights(user): name = user_highlight.owner_username self.context.log("Retrieving highlights \"{}\" from profile {}".format(user_highlight.title, name)) totalcount = user_highlight.itemcount diff --git a/instaloader/structures.py b/instaloader/structures.py index eee5472..96de483 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -870,8 +870,9 @@ class Story: class Highlight(Story): - def __init__(self, context: InstaloaderContext, node: Dict[str, Any]): + def __init__(self, context: InstaloaderContext, node: Dict[str, Any], owner: Optional[Profile] = None): super().__init__(context, node) + self._owner_profile = owner self._items = None def __repr__(self): From 2e517e972fde3aef007d31ecf5c679f4f4214c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Thu, 23 Aug 2018 22:06:35 +0200 Subject: [PATCH 14/27] Extend _PostPathFormatter to replace more chars On Windows, all forbidden characters now get replaced with similar looking unicode chars. --- instaloader/instaloader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index 87dab74..277ace7 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -70,7 +70,10 @@ class _PostPathFormatter(_ArbitraryItemFormatter): def vformat(self, format_string, args, kwargs): """Override :meth:`string.Formatter.vformat` for character substitution in paths for Windows, see issue #84.""" ret = super().vformat(format_string, args, kwargs) - return ret.replace(':', '\ua789') if platform.system() == 'Windows' else ret + if platform.system() == 'Windows': + ret = ret.replace(':', '\ua789').replace('<', '\ufe64').replace('>', '\ufe65').replace('\"', '\uff02') + ret = ret.replace('\\', '\uff3c').replace('|', '\uff5c').replace('?', '\ufe16').replace('*', '\uff0a') + return ret class Instaloader: From cbdd85ef07d49938395a9a6cf2abfcfeccf98884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Thu, 23 Aug 2018 23:17:48 +0200 Subject: [PATCH 15/27] Highlights downloadable through CLI By using --highlights all available highlight stories of target profiles will get downloaded. Closes #162. --- instaloader/__main__.py | 11 +++++++---- instaloader/instaloader.py | 9 ++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/instaloader/__main__.py b/instaloader/__main__.py index baf50ca..0123826 100644 --- a/instaloader/__main__.py +++ b/instaloader/__main__.py @@ -59,8 +59,8 @@ def filterstr_to_filterfunc(filter_str: str, item_type: type): def _main(instaloader: Instaloader, targetlist: List[str], username: Optional[str] = None, password: Optional[str] = None, sessionfile: Optional[str] = None, - download_profile_pic: bool = True, - download_posts=True, download_stories: bool = False, download_tagged: bool = False, + download_profile_pic: bool = True, download_posts=True, + download_stories: bool = False, download_highlights: bool = False, download_tagged: bool = False, fast_update: bool = False, max_count: Optional[int] = None, post_filter_str: Optional[str] = None, storyitem_filter_str: Optional[str] = None) -> None: @@ -167,8 +167,8 @@ def _main(instaloader: Instaloader, targetlist: List[str], instaloader.context.log("Downloading {} profiles: {}".format(len(profiles), ' '.join([p.username for p in profiles]))) instaloader.download_profiles(profiles, - download_profile_pic, download_posts, download_tagged, download_stories, - fast_update, post_filter, storyitem_filter) + download_profile_pic, download_posts, download_tagged, download_highlights, + download_stories, fast_update, post_filter, storyitem_filter) if anonymous_retry_profiles: instaloader.context.log("Downloading anonymously: {}" .format(' '.join([p.username for p in anonymous_retry_profiles]))) @@ -263,6 +263,8 @@ def main(): help='Also download stories of each profile that is downloaded. Requires --login.') g_prof.add_argument('--stories-only', action='store_true', help=SUPPRESS) + g_prof.add_argument('--highlights', action='store_true', + help='Also download highlights of each profile that is downloaded. Requires --login.') g_prof.add_argument('--tagged', action='store_true', help='Also download posts where each profile is tagged.') @@ -378,6 +380,7 @@ def main(): download_profile_pic=download_profile_pic, download_posts=download_posts, download_stories=download_stories, + download_highlights=args.highlights, download_tagged=args.tagged, fast_update=args.fast_update, max_count=int(args.count) if args.count is not None else None, diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index 277ace7..bcd2228 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -766,7 +766,8 @@ class Instaloader: raise ProfileNotExistsException("Profile {0} does not exist.".format(profile_name)) def download_profiles(self, profiles: Set[Profile], - profile_pic: bool = True, posts: bool = True, tagged: bool = False, stories: bool = False, + profile_pic: bool = True, posts: bool = True, + tagged: bool = False, highlights: bool = False, stories: bool = False, fast_update: bool = False, post_filter: Optional[Callable[[Post], bool]] = None, storyitem_filter: Optional[Callable[[Post], bool]] = None): @@ -776,6 +777,7 @@ class Instaloader: :param profile_pic: not :option:`--no-profile-pic`. :param posts: not :option:`--no-posts`. :param tagged: :option:`--tagged`. + :param highlights: :option: `--highlights`. :param stories: :option:`--stories`. :param fast_update: :option:`--fast-update`. :param post_filter: :option:`--post-filter`. @@ -811,6 +813,11 @@ class Instaloader: with self.context.error_catcher('Download tagged of {}'.format(profile_name)): self.download_tagged(profile, fast_update=fast_update, post_filter=post_filter) + # Download highlights, if requested + if highlights: + with self.context.error_catcher('Download highlights of {}'.format(profile_name)): + self.download_highlights(profile, fast_update=fast_update, storyitem_filter=storyitem_filter) + # Iterate over pictures and download them if posts: self.context.log("Retrieving posts from profile {}.".format(profile_name)) From 584c69d93c7d5cae2f25773789ae860a8472aca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Fri, 24 Aug 2018 00:23:19 +0200 Subject: [PATCH 16/27] Update docs and docstrings concerning highlights --- docs/cli-options.rst | 7 +++++++ instaloader/instaloader.py | 8 ++++++-- instaloader/structures.py | 22 ++++++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/cli-options.rst b/docs/cli-options.rst index 34d3511..afc971a 100644 --- a/docs/cli-options.rst +++ b/docs/cli-options.rst @@ -115,6 +115,13 @@ What to Download of each Profile Also **download stories** of each profile that is downloaded. Requires :option:`--login`. +.. option:: --highlights + + Also **download highlights** of each profile that is downloaded. Requires + :option:`--login`. + + .. versionadded:: 4.1 + .. option:: --tagged Also download posts where each profile is tagged. diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index bcd2228..f5ac2eb 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -494,7 +494,9 @@ class Instaloader: @_requires_login def get_highlights(self, user: Union[int, Profile]) -> Iterator[Highlight]: """Get all highlights from a user. - To use this, one needs to be logged in + To use this, one needs to be logged in. + + .. versionadded:: 4.1 :param user: ID or Profile of the user whose highlights should get fetched. """ @@ -517,7 +519,9 @@ class Instaloader: storyitem_filter: Optional[Callable[[StoryItem], bool]] = None) -> None: """ Download available highlights from a user whose ID is given. - To use this, one needs to be logged in + To use this, one needs to be logged in. + + .. versionadded:: 4.1 :param user: ID or Profile of the user whose highlights should get downloaded. :param fast_update: If true, abort when first already-downloaded picture is encountered diff --git a/instaloader/structures.py b/instaloader/structures.py index 96de483..69bdcb8 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -869,6 +869,28 @@ class Story: class Highlight(Story): + """ + Structure representing a user's highlight with its associated story items. + + Provides methods for accessing highlight properties, as well as :meth:`Highlight.get_items` to request associated + :class:`StoryItem` nodes. Highlights are returned by :meth:`Instaloader.get_highlights`. + + With a logged-in :class:`Instaloader` instance `L`, you may download all highlights of a :class:`Profile` instance + USER with:: + + for highlight in L.get_highlights(USER): + # highlight is a Highlight object + for item in highlight.get_items(): + # item is a StoryItem object + L.download_storyitem(item, '{}/{}'.format(highlight.owner_username, highlight.title)) + + This class implements == and is hashable. + + :param context: :class:`InstaloaderContext` instance used for additional queries if necessary. + :param node: Dictionary containing the available information of the highlight as returned by Instagram. + :param owner: :class:`Profile` instance representing the owner profile of the highlight. + + .. versionadded:: 4.1""" def __init__(self, context: InstaloaderContext, node: Dict[str, Any], owner: Optional[Profile] = None): super().__init__(context, node) From b443cc6654a63d9772c4ab4709219e9e82f01d3f Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 24 Aug 2018 11:38:45 +0200 Subject: [PATCH 17/27] Minor doc fixes and improvements --- README.rst | 2 +- docs/_static/style.css | 4 ++++ docs/as-module.rst | 2 ++ docs/basic-usage.rst | 13 ++++++++++--- docs/cli-options.rst | 2 ++ docs/index.rst | 2 +- instaloader/__main__.py | 2 +- instaloader/instaloader.py | 4 +++- instaloader/structures.py | 7 ++++--- 9 files changed, 28 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index cdebeb5..64a9e82 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ :: - instaloader [--comments] [--geotags] [--stories] [--tagged] + instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged] [--login YOUR-USERNAME] [--fast-update] profile | "#hashtag" | :stories | :feed | :saved diff --git a/docs/_static/style.css b/docs/_static/style.css index 971da77..ee4203f 100644 --- a/docs/_static/style.css +++ b/docs/_static/style.css @@ -22,3 +22,7 @@ a:hover { body { font-size: 16px; } + +.versionmodified { + font-size: 14px; +} diff --git a/docs/as-module.rst b/docs/as-module.rst index b4d156c..44a8cce 100644 --- a/docs/as-module.rst +++ b/docs/as-module.rst @@ -144,6 +144,8 @@ Highlights Bases: :class:`Story` + .. versionadded:: 4.1 + Profiles """""""" diff --git a/docs/basic-usage.rst b/docs/basic-usage.rst index ef7d6d1..96dac34 100644 --- a/docs/basic-usage.rst +++ b/docs/basic-usage.rst @@ -71,9 +71,16 @@ Instaloader supports the following targets: finds it by its unique ID and renames the folder accordingly. Besides the profile's posts, its current profile picture is downloaded. For - each profile you download, :option:`--stories` instructs Instaloader to also - **download the user's stories**, and :option:`--tagged` to **download posts - where the user is tagged**. + each profile you download, + + - :option:`--stories` + instructs Instaloader to also **download the user's stories**, + + - :option:`--highlights` + to **download highlights of each profile that is downloaded**, and + + - :option:`--tagged` + to **download posts where the user is tagged**. - ``"#hashtag"`` Posts with a certain **hashtag** (the quotes are usually necessary), diff --git a/docs/cli-options.rst b/docs/cli-options.rst index afc971a..1b8375b 100644 --- a/docs/cli-options.rst +++ b/docs/cli-options.rst @@ -106,6 +106,8 @@ What to Download of each Profile Do not download regular posts. + .. versionadded:: 4.1 + .. option:: --no-profile-pic Do not download profile picture. diff --git a/docs/index.rst b/docs/index.rst index d55e7c8..ce2d0ad 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -41,7 +41,7 @@ See :ref:`install` for more options on how to install Instaloader. :: - instaloader [--comments] [--geotags] [--stories] [--tagged] + instaloader [--comments] [--geotags] [--stories] [--highlights] [--tagged] [--login YOUR-USERNAME] [--fast-update] profile | "#hashtag" | :stories | :feed | :saved diff --git a/instaloader/__main__.py b/instaloader/__main__.py index 0123826..ca54e88 100644 --- a/instaloader/__main__.py +++ b/instaloader/__main__.py @@ -18,7 +18,7 @@ def usage_string(): argv0 = os.path.basename(sys.argv[0]) argv0 = "instaloader" if argv0 == "__main__.py" else argv0 return """ -{0} [--comments] [--geotags] [--stories] [--tagged] +{0} [--comments] [--geotags] [--stories] [--highlights] [--tagged] {2:{1}} [--login YOUR-USERNAME] [--fast-update] {2:{1}} profile | "#hashtag" | :stories | :feed | :saved {0} --help""".format(argv0, len(argv0), '') diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index f5ac2eb..acfadfe 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -717,6 +717,8 @@ class Instaloader: def save_profile_id(self, profile: Profile): """ Store ID of profile locally. + + .. versionadded:: 4.0.6 """ os.makedirs(self.dirname_pattern.format(profile=profile.username, target=profile.username), exist_ok=True) @@ -781,7 +783,7 @@ class Instaloader: :param profile_pic: not :option:`--no-profile-pic`. :param posts: not :option:`--no-posts`. :param tagged: :option:`--tagged`. - :param highlights: :option: `--highlights`. + :param highlights: :option:`--highlights`. :param stories: :option:`--stories`. :param fast_update: :option:`--fast-update`. :param post_filter: :option:`--post-filter`. diff --git a/instaloader/structures.py b/instaloader/structures.py index 69bdcb8..2c66899 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -574,7 +574,9 @@ class Profile: @property def profile_pic_url(self) -> str: - """Return URL of profile picture""" + """Return URL of profile picture + + .. versionadded:: 4.0.3""" try: return self._iphone_struct['hd_profile_pic_url_info']['url'] except (InstaloaderException, KeyError) as err: @@ -889,8 +891,7 @@ class Highlight(Story): :param context: :class:`InstaloaderContext` instance used for additional queries if necessary. :param node: Dictionary containing the available information of the highlight as returned by Instagram. :param owner: :class:`Profile` instance representing the owner profile of the highlight. - - .. versionadded:: 4.1""" + """ def __init__(self, context: InstaloaderContext, node: Dict[str, Any], owner: Optional[Profile] = None): super().__init__(context, node) From f988762cb1040199d5d243c428b7d392408a5bde Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 24 Aug 2018 12:21:45 +0200 Subject: [PATCH 18/27] download_profiles: raise_errors parameter With raise_errors=True it behaves like now-deprecated download_profile(). --- instaloader/instaloader.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index acfadfe..1431f90 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -776,7 +776,8 @@ class Instaloader: tagged: bool = False, highlights: bool = False, stories: bool = False, fast_update: bool = False, post_filter: Optional[Callable[[Post], bool]] = None, - storyitem_filter: Optional[Callable[[Post], bool]] = None): + storyitem_filter: Optional[Callable[[Post], bool]] = None, + raise_errors: bool = False): """High-level method to download set of profiles. :param profiles: Set of profiles to download. @@ -788,11 +789,19 @@ class Instaloader: :param fast_update: :option:`--fast-update`. :param post_filter: :option:`--post-filter`. :param storyitem_filter: :option:`--post-filter`. + :param raise_errors: + Whether :exc:`LoginRequiredException` and :exc:`PrivateProfileNotFollowedException` should be raised or + catched and printed with :meth:`InstaloaderContext.error_catcher`. .. versionadded:: 4.1""" + def _error_raiser(_str): + yield + + error_handler = _error_raiser if raise_errors else self.context.error_catcher + for profile in profiles: - with self.context.error_catcher(profile.username): + with error_handler(profile.username): profile_name = profile.username # Save metadata as JSON if desired. From 60d47be2f3230665cce08b5089dffdec64e71604 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 24 Aug 2018 12:33:28 +0200 Subject: [PATCH 19/27] Unit test for Instaloader.get_highlights() --- test/instaloader_unittests.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/instaloader_unittests.py b/test/instaloader_unittests.py index 5f2a0fb..aece971 100644 --- a/test/instaloader_unittests.py +++ b/test/instaloader_unittests.py @@ -8,6 +8,7 @@ from itertools import islice import instaloader +PROFILE_WITH_HIGHLIGHTS = 325732271 PUBLIC_PROFILE = "selenagomez" PUBLIC_PROFILE_ID = 460563723 HASHTAG = "kitten" @@ -102,6 +103,14 @@ class TestInstaloaderLoggedIn(TestInstaloaderAnonymously): for item in user_story.get_items(): print(item) + def test_highlights_paging(self): + for user_highlight in self.L.get_highlights(PROFILE_WITH_HIGHLIGHTS): + print("Retrieving {} highlights \"{}\" from profile {}".format(user_highlight.itemcount, + user_highlight.title, + user_highlight.owner_username)) + for item in user_highlight.get_items(): + print(item) + def test_private_profile_paging(self): self.post_paging_test(instaloader.Profile.from_username(self.L.context, PRIVATE_PROFILE).get_posts()) From 1394e8e9f5e5aa119773561628efe0b244cd2482 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 24 Aug 2018 12:49:18 +0200 Subject: [PATCH 20/27] First Alpha Release for Version 4.1 --- instaloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instaloader/__init__.py b/instaloader/__init__.py index d39eae5..b91d3ae 100644 --- a/instaloader/__init__.py +++ b/instaloader/__init__.py @@ -1,7 +1,7 @@ """Download pictures (or videos) along with their captions and other metadata from Instagram.""" -__version__ = '4.0.8' +__version__ = '4.1a1' try: From 567a04a661cb59bc82848309b0246878daea93ea Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 30 Aug 2018 09:13:49 +0200 Subject: [PATCH 21/27] format_filename method to apply filename pattern --- instaloader/instaloader.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index 1431f90..fb2e12c 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -338,6 +338,12 @@ class Instaloader: :raises ConnectionException: If connection to Instagram failed.""" self.context.login(user, passwd) + def format_filename(self, item: Union[Post, StoryItem], target: Optional[str] = None): + """Format filename of a :class:`Post` or :class:`StoryItem` according to ``filename-pattern`` parameter. + + .. versionadded:: 4.1""" + return _PostPathFormatter(item).format(self.filename_pattern, target=target) + def download_post(self, post: Post, target: str) -> bool: """ Download everything associated with one instagram post node, i.e. picture, caption and video. @@ -348,7 +354,7 @@ class Instaloader: """ dirname = _PostPathFormatter(post).format(self.dirname_pattern, target=target) - filename = dirname + '/' + _PostPathFormatter(post).format(self.filename_pattern, target=target) + filename = dirname + '/' + self.format_filename(post, target=target) os.makedirs(os.path.dirname(filename), exist_ok=True) # Download the image(s) / video thumbnail and videos within sidecars if desired @@ -473,7 +479,7 @@ class Instaloader: date_local = item.date_local dirname = _PostPathFormatter(item).format(self.dirname_pattern, target=target) - filename = dirname + '/' + _PostPathFormatter(item).format(self.filename_pattern, target=target) + filename = dirname + '/' + self.format_filename(item, target=target) os.makedirs(os.path.dirname(filename), exist_ok=True) downloaded = False if not item.is_video or self.download_video_thumbnails is True: From 0e534ba5194f96b6c9742b9f910d6840544a6c23 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 30 Aug 2018 09:57:42 +0200 Subject: [PATCH 22/27] Use username/:tagged as target for tagged posts Subdirs in profile folders were introduced with --highlights, and moving :tagged posts there allows to further call instaloader as instaloader [flags] */ (#154) --- instaloader/instaloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index fb2e12c..f7e8a9e 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -699,7 +699,7 @@ class Instaloader: .. versionadded:: 4.1""" if target is None: - target = profile.username + ':tagged' + target = profile.username + '/:tagged' self.context.log("Retrieving tagged posts for profile {}.".format(profile.username)) count = 1 for post in profile.get_tagged_posts(): From cc15cb585735411a03b23a2319a24a0870eedfae Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 30 Aug 2018 13:51:55 +0200 Subject: [PATCH 23/27] Cache and reuse profiles for Profile.from_id() --- instaloader/instaloadercontext.py | 3 +++ instaloader/structures.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/instaloader/instaloadercontext.py b/instaloader/instaloadercontext.py index d17ce43..9571a20 100644 --- a/instaloader/instaloadercontext.py +++ b/instaloader/instaloadercontext.py @@ -69,6 +69,9 @@ class InstaloaderContext: # Can be set to True for testing, disables supression of InstaloaderContext._error_catcher self.raise_all_errors = False + # Cache profile from id (mapping from id to Profile) + self.profile_id_cache = dict() + @contextmanager def anonymous_copy(self): session = self._session diff --git a/instaloader/structures.py b/instaloader/structures.py index 2c66899..5c8a6f2 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -402,6 +402,8 @@ class Profile: :param profile_id: userid :raises: :class:`ProfileNotExistsException`, :class:`ProfileHasNoPicsException` """ + if profile_id in context.profile_id_cache: + return context.profile_id_cache[profile_id] data = context.graphql_query("472f257a40c653c64c666ce877d59d2b", {'id': str(profile_id), 'first': 1}, rhx_gis=context.root_rhx_gis)['data']['user'] @@ -415,7 +417,9 @@ class Profile: raise ProfileHasNoPicsException("Profile with ID {0}: no pics found.".format(str(profile_id))) else: raise LoginRequiredException("Login required to determine username (ID: " + str(profile_id) + ").") - return Post(context, data['edges'][0]['node']).owner_profile + profile = Post(context, data['edges'][0]['node']).owner_profile + context.profile_id_cache[profile_id] = profile + return profile def _asdict(self): json_node = self._node.copy() From 862c51fa81aca8a20d4b697e486aacd9531623f4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 30 Aug 2018 13:58:02 +0200 Subject: [PATCH 24/27] Profile.from_id graphql query is not rate limited --- instaloader/instaloadercontext.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/instaloader/instaloadercontext.py b/instaloader/instaloadercontext.py index 9571a20..1f0e263 100644 --- a/instaloader/instaloadercontext.py +++ b/instaloader/instaloadercontext.py @@ -248,7 +248,8 @@ class InstaloaderContext: is_graphql_query = 'query_hash' in params and 'graphql/query' in path # some queries are not rate limited if invoked anonymously: query_not_limited = is_graphql_query and not self.is_logged_in \ - and params['query_hash'] in ['9ca88e465c3f866a76f7adee3871bdd8'] + and params['query_hash'] in ['9ca88e465c3f866a76f7adee3871bdd8', + '472f257a40c653c64c666ce877d59d2b'] if is_graphql_query and not query_not_limited: waittime = graphql_query_waittime() if waittime > 0: From 1f311d3e1e1d8ce5bd0bd8f3bead105c315f7117 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 31 Aug 2018 12:11:43 +0200 Subject: [PATCH 25/27] First Beta Release for Version 4.1 --- instaloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instaloader/__init__.py b/instaloader/__init__.py index b91d3ae..28d0f97 100644 --- a/instaloader/__init__.py +++ b/instaloader/__init__.py @@ -1,7 +1,7 @@ """Download pictures (or videos) along with their captions and other metadata from Instagram.""" -__version__ = '4.1a1' +__version__ = '4.1b1' try: From bb8749b753ae53b510a057b4cb3138983c0cdd9f Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 31 Aug 2018 17:15:57 +0200 Subject: [PATCH 26/27] Adjust rate control to current rate limits --- instaloader/instaloadercontext.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/instaloader/instaloadercontext.py b/instaloader/instaloadercontext.py index 1f0e263..8601960 100644 --- a/instaloader/instaloadercontext.py +++ b/instaloader/instaloadercontext.py @@ -57,7 +57,7 @@ class InstaloaderContext: self.quiet = quiet self.max_connection_attempts = max_connection_attempts self._graphql_page_length = 50 - self.graphql_count_per_slidingwindow = graphql_count_per_slidingwindow or 20 + self.graphql_count_per_slidingwindow = graphql_count_per_slidingwindow or 200 self._root_rhx_gis = None # error log, filled with error() and printed at the end of Instaloader.main() @@ -221,7 +221,7 @@ class InstaloaderContext: def _sleep(self): """Sleep a short time if self.sleep is set. Called before each request to instagram.com.""" if self.sleep: - time.sleep(min(random.expovariate(0.6), 5.0)) + time.sleep(min(random.expovariate(0.7), 5.0)) def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com', session: Optional[requests.Session] = None, _attempt=1) -> Dict[str, Any]: @@ -248,8 +248,7 @@ class InstaloaderContext: is_graphql_query = 'query_hash' in params and 'graphql/query' in path # some queries are not rate limited if invoked anonymously: query_not_limited = is_graphql_query and not self.is_logged_in \ - and params['query_hash'] in ['9ca88e465c3f866a76f7adee3871bdd8', - '472f257a40c653c64c666ce877d59d2b'] + and params['query_hash'] in ['9ca88e465c3f866a76f7adee3871bdd8'] if is_graphql_query and not query_not_limited: waittime = graphql_query_waittime() if waittime > 0: From 1a239e17cb8db5abc664fc4958c4ce44b0ce8305 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 31 Aug 2018 17:42:59 +0200 Subject: [PATCH 27/27] First Release Candidate for Version 4.1 --- instaloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instaloader/__init__.py b/instaloader/__init__.py index 28d0f97..7ced198 100644 --- a/instaloader/__init__.py +++ b/instaloader/__init__.py @@ -1,7 +1,7 @@ """Download pictures (or videos) along with their captions and other metadata from Instagram.""" -__version__ = '4.1b1' +__version__ = '4.1rc1' try: