From 4a944fbbdba602904b74d063c131998fff3e20f3 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Mar 2018 11:32:53 +0100 Subject: [PATCH 01/11] doc: meta tag to redirect to new documentation URL --- docs/_templates/layout.html | 3 +++ docs/conf.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 1ce3046..d22dc3b 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -2,4 +2,7 @@ {% block extrahead %} +{% if READTHEDOCS %} + +{% endif %} {% endblock %} diff --git a/docs/conf.py b/docs/conf.py index a9d5c9d..5f2d1dc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -361,4 +361,4 @@ texinfo_documents = [ current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).decode("ascii")[1:-1] current_release_date = subprocess.check_output(["git", "log", "-1", "--tags", "--format=%ad", "--date=format:%e %b %Y"]).decode("ascii")[:-1] -html_context = {'current_release': current_release, 'current_release_date': current_release_date} +html_context = {'current_release': current_release, 'current_release_date': current_release_date, 'READTHEDOCS': bool(os.environ.get('READTHEDOCS'))} From 36110c87274e914c6b01bd29eeafda26ee9e7e57 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Mar 2018 12:37:24 +0100 Subject: [PATCH 02/11] Disable search engine indexing on old doc hoster --- docs/_templates/layout.html | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index d22dc3b..fba9403 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -4,5 +4,6 @@ {% if READTHEDOCS %} + {% endif %} {% endblock %} From 71331d37e49ca1ad8aee139dfb12f99204275ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Sat, 24 Mar 2018 19:54:07 +0100 Subject: [PATCH 03/11] Removed modifying of profilepic url Since Instagram does not allow anymore to download pics without a valid signature given in the URL, Instaloader has to be content with a resolution for the profile pic that is provided in the JSON data. Closes #85. --- instaloader.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/instaloader.py b/instaloader.py index 2c8cd06..4548219 100755 --- a/instaloader.py +++ b/instaloader.py @@ -919,13 +919,7 @@ class Instaloader: if os.path.isfile(filename): self._log(filename + ' already exists') return None - url_best = re.sub(r'/s([1-9][0-9]{2})x\1/', '/s2048x2048/', url) - url_best = re.sub(r'/vp/[a-f0-9]{32}/[A-F0-9]{8}/', '/', url_best) # remove signature - try: - self._get_and_write_raw(url_best, filename) - except (QueryReturnedForbiddenException, QueryReturnedNotFoundException) as err: - self.error('{} Retrying with lower quality version.'.format(err)) - self._get_and_write_raw(url, filename) + self._get_and_write_raw(url, filename) os.utime(filename, (datetime.now().timestamp(), date_object.timestamp())) self._log('') # log output of _get_and_write_raw() does not produce \n From 539273b963d5bb63fbece2018f2bbe8b356a7acf Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Mar 2018 21:46:21 +0100 Subject: [PATCH 04/11] make doc more mobile-friendly --- docs/_templates/layout.html | 8 ++++++++ docs/conf.py | 3 +-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index fba9403..1d5c904 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -1,9 +1,17 @@ {% extends "!layout.html" %} {% block extrahead %} +{{ super() }} {% if READTHEDOCS %} {% endif %} + {% endblock %} diff --git a/docs/conf.py b/docs/conf.py index 5f2d1dc..b5ea5ce 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -141,8 +141,7 @@ html_theme = 'alabaster' # html_theme_options = { 'show_powered_by': False, - 'sidebar_width': '290px', - 'page_width': '935px' } + 'sidebar_width': '290px' } # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] From d533c6082411912ababf7617d39574e7364eff94 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 28 Mar 2018 19:39:58 +0200 Subject: [PATCH 05/11] doc: Switch to mobile-friendlier bootstrap theme It seems to be better supported than Alabaster and is easier to be configured to look good on both a phone and a desktop. --- docs/_static/style.css | 3 ++ docs/_templates/caption.html | 2 -- docs/_templates/layout.html | 20 ++++-------- docs/_templates/links.html | 23 ------------- docs/_templates/navbar.html | 58 +++++++++++++++++++++++++++++++++ docs/_templates/rtdmessage.html | 3 -- docs/conf.py | 25 +++++++++----- docs/requirements.txt | 1 + 8 files changed, 86 insertions(+), 49 deletions(-) create mode 100644 docs/_static/style.css delete mode 100644 docs/_templates/caption.html delete mode 100644 docs/_templates/links.html create mode 100644 docs/_templates/navbar.html delete mode 100644 docs/_templates/rtdmessage.html diff --git a/docs/_static/style.css b/docs/_static/style.css new file mode 100644 index 0000000..f34d322 --- /dev/null +++ b/docs/_static/style.css @@ -0,0 +1,3 @@ +code { + color: #222; +} \ No newline at end of file diff --git a/docs/_templates/caption.html b/docs/_templates/caption.html deleted file mode 100644 index a4527b5..0000000 --- a/docs/_templates/caption.html +++ /dev/null @@ -1,2 +0,0 @@ -

Instaloader

-

Download pictures (or videos) along with their captions and other metadata from Instagram.

diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 1d5c904..e722476 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -1,17 +1,11 @@ {% extends "!layout.html" %} {% block extrahead %} {{ super() }} - - -{% if READTHEDOCS %} - - -{% endif %} - + + {% if pagename == "index" %} + + {% else %} + + {% endif %} + {% endblock %} diff --git a/docs/_templates/links.html b/docs/_templates/links.html deleted file mode 100644 index b06cba1..0000000 --- a/docs/_templates/links.html +++ /dev/null @@ -1,23 +0,0 @@ -{% if next %} -

Next

- -{% endif %} -

Current Release

- -

Links

- - diff --git a/docs/_templates/navbar.html b/docs/_templates/navbar.html new file mode 100644 index 0000000..ecb9852 --- /dev/null +++ b/docs/_templates/navbar.html @@ -0,0 +1,58 @@ +
+
+
+ + + + {%- block sidebarlogo %} + {%- if logo %}{%- endif %} + {%- endblock %} + {% if theme_navbar_title -%}{{ theme_navbar_title|e }}{%- else -%}{{ project|e }}{%- endif -%} + + + v{{current_release}} + +
+ +
+
    + {% if theme_navbar_links %} + {%- for link in theme_navbar_links %} +
  • {{ link[0] }}
    • + {%- endfor %} + {% endif %} + {% block navbartoc %} + {% include "globaltoc.html" %} + {% if theme_navbar_pagenav %} + {% include "navbartoc.html" %} + {% endif %} + {% endblock %} + {% if theme_navbar_sidebarrel %} + {% block sidebarrel %} + {% include "relations.html" %} + {% endblock %} + {% endif %} +
+ +
+
+
diff --git a/docs/_templates/rtdmessage.html b/docs/_templates/rtdmessage.html deleted file mode 100644 index 6287a02..0000000 --- a/docs/_templates/rtdmessage.html +++ /dev/null @@ -1,3 +0,0 @@ -

New Documentation URL

-

Instaloader's Documentation has been moved to - https://instaloader.github.io.

\ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index b5ea5ce..2fbd999 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,6 +20,9 @@ import os import subprocess import sys + +import sphinx_bootstrap_theme + sys.path.insert(0, os.path.abspath('..')) # -- General configuration ------------------------------------------------ @@ -133,15 +136,20 @@ todo_include_todos = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = 'bootstrap' +html_theme_path = sphinx_bootstrap_theme.get_html_theme_path() # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = { - 'show_powered_by': False, - 'sidebar_width': '290px' } + 'navbar_site_name': 'Site Contents', + 'navbar_pagenav_name': 'Page Contents', + 'navbar_pagenav': True, + 'navbar_sidebarrel': True, + 'nosidebar': True, + } # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] @@ -190,10 +198,8 @@ html_static_path = ['_static'] # Custom sidebar templates, maps document names to template names. # -if not os.environ.get("READTHEDOCS"): - html_sidebars = {'**': ["caption.html", "globaltoc.html", "relations.html", "links.html"] } -else: - html_sidebars = {'**': ["caption.html", "rtdmessage.html", "globaltoc.html", "relations.html", "links.html"] } +#html_sidebars = {'**': ["relations.html", "links.html"] } +html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. @@ -360,4 +366,7 @@ texinfo_documents = [ current_release = subprocess.check_output(["git", "describe", "--abbrev=0"]).decode("ascii")[1:-1] current_release_date = subprocess.check_output(["git", "log", "-1", "--tags", "--format=%ad", "--date=format:%e %b %Y"]).decode("ascii")[:-1] -html_context = {'current_release': current_release, 'current_release_date': current_release_date, 'READTHEDOCS': bool(os.environ.get('READTHEDOCS'))} +html_context = {'current_release': current_release, 'current_release_date': current_release_date} + +def setup(app): + app.add_stylesheet("style.css") \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index 914f619..2827417 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ requests sphinx sphinx-autodoc-typehints +sphinx-bootstrap-theme From 5b6e8f1a3148d0c8bc8bf6dbccc3b4d00903a0f4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 29 Mar 2018 11:19:15 +0200 Subject: [PATCH 06/11] Fix indent in USAGE_STRING message --- instaloader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/instaloader.py b/instaloader.py index 4548219..fcce2ba 100755 --- a/instaloader.py +++ b/instaloader.py @@ -35,9 +35,9 @@ __version__ = '3.3.3' # NOTE: duplicated in README.rst and docs/index.rst USAGE_STRING = """ {0} [--comments] [--geotags] [--stories] - [--login YOUR-USERNAME] [--fast-update] - profile | "#hashtag" | :stories | :feed | :saved -{0} --help""".format(sys.argv[0]) +{2:{1}} [--login YOUR-USERNAME] [--fast-update] +{2:{1}} profile | "#hashtag" | :stories | :feed | :saved +{0} --help""".format(sys.argv[0], len(sys.argv[0]), '') try: # pylint:disable=wrong-import-position From d9b5c77e3a8cb33fda45a243bc060e3afdb6714d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Thu, 29 Mar 2018 19:33:05 +0200 Subject: [PATCH 07/11] Implement high quality profile pic hack Use 'i.instagram.com' API to retrieve the best quality version of the profile pic. Related to #85. --- instaloader.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/instaloader.py b/instaloader.py index fcce2ba..1e20af5 100755 --- a/instaloader.py +++ b/instaloader.py @@ -588,12 +588,13 @@ class Instaloader: self.error("[skipped by user]", repeat_at_end=False) raise ConnectionException(error_string) - def get_json(self, url: str, params: Dict[str, Any], + def get_json(self, path: str, params: Dict[str, Any], host: str = 'www.instagram.com', session: Optional[requests.Session] = None, _attempt = 1) -> Dict[str, Any]: """JSON request to Instagram. - :param url: URL, relative to www.instagram.com/ + :param path: URL, relative to the given domain which defaults to www.instagram.com/ :param params: GET parameters + :param host: Domain part of the URL from where to download the requested JSON; defaults to www.instagram.com :param session: Session to use, or None to use self.session :return: Decoded response dictionary :raises QueryReturnedNotFoundException: When the server responds with a 404. @@ -610,7 +611,7 @@ class Instaloader: if len(timestamps) < 100 and not untracked_queries: return 0 return round(min(timestamps) + sliding_window - current_time) + 6 - is_graphql_query = 'query_id' in params and 'graphql/query' in url + is_graphql_query = 'query_id' in params and 'graphql/query' in path if is_graphql_query: query_id = params['query_id'] waittime = graphql_query_waittime(query_id) @@ -625,7 +626,7 @@ class Instaloader: sess = session if session else self.session try: self._sleep() - resp = sess.get('https://www.instagram.com/' + url, params=params) + resp = sess.get('https://{0}/{1}'.format(host, path), params=params) if resp.status_code == 404: raise QueryReturnedNotFoundException("404") if resp.status_code == 429: @@ -641,7 +642,7 @@ class Instaloader: raise ConnectionException("Returned \"{}\" status.".format(resp_json['status'])) return resp_json except (ConnectionException, json.decoder.JSONDecodeError, requests.exceptions.RequestException) as err: - error_string = "JSON Query to {}: {}".format(url, err) + error_string = "JSON Query to {}: {}".format(path, err) if _attempt == self.max_connection_attempts: raise ConnectionException(error_string) self.error(error_string + " [retrying; skip with ^C]", repeat_at_end=False) @@ -657,7 +658,7 @@ class Instaloader: self._log('The request will be retried in {} seconds.'.format(waittime)) time.sleep(waittime) self._sleep() - return self.get_json(url, params, sess, _attempt + 1) + return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1) except KeyboardInterrupt: self.error("[skipped by user]", repeat_at_end=False) raise ConnectionException(error_string) @@ -900,8 +901,14 @@ class Instaloader: def download_profilepic(self, name: str, profile_metadata: Dict[str, Any]) -> None: """Downloads and saves profile pic.""" - url = profile_metadata["user"]["profile_pic_url_hd"] if "profile_pic_url_hd" in profile_metadata["user"] \ - else profile_metadata["user"]["profile_pic_url"] + try: + data = self.get_json(path='api/v1/users/{0}/info/'.format(profile_metadata["user"]["id"]), params={}, + host='i.instagram.com') + url = data["user"]["hd_profile_pic_url_info"]["url"] + except (InstaloaderException, KeyError) as err: + self.error('{} Unable to fetch high quality profile pic.'.format(err)) + url = profile_metadata["user"]["profile_pic_url_hd"] if "profile_pic_url_hd" in profile_metadata["user"] \ + else profile_metadata["user"]["profile_pic_url"] def _epoch_to_string(epoch: datetime) -> str: return epoch.strftime('%Y-%m-%d_%H-%M-%S') From 40a87761c9a02ea324c8e8191f73cbe502399c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Thu, 29 Mar 2018 19:47:35 +0200 Subject: [PATCH 08/11] Mention Windows standalone exe in documentation --- docs/installation.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 4c14c4f..2c5c54e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -28,7 +28,7 @@ will be installed automatically, if it is not already installed. - If you do not want to use pip, even though it is highly recommended, and prefer to **install Instaloader manually**, - `Download the Source `__, + `Download the Source `__, extract the Zip or Tarball and execute ``instaloader.py`` from there. - On **Arch Linux**, you may install Instaloader using the @@ -36,3 +36,6 @@ will be installed automatically, if it is not already installed. - On **Gentoo Linux**, you may install Instaloader using the `Instaloader Ebuild `__. + +- On **Windows 10**, you may download the standalone executable from the + `current release page `__. From c5b60b8a127a311a2e1c171b95cb8ebab0199142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Thu, 29 Mar 2018 20:31:39 +0200 Subject: [PATCH 09/11] GraphQL for post locations --- instaloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instaloader.py b/instaloader.py index 1e20af5..7fd2126 100755 --- a/instaloader.py +++ b/instaloader.py @@ -411,7 +411,7 @@ class Post: if loc_dict is not None: location_json = self._instaloader.get_json("explore/locations/{0}/".format(loc_dict["id"]), params={'__a': 1}) - return location_json["location"] + return location_json["location"] if "location" in location_json else location_json['graphql']['location'] @staticmethod def json_encoder(obj) -> Dict[str, Any]: From dae18a05c6330245c91050c315e042694d94565f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Thu, 29 Mar 2018 20:33:20 +0200 Subject: [PATCH 10/11] Use anonymous session for profile pic query --- instaloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instaloader.py b/instaloader.py index 7fd2126..96aafe3 100755 --- a/instaloader.py +++ b/instaloader.py @@ -903,7 +903,7 @@ class Instaloader: try: data = self.get_json(path='api/v1/users/{0}/info/'.format(profile_metadata["user"]["id"]), params={}, - host='i.instagram.com') + host='i.instagram.com', session=self._get_anonymous_session()) url = data["user"]["hd_profile_pic_url_info"]["url"] except (InstaloaderException, KeyError) as err: self.error('{} Unable to fetch high quality profile pic.'.format(err)) From edd1fbe49c54a25903606cae40f52a00f5b9c98c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Thu, 29 Mar 2018 20:36:53 +0200 Subject: [PATCH 11/11] Release of version 3.3.4 --- instaloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instaloader.py b/instaloader.py index 96aafe3..487713c 100755 --- a/instaloader.py +++ b/instaloader.py @@ -30,7 +30,7 @@ import requests.utils import urllib3 -__version__ = '3.3.3' +__version__ = '3.3.4' # NOTE: duplicated in README.rst and docs/index.rst USAGE_STRING = """