REWise/tests/extended/dl.py

import os
import urllib
import urllib.error
import urllib.request
from urllib.parse import urlparse, quote, unquote
from http.client import RemoteDisconnected


def encodeUrl(url):
  parsed = urlparse(url)
  parsed = parsed._replace(path=quote(unquote(parsed.path)),
                           query=quote(unquote(parsed.query)))
  return parsed.geturl()


def downloadFile(url, outputFile):
  url = encodeUrl(url)

  # Create needed dirs
  outputDir = os.path.dirname(outputFile)
  if not os.path.isdir(outputDir):
    os.makedirs(outputDir)

  # Download ..
  try:
    urllib.request.urlretrieve(url, outputFile)
  except (urllib.error.ContentTooShortError, urllib.error.URLError, RemoteDisconnected) as e:
    print(e, url)
    return str(e)
  return None