def download_images(course, threads_folder, thread):
    """
    Download images in given thread.
    The given thread object will be mutated.
    """
    posts = thread['posts']
    comments = thread['comments']

    thread_id = thread['id']
    thread_page = thread['start_page']

    images = []
    last_post_is_full = False
    for post in reversed(posts):
        if 'post_text' in post:
            text = post['post_text']
            text = find_images(text, images, thread_id, thread_page)
            post['post_text'] = text
            last_post_is_full = True
        elif last_post_is_full:
            break

    for comment in comments:
        text = comment['comment_text']
        text = find_images(text, images, thread_id, thread_page)
        comment['comment_text'] = text

    for url, path in images:
        path = '{}/{}'.format(threads_folder, path)
        util.download(url, path, course.get_cookie_file(), resume=True)

def download_thread(course, threads_folder, thread_id, page=1, post_id=None):
    """
    Download a thread.
    """
    # Download 1st page
    url = '{}/api/forum/threads/{}'.format(course.get_url(), thread_id)
    if post_id:
        url = '{}?post_id={}&position=after'.format(url, post_id)

    path = '{}/{}/{}.json'.format(threads_folder, thread_id, page)
    util.download(url, path, course.get_cookie_file())

    thread = util.read_json(path)
    download_images(course, threads_folder, thread)

    util.write_json(path, thread)

    # Download rest pages
    page = thread['start_page']
    num_page = thread['num_pages']

    if page < num_page:
        page += 1
        print 'thread page {}/{}'.format(page, num_page)

        post_id = get_next_post_id(thread['posts'])
        if post_id:
            download_thread(course, threads_folder, thread_id, page, post_id)

def _find_files(url, folder, cookie):
    """
    Recursively find all files in current page.
    :param url: A URL to given page.
    :param folder: A destination folder for this page.
    :param cookie: A cookie file used for downloading.
    :return: A list of files (URL, path) in current page.
    """
    files = []

    path = '{}/temp.html'.format(folder)
    util.download(url, path, cookie)

    page = util.read_file(path)
    util.remove(path)

    # recursively find all files in sub-folders
    pattern = r'<tr><td colspan="4"><a href="(.*?)">(.*?)</a>'
    for find in re.finditer(pattern, page, re.DOTALL):
        url = find.group(1)
        sub_folder = '{}/{}'.format(folder, find.group(2))
        files += _find_files(url, sub_folder, cookie)

    # find all files in this page
    pattern = r'<tr><td>(.*?)</td>.*?Embed.*?<a href="(.*?)\?.*?">Download</a>'
    for find in re.finditer(pattern, page, re.DOTALL):
        url = find.group(2)
        file_name = find.group(1)
        path = u'{}/{}'.format(folder, file_name)
        files.append((url, path))

    return files

def find_threads(course, forum_folder, forum_id):
    """
    Find all threads in current forum.
    Note: forum 0 has every thread!
    """
    # download the 1st page of given forum
    query = 'sort=firstposted&page=1'
    url = '{}/api/forum/forums/{}/threads?{}'
    url = url.format(course.get_url(), forum_id, query)
    path = forum_folder + '/temp.json'
    util.download(url, path, course.get_cookie_file())

    # download a huge page with all threads
    forum = util.read_json(path)
    num_threads = forum['total_threads']
    url += '&page_size={}'.format(num_threads)
    util.download(url, path, course.get_cookie_file())

    # add each thread's id to forum info
    threads = util.read_json(path)['threads']
    util.remove(path)

    path = forum_folder + '/info.json'
    forum = util.read_json(path)

    forum_threads = []
    for thread in reversed(threads):
        forum_threads.append({'id': thread['id']})

    forum['num_threads'] = num_threads
    forum['threads'] = forum_threads

    util.write_json(path, forum)

 def prepare(self):
     if "version" in self.settings:
         version = self.settings["version"]
         download(self.url % (version, version), self.zipfile)
         unzip(self.zipfile, 'temp')
     else:
         git_clone(self.repo, 'master', 'src')

def _download_old_quizzes(course, item, path):
    """
    Download old version in-video quizzes.
    """
    url = '{}/admin/quiz/quiz_load?quiz_id={}'
    url = url.format(course.get_url(), item['quiz']['parent_id'])
    util.download(url, path, course.get_cookie_file())
    util.write_json(path, util.read_json(path))

    def _check_setuptools(self):

        if self.check_module("setuptools"):
            return

        url, name = URLS['ez_setup']
        util.download(url)

        self.run_py(name)

    def download_stats(self):
        url = self.url + '/data/stats'
        path = self.info_folder + '/stats.html'
        util.download(url, path, self.cookie_file)

        content = util.read_file(path)
        pattern = r'<h1.*?</table>'
        content = re.search(pattern, content, re.DOTALL).group(0)
        util.write_file(path, content)

    def _check_pyparsing(self):

        if self.check_module("pyparsing"):
            return

        url, name = URLS['pyparsing']
        util.download(url, name)

        self.run_ez(name)

def prepare_negative_dataset(dataset_directory):
    negative_dataset_url = \
        'http://www.ics.uci.edu/~dramanan/papers/parse/people.zip'
    data_filepath = os.path.join(dataset_root,
                                 os.path.basename(negative_dataset_url))
    if not(os.path.exists(data_filepath)):
        download(negative_dataset_url, path=data_filepath)
    unzip(data_filepath, dataset_root)

    shutil.move(os.path.join(dataset_root, 'people_all'), dataset_directory)

    def _check_python(self):

        if os.path.exists(os.path.join(PYDIR, "python.exe")):
            return True

        url, name = URLS['python']
        util.download(url)

        print "Extracting the the python installer..."
        os.system('msiexec /a %s /qn TARGETDIR="%s"' % (name, PYDIR))

 def test_download(self):
   # Create a temporary file name, then delete it...
   # Of course, never do this in non-testing code!!
   f = tempfile.NamedTemporaryFile()
   f.close()
   # Make sure it doesn't exist.
   assert not os.path.exists(f.name)
   # Now download, using the deleted temporary file name.
   util.download(self.url, f.name)
   assert util.sha1file(f.name) == self.sha1_gtfs

def _load_from_url(url, to_filename):
    """ First downloads the connectome file to a file to_filename
    load it and return the reference to the connectome object
    
    Not tested.
    """
    
    from util import download    
    download(url, to_filename)
    return _load_from_cff(to_filename)

def download(url,name):
	downloads = __addon__.getSetting('downloads')
	if '' == downloads:
		xbmcgui.Dialog().ok(__scriptname__,__language__(30031))
		return
	stream = resolve(url)
	if stream:
		util.reportUsage(__scriptid__,__scriptid__+'/download')
		name+='.flv'
		util.download(__addon__,name,stream['url'],os.path.join(downloads,name))

 def prepare(self):
     if "version" in self.settings:
         version = self.settings["version"]
         download(self.url % (version), self.zipfile)
         unzip(self.zipfile, 'temp')
         cp('temp/imgui-%s/' % (version), 'temp/') # TODO: mv would be cleaner
     else:
         git_clone(self.repo, 'master', 'temp')
     if "patch" in self.settings:
         with cd('temp/'):
             patch(self.settings["patch"])

    def download_email_blacklist(self):
        url = self.url + '/data/export/pii'
        path = self.info_folder + '/temp.html'
        util.download(url, path, self.cookie_file)

        content = util.read_file(path)
        pattern = r'href="(https://coursera-reports.*?)"'
        url = re.search(pattern, content).group(1)

        util.remove(path)
        path = self.info_folder + '/email_blacklist.csv'
        util.download(url, path, self.cookie_file)

    def download_info(self):
        url = self.url
        temp = self.info_folder + '/temp.html'
        util.download(url, temp, self.cookie_file)
        page_html = util.read_file(temp)
        util.remove(temp)

        info_files = ['user.json', 'course.json', 'sidebar.json']
        matches = re.findall(r'JSON\.parse\("(.*?)"\);', page_html)
        for match, info_file in zip(matches, info_files)[1:]:
            info = util.unicode_unescape(match).replace('\\\\', '')
            path = '{}/{}'.format(self.info_folder, info_file)
            util.write_json(path, util.read_json(info, True))

def pip_download_install():
    url = 'https://pypi.python.org/packages/source/p/pip/pip-6.0.8.tar.gz'
    target = 'pip-6.0.8.tar.gz'
    targetdir = 'pip-6.0.8'
    print('============ downloading ' + target + ' from:' + url)
    util.download(url,target)
    print('============ extracting ' + target)
    util.decompress(target,'.')
    os.chdir(targetdir)
    print('============ installing pip')
    cmdResult = os.popen('python setup.py install').readlines()
    util.printCommandResult(cmdResult)
    print('============ installed,plese add pip to your path')

def run():
    parser = ArgumentParser(prog='download_file',
                            description='download and cache the jar, then create a hard link specified by \'-o\'.' +
                                        ' Background: with the jar to create genrule used for prebuilt_jar.'
                            )
    parser.add_argument('--repo',
                        nargs='?',
                        type=lambda alias: map_of_sites[alias] if alias in map_of_sites.keys() else alias,
                        default=npmjs_defs.MAVEN,
                        help='web site from where to download (default: %s). Can be one of the key of %s' % (
                            npmjs_defs.MAVEN, __builtin__.str(map_of_sites)),
                        metavar='web site'
                        )
    parser.add_argument('--cache-path',
                        nargs='?',
                        type=util.path_of,
                        default=util.path_of(npmjs_defs.CACHED_PATH),
                        help='path to cached jar (default: %s) ' % npmjs_defs.CACHED_PATH,
                        metavar='cached path'
                        )
    required = parser.add_argument_group('required input')
    required.add_argument('-u',
                          required=True,
                          help='part of URL work with, not including the content of \'-w\'',
                          metavar='tail of URL'
                          )
    required.add_argument('-o',
                          required=True,
                          help='the local hard link name',
                          metavar='file name'
                          )
    parser.add_argument('--sha1',
                        help='jar\'s SHA-1, with it to verify content integration',
                        metavar='SHA-1'
                        )
    args = parser.parse_args()

    cache_entry = cached_file(args.sha1, args.o, args.u, args.cache_path)

    url = npmjs_defs.get_url(args.repo, args.u)
    if not os.path.isfile(cache_entry):
        util.download(url, cache_entry, True)

    if args.sha1 and util.is_integrated(cache_entry, args.sha1) is False:
        print('error download %s' % url, file=sys.stderr)
        delete_wrong_file(cache_entry)
        sys.exit(1)
    cache_entry = npmjs_defs.make_sure_deps(args.u, cache_entry, args.repo)
    check_dir_of(args.o)
    hard_link(cache_entry, args.o)
    return 0

    def _check_pywin32(self):

        if self.check_module("pywintypes"):
            return

        url, name = URLS['pywin32']
        util.download(url, name)

        util.unzip(name, 'tmp_pyw32')

        os.system("xcopy /q /y /e tmp_pyw32\\PLATLIB\\* \"%s\\Lib\\site-packages\"" % PYDIR)
        os.system("copy /y \"%s\\Lib\\site-packages\\pywin32_system32\\*\" \"%s\"" % (PYDIR, PYDIR))
        os.system("copy /y \"%s\\Lib\\site-packages\\win32\\*.exe\" \"%s\"" % (PYDIR, PYDIR))
        os.system("copy /y \"%s\\Lib\\site-packages\\win32\\*.dll\" \"%s\"" % (PYDIR, PYDIR))
        os.system("rmdir /s /q tmp_pyw32")