def url_download():
    url = request.forms.get('url')
    tags = request.forms.get('tag_list')
    tags = "url,"+tags
    if request.forms.get("tor"):
        upload = network.download(url,tor=True)
    else:
        upload = network.download(url,tor=False)
    if upload == None:
        return template('error.tpl', error="server can't download from URL")
    # Set Project
    project = 'Main'
    db = Database()
    tf = tempfile.NamedTemporaryFile()
    tf.write(upload)
    if tf == None:
        return template('error.tpl', error="server can't download from URL")
    tf.flush()
    tf_obj = File(tf.name)
    tf_obj.name = tf_obj.sha256
    new_path = store_sample(tf_obj)
    success = False
    if new_path:
        # Add file to the database.
        success = db.add(obj=tf_obj, tags=tags)

    if success:
        #redirect("/project/{0}".format(project))
        redirect("/file/Main/"+tf_obj.sha256)
    else:
        return template('error.tpl', error="Unable to Store The File,already in database")

        def add_file(obj, tags=None):
            if get_sample_path(obj.sha256):
                self.log('warning', "Skip, file \"{0}\" appears to be already stored".format(obj.name))
                return False

            if __sessions__.is_attached_misp(quiet=True):
                if tags is not None:
                    tags += ',misp:{}'.format(__sessions__.current.misp_event.event.id)
                else:
                    tags = 'misp:{}'.format(__sessions__.current.misp_event.event.id)

            # Try to store file object into database.
            status = db.add(obj=obj, tags=tags)
            if status:
                # If succeeds, store also in the local repository.
                # If something fails in the database (for example unicode strings)
                # we don't want to have the binary lying in the repository with no
                # associated database record.
                new_path = store_sample(obj)
                self.log("success", "Stored file \"{0}\" to {1}".format(obj.name, new_path))

            else:
                return False

            # Delete the file if requested to do so.
            if args.delete:
                try:
                    os.unlink(obj.path)
                except Exception as e:
                    self.log('warning', "Failed deleting file: {0}".format(e))

            return True

        def add_file(obj, tags=None):
            if get_sample_path(obj.sha256):
                self.log("warning", 'Skip, file "{0}" appears to be already stored'.format(obj.name))
                return False

            # Try to store file object into database.
            status = self.db.add(obj=obj, tags=tags)
            if status:
                # If succeeds, store also in the local repository.
                # If something fails in the database (for example unicode strings)
                # we don't want to have the binary lying in the repository with no
                # associated database record.
                new_path = store_sample(obj)
                self.log("success", 'Stored file "{0}" to {1}'.format(obj.name, new_path))
            else:
                return False

            # Delete the file if requested to do so.
            if args.delete:
                try:
                    os.unlink(obj.path)
                except Exception as e:
                    self.log("warning", "Failed deleting file: {0}".format(e))

            return True

 def parse_message(self, message_folder):
     db = Database()
     email_header = os.path.join(message_folder, 'InternetHeaders.txt')
     email_body = os.path.join(message_folder, 'Message.txt')
     attachments = []
     
     envelope = headers = email_text = ''
     if os.path.exists(email_header):
         envelope, headers = self.email_headers(email_header)
     if os.path.exists(email_body):
         email_text = open(email_body, 'rb').read()
     
     tags = 'pst, {0}'.format(message_folder)
     if os.path.exists(os.path.join(message_folder, 'Attachments')):
         for filename in os.listdir(os.path.join(message_folder, 'Attachments')):
             if os.path.isfile(os.path.join(message_folder, 'Attachments', filename)):
                 obj = File(os.path.join(message_folder, 'Attachments', filename))
                 sha256 = hashlib.sha256(open(os.path.join(message_folder, 'Attachments', filename), 'rb').read()).hexdigest()
                 new_path = store_sample(obj)
                 success = False
                 if new_path:
                         # Add file to the database.
                         success = db.add(obj=obj, tags=tags)
                 # Add Email Details as a Note
                 # To handle duplicates we use multiple notes
                 headers_body = 'Envelope: \n{0}\nHeaders: \n{1}\n'.format(envelope, headers)
                 db.add_note(sha256, 'Headers', headers_body)
                 
                 # Add a note with email body
                 db.add_note(sha256, 'Email Body', string_clean(email_text))

def add_file():
    tags = request.forms.get('tags')
    upload = request.files.get('file')

    tf = tempfile.NamedTemporaryFile()
    tf.write(upload.file.read())
    tf.flush()
    
    # Added to process zip files
    if request.headers.get('compression') == 'zip' or request.headers.get('compression') == 'ZIP':
        with upload_temp() as temp_dir:
            with ZipFile(tf.name) as zf:
                zf.extractall(temp_dir, pwd=request.headers.get('compression_password'))

            stored_files = []
   
            for root, dirs, files in os.walk(temp_dir, topdown=False):
                for name in files:
                    if not name == upload.filename:
                        tf_obj=File(os.path.join(root,name))
                        new_path = store_sample(tf_obj)
                        success = False
                        
                        if new_path:
                            success = db.add(obj=tf_obj, tags=tags)
                       
                        if success:
                            stored_files.append(name)

            if stored_files:
                return jsonize({'message': 'Files added: %s' % ','.join(stored_files)})
    else:
        tf_obj = File(tf.name)
        tf_obj.name = upload.filename

        new_path = store_sample(tf_obj)

        success = False
        if new_path:
            # Add file to the database.
            success = db.add(obj=tf_obj, tags=tags)

        if success:
            return jsonize({'message' : 'added'})
        else:
            response.status = 500
            return jsonize({'message':'Unable to store file'})

 def add_file(self, file_path, tags, parent):
     obj = File(file_path)
     new_path = store_sample(obj)
     if new_path:
         # Add file to the database.
         db = Database()
         db.add(obj=obj, tags=tags, parent_sha=parent)
         return obj.sha256

def _add_file(file_path, name, tags, parent_sha):
    obj = File(file_path)
    new_path = store_sample(obj)
    if new_path:
        db = Database()
        db.add(obj=obj, name=name, tags=tags, parent_sha=parent_sha)
        return obj.sha256
    else:
        return None

        def add_file(obj, tags=None):
            # Store file to the local repository.
            new_path = store_sample(obj)
            if new_path:
                # Add file to the database.
                status = self.db.add(obj=obj, tags=tags)
                print_success("Stored to: {0}".format(new_path))

            # Delete the file if requested to do so.
            if do_delete:
                try:
                    os.unlink(obj.path)
                except Exception as e:
                    print_warning("Failed deleting file: {0}".format(e))

def add_file():
    tags = request.forms.get('tag_list')
    upload = request.files.get('file')

    # Set Project
    project = request.forms.get('project')
    if project in project_list():
        __project__.open(project)
    else:
        __project__.open('../')
        project = 'Main'
    db = Database()    

    # Write temp file to disk
    with upload_temp() as temp_dir:
        file_path = os.path.join(temp_dir, upload.filename)
        with open(file_path, 'w') as tmp_file:
            tmp_file.write(upload.file.read())
        file_list = []
        # Zip Files
        if request.forms.get('unzip'):
            zip_pass = request.forms.get('zip_pass')
            try:
                with ZipFile(file_path) as zf:
                    zf.extractall(temp_dir, pwd=zip_pass)            
                for root, dirs, files in os.walk(temp_dir, topdown=False):
                    for name in files:
                        if not name == upload.filename:
                            file_list.append(os.path.join(root, name))
            except Exception as e:
                return template('error.tpl', error="Error with zipfile - {0}".format(e))
        # Non zip files
        else:
            file_list.append(file_path)
        
        # Add each file
        for new_file in file_list:
            obj = File(new_file)
            new_path = store_sample(obj)
            success = False
            if new_path:
                # Add file to the database.
                success = db.add(obj=obj, tags=tags)
    if success:
        redirect("/project/{0}".format(project))
    else:
        return template('error.tpl', error="Unable to Store The File")

    def _process_uploaded(db, uploaded_file_path, file_name, tag_list=None, note_title=None, note_body=None):
        """_process_uploaded add one uploaded file to database and to storage then remove uploaded file"""

        log.debug("adding: {} as {}".format(uploaded_file_path, file_name))

        malware = File(uploaded_file_path)
        malware.name = file_name

        if get_sample_path(malware.sha256):
            error = {"error": {"code": "DuplicateFileHash",
                               "message": "File hash exists already: {} (sha256: {})".format(malware.name, malware.sha256)}}
            log.error("adding failed: {}".format(error))
            raise ValidationError(detail=error)  # TODO(frennkie) raise more specific error?! so that we can catch it..?!
        # Try to store file object into database
        if db.add(obj=malware, tags=tag_list):
            # If succeeds, store also in the local repository.
            # If something fails in the database (for example unicode strings)
            # we don't want to have the binary lying in the repository with no
            # associated database record.
            malware_stored_path = store_sample(malware)

            # run autoruns on the stored sample
            if cfg.get('autorun').enabled:
                autorun_module(malware.sha256)

            log.debug("added file \"{0}\" to {1}".format(malware.name, malware_stored_path))

            if note_body and note_title:
                db.add_note(malware.sha256, note_title, note_body)
                log.debug("added note: \"{0}\"".format(note_title))

        else:
            error = {"error": {"code": "DatabaseAddFailed",
                               "message": "Adding File to Database failed: {} (sha256: {})".format(malware.name, malware.sha256)}}
            log.error("adding failed: {}".format(error))
            raise ValidationError(detail=error)

        # clean up
        try:
            os.remove(uploaded_file_path)
        except OSError as err:
            log.error("failed to delete temporary file: {}".format(err))

        return malware

def add_file():
    tags = request.forms.get('tags')
    upload = request.files.get('file')

    tf = tempfile.NamedTemporaryFile()
    tf.write(upload.file.read())
    tf_obj = File(tf.name)
    tf_obj.name = upload.filename

    new_path = store_sample(tf_obj)

    success = False
    if new_path:
        # Add file to the database.
        success = db.add(obj=tf_obj, tags=tags)

    if success:
        return jsonize({'message' : 'added'})
    else:
        return HTTPError(500, 'Unable to store file')

        def add_file(obj, tags=None):
            if get_sample_path(obj.sha256):
                print_warning('Skip, file "{0}" appears to be already stored'.format(obj.name))
                return False

            # Store file to the local repository.
            new_path = store_sample(obj)
            if new_path:
                # Add file to the database.
                status = self.db.add(obj=obj, tags=tags)
                print_success('Stored file "{0}" to {1}'.format(obj.name, new_path))

            # Delete the file if requested to do so.
            if arg_delete:
                try:
                    os.unlink(obj.path)
                except Exception as e:
                    print_warning("Failed deleting file: {0}".format(e))

            return True

def add_file(file_path, tags, parent):
    obj = File(file_path)
    new_path = store_sample(obj)
    print new_path
    success = True
    if new_path:
        # Add file to the database.
        db = Database()
        success = db.add(obj=obj, tags=tags, parent_sha=parent)

        # AutoRun Modules
        if cfg.autorun.enabled:
            autorun_module(obj.sha256)
            # Close the open session to keep the session table clean
            __sessions__.close()
        return obj.sha256

    else:
        # ToDo Remove the stored file if we cant write to DB
        return

    def copy(self, id, src_project, dst_project,
             copy_analysis=True, copy_notes=True, copy_tags=True, copy_children=True, _parent_sha256=None):  # noqa
        session = self.Session()

        # make sure to open source project
        __project__.open(src_project)

        # get malware from DB
        malware = session.query(Malware). \
            options(subqueryload(Malware.analysis)). \
            options(subqueryload(Malware.note)). \
            options(subqueryload(Malware.parent)). \
            options(subqueryload(Malware.tag)). \
            get(id)

        # get path and load file from disk
        malware_path = get_sample_path(malware.sha256)
        sample = File(malware_path)
        sample.name = malware.name

        log.debug("Copying ID: {} ({}): from {} to {}".format(malware.id, malware.name, src_project, dst_project))
        # switch to destination project, add to DB and store on disk
        __project__.open(dst_project)
        dst_db = Database()
        dst_db.add(sample)
        store_sample(sample)
        print_success("Copied: {} ({})".format(malware.sha256, malware.name))

        if copy_analysis:
            log.debug("copy analysis..")
            for analysis in malware.analysis:
                dst_db.add_analysis(malware.sha256, cmd_line=analysis.cmd_line, results=analysis.results)

        if copy_notes:
            log.debug("copy notes..")
            for note in malware.note:
                dst_db.add_note(malware.sha256, title=note.title, body=note.body)

        if copy_tags:
            log.debug("copy tags..")
            dst_db.add_tags(malware.sha256, [x.tag for x in malware.tag])

        if copy_children:
            children = session.query(Malware).filter(Malware.parent_id == malware.id).all()
            if not children:
                pass
            else:
                _parent_sha256 = malware.sha256  # set current recursion item as parent
                for child in children:
                    self.copy(child.id,
                              src_project=src_project, dst_project=dst_project,
                              copy_analysis=copy_analysis, copy_notes=copy_notes, copy_tags=copy_tags,
                              copy_children=copy_children, _parent_sha256=_parent_sha256)
                    # restore parent-child relationships
                    log.debug("add parent {} to child {}".format(_parent_sha256, child.sha256))
                    if _parent_sha256:
                        dst_db.add_parent(child.sha256, _parent_sha256)

        # switch back to source project
        __project__.open(src_project)

        # store tuple of ID (in source project) and sha256 of copied samples
        self.copied_id_sha256.append((malware.id, malware.sha256))

        return True

def add_file():
    tags = request.forms.get('tag_list')
    uploads = request.files.getlist('file')
    
    # Set Project
    project = request.forms.get('project')
    if project in project_list():
        __project__.open(project)
    else:
        __project__.open('../')
        project = 'Main'
    db = Database()    
    file_list = []
    # Write temp file to disk
    with upload_temp() as temp_dir:
        for upload in uploads:
            file_path = os.path.join(temp_dir, upload.filename)
            with open(file_path, 'w') as tmp_file:
                tmp_file.write(upload.file.read())
            # Zip Files
            if request.forms.get('compression') == 'zip':
                zip_pass = request.forms.get('zip_pass')
                try:
                    with ZipFile(file_path) as zf:
                        zf.extractall(temp_dir, pwd=zip_pass)            
                    for root, dirs, files in os.walk(temp_dir, topdown=False):
                        for name in files:
                            if not name == upload.filename:
                                file_list.append(os.path.join(root, name))
                except Exception as e:
                    return template('error.tpl', error="Error with zipfile - {0}".format(e))
            # GZip Files
            elif request.forms.get('compression') == 'gz':
                try:
                    gzf = GzipFile(file_path, 'rb')
                    decompress = gzf.read()
                    gzf.close()
                    with open(file_path[:-3],"wb") as df:
                        df.write(decompress)
                    file_list.append(file_path[:-3])
                except Exception as e:
                    return template('error.tpl', error="Error with gzipfile - {0}".format(e))
            # BZip2 Files
            elif request.forms.get('compression') == 'bz2':
                try:
                    bz2f = BZ2File(file_path, 'rb')
                    decompress = bz2f.read()
                    bz2f.close()
                    with open(file_path[:-3],"wb") as df:
                        df.write(decompress)
                    file_list.append(file_path[:-3])
                except Exception as e:
                    return template('error.tpl', error="Error with bzip2file - {0}".format(e))
            # Tar Files (any, including tar.gz tar.bz2)
            elif request.forms.get('compression') == 'tar':
                try:
                    if not tarfile.is_tarfile(file_path):
                        return template('error.tpl', error="This is not a tar file")
                    with tarfile.open(file_path,'r:*') as tarf:
                        tarf.extractall(temp_dir)
                    for root, dirs, files in os.walk(temp_dir, topdown=False):
                        for name in files:
                            if not name == upload.filename:
                                file_list.append(os.path.join(root, name))
                except Exception as e:
                    return template('error.tpl', error="Error with tarfile - {0}".format(e))
            # Non zip files
            elif request.forms.get('compression') == 'none':
                file_list.append(file_path)
            
        # Add each file
        for new_file in file_list:
            print new_file
            obj = File(new_file)
            new_path = store_sample(obj)
            success = True
            if new_path:
                # Add file to the database.
                success = db.add(obj=obj, tags=tags)
                if not success:
                    return template('error.tpl', error="Unable to Store The File: {0}".format(upload.filename))
    redirect("/project/{0}".format(project))