def get_or_create_tool_shed_repository( self, tool_shed, name, owner, changeset_revision ):
     """
     Return a tool shed repository database record defined by the combination of
     tool shed, repository name, repository owner and changeset_revision or
     installed_changeset_revision.  A new tool shed repository record will be
     created if one is not located.
     """
     install_model = self.app.install_model
     # We store the port in the database.
     tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed )
     # This method is used only in Galaxy, not the tool shed.
     repository = suc.get_repository_for_dependency_relationship( self.app, tool_shed, name, owner, changeset_revision )
     if not repository:
         tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed )
         repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name )
         ctx_rev = suc.get_ctx_rev( self.app, tool_shed_url, name, owner, changeset_revision )
         repository = suc.create_or_update_tool_shed_repository( app=self.app,
                                                                 name=name,
                                                                 description=None,
                                                                 installed_changeset_revision=changeset_revision,
                                                                 ctx_rev=ctx_rev,
                                                                 repository_clone_url=repository_clone_url,
                                                                 metadata_dict={},
                                                                 status=install_model.ToolShedRepository.installation_status.NEW,
                                                                 current_changeset_revision=None,
                                                                 owner=owner,
                                                                 dist_to_shed=False )
     return repository

 def from_workflow_step(Class, trans, step, **kwds):
     tool_id = trans.app.toolbox.get_tool_id(step.tool_id) or step.tool_id
     tool_version = step.tool_version
     module = super(ToolModule, Class).from_workflow_step(trans, step, tool_id=tool_id, tool_version=tool_version)
     module.workflow_outputs = step.workflow_outputs
     module.post_job_actions = {}
     for pja in step.post_job_actions:
         module.post_job_actions[pja.action_type] = pja
     if module.tool:
         message = ""
         if step.tool_id != module.tool_id:  # This means the exact version of the tool is not installed. We inform the user.
             old_tool_shed = step.tool_id.split("/repos/")[0]
             if old_tool_shed not in tool_id:  # Only display the following warning if the tool comes from a different tool shed
                 old_tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(trans.app, old_tool_shed)
                 if not old_tool_shed_url:  # a tool from a different tool_shed has been found, but the original tool shed has been deactivated
                     old_tool_shed_url = "http://" + old_tool_shed  # let's just assume it's either http, or a http is forwarded to https.
                 old_url = old_tool_shed_url + "/view/%s/%s/" % (module.tool.repository_owner, module.tool.repository_name)
                 new_url = module.tool.sharable_url + '/%s/' % module.tool.changeset_revision
                 new_tool_shed_url = new_url.split("/view")[0]
                 message += "The tool \'%s\', version %s by the owner %s installed from <a href=\"%s\" target=\"_blank\">%s</a> is not available. " % (module.tool.name, tool_version, module.tool.repository_owner, old_url, old_tool_shed_url)
                 message += "A derivation of this tool installed from <a href=\"%s\" target=\"_blank\">%s</a> will be used instead. " % (new_url, new_tool_shed_url)
         if step.tool_version and (step.tool_version != module.tool.version):
             message += "<span title=\"tool id '%s'\">Using version '%s' instead of version '%s' specified in this workflow. " % (tool_id, module.tool.version, step.tool_version)
         if message:
             log.debug(message)
             module.version_changes.append(message)
     else:
         log.warning("The tool '%s' is missing. Cannot build workflow module." % tool_id)
     return module

 def create_temporary_tool_dependencies_config( self, tool_shed_url, name, owner, changeset_revision ):
     """Make a call to the tool shed to get the required repository's tool_dependencies.xml file."""
     tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed_url )
     if tool_shed_url is None or name is None or owner is None or changeset_revision is None:
         message = "Unable to retrieve required tool_dependencies.xml file from the Tool Shed because one or more of the "
         message += "following required parameters is None: tool_shed_url: %s, name: %s, owner: %s, changeset_revision: %s " % \
             ( str( tool_shed_url ), str( name ), str( owner ), str( changeset_revision ) )
         raise Exception( message )
     params = dict( name=name,
                    owner=owner,
                    changeset_revision=changeset_revision )
     pathspec = [ 'repository', 'get_tool_dependencies_config_contents' ]
     text = url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
     if text:
         # Write the contents to a temporary file on disk so it can be reloaded and parsed.
         fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-cttdc"  )
         tmp_filename = fh.name
         fh.close()
         fh = open( tmp_filename, 'wb' )
         fh.write( text )
         fh.close()
         return tmp_filename
     else:
         message = "Unable to retrieve required tool_dependencies.xml file from the Tool Shed for revision "
         message += "%s of installed repository %s owned by %s." % ( str( changeset_revision ), str( name ), str( owner ) )
         raise Exception( message )
         return None

 def check_for_tool_dependencies( self, trans, migration_stage ):
     # Get the 000x_tools.xml file associated with migration_stage.
     tools_xml_file_path = os.path.abspath( os.path.join( trans.app.config.root, 'scripts', 'migrate_tools', '%04d_tools.xml' % migration_stage ) )
     tree = galaxy.util.parse_xml( tools_xml_file_path )
     root = tree.getroot()
     tool_shed = root.get( 'name' )
     shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed )
     repo_name_dependency_tups = []
     if shed_url:
         for elem in root:
             if elem.tag == 'repository':
                 tool_dependencies = []
                 tool_dependencies_dict = {}
                 repository_name = elem.get( 'name' )
                 changeset_revision = elem.get( 'changeset_revision' )
                 params = dict( name=repository_name, owner='devteam', changeset_revision=changeset_revision )
                 pathspec = [ 'repository', 'get_tool_dependencies' ]
                 text = url_get( shed_url, password_mgr=self.app.tool_shed_registry.url_auth( shed_url ), pathspec=pathspec, params=params )
                 if text:
                     tool_dependencies_dict = encoding_util.tool_shed_decode( text )
                     for dependency_key, requirements_dict in tool_dependencies_dict.items():
                         tool_dependency_name = requirements_dict[ 'name' ]
                         tool_dependency_version = requirements_dict[ 'version' ]
                         tool_dependency_type = requirements_dict[ 'type' ]
                         tool_dependency_readme = requirements_dict.get( 'readme', '' )
                         tool_dependencies.append( ( tool_dependency_name, tool_dependency_version, tool_dependency_type, tool_dependency_readme ) )
                 repo_name_dependency_tups.append( ( repository_name, tool_dependencies ) )
     return repo_name_dependency_tups

    def category(self, trans, **kwd):
        """
        GET /api/tool_shed/category

        Display a list of repositories in the selected category.

        :param tool_shed_url: the url of the toolshed to get repositories from
        :param category_id: the category to get repositories from
        """
        tool_shed_url = urlunquote(kwd.get('tool_shed_url', ''))
        category_id = kwd.get('category_id', '')
        params = dict(installable=True)
        tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(trans.app, tool_shed_url)
        url = util.build_url(tool_shed_url, pathspec=['api', 'categories', category_id, 'repositories'], params=params)
        repositories = []
        return_json = json.loads(util.url_get(url))
        for repository in return_json['repositories']:
            api_url = web.url_for(controller='api/tool_shed',
                                  action='repository',
                                  tool_shed_url=urlquote(tool_shed_url),
                                  repository_id=repository['id'],
                                  qualified=True)
            repository['url'] = api_url
            repositories.append(repository)
        return_json['repositories'] = repositories
        return return_json

 def get_sharable_url( self, app ):
     tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, self.tool_shed )
     if tool_shed_url:
         # Append a slash to the tool shed URL, because urlparse.urljoin will eliminate
         # the last part of a URL if it does not end with a forward slash.
         tool_shed_url = '%s/' % tool_shed_url
         return urljoin( tool_shed_url, 'view/%s/%s' % ( self.owner, self.name ) )
     return tool_shed_url

def get_repository_type_from_tool_shed(app, tool_shed_url, name, owner):
    """
    Send a request to the tool shed to retrieve the type for a repository defined by the
    combination of a name and owner.
    """
    tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(app, tool_shed_url)
    params = dict(name=name, owner=owner)
    pathspec = ['repository', 'get_repository_type']
    repository_type = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params)
    return repository_type

def get_ctx_rev(app, tool_shed_url, name, owner, changeset_revision):
    """
    Send a request to the tool shed to retrieve the ctx_rev for a repository defined by the
    combination of a name, owner and changeset revision.
    """
    tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(app, tool_shed_url)
    params = dict(name=name, owner=owner, changeset_revision=changeset_revision)
    pathspec = ['repository', 'get_ctx_rev']
    ctx_rev = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params)
    return ctx_rev

def get_tool_dependency_definition_metadata_from_tool_shed(app, tool_shed_url, name, owner):
    """
    Send a request to the tool shed to retrieve the current metadata for a
    repository of type tool_dependency_definition defined by the combination
    of a name and owner.
    """
    tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(app, tool_shed_url)
    params = dict(name=name, owner=owner)
    pathspec = ['repository', 'get_tool_dependency_definition_metadata']
    metadata = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params)
    return metadata

    def shed_category( self, trans, **kwd ):
        """
        GET /api/tool_shed_repositories/shed_category

        Display a list of repositories in the selected category.

        :param tool_shed_url: the url of the toolshed to get repositories from
        :param category_id: the category to get repositories from
        """
        tool_shed_url = kwd.get( 'tool_shed_url', '' )
        category_id = kwd.get( 'category_id', '' )
        tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url )
        url = util.build_url( tool_shed_url, pathspec=[ 'api', 'categories', category_id, 'repositories' ] )
        category = json.loads( util.url_get( url ) )
        return category

def get_updated_changeset_revisions_from_tool_shed( app, tool_shed_url, name, owner, changeset_revision ):
    """
    Get all appropriate newer changeset revisions for the repository defined by
    the received tool_shed_url / name / owner combination.
    """
    tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
    if tool_shed_url is None or name is None or owner is None or changeset_revision is None:
        message = "Unable to get updated changeset revisions from the Tool Shed because one or more of the following "
        message += "required parameters is None: tool_shed_url: %s, name: %s, owner: %s, changeset_revision: %s " % \
            ( str( tool_shed_url ), str( name ), str( owner ), str( changeset_revision ) )
        raise Exception( message )
    params = dict( name=name, owner=owner, changeset_revision=changeset_revision )
    pathspec = [ 'repository', 'updated_changeset_revisions' ]
    text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
    return text

 def get_repository_dependencies_for_installed_tool_shed_repository( self, app, repository ):
     """
     Send a request to the appropriate tool shed to retrieve the dictionary of repository dependencies defined
     for the received repository which is installed into Galaxy.  This method is called only from Galaxy.
     """
     tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, str( repository.tool_shed ) )
     params = dict( name=str( repository.name ),
                    owner=str( repository.owner ),
                    changeset_revision=str( repository.changeset_revision ) )
     pathspec = [ 'repository', 'get_repository_dependencies' ]
     try:
         raw_text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
     except Exception, e:
         log.error("The URL\n%s\nraised the exception:\n%s\n", common_util.url_join( tool_shed_url, pathspec=pathspec, params=params ), str( e ) )
         return ''

 def load_from_element( self, elem, tool_path ):
     assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % ( elem.tag )
     self.declared_id = elem.get( 'id', None )
     self.guid = elem.get( 'guid', None )
     path = elem.get( 'tool_file', None )
     self.version = elem.get( 'version', self.version )
     tool_shed_repository_id = None
     tool_guid = None
     if path is None:
         tool_elem = elem.find( 'tool' )
         assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string( elem ) )
         path = tool_elem.get( "file", None )
         tool_guid = tool_elem.get( "guid", None )
         # need to determine repository info so that dependencies will work correctly
         tool_shed_url = tool_elem.find( 'tool_shed' ).text
         # Handle protocol changes.
         tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.data_managers.app, tool_shed_url )
         # The protocol is not stored in the database.
         tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url )
         repository_name = tool_elem.find( 'repository_name' ).text
         repository_owner = tool_elem.find( 'repository_owner' ).text
         installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text
         self.tool_shed_repository_info_dict = dict( tool_shed=tool_shed,
                                                     name=repository_name,
                                                     owner=repository_owner,
                                                     installed_changeset_revision=installed_changeset_revision )
         tool_shed_repository = \
             suc.get_installed_repository( self.data_managers.app,
                                           tool_shed=tool_shed,
                                           name=repository_name,
                                           owner=repository_owner,
                                           installed_changeset_revision=installed_changeset_revision )
         if tool_shed_repository is None:
             log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' )
             # we'll set tool_path manually here from shed_conf_file
             tool_shed_repository_id = None
             try:
                 tool_path = util.parse_xml( elem.get( 'shed_conf_file' ) ).getroot().get( 'tool_path', tool_path )
             except Exception, e:
                 log.error( 'Error determining tool_path for Data Manager during testing: %s', e )
         else:
             tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id )
         # use shed_conf_file to determine tool_path
         shed_conf_file = elem.get( "shed_conf_file", None )
         if shed_conf_file:
             shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename( shed_conf_file, None )
             if shed_conf:
                 tool_path = shed_conf.get( "tool_path", tool_path )

 def get_repository_dependencies_for_installed_tool_shed_repository( self, app, repository ):
     """
     Send a request to the appropriate tool shed to retrieve the dictionary of repository dependencies defined
     for the received repository which is installed into Galaxy.  This method is called only from Galaxy.
     """
     tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, str( repository.tool_shed ) )
     params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( repository.name ),
                                                            str( repository.owner ),
                                                            str( repository.changeset_revision ) )
     url = common_util.url_join( tool_shed_url,
                                 'repository/get_repository_dependencies%s' % params )
     try:
         raw_text = common_util.tool_shed_get( app, tool_shed_url, url )
     except Exception, e:
         print "The URL\n%s\nraised the exception:\n%s\n" % ( url, str( e ) )
         return ''

def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ):
    """
    Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section
    selection page.
    """
    name = next(iter(repo_info_dict))
    repo_info_tuple = repo_info_dict[ name ]
    description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
        repository_util.get_repo_info_tuple_contents( repo_info_tuple )
    # Handle changing HTTP protocols over time.
    tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
    params = dict( name=name, owner=repository_owner, changeset_revision=changeset_revision )
    pathspec = [ 'repository', 'get_readme_files' ]
    raw_text = url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
    readme_files_dict = json.loads( raw_text )
    return readme_files_dict

def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ):
    """
    Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section
    selection page.
    """
    name = repo_info_dict.keys()[ 0 ]
    repo_info_tuple = repo_info_dict[ name ]
    description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
        suc.get_repo_info_tuple_contents( repo_info_tuple )
    # Handle changing HTTP protocols over time.
    tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
    params = '?name=%s&owner=%s&changeset_revision=%s' % ( name, repository_owner, changeset_revision )
    url = common_util.url_join( tool_shed_url,
                                'repository/get_readme_files%s' % params )
    raw_text = common_util.tool_shed_get( app, tool_shed_url, url )
    readme_files_dict = json.from_json_string( raw_text )
    return readme_files_dict

    def shed_categories( self, trans, **kwd ):
        """
        GET /api/tool_shed_repositories/shed_categories

        Display a list of categories in the selected toolshed.

        :param tool_shed_url: the url of the toolshed to get categories from
        """
        tool_shed_url = kwd.get( 'tool_shed_url', '' )
        tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url )
        url = util.build_url( tool_shed_url, pathspec=[ 'api', 'categories' ] )
        categories = json.loads( util.url_get( url ) )
        repositories = []
        url = util.build_url( tool_shed_url, pathspec=[ 'api', 'repositories' ] )
        for repo in json.loads( util.url_get( url ) ):
            repositories.append( dict( value=repo[ 'id' ], label='%s/%s' % ( repo[ 'owner' ], repo[ 'name' ] ) ) )
        return { 'categories': categories, 'repositories': repositories }

 def get_update_to_changeset_revision_and_ctx_rev( self, repository ):
     """Return the changeset revision hash to which the repository can be updated."""
     changeset_revision_dict = {}
     tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, str( repository.tool_shed ) )
     params = dict( name=str( repository.name ),
                    owner=str( repository.owner ),
                    changeset_revision=str( repository.installed_changeset_revision ) )
     pathspec = [ 'repository', 'get_changeset_revision_and_ctx_rev' ]
     try:
         encoded_update_dict = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
         if encoded_update_dict:
             update_dict = encoding_util.tool_shed_decode( encoded_update_dict )
             includes_data_managers = update_dict.get( 'includes_data_managers', False )
             includes_datatypes = update_dict.get( 'includes_datatypes', False )
             includes_tools = update_dict.get( 'includes_tools', False )
             includes_tools_for_display_in_tool_panel = update_dict.get( 'includes_tools_for_display_in_tool_panel', False )
             includes_tool_dependencies = update_dict.get( 'includes_tool_dependencies', False )
             includes_workflows = update_dict.get( 'includes_workflows', False )
             has_repository_dependencies = update_dict.get( 'has_repository_dependencies', False )
             has_repository_dependencies_only_if_compiling_contained_td = update_dict.get( 'has_repository_dependencies_only_if_compiling_contained_td', False )
             changeset_revision = update_dict.get( 'changeset_revision', None )
             ctx_rev = update_dict.get( 'ctx_rev', None )
         changeset_revision_dict[ 'includes_data_managers' ] = includes_data_managers
         changeset_revision_dict[ 'includes_datatypes' ] = includes_datatypes
         changeset_revision_dict[ 'includes_tools' ] = includes_tools
         changeset_revision_dict[ 'includes_tools_for_display_in_tool_panel' ] = includes_tools_for_display_in_tool_panel
         changeset_revision_dict[ 'includes_tool_dependencies' ] = includes_tool_dependencies
         changeset_revision_dict[ 'includes_workflows' ] = includes_workflows
         changeset_revision_dict[ 'has_repository_dependencies' ] = has_repository_dependencies
         changeset_revision_dict[ 'has_repository_dependencies_only_if_compiling_contained_td' ] = has_repository_dependencies_only_if_compiling_contained_td
         changeset_revision_dict[ 'changeset_revision' ] = changeset_revision
         changeset_revision_dict[ 'ctx_rev' ] = ctx_rev
     except Exception as e:
         log.debug( "Error getting change set revision for update from the tool shed for repository '%s': %s" % ( repository.name, str( e ) ) )
         changeset_revision_dict[ 'includes_data_managers' ] = False
         changeset_revision_dict[ 'includes_datatypes' ] = False
         changeset_revision_dict[ 'includes_tools' ] = False
         changeset_revision_dict[ 'includes_tools_for_display_in_tool_panel' ] = False
         changeset_revision_dict[ 'includes_tool_dependencies' ] = False
         changeset_revision_dict[ 'includes_workflows' ] = False
         changeset_revision_dict[ 'has_repository_dependencies' ] = False
         changeset_revision_dict[ 'has_repository_dependencies_only_if_compiling_contained_td' ] = False
         changeset_revision_dict[ 'changeset_revision' ] = None
         changeset_revision_dict[ 'ctx_rev' ] = None
     return changeset_revision_dict

 def update_repository_record( self, repository, updated_metadata_dict, updated_changeset_revision, updated_ctx_rev ):
     """
     Update a tool_shed_repository database record with new information retrieved from the
     Tool Shed.  This happens when updating an installed repository to a new changeset revision.
     """
     repository.metadata = updated_metadata_dict
     tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, repository.tool_shed )
     suc.clean_dependency_relationships(self.app, updated_metadata_dict, repository, tool_shed_url)
     # Update the repository.changeset_revision column in the database.
     repository.changeset_revision = updated_changeset_revision
     repository.ctx_rev = updated_ctx_rev
     # Update the repository.tool_shed_status column in the database.
     tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( self.app, repository )
     if tool_shed_status_dict:
         repository.tool_shed_status = tool_shed_status_dict
     else:
         repository.tool_shed_status = None
     self.app.install_model.context.add( repository )
     self.app.install_model.context.flush()
     self.app.install_model.context.refresh( repository )
     return repository

    def show(self, trans, **kwd):
        """
        GET /api/tool_shed/contents

        Display a list of categories in the selected toolshed.

        :param tool_shed_url: the url of the toolshed to get categories from
        """
        tool_shed_url = urlunquote(kwd.get('tool_shed_url', ''))
        tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(trans.app, tool_shed_url)
        url = util.build_url(tool_shed_url, pathspec=['api', 'categories'])
        categories = []
        for category in json.loads(util.url_get(url)):
            api_url = web.url_for(controller='api/tool_shed',
                                  action='category',
                                  tool_shed_url=urlquote(tool_shed_url),
                                  category_id=category['id'],
                                  qualified=True)
            category['url'] = api_url
            categories.append(category)
        return categories