def test_launch_burst_invalid_portlet_analyzer_data(self):
        """
        Test that burst is marked as error if invalid data is passed to the first step.
        """
        algo_id = self.flow_service.get_algorithm_by_module_and_class('tvb.tests.framework.adapters.testadapter1',
                                                                      'TestAdapter1').id
        #Adapter tries to do an int(test1_val1) and int(test1_val2) so this should be valid
        burst_config = self.burst_service.new_burst_configuration(self.test_project.id)
        kwargs_replica = {'test1_val1': '1', 'test1_val2': '0'}
        burst_config.update_simulator_configuration(kwargs_replica)

        test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
        portlet_configuration = self.burst_service.new_portlet_configuration(test_portlet.id)
        #Portlet analyzer tries to do int(input) which should fail
        declared_overwrites = {ADAPTER_PREFIX_ROOT + '0test_non_dt_input': 'asa'}
        self.burst_service.update_portlet_configuration(portlet_configuration, declared_overwrites)
        burst_config.tabs[0].portlets[0] = portlet_configuration

        burst_id, _ = self.burst_service.launch_burst(burst_config, 0, algo_id, self.test_user.id)
        burst_config = dao.get_burst_by_id(burst_id)
        #Wait maximum x seconds for burst to finish
        burst_config = self._wait_for_burst(burst_config, error_expected=True)

        burst_wf = dao.get_workflows_for_burst(burst_config.id)[0]
        wf_steps = dao.get_workflow_steps(burst_wf.id)
        self.assertTrue(len(wf_steps) == 2,
                        "Should have exactly 2 wf steps. One for 'simulation' one for portlet analyze operation.")
        simulator_op = dao.get_operation_by_id(wf_steps[0].fk_operation)
        self.assertEqual(model.STATUS_FINISHED, simulator_op.status,
                         "First operation should be simulator which should have 'finished' status.")
        portlet_analyze_op = dao.get_operation_by_id(wf_steps[1].fk_operation)
        self.assertEqual(portlet_analyze_op.status, model.STATUS_ERROR,
                         "Second operation should be portlet analyze step which should have 'error' status.")

    def test_launch_group_burst_no_metric(self):
        """
        Test the launch burst method from burst service. Try to launch a burst with test adapter which has
        no metrics associated. This should fail.
        """
        burst_config = self.burst_service.new_burst_configuration(self.test_project.id)

        algo_id = self.flow_service.get_algorithm_by_module_and_class('tvb.tests.framework.adapters.testadapter1',
                                                                      'TestAdapter1').id
        kwargs_replica = {'test1_val1': '[0, 1, 2]', 'test1_val2': '0', model.RANGE_PARAMETER_1: 'test1_val1'}
        test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
        tab_config = {test_portlet.id: [(0, 0), (0, 1), (1, 0)]}
        self._add_portlets_to_burst(burst_config, tab_config)
        burst_config.update_simulator_configuration(kwargs_replica)
        burst_id, _ = self.burst_service.launch_burst(burst_config, 0, algo_id, self.test_user.id)
        burst_config = dao.get_burst_by_id(burst_id)
        # Wait maximum x seconds for burst to finish
        self._wait_for_burst(burst_config, error_expected=True)

        launched_workflows = dao.get_workflows_for_burst(burst_id, is_count=True)
        self.assertEqual(3, launched_workflows, "3 workflows should have been launched due to group parameter.")

        op_groups = self.count_all_entities(model.OperationGroup)
        dt_groups = self.count_all_entities(model.DataTypeGroup)
        self.assertEqual(5, op_groups, "An operation group should have been created for each step.")
        self.assertEqual(5, dt_groups, "An dataType group should have been created for each step.")

 def test_load_tab_configuration(self):
     """
     Create a burst with some predefined portlets in some known positions. Check that the
     load_tab_configuration method does what it is expected, and we get the portlets in the
     corresponding tab positions.
     """
     burst_config = self.burst_service.new_burst_configuration(self.test_project.id)
     SIMULATOR_MODULE = 'tvb.tests.framework.adapters.testadapter1'
     SIMULATOR_CLASS = 'TestAdapter1'
     algo_id = self.flow_service.get_algorithm_by_module_and_class(SIMULATOR_MODULE, SIMULATOR_CLASS).id
     kwargs_replica = {'test1_val1': '0', 'test1_val2': '0'}
     test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
     # Add test_portlet to positions (0,0), (0,1) and (1,0)
     tab_config = {test_portlet.id: [(0, 0), (0, 1), (1, 0)]}
     self._add_portlets_to_burst(burst_config, tab_config)
     burst_config.update_simulator_configuration(kwargs_replica)
     burst_id, _ = self.burst_service.launch_burst(burst_config, 0, algo_id, self.test_user.id)
     burst_config = dao.get_burst_by_id(burst_id)
     burst_config = self._wait_for_burst(burst_config)
     burst_wf = dao.get_workflows_for_burst(burst_config.id)[0]
     wf_step = dao.get_workflow_steps(burst_wf.id)[0]
     burst_config.prepare_after_load()
     for tab in burst_config.tabs:
         for portlet in tab.portlets:
             self.assertTrue(portlet is None, "Before loading the tab configuration all portlets should be none.")
     burst_config = self.burst_service.load_tab_configuration(burst_config, wf_step.fk_operation)
     for tab_idx, tab in enumerate(burst_config.tabs):
         for portlet_idx, portlet in enumerate(tab.portlets):
             if (tab_idx == 0 and portlet_idx in [0, 1]) or (tab_idx == 1 and portlet_idx == 0):
                 self.assertTrue(portlet is not None, "portlet gonfiguration not set")
                 self.assertEqual(test_portlet.id, portlet.portlet_id, "Unexpected portlet entity loaded.")
             else:
                 self.assertTrue(portlet is None, "Before loading the tab configuration all portlets should be none")

    def test_update_portlet_config(self):
        """
        Test if a portlet configuration parameters are updated accordingly with a set
        of overwrites that would normally come from UI. Make sure to restart only if 
        analyzer parameters change.
        """


        def __update_params(declared_overwrites, expected_result):
            """
            Do the update and check that we get indeed the expected_result.
            :param declared_overwrites: a input dictionary in the form {'$$name$$' : '$$value$$'}. Make
                sure $$name$$ has the prefix that is added in case of portlet parameters,
                namely ADAPTER_PREFIX_ROOT + step_index + actual_name
            :param expected_result: boolean which should represent if we need or not to restart. (Was a
                visualizer parameter change or an analyzer one)
            """
            result = self.burst_service.update_portlet_configuration(portlet_configuration, declared_overwrites)
            self.assertEqual(expected_result, result,
                             "After update expected %s as 'need_restart' but got %s." % (expected_result, result))


        test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
        portlet_configuration = self.burst_service.new_portlet_configuration(test_portlet.id)
        previous_entry = portlet_configuration.analyzers[0].static_param['test_non_dt_input']
        declared_overwrites = {ADAPTER_PREFIX_ROOT + '0test_non_dt_input': previous_entry}
        __update_params(declared_overwrites, False)
        declared_overwrites = {ADAPTER_PREFIX_ROOT + '1test2': 'new_value'}
        __update_params(declared_overwrites, False)
        declared_overwrites = {ADAPTER_PREFIX_ROOT + '0test_non_dt_input': '1'}
        __update_params(declared_overwrites, True)

 def test_launch_burst(self):
     """
     Test the launch burst method from burst service.
     """
     first_step_algo = self.flow_service.get_algorithm_by_module_and_class(
         'tvb.tests.framework.adapters.testadapter1', 'TestAdapter1')
     adapter_interface = self.flow_service.prepare_adapter(self.test_project.id, first_step_algo)
     ui_submited_simulator_iface_replica = {}
     kwargs_replica = {}
     for entry in adapter_interface:
         ui_submited_simulator_iface_replica[entry[ABCAdapter.KEY_NAME]] = {model.KEY_PARAMETER_CHECKED: True,
                                                                            model.KEY_SAVED_VALUE: entry[
                                                                                ABCAdapter.KEY_DEFAULT]}
         kwargs_replica[entry[ABCAdapter.KEY_NAME]] = entry[ABCAdapter.KEY_DEFAULT]
     burst_config = self.burst_service.new_burst_configuration(self.test_project.id)
     burst_config.simulator_configuration = ui_submited_simulator_iface_replica
     test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
     tab_config = {test_portlet.id: [(0, 0), (0, 1), (1, 0)]}
     self._add_portlets_to_burst(burst_config, tab_config)
     burst_config.update_simulator_configuration(kwargs_replica)
     burst_id, _ = self.burst_service.launch_burst(burst_config, 0, first_step_algo.id, self.test_user.id)
     burst_config = dao.get_burst_by_id(burst_id)
     self.assertTrue(burst_config.status in (BurstConfiguration.BURST_FINISHED, BurstConfiguration.BURST_RUNNING),
                     "Burst not launched successfully!")
     # Wait maximum x seconds for burst to finish
     self._wait_for_burst(burst_config)

 def get_portlet(self):
     """
     Return the portlet that was saved on this entity. Should only be called if 
     beforehand 'set_portlet' was called on this entity.
     """
     if self.PORTLET_IDENT not in self._data_dict:
         return None
     return dao.get_portlet_by_identifier(self._data_dict[self.PORTLET_IDENT])

 def test_build_portlet_interface_invalid(self):
     """
     Test that a proper exception is raised in case an invalid portlet configuration is provided.
     """
     test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
     portlet_configuration = self.burst_service.new_portlet_configuration(test_portlet.id)
     portlet_configuration.portlet_id = "this-is-invalid"
     self.assertRaises(InvalidPortletConfiguration, self.burst_service.build_portlet_interface,
                       portlet_configuration, self.test_project.id)

 def test_portlet_configurable_interface(self):
     """
     A simple test for the get configurable interface method.
     """        
     test_portlet = dao.get_portlet_by_identifier("TA1TA2")
     
     result = PortletConfigurer(test_portlet).get_configurable_interface()
     self.assertEqual(len(result), 2, "Length of the resulting interface not as expected")
     for one_entry in result:
         for entry in one_entry.interface:
             if entry['name'] == 'test1':
                 self.assertTrue(entry['default'] == 'step_0[0]', "Overwritten default not in effect.")
             if entry['name'] == 'test2':
                 self.assertTrue(entry['default'] == '0', "Value that was not overwritten changed.")

 def test_portlet_tab_display(self):
     """
     Update the default portlet configuration, by storing a TimeSeries
     portlet for all postions. Then check that we get the same configuration.
     """
     self.burst_c.index()
     portlet_id = dao.get_portlet_by_identifier("TimeSeries").id
     one_tab = [[portlet_id, "TimeSeries"] for _ in range(NUMBER_OF_PORTLETS_PER_TAB)]
     full_tabs = [one_tab for _ in range(BurstConfiguration.nr_of_tabs)]
     data = {'tab_portlets_list': json.dumps(full_tabs)}
     result = self.burst_c.portlet_tab_display(**data)
     selected_portlets = result['portlet_tab_list']
     for entry in selected_portlets:
         self.assertEqual(entry.id, portlet_id)

 def test_get_portlet_session_configuration(self):
     """
     Test that the default portlet session sonciguration is generated
     as expected, with a default TimeSeries portlet and rest empty.
     """
     self.burst_c.index()
     result = json.loads(self.burst_c.get_portlet_session_configuration())
     portlet_id = dao.get_portlet_by_identifier("TimeSeries").id
     for tab_idx, tab in enumerate(result):
         for index_in_tab, value in enumerate(tab):
             if tab_idx == 0 and index_in_tab == 0:
                 self.assertEqual(value, [portlet_id, "TimeSeries"])
             else:
                 self.assertEqual(value, [-1, "None"])

    def test_update_portlet_config_invalid_data(self):
        """
        Trying an update on a portlet configuration with invalid data
        should not change the configuration instance in any way.
        """
        test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
        portlet_configuration = self.burst_service.new_portlet_configuration(test_portlet.id)

        invalid_overwrites = {'this_is_not_a_valid_key': 'for_test_portlet_update'}
        before_update = copy.deepcopy(portlet_configuration)
        self.burst_service.update_portlet_configuration(portlet_configuration, invalid_overwrites)
        self.assertEqual(set(dir(before_update)), set(dir(portlet_configuration)))
        #An update with invalid input data should have no effect on the configuration, but attributes changed
        for key in portlet_configuration.__dict__.keys():
            if hasattr(getattr(portlet_configuration, key), '__call__'):
                self.assertEqual(getattr(before_update, key), getattr(portlet_configuration, key),
                                 "The value of attribute %s changed by a update with invalid data "
                                 "when it shouldn't have." % key)

    def test_new_portlet_configuration(self):
        """
        Test that the correct portlet configuration is generated for the test portlet.
        """
        # Passing an invalid portlet ID should fail and raise an InvalidPortletConfiguration exception.
        self.assertRaises(InvalidPortletConfiguration, self.burst_service.new_portlet_configuration, -1)

        # Now the happy flow
        test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
        portlet_configuration = self.burst_service.new_portlet_configuration(test_portlet.id)
        analyzers = portlet_configuration.analyzers
        self.assertEqual(len(analyzers), 1, "Portlet configuration not build properly."
                                            " Portlet's analyzers list has unexpected number of elements.")
        self.assertEqual(analyzers[0].dynamic_param, {u'test_dt_input': {wf_cfg.DATATYPE_INDEX_KEY: 0,
                                                                         wf_cfg.STEP_INDEX_KEY: 0}},
                         "Dynamic parameters not loaded properly")
        visualizer = portlet_configuration.visualizer
        self.assertEqual(visualizer.dynamic_param, {}, "Dynamic parameters not loaded properly")
        self.assertEqual(visualizer.static_param, {u'test2': u'0'}, 'Static parameters not loaded properly')

 def test_index(self):
     """
     Test that index returns a dict with all required keys. Also check
     that the default portlets are populated, with only the first being
     the TimeSeries portlet and the rest are empty.
     """
     result_dict = self.burst_c.index()
     self.assertTrue('burst_list' in result_dict and result_dict['burst_list'] == [])
     self.assertTrue('available_metrics' in result_dict and isinstance(result_dict['available_metrics'], list))
     self.assertTrue('portletList' in result_dict and isinstance(result_dict['portletList'], list))
     self.assertEqual(result_dict[common.KEY_SECTION], "burst")
     self.assertTrue('burstConfig' in result_dict and isinstance(result_dict['burstConfig'], BurstConfiguration))
     portlets = json.loads(result_dict['selectedPortlets'])
     portlet_id = dao.get_portlet_by_identifier("TimeSeries").id
     for tab_idx, tab in enumerate(portlets):
         for index_in_tab, value in enumerate(tab):
             if tab_idx == 0 and index_in_tab == 0:
                 self.assertEqual(value, [portlet_id, "TimeSeries"])
             else:
                 self.assertEqual(value, [-1, "None"])
     self.assertTrue(result_dict['draw_hidden_ranges'])

 def test_build_portlet_interface(self):
     """
     Test that the portlet interface is build properly, splitted by steps and prefixed.
     """
     test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
     portlet_configuration = self.burst_service.new_portlet_configuration(test_portlet.id)
     actual_interface = self.burst_service.build_portlet_interface(portlet_configuration, self.test_project.id)
     #The expected portlet steps and interface in correspondace to the xml declaration
     #from tvb.tests.framework/core/portlets/test_portlet.xml
     expected_steps = [{'ui_name': 'TestAdapterDatatypeInput'},
                       {'ui_name': 'TestAdapter2'}]
     expected_interface = [{ABCAdapter.KEY_DEFAULT: 'step_0[0]', ABCAdapter.KEY_DISABLED: True,
                            KEY_DYNAMIC: True, ABCAdapter.KEY_NAME: ADAPTER_PREFIX_ROOT + '0test_dt_input'},
                           {ABCAdapter.KEY_DEFAULT: '0', ABCAdapter.KEY_DISABLED: False,
                            KEY_DYNAMIC: False, ABCAdapter.KEY_NAME: ADAPTER_PREFIX_ROOT + '1test2'}]
     for idx, entry in enumerate(expected_steps):
         step = actual_interface[idx]
         for key in entry:
             self.assertEqual(entry.get(key), getattr(step, key))
         for key in expected_interface[idx]:
             self.assertEqual(expected_interface[idx].get(key, False), step.interface[0].get(key, False))

    def _prepare_and_launch_sync_burst(self):
        """
        Private method to launch a dummy burst. Return the burst loaded after the launch finished
        as well as the workflow steps that initially formed the burst.
        NOTE: the burst launched by this method is a `dummy` one, meaning we do not use an actual
        simulation, but instead test adapters.
        """
        burst_config = TestFactory.store_burst(self.test_project.id)

        workflow_step_list = []
        test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)

        stored_dt = datatypes_factory.DatatypesFactory()._store_datatype(Datatype1())
        first_step_algorithm = self.flow_service.get_algorithm_by_module_and_class(
            "tvb.tests.framework.adapters.testadapter1", "TestAdapterDatatypeInput")
        metadata = {DataTypeMetaData.KEY_BURST: burst_config.id}
        kwargs = {"test_dt_input": stored_dt.gid, 'test_non_dt_input': '0'}
        operations, group = self.operation_service.prepare_operations(self.test_user.id, self.test_project.id,
                                                                      first_step_algorithm,
                                                                      first_step_algorithm.algorithm_category,
                                                                      metadata, **kwargs)
        view_step = TestFactory.create_workflow_step("tvb.tests.framework.adapters.testadapter2", "TestAdapter2",
                                                     {"test2": 2}, {}, 0, 0, 0, 0, is_view_step=True)
        view_step.fk_portlet = test_portlet.id
        workflow_step_list.append(view_step)

        workflows = self.workflow_service.create_and_store_workflow(self.test_project.id, burst_config.id, 0,
                                                                    first_step_algorithm.id, operations)
        self.operation_service.prepare_operations_for_workflowsteps(workflow_step_list, workflows, self.test_user.id,
                                                                    burst_config.id, self.test_project.id, group,
                                                                    operations)
        ### Now fire the workflow and also update and store the burst configuration ##
        self.operation_service.launch_operation(operations[0].id, False)
        loaded_burst, _ = self.burst_service.load_burst(burst_config.id)
        import_operation = dao.get_operation_by_id(stored_dt.fk_from_operation)
        dao.remove_entity(import_operation.__class__, import_operation.id)
        dao.remove_datatype(stored_dt.gid)
        return loaded_burst, workflow_step_list

 def get_portlet_by_identifier(portlet_identifier):
     """
     :returns: the portlet entity with the algorithm identifier [email protected]_identifier
     """
     return dao.get_portlet_by_identifier(portlet_identifier)

        self.logger.debug("Refreshing portlets from xml declarations.")
        stored_portlets = dao.get_available_portlets()
        #First update old portlets from DB
        for stored_portlet in stored_portlets:
            for verified_portlet in portlets_list:
                if stored_portlet.algorithm_identifier == verified_portlet.algorithm_identifier:
                    stored_portlet.xml_path = verified_portlet.xml_path
                    stored_portlet.last_introspection_check = datetime.datetime.now()
                    stored_portlet.name = verified_portlet.name
                    dao.store_entity(stored_portlet)
                    break

        #Now add portlets that were not in DB at previous run but are valid now
        for portlet in portlets_list:
            db_entity = dao.get_portlet_by_identifier(portlet.algorithm_identifier)
            if db_entity is None:
                self.logger.debug("Will now store portlet %s" % (str(portlet),))
                dao.store_entity(portlet)

    @staticmethod
    def _is_concrete_subclass(clz, super_cls):
        return inspect.isclass(clz) and not inspect.isabstract(clz) and issubclass(clz, super_cls)


    def __get_datatypes(self, path_types):
        """
        Imports each DataType to update the DB model, by creating a new table for each DataType.
        """
        for my_type in Introspector.__read_module_variable(path_types):
            try:

    def __get_portlets(self, path_portlets):
        """
        Given a path in the form of a python package e.g.: "tvb.portlets', import
        the package, get it's folder and look for all the XML files defined 
        there, then read all the portlets defined there and store them in DB.
        """
        portlet_package = __import__(path_portlets, globals(), locals(), ["__init__"])
        portlet_folder = os.path.dirname(portlet_package.__file__)
        portlets_list = []
        for file_n in os.listdir(portlet_folder):
            try:
                if file_n.endswith('.xml'):
                    complete_file_path = os.path.join(portlet_folder, file_n)
                    portlet_reader = XMLPortletReader(complete_file_path)
                    portlet_list = portlet_reader.get_algorithms_dictionary()
                    self.logger.debug("Starting to verify currently declared portlets in %s." % (file_n,))
                    for algo_identifier in portlet_list:
                        adapters_chain = portlet_reader.get_adapters_chain(algo_identifier)
                        is_valid = True
                        for adapter in adapters_chain:
                            class_name = adapter[ABCAdapter.KEY_TYPE].split('.')[-1]
                            module_name = adapter[ABCAdapter.KEY_TYPE].replace('.' + class_name, '')
                            try:
                                #Check that module is properly declared
                                module = __import__(module_name, globals(), fromlist=[class_name])
                                if type(module) != ModuleType:
                                    is_valid = False
                                    self.logger.error("Wrong module %s in portlet %s" % (module_name, algo_identifier))
                                    continue
                                #Check that class is properly declared
                                if not hasattr(module, class_name):
                                    is_valid = False
                                    self.logger.error("Wrong class %s in portlet %s." % (class_name, algo_identifier))
                                    continue
                                #Check inputs that refers to this adapter
                                portlet_inputs = portlet_list[algo_identifier][ELEM_INPUTS]
                                adapter_instance = PortletConfigurer.build_adapter_from_declaration(adapter)
                                if adapter_instance is None:
                                    is_valid = False
                                    self.logger.warning("No group having class=%s stored for "
                                                        "portlet %s." % (class_name, algo_identifier))
                                    continue
                                adapter_input_names = [entry[ABCAdapter.KEY_NAME] for entry
                                                       in adapter_instance.flaten_input_interface()]
                                for input_entry in portlet_inputs.values():
                                    if input_entry[ATT_OVERWRITE] == adapter[ABCAdapter.KEY_NAME]:
                                        if input_entry[ABCAdapter.KEY_NAME] not in adapter_input_names:
                                            self.logger.error("Invalid input %s for adapter %s" % (
                                                input_entry[ABCAdapter.KEY_NAME], adapter_instance))
                                            is_valid = False
                            except ImportError:
                                self.logger.error("Invalid adapter declaration %s in portlet %s" % (
                                                  adapter[ABCAdapter.KEY_TYPE], algo_identifier))
                                is_valid = False
                        if is_valid:
                            portlets_list.append(model.Portlet(algo_identifier, complete_file_path,
                                                               portlet_list[algo_identifier]['name']))
            except XmlParserException as excep:
                self.logger.exception(excep)
                self.logger.error("Invalid Portlet description File " + file_n + " will continue without it!!")

        self.logger.debug("Refreshing portlets from xml declarations.")
        stored_portlets = dao.get_available_portlets()
        #First update old portlets from DB
        for stored_portlet in stored_portlets:
            for verified_portlet in portlets_list:
                if stored_portlet.algorithm_identifier == verified_portlet.algorithm_identifier:
                    stored_portlet.xml_path = verified_portlet.xml_path
                    stored_portlet.last_introspection_check = datetime.datetime.now()
                    stored_portlet.name = verified_portlet.name
                    dao.store_entity(stored_portlet)
                    break

        #Now add portlets that were not in DB at previous run but are valid now
        for portlet in portlets_list:
            db_entity = dao.get_portlet_by_identifier(portlet.algorithm_identifier)
            if db_entity is None:
                self.logger.debug("Will now store portlet %s" % (str(portlet),))
                dao.store_entity(portlet)