def startService(self):
     "Start monitoring service"
     twisted_logger.writeLog(self.logPrefix, self.logName, "Starting '%s' service..." % self.name)
     if scheduleChecks:
         # Start server connection checks (monitoring)
         self._startServerChecks()
     return service.Service.startService(self)

    def test_sendRequest(self):
        "Test that _SCSClientService.sendRequest() successfully sends request to external server"
        self.logPrefix = "SCSClientServiceTest.test_sendRequest"
        testRequest = {'scs_jobid': 5, 'request': 'I want to run marathon'}
        
        def requestReceivedHandler(data):
            twisted_logger.writeLog(self.logPrefix, None, "Server received request: '%s'" % data)
            
        def checkResults(protocol):
            twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
            self.assertEquals(isinstance(protocol.jobRequest, client.client_request.JobRequest), True)
            self.assertEquals(protocol.jobRequest.request['type'], 'REQUEST')
            self.assertEquals(protocol.jobRequest.scs_jobid, testRequest['scs_jobid'])
            self.assertEquals(protocol.jobRequest.request['request'], testRequest['request'])
            twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")
        
        # Start test server
        factory = protocol.ServerFactory()
        factory.protocol = ServerProtocol
        factory.protocol.dataRcvdDeferred = defer.Deferred().addCallback(requestReceivedHandler)
        self.listeningPort = reactor.listenTCP(testPort, factory)

        self.assertEquals(self.clientService.clientProtocols, [])
        deferred = defer.Deferred()            
        twisted_logger.writeLog(self.logPrefix, None, "Sending test request to the server...")
        return self.clientService.sendRequest(testRequest, deferred).addCallback(checkResults)

 def checkResults(nothing):
     twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
     self.assertEquals(self.controller.namedServices.has_key('ServerManager'), True)
     self.assertEquals(self.controller.namedServices.has_key('ClientManager'), True)
     self.assertEquals(self.controller.namedServices.has_key('WorkflowManager'), True)
     self.assertEquals(self.controller.namedServices['ServerManager'].running, True)
     self.assertEquals(self.controller.namedServices['ClientManager'].running, True)
     self.assertEquals(self.controller.namedServices['WorkflowManager'].running, True)
     
     # Verify that when Workflow entities have any steps (clients) - corresponding ClientService 
     # entities have non-empty onlineDeferreds and offlineDeferreds dictionaries
     for srv in self.controller.namedServices['WorkflowManager']:
         if isinstance(srv, SCS_controller.workflow._WorkflowService):
             for step in srv.workflow.steps:
                 clientService = step['client']
                 self.assertNotEquals(clientService.onlineDeferreds, {})
                 self.assertNotEquals(clientService.offlineDeferreds, {})
     
                 for srv1 in clientService.services:
                     if isinstance(srv1, SCS_controller.client._SCSServerMonitor):
                         monitorService = srv1
                         self.assertEquals(monitorService.onlineDeferreds, clientService.onlineDeferreds)
                         self.assertEquals(monitorService.offlineDeferreds, clientService.offlineDeferreds)
                                 
     twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")

    def runJobStep(self, result, step_no = 0):
        """Start asynchronous job step (<step_no>) execution
        
        @return: deferred which will 'fire' when given JobStep instance has started execution
        """
        def updateJobStatus(res):
            "Update job status to 'RUNNING'"
            self.status = 'RUNNING'
            stmt = "update scs.job set status = '%s', text_input = '%s', start_time = now() " \
                    "where job_id = %d" % (self.status, self.input.replace("'", "''"), self.jobID)
            self.db.execute(stmt).addErrback(self.__handleDBFailure, stmt, 'update')
            
            # Schedule job timeout check using workflow's <timeout> attribute
            if timeoutFlag and self.workflow.timeout:
                self.callID = reactor.callLater(self.workflow.timeout, self._timeout)
                                    
        step = self.steps[step_no]
        if step_no == 0:
            request = self.input
        else:
#            if isinstance(self.steps[step.inputSource - 1].output, dict):
#                request = eval(self.steps[step.inputSource - 1].output)
#            else:
            request = self.steps[step.inputSource - 1].output
                
        # Add <self.failureHandler()> errback method to given job step's deferred
        step.deferred.addErrback(self.failureHandler)

        # Execute job step's run() and add <self.failureHandler()> errback to it - to handle job failure event
        twisted_logger.writeLog(self.logPrefix, self.logName, "Executing job step #%d... " % (step_no + 1))
        if step_no == 0:
            return step.run(request).addCallback(updateJobStatus).addErrback(self.failureHandler)
        else:
            return step.run(request).addErrback(self.failureHandler)

 def stopService(self):
     """Stop SCSServer service
     
     @return: deferred which will 'fire' when all of the service termination tasks have completed
     """
     if self.running:
         twisted_logger.writeLog(self.logPrefix, self.logName, "Stopping '%s' service..." % self.name)
         actions = []
         
         if not self.listeningPort or not self.listeningPort.connected:
             err_msg = "Unable to stop '%s' server: server not listening on localhost:%d port" % (self.name, self.serverInfo['port'])
             twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
         else:
             # Stop running SCS server
             msg = "'%s' server service shutdown in progress..." % self.name
             twisted_logger.writeLog(self.logPrefix, self.logName, msg)
             actions.append(self.listeningPort.stopListening().addCallback(self._stopSuccessHandler).addErrback(self._stopFailureHandler))
         
         for srv in self.services[:]:
             try:
                 deferred = self.removeService(srv)
                 if deferred:
                     actions.append(deferred)
             except Exception, err:
                 twisted_logger.writeErr(self.logPrefix, self.logName, "Failure stopping '%s' service: %s" % (srv.name, str(err)))
                 
         service.Service.stopService(self)    

    def processResponse(self, result, sendResponse = True):
        "Process job completion, timeout or failure event"
        self.status = jobStatusMap[self.job.status]
        self.error = self.job.error
        # Update server factory's jobInfos dictionary with current job info
        self.channel.factory.jobInfos[self.job.jobID] = {'status': self.job.status, 'error': self.job.error,
                                                          'input': self.job.input, 'output': self.job.output}

        if isinstance(result, failure.Failure):
            # Generate job failure message to be written to log
            msg = "Job #%d has failed: %s" % (self.job.jobID, self.error)
            # Write message to event log
            twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, msg)
            if sendResponse:
                # Send job failure response to the client
                self._sendError(http.INTERNAL_SERVER_ERROR, msg)
        else:
            # Generate job success message to be written to log
            msg = "Job #%d has successfully completed" % self.job.jobID
            # Write message to event log
            twisted_logger.writeLog(self.channel.logPrefix, self.channel.logName, msg)
            if sendResponse:
                # Send job results to the client
                self._sendResponse(result, http.OK)
        
        # Delete original job entity
        del self.job
        self.job = None

        if not sendResponse:
            self.finished = True
            
        self.finish()

 def checkResults(nothing):
     twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
     self.assertEquals(clientManager.running, 1)
     self.assertEquals(clientManager.namedServices.has_key(testClientName), True)
     clientService = clientManager.namedServices[testClientName]
     self.assertEquals(clientService.running, 1)        
     twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")

 def checkResults(nothing):
     twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
     self.assertEquals(serverManager.running == 1, True)
     self.assertEquals(serverManager.namedServices.has_key(testServerName), True)
     self.assertEquals(serverManager.namedServices[testServerName].running, True)        
     self.assertEquals(serverManager.namedServices[testServerName].listeningPort.connected, True)
     twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")

    def _stop(self):
        """JobRequest termination: 
            1. fire callback or errback method associated with request's deferred        
            2. perform cleanup
        """
        if not self.stopped:
            # Stop job status requests and remove such requests from factory's cachedRequests list (if any)
            self.statusRequest._stop()
            # Delete JobStatusRequest entity - to free up memory
            del self.statusRequest            
            self.stopped = True

            # Execute deferred's callback() or errback() to notify JobStep (and Job) instance of job step completion
            if hasattr(self, 'deferred') and self.deferred.called == 0:
                if self.status == 'FAILURE' and self.error:
                    msg = "Executing deferred's errback() to notify of job step failure"
                    twisted_logger.writeErr(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
                    # Execute deferred's errback() to notify of job's failure
                    self.deferred.errback(failure.Failure(self.error, RuntimeError))
                elif self.status == 'SUCCESS':
                    msg = "Executing deferred's callback() to notify of job step success"
                    twisted_logger.writeLog(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
                    # Execute deferred's callback() to notify of job's success
                    self.deferred.callback(self.response)
                else:
                    self.deferred.called = 1
                    
            else:
                msg = "Deferred's callback/errback has been canceled"
                twisted_logger.writeLog(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)

    def stopService(self):
        "Stop SCSClient service"
        twisted_logger.writeLog(self.logPrefix, self.logName, "Stopping '%s' service..." % self.name)
        actions = []

        if self.running:
            for clientProtocol in self.clientProtocols[:]:
                if hasattr(clientProtocol, "jobRequest"):
                    clientProtocol._stopJobRequest()
                if clientProtocol.connected == 1:
                    msg = "Client is being disconnected from %s server..." % self.peer
                    twisted_logger.writeLog(self.logPrefix, self.logName, msg)
                    actions.append(self.stopClient(clientProtocol))

            for srv in self.services[:]:
                try:
                    deferred = self.removeService(srv)
                    if deferred:
                        actions.append(deferred)
                except Exception, err:
                    twisted_logger.writeErr(
                        self.logPrefix, self.logName, "Failure stopping '%s' service: %s" % (srv.name, str(err))
                    )

            service.Service.stopService(self)

    def __jobStartHandler(self, result):
        "Report job start success or failure to the client that sent this job request"        

        if isinstance(result, failure.Failure):
            # Job start failure
            self.status = jobStatusMap['FAILURE']
            self.error = "Job start failure"
            err_msg = "%s: %s" % (self.error, result.getErrorMessage())
            twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, err_msg)
            # Trap failure event
            result.trap(Exception)
            # Delete job entity (it is not going to be processed anyway)
            del self.job
            self.job = None

            # Send failure response to external client
            self._sendError(http.INTERNAL_SERVER_ERROR, self.error)
            self.finish()
        else:
            msg = "New job #%d has been created. Status: '%s'" % (self.job.jobID, self.job.status)
            twisted_logger.writeLog(self.channel.logPrefix, self.channel.logName, msg)
            # Add job info to server factory's jobInfos dictionary
            self.channel.factory.jobInfos[self.job.jobID] = {'status': self.job.status, 'error': self.job.error,
                                                             'input': self.job.input, 'output': self.job.output}
            return "ok"

 def sendRequest(self, request, deferred=None):
     """Send request to external server
     
     @param request: request message (dictionary) to be sent to external server
     @param deferred: deferred to be associated with the given job (for job request only)
     @type request: dictionary containing {'scs_jobid': <jobid>, 
                                           'request': <request - text>}
     @type deferred: defer.Deferred instance 
     """
     # Reset <self.logPrefix> to include 'scs_jobid'
     self.logPrefix += " JOBID #%d" % request["scs_jobid"]
     try:
         # Create JobRequest instance that will be responsible for:
         # 1. processing job request
         # 2. scheduling regular job 'STATUS' requests
         # 3. Handling job response - when it comes from the server
         self.jobRequest = client_request.JobRequest(request, deferred, self)
         # Process job request
         self.jobRequest.processRequest()
     except RuntimeError, err:
         twisted_logger.writeErr(self.logPrefix, self.logName, str(err))
         if isinstance(deferred.__class__, defer.Deferred):
             twisted_logger.writeLog(
                 self.logPrefix,
                 self.logName,
                 "Executing external job step deferred's errback() to report job start failure",
             )
             deferred.errback(failure.Failure(str(err)), RuntimeError)

 def handleWorkflowReconnect(result, wf):
     global reconnectHandled, reconnectCount
     twisted_logger.writeLog(self.logPrefix, None, "Workflow.handleClientReconnect() has been called back ...")
     reconnectCount += 1
     reconnectHandled = True
     err_msg = "Workflow.handleClientReconnect() resets workflow status to online when other client(s) are still offline"
     raise self.failureException, err_msg

 def enable(self, workflowName):
     """Enable workflow. SCSServer entity that 'owns' this workflow will start accepting external client's 
     job requests - provided that workflow is also 'online' (all SCSClient entities used by the workflow 
     are connected to their external servers)"""
     if workflowName in self.namedServices.keys():            
         msg = "Enabling '%s' workflow..." % workflowName
         twisted_logger.writeLog(self.logPrefix, self.logName, msg)
         self.namedServices[workflowName].workflow.enable()

 def checkResults(data, response):
     twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
     self.assertEquals(data, self.request['request'])
     self.assertEquals(self.jobRequest.status, 'SUCCESS')
     self.assertEquals(self.jobRequest.request['request'], self.request['request'])
     self.assertEquals(self.jobRequest.response, response['result_set'])
     self.assertEquals(self.jobRequest.error, None)
     twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")

 def checkResults(result, wf):
     "Verify that workflow is offline and that workflow reconnect event has not occured"
     global reconnectHandled, reconnectCount
     twisted_logger.writeLog(self.logPrefix, None, "Checking results...")
     self.assertEquals(wf.online, False)
     self.assertEquals(reconnectHandled, False)
     self.assertEquals(reconnectCount, 0)
     twisted_logger.writeLog(self.logPrefix, None, "Success...")

 def checkAfterStartUp(nothing, origEnabled):
     twisted_logger.writeLog(self.logPrefix, None, "Checking test results after startup...")
     for clientname in clientManager.clientInfo.keys():
         self.assertEquals(clientManager.namedServices.has_key(clientname), True)
         self.assertEquals(clientManager.namedServices[clientname].running == 1, True)
         clientManager.clientInfo[clientname]['enabled'] = origEnabled[clientname]
     twisted_logger.writeLog(self.logPrefix, None, "All client services have been started")
     return clientManager.shutDown().addCallback(checkAfterShutDown)

 def test__init__(self):
     "Test that JobRequest.__init__() correctly initializes JobRequest instance"
     self.logPrefix = "test__init__"
     twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
     self.assertEquals(self.httpRequest.job, None)
     self.assertEquals(self.httpRequest.status, None)
     self.assertEquals(self.httpRequest.response, None)
     twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")

 def _success(self):
     "Process job success event. Executes self._stop() that fires self.deferred.callback()"
     if self.status is None:
         self.status = 'SUCCESS'
         msg = "Job step (job #%d) has successfully completed" % self.scs_jobid
         twisted_logger.writeLog(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
         # Stop job request's processing
         self._stop()

 def restartService(nothing):
     twisted_logger.writeLog(self.logPrefix, None, "Verifying that '%s' service has stopped..." % testClientName)
     self.assertEquals(clientManager.running, True)
     # Verify that server is deleted from ServerManager.services list and ServerManager.namedServices dictionary
     self.assertEquals(clientManager.namedServices.has_key(testClientName), False)
     twisted_logger.writeLog(self.logPrefix, None, "Service has stopped. Restarting the service...")            
     # Restart the server
     return clientManager.start(testClientName).addCallback(checkResults)