Commit 70332624 authored by Michael Salim's avatar Michael Salim
Browse files

Merge remote-tracking branch 'origin/develop'

parents c3e0845e badd48f6
......@@ -2,7 +2,7 @@ from django.conf.urls import url
import logging
logger = logging.getLogger(__name__)
from argo import views
from balsam.argo import views
local_urlpatterns = [
url(r'^$', views.index, name='index'),
......
import os,logging,sys
from django.shortcuts import render,get_object_or_404
from argo.html_forms import JobDisplayForm
from balsam.argo.html_forms import JobDisplayForm
logger = logging.getLogger(__name__)
from argo import models
from balsam.argo import models
# Create your views here.
......
from common.file_tools import delete_old_files_directories
from balsam.common.file_tools import delete_old_files_directories
import time
class DirCleaner:
......
import logging
logger = logging.getLogger(__name__)
class MessageInterface:
'''These are the public methods to be implemented by MessageInterfaces like
PikaMessageInterface. All protocol-specfic methods should be hidden'''
def __init__(self, settings):
raise NotImplementedError
def setup_send(self):
raise NotImplementedError
def setup_receive(self, consume_msg=None):
raise NotImplementedError
def send_msg(self, message_body):
raise NotImplementedError
def receive_msg(self):
raise NotImplementedError
def start_receive_loop(self):
raise NotImplementedError
def stop_receive_loop(self):
raise NotImplementedError
def close(self):
raise NotImplementedError
from balsam.common import PikaMessageInterface, NoMessageInterface
from django.conf import settings
import logging,sys,multiprocessing,time,os
logger = logging.getLogger(__name__)
RECEIVER_MAP = {
'pika' : PikaMessageInterface.PikaMessageInterface,
'no_message' : NoMessageInterface.NoMessageInterface,
'zmq' : ZMQMessageInterface.ZMQMessageInterface
}
class MessageReceiver(multiprocessing.Process):
''' subscribes to a queue and executes the given callback'''
def __init__(self, settings):
# execute multiprocessing.Process superconstructor
super(MessageReceiver,self).__init__()
receiver_mode = settings['mode']
MessageClass = RECEIVER_MAP[receiver_mode]
self.messageInterface = MessageClass(settings)
self.consume_msg = getattr(self, '%s_consume_msg' % receiver_mode)
def handle_msg(self, msg_body):
'''This handles the message in a protocol-independent way'''
raise NotImplementedError
def run(self):
logger.debug(' in run ')
self.messageInterface.setup_receive(self.consume_msg)
self.messageInterface.start_receive_loop()
def pika_consume_msg(self,channel,method_frame,header_frame,body):
logger.debug('in pika_consume_msg' )
if body is not None:
logger.debug(' received message: ' + body )
try:
self.handle_msg(body)
except Exception as e:
logger.exception('failed to handle_msg. not continuing with this job')
channel.basic_ack(method_frame.delivery_tag)
return
else:
logger.error(' consume_msg called, but body is None ')
# should be some failure notice to argo
# acknowledge receipt of message
channel.basic_ack(method_frame.delivery_tag)
def no_message_consume_msg(self):
pass
def zmq_consume_msg(self, body):
logger.debug(' in zmq_message_consume_msg')
if body:
logger.debug(' received ZMQmessage: ' + body)
try:
self.handle_msg(body)
except Exception as e:
logger.exception('failed to handle_msg. not continuing with this job')
return
else:
logger.error(' consume_msg called, but body is empty or None'))
def shutdown(self):
logger.debug(' stopping message consumer ')
self.messageInterface.stop_receive_loop()
import logging
import time
import threading
logger = logging.getLogger(__name__)
from balsam.common import MessageInterface
class NoMessageInterface(MessageInterface.MessageInterface):
def __init__(self, settings):
self.alive = True
self.thread = None
def setup_send(self):
pass
def setup_receive(self, consume_msg=None):
self.thread = threading.Thread(target=self._fake_ioloop)
def send_msg(self, message_body):
pass
def receive_msg(self):
pass
def start_receive_loop(self):
try:
logger.debug('starting dummy receiver ioloop')
self.thread.start()
self.thread.join()
except Exception:
logger.exception('failed to start dummy receiver ioloop')
def stop_receive_loop(self):
self.alive = False
self.thread.join()
def close(self):
pass
def _fake_ioloop(self):
logger.debug('Thread: inside fake_ioloop')
try:
while self.alive:
logger.debug(' Thread: Idling Receiver')
time.sleep(15)
except Exception:
logger.debug('The ioloop failed')
import sys,os,ssl
import pika,time
import logging
logger = logging.getLogger(__name__)
logging.getLogger('pika').setLevel(logging.WARNING)
from balsam.common import MessageInterface
class PikaMessageInterface(MessageInterface.MessageInterface):
def __init__(self, settings):
self.username = settings['username']
self.password = settings['password']
self.host = settings['host']
self.port = settings['port']
self.virtual_host = settings['virtual_host']
self.socket_timeout = settings['socket_timeout']
self.exchange_name = settings['exchange_name']
self.exchange_type = settings['exchange_type']
self.exchange_durable = settings['exchange_durable']
self.exchange_auto_delete = settings['exchange_auto_delete']
self.queue_name = settings['queue_name']
self.queue_is_durable = settings['queue_is_durable']
self.queue_is_exclusive = settings['queue_is_exclusive']
self.queue_is_auto_delete = settings['queue_is_auto_delete']
self.default_routing_key = settings['default_routing_key']
self.ssl_cert = settings['ssl_cert']
self.ssl_key = settings['ssl_key']
self.ssl_ca_certs = settings['ssl_ca_certs']
self.credentials = None
self.parameters = None
self.connection = None
self.channel = None
self.consume_msg = None
def send_msg(self, message_body, routing_key=None):
exchange_name = self.exchange_name
message_headers = {}
priority = 0
delivery_mode = 2
if routing_key is None:
routing_key = self.default_routing_key
timestamp = time.time()
# create the message properties
properties = pika.BasicProperties(
delivery_mode = delivery_mode,
priority = priority,
timestamp = timestamp,
headers = message_headers,
)
logger.debug("sending message body:\n" + str(message_body))
logger.debug('sending message to exchange: ' + self.exchange_name)
logger.debug('sending message with routing key: ' + routing_key)
try:
self.channel.basic_publish(
exchange = exchange_name,
routing_key = routing_key,
body = message_body,
properties = properties,
)
except Exception as e:
logger.exception('exception received while trying to send message')
raise Exception('exception received while trying to send message' + str(e))
def receive_msg(self):
method, properties, body = self.channel.basic_get(queue=self.queue_name)
return body
def close(self):
self.channel = None
self.connection = None
def setup_send(self):
self._open_blocking_connection()
def setup_receive(self, consume_msg=None, routing_key=None):
if routing_key is None:
self.routing_key = self.default_routing_key
if consume_msg is not None:
self.consume_msg = consume_msg
# setup receiving queue and exchange
logger.debug( ' open blocking connection to setup queue ' )
self._open_blocking_connection()
self._create_queue(self.queue_name,self.routing_key)
self._close()
logger.debug( ' open select connection ' )
try:
self._open_select_connection(self._on_connection_open)
except:
logger.exception(' Received exception while opening select connection: ' + str(sys.exc_info()[1]))
raise
def start_receive_loop(self):
logger.debug( ' start message consumer ' )
try:
self.connection.ioloop.start()
except:
logger.exception(' Received exception while starting ioloop for message consumer: ' + str(sys.exc_info()[1]))
raise
def stop_receive_loop(self):
try:
logger.debug(' message connection: ' + str(self.connection) )
logger.debug(' message ioloop: ' + str(self.connection.ioloop) )
self.connection.ioloop.stop()
logger.debug( ' after stopping message consumer ')
except:
logger.exception(' Received exception while stopping ioloop for the message consumer: ' + str(sys.exc_info()[1]))
raise
def _open_blocking_connection(self):
logger.debug("open blocking connection")
self._create_connection_parameters()
try:
self.connection = pika.BlockingConnection(self.parameters)
except:
logger.exception(' Exception received while trying to open blocking connection')
raise
try:
self.channel = self.connection.channel()
except:
logger.exception(' Exception received while trying to open a channel')
raise
logger.debug("create exchange, name = " + self.exchange_name)
self.channel.exchange_declare(
exchange = self.exchange_name,
exchange_type = self.exchange_type,
durable = self.exchange_durable,
auto_delete = self.exchange_auto_delete,
)
def _open_select_connection(self,
on_open_callback = None,
on_open_error_callback = None,
on_close_callback = None,
stop_ioloop_on_close = True,
):
logger.debug("create select connection")
self._create_connection_parameters()
# open the connection
if on_open_callback is not None:
try:
self.connection = pika.SelectConnection(self.parameters,
on_open_callback,
on_open_error_callback,
on_close_callback,
stop_ioloop_on_close,
)
except:
logger.error(' Exception received while trying to open select connection'
+ str(sys.exc_info()))
raise
def _create_connection_parameters(self):
logger.debug("create connection parameters, server = " + self.host + " port = " + str(self.port))
# need to set credentials to login to the message server
#self.credentials = pika.PlainCredentials(self.username,self.password)
self.credentials = pika.credentials.ExternalCredentials()
ssl_options_dict = {
"certfile": self.ssl_cert,
"keyfile": self.ssl_key,
"ca_certs": self.ssl_ca_certs,
"cert_reqs": ssl.CERT_REQUIRED,
}
#logger.debug(str(ssl_options_dict))
# setup our connection parameters
self.parameters = pika.ConnectionParameters(
host = self.host,
port = self.port,
virtual_host = self.virtual_host,
credentials = self.credentials,
socket_timeout = self.socket_timeout,
ssl = True,
ssl_options = ssl_options_dict,
)
def _create_queue(self,name,routing_key):
# declare a random queue which this job will use to receive messages
# durable = survive reboots of the broker
# exclusive = only current connection can access this queue
# auto_delete = queue will be deleted after connection is closed
self.channel.queue_declare(
queue = str(name),
durable = self.queue_is_durable,
exclusive = self.queue_is_exclusive,
auto_delete = self.queue_is_auto_delete
)
# now bind this queue to the exchange, using a routing key
# any message submitted to the echange with the
# routing key will appear on this queue
self.channel.queue_bind(exchange=self.exchange_name,
queue=str(name),
routing_key=str(routing_key)
)
def _purge_queue(self):
self.channel.queue_purge(queue=self.queue_name)
def _on_connection_open(self,connection):
logger.debug(' in on_connection_open')
try:
connection.channel(self._on_channel_open)
except:
logger.exception(' Received exception while opening connection to message server: ' + str(sys.exc_info()[1]))
raise
def _on_channel_open(self,channel):
logger.debug(' in on_channel_open')
try:
channel.basic_consume(self.consume_msg, self.queue_name)
except:
logger.exception(' Received exception while creating message consumer: ' + str(sys.exc_info()[1]))
raise
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment