user_settings.py 9.25 KB
Newer Older
jtchilders's avatar
jtchilders committed
1
2
import os,sys,logging
import socket
3
from django.core.management import call_command
4
logger = logging.getLogger('console')
5
    
jtchilders's avatar
jtchilders committed
6
try:
7
   INSTALL_PATH = os.environ['ARGOBALSAM_INSTALL_PATH']
8
   LOGGING_DIRECTORY = os.path.join(INSTALL_PATH, 'log') # where to store log files
Thomas Uram's avatar
Thomas Uram committed
9
except KeyError as e:
jtchilders's avatar
jtchilders committed
10
11
12
   logger.error('Environment not setup: ' + str(e))
   raise

13

jtchilders's avatar
jtchilders committed
14
15
16
#------------------------------
# DATABASE CONFIG INFO
#------------------------------
17
CONCURRENCY_ENABLED = True
jtchilders's avatar
jtchilders committed
18
19
20
21
22
23
24
25
26
27
28
29
USING_DB_LOGIN                      = False
DBUSER                              = ''
DBPASS                              = ''
if USING_DB_LOGIN:
   DBUSER                           = os.environ['ARGOBALSAM_DBUSER']
   DBPASS                           = os.environ['ARGOBALSAM_DBPASS']

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases
default_db = {}
default_db['ENGINE'] = 'django.db.backends.sqlite3'
default_db['NAME'] = os.path.join(INSTALL_PATH,'db.sqlite3')
30
default_db['OPTIONS'] = {'timeout' : 500000.0}
jtchilders's avatar
jtchilders committed
31
32
33
34
if USING_DB_LOGIN:
   default_db['USER'] = DBUSER
   default_db['PASSWORD'] = DBPASS

35
36
37
38
39
40
41
42
43
44
45
46

testing = os.environ.get('BALSAM_TEST', '')
if testing:
    INSTALL_PATH = os.environ['BALSAM_TEST_DIRECTORY']
    db_test = default_db.copy()
    db_test['NAME'] = os.path.join(INSTALL_PATH, 'test_db.sqlite3')
    DATABASES = {'default':db_test}
else:
    DATABASES = {'default':default_db}

DATA_PATH = os.path.join(INSTALL_PATH,'data')
ALLOWED_EXE_PATH = os.path.join(INSTALL_PATH,'exe')
jtchilders's avatar
jtchilders committed
47
48
49
50
51
52
53
54

#------------------------------
# BALSAM CONFIG INFO
#------------------------------
BALSAM_WORK_DIRECTORY               = os.path.join(DATA_PATH,'balsamjobs') # where to store local job data used for submission
BALSAM_DELETE_OLD_WORK              = True # enable deletion of old folders in BALSAM_WORK_DIRECTORY at a period of BALSAM_DELETE_OLD_WORK_PERIOD
BALSAM_DELETE_OLD_WORK_PERIOD       = 86400 # once a day check for old work folders older than the BALSAM_DELETE_OLD_WORK_AGE
BALSAM_DELETE_OLD_WORK_AGE          = 86400 * 31 # delete work folders that are older than 31 days
55
56
57
BALSAM_SCHEDULER_SUBMIT_EXE         = '/usr/bin/qsub'
BALSAM_SCHEDULER_STATUS_EXE         = '/usr/bin/qstat'
BALSAM_SCHEDULER_HISTORY_EXE        = '/usr/bin/'
58
BALSAM_SERVICE_PERIOD               = 1 # seconds between service loop execution
jtchilders's avatar
jtchilders committed
59
60
BALSAM_MAX_QUEUED                   = 20 # the maximum number of jobs allowed on the local queue
BALSAM_SUBMIT_JOBS                  = True # submit jobs to queue, turn off when testing
61
62
BALSAM_DEFAULT_QUEUE                = 'debug-cache-quad' # default local queue name
BALSAM_DEFAULT_PROJECT              = 'datascience' # default local project name
jtchilders's avatar
jtchilders committed
63
BALSAM_ALLOWED_EXECUTABLE_DIRECTORY = ALLOWED_EXE_PATH # path to allowed executables
64
BALSAM_SITE                         = 'theta' # local balsam site name
65
BALSAM_SCHEDULER_CLASS              = 'CobaltScheduler' # local scheduler in use
66
BALSAM_MAX_CONCURRENT_TRANSITIONS   = 5 # maximum number of sub threads spawned by Balsam
Michael Salim's avatar
Michael Salim committed
67
BALSAM_MAX_CONCURRENT_RUNNERS       = 50 # maximum number of background 'mpirun' subprocesses
jtchilders's avatar
jtchilders committed
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

#------------------------------
# ARGO CONFIG INFO
#------------------------------
ARGO_SERVICE_PERIOD                 = 10 # seconds between service loop execution
ARGO_WORK_DIRECTORY                 = os.path.join(DATA_PATH,'argojobs')
ARGO_DELETE_OLD_WORK                = True # enable deletion of old folders in BALSAM_WORK_DIRECTORY at a period of ARGO_DELETE_OLD_WORK_PERIOD
ARGO_DELETE_OLD_WORK_PERIOD         = 86400 # in seconds,  once a day check for old work folders older than
ARGO_DELETE_OLD_WORK_AGE            = 86400 * 31 # in seconds,  delete work folders that are older than 31 days
ARGO_MAX_CONCURRENT_TRANSITIONS     = 5 # maximum number of sub threads spawned by ARGO


#------------------------------
# GRID FTP SERVER INFO
#------------------------------

84
GRIDFTP_BIN                         = '/soft/data-transfer/globus/bin'
jtchilders's avatar
jtchilders committed
85
86
87
88
GRIDFTP_GLOBUS_URL_COPY             = os.path.join(GRIDFTP_BIN,'globus-url-copy')
GRIDFTP_PROXY_INFO                  = os.path.join(GRIDFTP_BIN,'grid-proxy-info')
GRIDFTP_PROXY_INIT                  = os.path.join(GRIDFTP_BIN,'grid-proxy-init')
GRIDFTP_PROTOCOL                    = 'gsiftp://'
89
GRIDFTP_SERVER                      = ''
jtchilders's avatar
jtchilders committed
90
91

#------------------------------
92
93
94
# MESSAGING CONFIG
# Specify everything in a SENDER_CONFIG for BalsamStatusSender
# And in RECEIVER_CONFIG for BalsamJobReceiver
jtchilders's avatar
jtchilders committed
95
#------------------------------
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
#SENDER_CONFIG = {
#    "mode"                          : "pika",
#    "username"                      : '',
#    "password"                      : '',
#    "host"                          : '',
#    "port"                          : '',
#    "virtual_host"                  : '',
#    "socket_timeout"                : '',
#    "exchange_name"                 : '',
#    "exchange_type"                 : '',
#    "exchange_durable"              : '',
#     "exchange_auto_delete"         : '',
#    "queue_name"                    : '',
#    "queue_is_durable"              : '',
#    "queue_is_exclusive"            : '',
#    "queue_is_auto_delete"          : '',
#    "default_routing_key"           : '',
#     "ssl_cert"                     : '',
#    "ssl_key"                       : '',
#     "ssl_ca_cer"                   : ''
#}
#RECEIVER_CONFIG = {
#    "mode"                          : "pika",
#    "username"                      : '',
#    "password"                      : '',
#    "host"                          : '',
#    "port"                          : '',
#    "virtual_host"                  : '',
#    "socket_timeout"                : '',
#    "exchange_name"                 : '',
#    "exchange_type"                 : '',
#    "exchange_durable"              : '',
#     "exchange_auto_delete"         : '',
#    "queue_name"                    : '',
#    "queue_is_durable"              : '',
#    "queue_is_exclusive"            : '',
#    "queue_is_auto_delete"          : '',
#    "default_routing_key"           : '',
#     "ssl_cert"                     : '',
#    "ssl_key"                       : '',
#     "ssl_ca_cer"                   : ''
#}

RECEIVER_CONFIG = {
    "mode" : "no_message"
}
jtchilders's avatar
jtchilders committed
142

143
144
145
SENDER_CONFIG = {
    "mode" : "no_message"
}
jtchilders's avatar
jtchilders committed
146
147
148
149
150
151
152
153


#------------------------------
# logging settings
#------------------------------
LOG_HANDLER_LEVEL = 'DEBUG'
LOG_BACKUP_COUNT = 5 # number of files worth of history
LOG_FILE_SIZE_LIMIT = 100 * 1024 * 1024 # file size at which to move to a new log file
154
155
156
157
158
159
160
161
162
163

if 'argoservice' in ' '.join(sys.argv):
    HANDLER_FILE = 'argo.log'
elif 'service' in ' '.join(sys.argv):
    HANDLER_FILE = 'balsam.log'
elif 'launcher' in ' '.join(sys.argv):
    HANDLER_FILE = 'launcher.log'
else:
    HANDLER_FILE = 'misc.log'
HANDLER_FILE = os.path.join(LOGGING_DIRECTORY, HANDLER_FILE)
jtchilders's avatar
jtchilders committed
164
165
166
167
168
169
170
171
172
173
174
175
176
177

LOGGING = {
   'version': 1,
   'disable_existing_loggers': False,
   'formatters': {
      'standard': {
      'format' : '%(asctime)s|%(process)d|%(levelname)8s|%(name)s:%(lineno)s] %(message)s',
      'datefmt' : "%d-%b-%Y %H:%M:%S"
      },
   },
   'handlers': {
      'console': {
         'class':'logging.StreamHandler',
         'formatter': 'standard',
178
          'level' : 'DEBUG'
jtchilders's avatar
jtchilders committed
179
180
181
182
      },
      'default': {
         'level':LOG_HANDLER_LEVEL,
         'class':'logging.handlers.RotatingFileHandler',
183
         'filename': HANDLER_FILE,
jtchilders's avatar
jtchilders committed
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
         'maxBytes': LOG_FILE_SIZE_LIMIT,
         'backupCount': LOG_BACKUP_COUNT,
         'formatter': 'standard',
      }
   },
   'loggers': {
      'django':{
         'handlers': ['default'],
         'level': 'DEBUG',
         'propagate': True,
      },
      'argo': {
         'handlers': ['default'],
         'level': 'DEBUG',
      },
      'common': {
         'handlers': ['default'],
         'level': 'DEBUG',
      },
      'console': {
         'handlers': ['console'],
         'level': 'INFO',
      },
      'balsam': {
         'handlers': ['default'],
         'level': 'DEBUG',
      },
211
212
213
214
      'balsamlauncher': {
         'handlers': ['default'],
         'level': 'DEBUG',
      },
jtchilders's avatar
jtchilders committed
215
216
217
218
219
220
221
222
223
224
      'django.db.backends': {
         'handlers':['default'],
         'level': 'WARNING',
      },
      'pika.adapters.base_connection': {
         'handlers':['default'],
         'level':'WARNING',
      },
   }
}
225
226
227
228
229
230
231
232
233
if 'argoservice' in ' '.join(sys.argv):
    logger = logging.getLogger('argo')
elif 'service' in sys.argv:
    logger = logging.getLogger('balsam')
elif 'launcher' in ' '.join(sys.argv):
    logger = logging.getLogger('balsamlauncher')
else:
    logger = logging.getLogger('console')

234

235
236
237
238
239
240
def log_uncaught_exceptions(exctype, value, tb,logger=logger):
   logger.error(f"Uncaught Exception {exctype}: {value}",exc_info=(exctype,value,tb))
   logger = logging.getLogger('console')
   logger.error(f"Uncaught Exception {exctype}: {value}",exc_info=(exctype,value,tb))

sys.excepthook = log_uncaught_exceptions
jtchilders's avatar
jtchilders committed
241
242
243
244
245
246
247
248
249
250

#------------------------------
# Sanity Checks
#------------------------------

# ensure that requisite paths exist
for d in [
      INSTALL_PATH,
      DATA_PATH,
      ALLOWED_EXE_PATH,
251
      LOGGING_DIRECTORY,
jtchilders's avatar
jtchilders committed
252
253
254
      BALSAM_WORK_DIRECTORY,
      BALSAM_ALLOWED_EXECUTABLE_DIRECTORY,
      ARGO_WORK_DIRECTORY,
255
256
257
258
259
260
      #GRIDFTP_GLOBUS_URL_COPY,
      #GRIDFTP_PROXY_INFO,
      #GRIDFTP_PROXY_INIT,
      #RABBITMQ_SSL_CERT,
      #RABBITMQ_SSL_KEY,
      #RABBITMQ_SSL_CA_CERTS,
jtchilders's avatar
jtchilders committed
261
262
   ]:
   if not os.path.exists(d):
263
      os.makedirs(d)
264
      logger.info(f'Created directory {d}')