reorg prob dir

README.md

@@ -51,4 +51,7 @@ Example
 =======
 ```
 mpiexec -np 2 python async-search.py --prob_dir=../benchmarks/qmcp --exp_dir=../experiments/ --exp_id=exp-01 --max_evals=10 --max_time=60 
-```
\ No newline at end of file
+```
+How to define your own autotuning problem
+=========================================
+

benchmarks/prob/evaluate.py

+from string import Template
+import re
+import os
+import sys
+import time
+import json
+import math
+import os
+import subprocess
+import csv
+
+def readResults(fname, evalnum):
+    pattern1 = re.compile("START TIME:", re.IGNORECASE)
+    pattern2 = re.compile("OUTPUT:", re.IGNORECASE)
+    pattern3 = re.compile("END TIME:", re.IGNORECASE)
+    pattern4 = re.compile("INPUT:", re.IGNORECASE)
+    resDict = {}
+    resDict['evalnum'] = evalnum
+    resDict['startTime'] = -1
+    resDict['endTime'] = -1
+    resDict['cost'] = sys.float_info.max
+    resDict['x'] = None
+    try:
+        while True:
+            with open(fname, 'rt') as in_file:
+                for linenum, line in enumerate(in_file):
+                    if pattern1.search(line) is not None:
+                        print(line)
+                        str1 = line.rstrip('\n')
+                        res = re.findall('START TIME:(.*)', str1)
+                        resDict['startTime'] = int(res[0])
+                    elif pattern2.search(line) is not None:
+                        print(line)
+                        str1 = line.rstrip('\n')
+                        res = re.findall('OUTPUT:(.*)', str1)
+                        rv = float(res[0])
+                        if math.isnan(rv):
+                            rv = sys.float_info.max
+                        resDict['cost'] = rv
+                    elif pattern3.search(line) is not None:
+                        print(line)
+                        str1 = line.rstrip('\n')
+                        res = re.findall('END TIME:(.*)', str1)
+                        resDict['endTime'] = int(res[0])
+                    elif pattern4.search(line) is not None:
+                        print(line)
+                        str1 = line.rstrip('\n')
+                        res = re.findall('INPUT:(.*)', str1)
+                        resDict['x'] = eval(res[0])
+                if len(resDict.keys()) == 5:
+                    key = os.path.basename(fname)
+                    resDict['key'] = key
+                    resDict['status'] = 0
+            if 'endTime' in resDict.keys():
+                    break
+            time.sleep(5)
+    except Exception:
+        print('Unexpected error:', sys.exc_info()[0])
+    print(resDict)
+    return(resDict)
+
+def commandLine(x, params):
+    cmd = ''
+    hlist = []
+    for p, v in zip(params, x):
+        cmd = cmd + ('--%s %s ') % (p, str(v))
+    print(cmd)
+    return(cmd)
+
+def evaluate(x, evalCounter, params, prob_dir, job_dir, result_dir):
+    cmd = commandLine(x, params)
+    jobfile = job_dir+'/%05d.job' % evalCounter
+    outputfile = result_dir+'/%05d.dat' % evalCounter
+    filein = open(prob_dir+'/job.tmpl')
+    src = Template(filein.read())
+    inpstr = str(x)
+    d = {'outputfile': outputfile, 'inpstr': inpstr, 'cmd': cmd, 'ompn':x[0]}
+    result = src.substitute(d)
+    with open(jobfile, "w") as jobFile:
+        jobFile.write(result)
+    status = subprocess.check_output('chmod +x %s' % jobfile, shell=True)
+    status = subprocess.call(' sh %s ' % jobfile, shell=True)
+    resDict = readResults(outputfile, evalCounter)
+    print(resDict)
+
+    return(resDict)
+
+    
\ No newline at end of file

benchmarks/prob/executable.py

+#!/usr/bin/env python
+from __future__ import print_function
+import re
+import os
+import sys
+import time
+import json
+import math
+import os
+import argparse
+seed = 12345
+
+def create_parser():
+    'command line parser for keras'
+
+    parser = argparse.ArgumentParser(add_help=True)
+    group = parser.add_argument_group('required arguments')
+
+    parser.add_argument('--p1', action='store', dest='p1',
+                        nargs='?', const=2, type=int, default='2',
+                        help='parameter p1 value')
+    parser.add_argument('--p2', action='store', dest='p2',
+                        nargs='?', const=2, type=int, default='2',
+                        help='parameter p2 value')
+    parser.add_argument('--p3', action='store', dest='p3',
+                        nargs='?', const=2, type=int, default='2',
+                        help='parameter p3 value')
+    parser.add_argument("--p4", nargs='?', type=str,
+                        default='a',
+                        help="parameter p4 value")
+
+    return(parser)
+
+parser = create_parser()
+cmdline_args = parser.parse_args()
+param_dict = vars(cmdline_args)
+
+p1 = param_dict['p1']
+p2 = param_dict['p2']
+p3 = param_dict['p3']
+p4 = param_dict['p4']
+
+if p4 == 'a':
+    pval = p1*p2*p3
+else:
+    pval = p1+p2+p3
+
+print('OUTPUT:%1.3f'%pval)

benchmarks/prob/job.tmpl

+#!/bin/bash -x
+#COBALT -n 1
+#COBALT -q debug-flat-quad
+#COBALT -A Performance
+#COBALT -t 30
+
+outputfile=$outputfile
+
+ut=$$(date '+%s')
+echo START TIME:$$ut > $outputfile
+echo INPUT:"$inpstr" >> $outputfile
+python ../benchmarks/prob/executable.py $cmd >> $outputfile
+ut=$$(date '+%s')
+echo END TIME:$$ut>> $outputfile
+echo 
+
+

benchmarks/qmcp/problem.py → benchmarks/prob/problem.py

@@ -2,14 +2,14 @@ from collections import OrderedDict
 class Problem():
     def __init__(self):
         space = OrderedDict()
-        
-        #bechmark specific parameters
-        space['OMP_NUM_THREADS'] = [2,4,6,8,10,12,14,16,18,20]
-        space['a'] = [8,16,32,64,128,256,512,1024,1536] 
-        space['w'] = range(8,41)
+        #problem specific parameters
+        space['p1'] = (2, 10)
+        space['p2'] = (8, 1024)
+        space['p3'] = [2 , 4, 8, 16, 32, 64, 128]
+        space['p4'] = ['a', 'b', 'c']
         self.space = space
         self.params = self.space.keys()
-        self.starting_point = [2, 8, 8]
+        self.starting_point = [2, 8, 2, 'c']
 
 if __name__ == '__main__':
     instance = Problem()

search/async-search.py

@@ -50,9 +50,6 @@ max_evals = param_dict['max_evals']
 max_time = param_dict['max_time']
 
 
-
-
-
 exp_dir = exp_dir+'/'+eid
 jobs_dir = exp_dir+'/jobs'
 results_dir = exp_dir+'/results'
@@ -61,6 +58,8 @@ results_csv_fname = exp_dir+'/'+eid+'_results.csv'
 
 sys.path.insert(0, prob_dir)
 import problem as problem
+from evaluate import evaluate
+
 instance = problem.Problem()
 spaceDict = instance.space
 params = instance.params
@@ -92,14 +91,16 @@ if rank == 0:
     num_workers = size - 1
     closed_workers = 0
     space = [spaceDict[key] for key in params]
+    print(space)
     eval_counter = 0
 
-
     parDict = {}
     evalDict = {}
     resultsList = []
-    parDict['kappa'] = 0
+    parDict['kappa'] = 1.96
     init_x = []
+
+
     opt = Optimizer(space, base_estimator='RF', acq_optimizer='sampling',
                     acq_func='LCB', acq_func_kwargs=parDict, random_state=seed)
     print("Master starting with %d workers" % num_workers)
@@ -161,7 +162,7 @@ else:
         tag = status.Get_tag()
         if tag == tags.START:
             print(task)
-            result = evaluatePoint(task['x'], task['eval_counter'], params, prob_dir, jobs_dir, results_dir)
+            result = evaluate(task['x'], task['eval_counter'], params, prob_dir, jobs_dir, results_dir)
             result['start_time'] = task['start_time']
             comm.send(result, dest=0, tag=tags.DONE)
         elif tag == tags.EXIT:

search/utils.py

@@ -8,55 +8,6 @@ import math
 import os
 import subprocess
 import csv
-def readResults(fname, evalnum):
-    pattern1 = re.compile("START TIME:", re.IGNORECASE)
-    pattern2 = re.compile("OUTPUT:", re.IGNORECASE)
-    pattern3 = re.compile("END TIME:", re.IGNORECASE)
-    pattern4 = re.compile("INPUT:", re.IGNORECASE)
-    resDict = {}
-    resDict['evalnum'] = evalnum
-    resDict['startTime'] = -1
-    resDict['endTime'] = -1
-    resDict['cost'] = sys.float_info.max
-    resDict['x'] = None
-    try:
-        while True:
-            with open(fname, 'rt') as in_file:
-                for linenum, line in enumerate(in_file):
-                    if pattern1.search(line) is not None:
-                        print(line)
-                        str1 = line.rstrip('\n')
-                        res = re.findall('START TIME:(.*)', str1)
-                        resDict['startTime'] = int(res[0])
-                    elif pattern2.search(line) is not None:
-                        print(line)
-                        str1 = line.rstrip('\n')
-                        res = re.findall('OUTPUT:(.*)', str1)
-                        rv = float(res[0])
-                        if math.isnan(rv):
-                            rv = sys.float_info.max
-                        resDict['cost'] = rv
-                    elif pattern3.search(line) is not None:
-                        print(line)
-                        str1 = line.rstrip('\n')
-                        res = re.findall('END TIME:(.*)', str1)
-                        resDict['endTime'] = int(res[0])
-                    elif pattern4.search(line) is not None:
-                        print(line)
-                        str1 = line.rstrip('\n')
-                        res = re.findall('INPUT:(.*)', str1)
-                        resDict['x'] = eval(res[0])
-                if len(resDict.keys()) == 5:
-                    key = os.path.basename(fname)
-                    resDict['key'] = key
-                    resDict['status'] = 0
-            if 'endTime' in resDict.keys():
-                    break
-            time.sleep(5)
-    except Exception:
-        print('Unexpected error:', sys.exc_info()[0])
-    print(resDict)
-    return(resDict)
 
 def saveResults(resultsList, json_fname, csv_fname):
     print(resultsList)
@@ -69,32 +20,3 @@ def saveResults(resultsList, json_fname, csv_fname):
         dict_writer = csv.DictWriter(output_file, keys)
         dict_writer.writeheader()
         dict_writer.writerows(resultsList)
-
-def commandLine(x, params):
-    cmd = ''
-    hlist = []
-    for p, v in zip(params, x):
-        if 'OMP_NUM_THREADS' not in p:
-            cmd = cmd + ('-%s %s ') % (p, str(v))
-        else:
-            hlist.append(v)
-    print(cmd)
-    return(cmd)
-
-def evaluatePoint(x, evalCounter, params, prob_dir, job_dir, result_dir):
-    cmd = commandLine(x, params)
-    jobfile = job_dir+'/%05d.job' % evalCounter
-    outputfile = result_dir+'/%05d.dat' % evalCounter
-    filein = open(prob_dir+'/theta.tmpl')
-    src = Template(filein.read())
-    inpstr = str(x)
-    d = {'outputfile': outputfile, 'inpstr': inpstr, 'cmd': cmd, 'ompn':x[0]}
-    result = src.substitute(d)
-    with open(jobfile, "w") as jobFile:
-        jobFile.write(result)
-    status = subprocess.check_output('chmod +x %s' % jobfile, shell=True)
-    status = subprocess.call(' sh %s ' % jobfile, shell=True)
-    resDict = readResults(outputfile, evalCounter)
-    print(resDict)
-
-    return(resDict)
\ No newline at end of file