Commit eb9826bd authored by Jonathan Jenkins's avatar Jonathan Jenkins
Browse files

rework config storage to a much tighter format

should do good things for codes-mapping lookup performance
parent 62e1153b
......@@ -16,26 +16,34 @@
#define CONFIGURATION_MAX_TYPES 10
#define CONFIGURATION_MAX_ANNOS 10
// NOTE: direct offsets are used when first building the structure as we're
// building up one big buffer to hold entity names. Pointers are set after the
// initialization process during configuration_load
typedef union {
char const * ptr;
int offset;
} config_name_u;
typedef struct config_lptype_s
{
char name[CONFIGURATION_MAX_NAME];
char anno[CONFIGURATION_MAX_NAME];
uint64_t count;
config_name_u name;
config_name_u anno;
int count;
} config_lptype_t;
typedef struct config_lpgroup_s
{
char name[CONFIGURATION_MAX_NAME];
uint64_t repetitions;
config_name_u name;
int repetitions;
int lptypes_count;
config_lptype_t lptypes[CONFIGURATION_MAX_TYPES];
uint64_t lptypes_count;
} config_lpgroup_t;
// mapping of lp type to the list of annotations used. Used for convenience when
// models are performing configuraiton code
typedef struct config_anno_map_s
{
char lp_name[CONFIGURATION_MAX_NAME];
config_name_u lp_name;
// only explicit annotations tracked here - use a flag to indicate a
// non-annotated LP type
int has_unanno_lp;
......@@ -43,10 +51,10 @@ typedef struct config_anno_map_s
// (annotation-ignoring) lookup semantics, provide a flag to determine if
// the unannotated lp type is first
int is_unanno_first;
uint64_t num_annos;
int num_annos;
// maintain the number of lps that have the particular annotation
uint64_t num_anno_lps[CONFIGURATION_MAX_ANNOS];
char * annotations[CONFIGURATION_MAX_ANNOS];
int num_anno_lps[CONFIGURATION_MAX_ANNOS];
config_name_u annotations[CONFIGURATION_MAX_ANNOS];
} config_anno_map_t;
typedef struct config_lpgroups_s
......
......@@ -280,17 +280,17 @@ int configuration_get_value_double (ConfigHandle *handle,
}
static void check_add_anno(
const char *anno,
int anno_offset,
config_anno_map_t *map){
if (anno[0] == '\0'){
if (anno_offset == -1) {
map->has_unanno_lp = 1;
if (!map->num_annos)
map->is_unanno_first = 1;
}
else{
uint64_t a = 0;
int a = 0;
for (; a < map->num_annos; a++){
if (strcmp(map->annotations[a], anno) == 0){
if (map->annotations[a].offset == anno_offset) {
map->num_anno_lps[a]++;
break;
}
......@@ -298,21 +298,21 @@ static void check_add_anno(
if (a == map->num_annos){
// we have a new anno!
assert(a < CONFIGURATION_MAX_ANNOS);
map->annotations[a] = strdup(anno);
map->annotations[a].offset = anno_offset;
map->num_annos++;
map->num_anno_lps[a] = 1;
} // else anno was already there, do nothing
}
}
static void check_add_lp_type_anno(
const char *lp_name,
const char *anno,
int lp_name_offset,
int anno_offset,
config_lpgroups_t *lpgroups){
uint64_t lpt_anno = 0;
for (; lpt_anno < lpgroups->lpannos_count; lpt_anno++){
config_anno_map_t *map = &lpgroups->lpannos[lpt_anno];
if (strcmp(map->lp_name, lp_name) == 0){
check_add_anno(anno, map);
if (map->lp_name.offset == lp_name_offset){
check_add_anno(anno_offset, map);
break;
}
}
......@@ -321,13 +321,13 @@ static void check_add_lp_type_anno(
assert(lpt_anno < CONFIGURATION_MAX_TYPES);
config_anno_map_t *map = &lpgroups->lpannos[lpt_anno];
// initialize this annotation map
strcpy(map->lp_name, lp_name);
map->lp_name.offset = lp_name_offset;
map->num_annos = 0;
map->has_unanno_lp = 0;
map->is_unanno_first = 0;
memset(map->num_anno_lps, 0,
CONFIGURATION_MAX_ANNOS*sizeof(*map->num_anno_lps));
check_add_anno(anno, map);
check_add_anno(anno_offset, map);
lpgroups->lpannos_count++;
}
}
......@@ -342,11 +342,11 @@ static void check_add_lp_type_anno(
} while (0)
/* helper for setting up the canonical name mapping */
static void check_add_uniq_str(
char const *** str_array_ref,
static int check_add_uniq_str(
int ** offset_array,
char ** str_buf,
int * num_strs,
int * str_array_cap, // buffer capacity for resizing
int * offset_array_cap, // buffer capacity for resizing
int * str_buf_len,
int * str_buf_cap, // buffer capacity for resizing
char const * str)
......@@ -354,20 +354,21 @@ static void check_add_uniq_str(
int slen = strlen(str);
for (int i = 0; i < *num_strs; i++) {
char const * b = (*str_array_ref)[i];
char const * b = (*str_buf) + (*offset_array)[i];
int blen = strlen(b);
if (slen == blen && memcmp(b, str, blen) == 0)
return;
return i;
}
REALLOC_IF(*str_array_cap, *num_strs, *str_array_ref);
REALLOC_IF(*offset_array_cap, *num_strs, *offset_array);
REALLOC_IF(*str_buf_cap, *str_buf_len + slen + 1, *str_buf);
// include null char
memcpy(*str_buf + *str_buf_len, str, slen+1);
(*str_array_ref)[*num_strs] = *str_buf + *str_buf_len;
(*offset_array)[*num_strs] = *str_buf_len;
*num_strs += 1;
*str_buf_len += slen+1;
return *num_strs-1;
}
int configuration_get_lpgroups (ConfigHandle *handle,
......@@ -382,6 +383,7 @@ int configuration_get_lpgroups (ConfigHandle *handle,
size_t subse_count = 10;
int i, j, lpt;
char data[256];
// buffer mgmt vars
int num_uniq_group_names = 0;
int group_names_buf_len = 0;
int lp_names_buf_len = 0;
......@@ -392,24 +394,25 @@ int configuration_get_lpgroups (ConfigHandle *handle,
int group_names_buf_cap = 1;
int lp_names_buf_cap = 1;
int anno_names_buf_cap = 1;
int name_pos;
memset (lpgroups, 0, sizeof(*lpgroups));
lpgroups->group_names =
malloc(sizeof(*lpgroups->group_names) * group_names_cap);
lpgroups->lp_names =
malloc(sizeof(*lpgroups->lp_names) * lp_names_cap);
lpgroups->anno_names =
malloc(sizeof(*lpgroups->anno_names) * anno_names_cap);
int *group_names_offsets =
malloc(sizeof(*group_names_offsets) * group_names_cap);
int *lp_names_offsets =
malloc(sizeof(*lp_names_offsets) * lp_names_cap);
int *anno_names_offsets =
malloc(sizeof(*anno_names_offsets) * anno_names_cap);
lpgroups->group_names_buf =
malloc(sizeof(*lpgroups->group_names_buf) * group_names_buf_cap);
lpgroups->lp_names_buf =
malloc(sizeof(*lpgroups->lp_names_buf) * lp_names_buf_cap);
lpgroups->anno_names_buf =
malloc(sizeof(*lpgroups->anno_names_buf) * anno_names_buf_cap);
assert(lpgroups->group_names != NULL);
assert(lpgroups->lp_names != NULL);
assert(lpgroups->anno_names != NULL);
assert(group_names_offsets != NULL);
assert(lp_names_offsets != NULL);
assert(anno_names_offsets != NULL);
assert(lpgroups->group_names_buf != NULL);
assert(lpgroups->lp_names_buf != NULL);
assert(lpgroups->anno_names_buf != NULL);
......@@ -438,12 +441,11 @@ int configuration_get_lpgroups (ConfigHandle *handle,
subse_count = 10;
cf_openSection(*handle, sh, se[i].name, &subsh);
cf_listSection(*handle, subsh, subse, &subse_count);
check_add_uniq_str(&lpgroups->group_names,
name_pos = check_add_uniq_str(&group_names_offsets,
&lpgroups->group_names_buf, &num_uniq_group_names,
&group_names_cap, &group_names_buf_len,
&group_names_buf_cap, se[i].name);
strncpy(lpgroups->lpgroups[i].name, se[i].name,
CONFIGURATION_MAX_NAME);
lpgroups->lpgroups[i].name.offset = group_names_offsets[name_pos];
lpgroups->lpgroups[i].repetitions = 1;
lpgroups->lpgroups_count++;
if (num_uniq_group_names != lpgroups->lpgroups_count)
......@@ -464,43 +466,47 @@ int configuration_get_lpgroups (ConfigHandle *handle,
}
else
{
size_t s = sizeof(lpgroups->lpgroups[i].lptypes[lpt].name);
char *nm = lpgroups->lpgroups[i].lptypes[lpt].name;
char *anno = lpgroups->lpgroups[i].lptypes[lpt].anno;
// assume these are lptypes and counts
strncpy(nm, subse[j].name, s-1);
lpgroups->lpgroups[i].lptypes[lpt].name[s-1] = '\0';
char *c = strchr(nm, '@');
if (c) {
strcpy(anno, c+1);
// to avoid copy, find a possible annotation, change the
// string in the config structure itself, then change
// back for posterity
char *c = strchr(subse[j].name, '@');
if (c != NULL) {
*c = '\0';
check_add_uniq_str(
&lpgroups->anno_names,
&lpgroups->anno_names_buf,
&lpgroups->num_uniq_annos,
&anno_names_cap,
&anno_names_buf_len,
&anno_names_buf_cap,
c+1);
}
else {
anno[0] = '\0';
name_pos = check_add_uniq_str(
&anno_names_offsets,
&lpgroups->anno_names_buf,
&lpgroups->num_uniq_annos,
&anno_names_cap,
&anno_names_buf_len,
&anno_names_buf_cap,
c+1);
lpgroups->lpgroups[i].lptypes[lpt].anno.offset =
anno_names_offsets[name_pos];
}
else
lpgroups->lpgroups[i].lptypes[lpt].anno.offset = -1;
check_add_uniq_str(
&lpgroups->lp_names,
name_pos = check_add_uniq_str(
&lp_names_offsets,
&lpgroups->lp_names_buf,
&lpgroups->num_uniq_lptypes,
&lp_names_cap,
&lp_names_buf_len,
&lp_names_buf_cap,
nm);
subse[j].name);
lpgroups->lpgroups[i].lptypes[lpt].name.offset =
lp_names_offsets[name_pos];
if (c != NULL)
*c = '@';
// add to anno map
check_add_lp_type_anno(nm, anno, lpgroups);
check_add_lp_type_anno(
lpgroups->lpgroups[i].lptypes[lpt].name.offset,
lpgroups->lpgroups[i].lptypes[lpt].anno.offset,
lpgroups);
CHECKED_STRTOL(lpgroups->lpgroups[i].lptypes[lpt].count,
nm, data);
lpgroups->lpgroups[i].lptypes[lpt].name, data);
lpgroups->lpgroups[i].lptypes_count++;
lpt++;
}
......@@ -510,15 +516,68 @@ int configuration_get_lpgroups (ConfigHandle *handle,
}
}
if (lpgroups->lpannos_count == 0) {
free(lpgroups->anno_names);
// set up the string pointers
lpgroups->group_names =
malloc(lpgroups->lpgroups_count * sizeof(*lpgroups->group_names));
lpgroups->lp_names =
malloc(lpgroups->num_uniq_lptypes * sizeof(*lpgroups->lp_names));
assert(lpgroups->group_names);
assert(lpgroups->lp_names);
for (int i = 0; i < lpgroups->lpgroups_count; i++) {
lpgroups->group_names[i] = lpgroups->group_names_buf +
group_names_offsets[i];
}
for (int i = 0; i < lpgroups->num_uniq_lptypes; i++) {
lpgroups->lp_names[i] = lpgroups->lp_names_buf +
lp_names_offsets[i];
}
if (lpgroups->num_uniq_annos == 0) {
free(lpgroups->anno_names_buf);
lpgroups->anno_names = NULL;
lpgroups->anno_names_buf = NULL;
}
else {
lpgroups->anno_names =
malloc(lpgroups->num_uniq_annos * sizeof(*lpgroups->anno_names));
assert(lpgroups->anno_names);
for (int i = 0; i < lpgroups->num_uniq_annos; i++) {
lpgroups->anno_names[i] = lpgroups->anno_names_buf +
anno_names_offsets[i];
}
}
// everything is set up in offset mode, now make a second pass and convert
// to pointers
for (int g = 0; g < lpgroups->lpgroups_count; g++) {
config_lpgroup_t *lpg = &lpgroups->lpgroups[g];
lpg->name.ptr = lpgroups->group_names_buf + lpg->name.offset;
for (int t = 0; t < lpg->lptypes_count; t++) {
config_lptype_t *lpt = &lpg->lptypes[t];
lpt->name.ptr = lpgroups->lp_names_buf + lpt->name.offset;
lpt->anno.ptr =
(lpt->anno.offset == -1)
? NULL
: lpgroups->anno_names_buf + lpt->anno.offset;
}
}
for (int m = 0; m < lpgroups->lpannos_count; m++) {
config_anno_map_t *map = &lpgroups->lpannos[m];
map->lp_name.ptr = lpgroups->lp_names_buf + map->lp_name.offset;
for (int a = 0; a < map->num_annos; a++) {
map->annotations[a].ptr =
(map->annotations[a].offset == -1)
? NULL
: lpgroups->anno_names_buf + map->annotations[a].offset;
}
}
cf_closeSection(*handle, sh);
free(group_names_offsets);
free(lp_names_offsets);
free(anno_names_offsets);
return 0;
}
......@@ -533,7 +592,7 @@ int configuration_get_annotation_index(const char * anno,
const config_anno_map_t * anno_map){
if (anno == NULL) return -1;
for (uint64_t i = 0; i < anno_map->num_annos; i++){
if (!strcmp(anno, anno_map->annotations[i])){
if (!strcmp(anno, anno_map->annotations[i].ptr)){
return (int)i;
}
}
......
......@@ -24,8 +24,9 @@ static int mini(int a, int b){ return a < b ? a : b; }
// compare passed in annotation strings (NULL or nonempty) against annotation
// strings in the config (empty or nonempty)
static int cmp_anno(const char * anno_user, const char * anno_config){
return anno_user == NULL ? anno_config[0]=='\0'
: !strcmp(anno_user, anno_config);
return anno_user == NULL ? anno_config == NULL
: (anno_config != NULL
&& !strcmp(anno_user, anno_config));
}
......@@ -91,7 +92,7 @@ int codes_mapping_get_group_reps(const char* group_name)
int grp;
for(grp = 0; grp < lpconf.lpgroups_count; grp++)
{
if(strcmp(lpconf.lpgroups[grp].name, group_name) == 0)
if(strcmp(lpconf.lpgroups[grp].name.ptr, group_name) == 0)
return lpconf.lpgroups[grp].repetitions;
}
return -1;
......@@ -113,14 +114,14 @@ int codes_mapping_get_lp_count(
const config_lpgroup_t *lpg = &lpconf.lpgroups[g];
// iterate over the lps if the group is null (count across all groups)
// or if the group names match
if (group_name == NULL || strcmp(lpg->name, group_name) == 0){
if (group_name == NULL || strcmp(lpg->name.ptr, group_name) == 0){
for (int l = 0; l < lpg->lptypes_count; l++){
const config_lptype_t *lpt = &lpg->lptypes[l];
if (strcmp(lp_type_name, lpt->name) == 0){
if (strcmp(lp_type_name, lpt->name.ptr) == 0){
// increment the count if we are ignoring annotations,
// query and entry are both unannotated, or if the
// annotations match
if (ignore_annos || cmp_anno(annotation, lpt->anno)){
if (ignore_annos || cmp_anno(annotation, lpt->anno.ptr)){
if (ignore_repetitions)
lp_type_ct_total += lpt->count;
else
......@@ -158,15 +159,15 @@ void codes_mapping_get_lp_id(
rep_count += lpg->lptypes[l].count;
}
// does group name match?
if (strcmp(lpg->name, group_name) == 0){
if (strcmp(lpg->name.ptr, group_name) == 0){
tw_lpid local_lp_count = 0;
// for each lp type
for (int l = 0; l < lpg->lptypes_count; l++){
const config_lptype_t *lpt = &lpg->lptypes[l];
// does lp name match?
if (strcmp(lpt->name, lp_type_name) == 0){
if (strcmp(lpt->name.ptr, lp_type_name) == 0){
// does annotation match (or are we ignoring annotations?)
if (ignore_anno || cmp_anno(annotation, lpt->anno)){
if (ignore_anno || cmp_anno(annotation, lpt->anno.ptr)){
// return if sane offset
if (offset >= (int) lpt->count){
goto ERROR;
......@@ -210,18 +211,18 @@ int codes_mapping_get_lp_relative_id(
local_lp_name, &lp_type_index, local_annotation, &rep_id, &offset);
const char * anno = (local_annotation[0]=='\0') ? NULL : local_annotation;
uint64_t group_lp_count = 0;
int group_lp_count = 0;
// if not group_specific, then count up LPs of all preceding groups
if (!group_wise){
for (int g = 0; g < group_index; g++){
uint64_t lp_count = 0;
int lp_count = 0;
const config_lpgroup_t *lpg = &lpconf.lpgroups[g];
for (int l = 0; l < lpg->lptypes_count; l++){
const config_lptype_t *lpt = &lpg->lptypes[l];
if (strcmp(local_lp_name, lpt->name) == 0){
if (strcmp(local_lp_name, lpt->name.ptr) == 0){
// increment the count if we are ignoring annotations,
// both LPs are unannotated, or if the annotations match
if (!annotation_wise || cmp_anno(anno, lpt->anno)){
if (!annotation_wise || cmp_anno(anno, lpt->anno.ptr)){
lp_count += lpt->count;
}
}
......@@ -232,12 +233,12 @@ int codes_mapping_get_lp_relative_id(
// count up LPs within my group occuring before me
// (the loop is necessary because different annotated LPs may exist in the
// same group)
uint64_t lp_count = 0;
uint64_t lp_pre_count = 0;
int lp_count = 0;
int lp_pre_count = 0;
for (int l = 0; l < lpconf.lpgroups[group_index].lptypes_count; l++){
const config_lptype_t *lpt = &lpconf.lpgroups[group_index].lptypes[l];
if (strcmp(local_lp_name, lpt->name) == 0){
if (!annotation_wise || cmp_anno(anno, lpt->anno)){
if (strcmp(local_lp_name, lpt->name.ptr) == 0){
if (!annotation_wise || cmp_anno(anno, lpt->anno.ptr)){
lp_count += lpt->count;
// special case: if we find an LP entry that matches, but is not
// the same entry where the input gid comes from, then increment
......@@ -267,15 +268,15 @@ tw_lpid codes_mapping_get_lpid_from_relative(
tw_lpid gid_count = 0;
for (int g = 0; g < lpconf.lpgroups_count; g++){
const config_lpgroup_t *lpg = &lpconf.lpgroups[g];
if (group_name == NULL || strcmp(group_name, lpg->name) == 0){
if (group_name == NULL || strcmp(group_name, lpg->name.ptr) == 0){
// consider this group for counting
tw_lpid local_gid_count = 0;
int local_rel_id_count = 0;
for (int l = 0; l < lpg->lptypes_count; l++){
const config_lptype_t *lpt = &lpg->lptypes[l];
local_gid_count += lpt->count;
if (strcmp(lp_type_name, lpt->name) == 0 &&
(!annotation_wise || cmp_anno(annotation, lpt->anno))){
if (strcmp(lp_type_name, lpt->name.ptr) == 0 &&
(!annotation_wise || cmp_anno(annotation, lpt->anno.ptr))){
local_rel_id_count += lpt->count;
}
}
......@@ -290,9 +291,9 @@ tw_lpid codes_mapping_get_lpid_from_relative(
// count up lps listed prior to this entry
for (int l = 0; l < lpg->lptypes_count; l++){
const config_lptype_t *lpt = &lpg->lptypes[l];
if ( strcmp(lp_type_name, lpt->name) == 0 &&
if ( strcmp(lp_type_name, lpt->name.ptr) == 0 &&
(!annotation_wise ||
cmp_anno(annotation, lpt->anno))){
cmp_anno(annotation, lpt->anno.ptr))){
if (rem < (int) lpt->count){
return gid + (tw_lpid) rem;
}
......@@ -358,7 +359,7 @@ void codes_mapping_get_lp_info(
// we've found the group
tw_lpid rem = gid - id_total;
if (group_name != NULL)
strncpy(group_name, lpg->name, MAX_NAME_LENGTH);
strncpy(group_name, lpg->name.ptr, MAX_NAME_LENGTH);
*group_index = g;
// find repetition within group
*rep_id = (int) (rem / num_id_per_rep);
......@@ -369,9 +370,13 @@ void codes_mapping_get_lp_info(
if (rem < num_id_per_rep + lpt->count){
// found the specific LP
if (lp_type_name != NULL)
strncpy(lp_type_name, lpt->name, MAX_NAME_LENGTH);
if (annotation != NULL)
strncpy(annotation, lpt->anno, MAX_NAME_LENGTH);
strncpy(lp_type_name, lpt->name.ptr, MAX_NAME_LENGTH);
if (annotation != NULL) {
if (lpt->anno.ptr == NULL)
annotation[0] = '\0';
else
strncpy(annotation, lpt->anno.ptr, MAX_NAME_LENGTH);
}
*offset = (int) (rem - num_id_per_rep);
*lp_type_index = l;
return; // done
......@@ -510,16 +515,13 @@ const char* codes_mapping_get_annotation_by_name(
const char * lp_type_name){
for (int g = 0; g < lpconf.lpgroups_count; g++){
const config_lpgroup_t *lpg = &lpconf.lpgroups[g];
if (strcmp(lpg->name, group_name) == 0){
if (strcmp(lpg->name.ptr, group_name) == 0){
// group found, iterate through types
for (int l = 0; l < lpg->lptypes_count; l++){
const config_lptype_t *lpt = &lpg->lptypes[l];
if (strcmp(lpt->name, lp_type_name) == 0){
if (strcmp(lpt->name.ptr, lp_type_name) == 0){
// type found, return the annotation
if (lpt->anno[0] == '\0')
return NULL;
else
return lpt->anno;
return lpt->anno.ptr;
}
}
}
......@@ -533,12 +535,7 @@ const char* codes_mapping_get_annotation_by_lpid(tw_lpid gid){
int group_index, lp_type_index, dummy;
codes_mapping_get_lp_info(gid, NULL, &group_index, NULL, &lp_type_index,
NULL, &dummy, &dummy);
const char * anno =
lpconf.lpgroups[group_index].lptypes[lp_type_index].anno;
if (anno[0] == '\0')
return NULL;
else
return anno;
return lpconf.lpgroups[group_index].lptypes[lp_type_index].anno.ptr;
}
/*
......@@ -549,7 +546,7 @@ const char* codes_mapping_get_annotation_by_lpid(tw_lpid gid){
const config_anno_map_t *
codes_mapping_get_lp_anno_map(const char *lp_name){
for (uint64_t i = 0; i < lpconf.lpannos_count; i++){
if (strcmp(lp_name, lpconf.lpannos[i].lp_name) == 0){
if (strcmp(lp_name, lpconf.lpannos[i].lp_name.ptr) == 0){
return &lpconf.lpannos[i];
}
}
......
......@@ -337,7 +337,7 @@ int lsm_get_num_priorities(
}
else {
for (int i = 0; i < anno_map->num_annos; i++) {
if (strcmp(anno_map->annotations[i], annotation) == 0)
if (strcmp(anno_map->annotations[i].ptr, annotation) == 0)
return models_anno[i].use_sched;
}
assert(0);
......@@ -796,7 +796,7 @@ void lsm_register(void)
}
// read the configuration file for a given annotation
static void read_config(ConfigHandle *ch, char * anno, disk_model_t *model)
static void read_config(ConfigHandle *ch, char const * anno, disk_model_t *model)
{
char **values;
size_t length;
......@@ -915,7 +915,7 @@ void lsm_configure(void)
}
for (uint64_t i = 0; i < anno_map->num_annos; i++){
char * anno = anno_map->annotations[i];
char const * anno = anno_map->annotations[i].ptr;
read_config(&config, anno, &models_anno[i]);
}
}
......
......@@ -440,11 +440,11 @@ void resource_lp_configure(){
}
for (uint64_t i = 0; i < anno_map->num_annos; i++){
ret = configuration_get_value_longint(&config, RESOURCE_LP_NM,
"available", anno_map->annotations[i], &avail);
"available", anno_map->annotations[i].ptr, &avail);
if (ret){
fprintf(stderr,
"Could not find section:resource value:available@%s for "