# -*- coding: utf-8 -*-
"""Developer convenience functions for ibs (detections).
TODO: need to split up into sub modules:
consistency_checks
feasibility_fixes
move the export stuff to dbio
then there are also convineience functions that need to be ordered at least
within this file
"""
import logging
from os.path import expanduser, join, abspath
import numpy as np
import vtool as vt
import utool as ut
import cv2
from wbia.control import controller_inject
import tqdm
# Inject utool functions
(print, rrr, profile) = ut.inject2(__name__, '[other.detectfuncs]')
logger = logging.getLogger('wbia')
SAMPLES = 1000
AP_SAMPLE_POINTS = [_ / float(SAMPLES) for _ in range(0, SAMPLES + 1)]
# Must import class before injection
CLASS_INJECT_KEY, register_ibs_method = controller_inject.make_ibs_register_decorator(
__name__
)
GLOBAL_GT_CACHE_DICT = {}
GLOBAL_PRED_CACHE_DICT = {}
def _resize(image, t_width=None, t_height=None, verbose=False):
if verbose:
logger.info('RESIZING WITH t_width = %r and t_height = %r' % (t_width, t_height))
height, width = image.shape[:2]
if t_width is None and t_height is None:
return image
elif t_width is not None and t_height is not None:
pass
elif t_width is None:
t_width = (width / height) * float(t_height)
elif t_height is None:
t_height = (height / width) * float(t_width)
t_width, t_height = float(t_width), float(t_height)
t_width, t_height = int(np.around(t_width)), int(np.around(t_height))
assert t_width > 0 and t_height > 0, 'target size too small'
assert (
t_width <= width * 10 and t_height <= height * 10
), 'target size too large (capped at 1000%)'
# interpolation = cv2.INTER_LANCZOS4
interpolation = cv2.INTER_LINEAR
return cv2.resize(image, (t_width, t_height), interpolation=interpolation)
[docs]@register_ibs_method
def get_species_nice_mapping(ibs, species):
from wbia.constants import SPECIES_MAPPING
species_code, species_nice = SPECIES_MAPPING.get(species, (species, species))
while species_code is None:
species_code, species_nice = SPECIES_MAPPING[species_nice]
return species_nice
[docs]def simple_code(label):
from wbia.constants import YAWALIAS, SPECIES_MAPPING, PARTS_MAPPING
if label == 'ignore':
return 'IGNORE'
for key in PARTS_MAPPING:
if key in label:
part_code, part_nice = PARTS_MAPPING[key]
while part_code is None:
part_code, part_nice = PARTS_MAPPING[part_nice]
assert part_code is not None
label = label.replace(key, part_code)
for key in SPECIES_MAPPING:
if key in label:
species_code, species_nice = SPECIES_MAPPING[key]
while species_code is None:
species_code, species_nice = SPECIES_MAPPING[species_nice]
assert species_code is not None
label = label.replace(key, species_code)
for key in sorted(YAWALIAS.keys(), key=len, reverse=True):
value = YAWALIAS[key]
label = label.replace(key, value)
return label
##########################################################################################
[docs]def general_precision_recall_algo(
ibs, label_list, confidence_list, category='positive', samples=SAMPLES, **kwargs
):
def errors(zipped, conf, category):
tp, tn, fp, fn = 0.0, 0.0, 0.0, 0.0
for index, (label, confidence) in enumerate(zipped):
if label == category:
if conf <= confidence:
tp += 1
else:
fn += 1
else:
if conf <= confidence:
fp += 1
else:
tn += 1
return tp, tn, fp, fn
zipped = list(zip(label_list, confidence_list))
conf_list = [_ / float(samples) for _ in range(0, int(samples) + 1)]
conf_dict = {}
for conf in conf_list:
conf_dict[conf] = errors(zipped, conf, category)
conf_list_ = [-1.0, -1.0]
pr_list = [1.0, 0.0]
re_list = [0.0, 1.0]
tpr_list = [0.0, 1.0]
fpr_list = [0.0, 1.0]
# conf_list_ = []
# pr_list = []
# re_list = []
# tpr_list = []
# fpr_list = []
for conf in sorted(conf_dict.keys(), reverse=True):
error_list = conf_dict[conf]
tp, tn, fp, fn = error_list
try:
pr = tp / (tp + fp)
re = tp / (tp + fn)
tpr = tp / (tp + fn)
fpr = fp / (fp + tn)
conf_list_.append(conf)
pr_list.append(pr)
re_list.append(re)
tpr_list.append(tpr)
fpr_list.append(fpr)
except ZeroDivisionError:
logger.info(
'Zero division error (%r) - tp: %r tn: %r fp: %r fn: %r'
% (conf, tp, tn, fp, fn)
)
return conf_list_, pr_list, re_list, tpr_list, fpr_list
[docs]def general_interpolate_precision_recall(conf_list, re_list, pr_list):
conf_list_, re_list_, pr_list_ = [], [], []
zipped = zip(re_list, conf_list, pr_list)
zipped = sorted(zipped, reverse=True)
max_pr = None
for re, conf, pr in zipped:
if max_pr is None or pr > max_pr:
if max_pr is not None:
conf_list_.append(np.nan)
re_list_.append(re)
pr_list_.append(max_pr)
max_pr = pr
if pr < max_pr:
pr = max_pr
conf_list_.append(conf)
re_list_.append(re)
pr_list_.append(pr)
return conf_list_, re_list_, pr_list_
[docs]def general_identify_operating_point(conf_list, x_list, y_list, target=(1.0, 1.0)):
best_length = np.inf
best_conf_list = []
best_x_list = []
best_y_list = []
tx, ty = target
for conf, x, y in sorted(zip(conf_list, x_list, y_list)):
x_ = x
y_ = y
x_ = x_ - tx
y_ = y_ - ty
length = np.sqrt(x_ * x_ + y_ * y_)
if length < best_length:
best_length = length
best_conf_list = [conf]
best_x_list = [x]
best_y_list = [y]
elif length == best_length:
flag_list = [abs(best_conf - conf) > 0.01 for best_conf in best_conf_list]
if False in flag_list:
continue
best_conf_list.append(conf)
best_x_list.append(x)
best_y_list.append(y)
return best_conf_list, best_x_list, best_y_list, best_length
[docs]def general_area_best_conf(
conf_list,
x_list,
y_list,
label='Unknown',
color='b',
marker='o',
plot_point=True,
interpolate=True,
target=(1.0, 1.0),
target_recall=None,
**kwargs,
):
import matplotlib.pyplot as plt
zipped = list(sorted(zip(x_list, y_list, conf_list)))
x_list = [_[0] for _ in zipped]
y_list = [_[1] for _ in zipped]
conf_list = [_[2] for _ in zipped]
if interpolate:
conf_list, x_list, y_list = general_interpolate_precision_recall(
conf_list, x_list, y_list
)
if interpolate:
ap_list = []
for AP_POINT in AP_SAMPLE_POINTS:
for re, pr in sorted(zip(x_list, y_list)):
if AP_POINT <= re:
ap_list.append(pr)
break
ap = sum(ap_list) / len(ap_list)
else:
ap = np.trapz(y_list, x=x_list)
tup1 = general_identify_operating_point(conf_list, x_list, y_list, target=target)
best_conf_list, best_x_list, best_y_list, best_length = tup1
tup2 = None
if target_recall is not None:
for x, y, conf in sorted(zip(x_list, y_list, conf_list)):
if target_recall <= x and not np.isnan(conf):
tup2 = [conf], [x], [y], None
break
if len(best_conf_list) > 1:
logger.info(
'WARNING: Multiple best operating points found %r' % (best_conf_list,)
)
assert len(best_conf_list) > 0
best_conf = best_conf_list[0]
if interpolate:
# label = '%s [AP = %0.02f, OP = %0.02f]' % (label, ap * 100.0, best_conf)
label = '%s [AP = %0.02f]' % (label, ap * 100.0)
else:
label = '%s [AUC = %0.02f]' % (label, ap * 100.0)
linestyle = '--' if kwargs.get('line_dotted', False) else '-'
plt.plot(x_list, y_list, color=color, linestyle=linestyle, label=label)
if plot_point:
plt.plot(best_x_list, best_y_list, color=color, marker=marker)
return ap, best_conf, tup1, tup2
[docs]def general_confusion_matrix_algo(
label_correct_list,
label_predict_list,
category_list,
category_mapping,
fig_,
axes_,
fuzzy_dict=None,
conf=None,
conf_list=None,
size=10,
**kwargs,
):
# import matplotlib.colors as colors
import matplotlib.pyplot as plt
suppressed_label = 'SUP'
if conf is not None:
assert conf_list is not None
category_list.append(suppressed_label)
index = len(category_list) - 1
category_mapping[suppressed_label] = index
if fuzzy_dict is not None:
fuzzy_dict[index] = set([])
if category_mapping is not None:
index_list = [category_mapping[category] for category in category_list]
zipped = list(sorted(zip(index_list, category_list)))
category_list = [_[1] for _ in zipped]
# Get the number of categories
num_categories = len(category_list)
# Build the confusion matrix
confusion_matrix = np.zeros((num_categories, num_categories))
zipped = zip(label_correct_list, label_predict_list)
suppressed = 0.0
suppressed_correct = 0.0
suppressed_fuzzy = 0.0
for index, (label_correct, label_predict) in enumerate(zipped):
if conf is not None:
conf_ = conf_list[index]
if conf_ < conf:
if label_correct != label_predict:
suppressed_correct += 1
if fuzzy_dict is not None:
x = category_mapping[label_correct]
y = category_mapping[label_predict]
if not (y in fuzzy_dict[x] or x in fuzzy_dict[y]):
suppressed_fuzzy += 1
label_predict = suppressed_label
suppressed += 1
# Perform any mapping that needs to be done
correct_ = category_mapping[label_correct]
predict_ = category_mapping[label_predict]
# Add to the confidence matrix
confusion_matrix[correct_][predict_] += 1
# Normalize the confusion matrix using the rows
row_normalizer = np.sum(confusion_matrix, axis=1)
confusion_normalized = np.array((confusion_matrix.T / row_normalizer).T)
# Draw the confusion matrix
res = axes_.imshow(confusion_normalized, cmap=plt.cm.jet, interpolation='nearest')
correct = suppressed_correct
fuzzy = suppressed_fuzzy
total = 0.0
for x in range(num_categories):
for y in range(num_categories):
number = int(confusion_matrix[x][y])
if x == y:
correct += number
if fuzzy_dict is not None and (y in fuzzy_dict[x] or x in fuzzy_dict[y]):
fuzzy += number
total += number
axes_.annotate(
str(number),
xy=(y, x),
horizontalalignment='center',
verticalalignment='center',
size=size,
)
cb = fig_.colorbar(res) # NOQA
# cb.set_clim(0.0, 1.0)
plt.xticks(np.arange(num_categories), category_list, rotation=90)
plt.yticks(np.arange(num_categories), category_list)
margin_small = 0.1
margin_large = 0.9
plt.subplots_adjust(
left=margin_small, right=margin_large, bottom=margin_small, top=margin_large
)
correct_rate = correct / total
fuzzy_rate = fuzzy / total
return correct_rate, fuzzy_rate
[docs]def general_intersection_over_union(bbox1, bbox2):
intersection_xtl = max(bbox1['xtl'], bbox2['xtl'])
intersection_ytl = max(bbox1['ytl'], bbox2['ytl'])
intersection_xbr = min(bbox1['xbr'], bbox2['xbr'])
intersection_ybr = min(bbox1['ybr'], bbox2['ybr'])
intersection_w = intersection_xbr - intersection_xtl
intersection_h = intersection_ybr - intersection_ytl
if intersection_w <= 0 or intersection_h <= 0:
return 0.0
intersection = intersection_w * intersection_h
union = (
(bbox1['width'] * bbox1['height'])
+ (bbox2['width'] * bbox2['height'])
- intersection
)
return intersection / union
[docs]def general_overlap(gt_list, pred_list):
overlap = np.zeros((len(gt_list), len(pred_list)), dtype=np.float32)
for i, gt in enumerate(gt_list):
for j, pred in enumerate(pred_list):
overlap[i, j] = general_intersection_over_union(gt, pred)
return overlap
[docs]def general_tp_fp_fn(gt_list, pred_list, min_overlap, **kwargs):
overlap = general_overlap(gt_list, pred_list)
num_gt, num_pred = overlap.shape
if num_gt == 0:
tp = 0.0
fp = num_pred
fn = 0.0
elif num_pred == 0:
tp = 0.0
fp = 0.0
fn = num_gt
else:
pred_index_list = range(num_pred)
gt_index_list = np.argmax(overlap, axis=0)
max_overlap_list = np.max(overlap, axis=0)
confidence_list = [pred.get('confidence', None) for pred in pred_list]
assert None not in confidence_list
zipped = zip(confidence_list, max_overlap_list, pred_index_list, gt_index_list)
pred_conf_list = [
(confidence, max_overlap, pred_index, gt_index)
for confidence, max_overlap, pred_index, gt_index in zipped
]
pred_conf_list = sorted(pred_conf_list, reverse=True)
assignment_dict = {}
for pred_conf, max_overlap, pred_index, gt_index in pred_conf_list:
if max_overlap > min_overlap:
if gt_index not in assignment_dict:
assignment_dict[gt_index] = pred_index
tp = len(assignment_dict.keys())
fp = num_pred - tp
fn = num_gt - tp
assert tp >= 0
assert fp >= 0
assert fn >= 0
return tp, fp, fn
[docs]def general_get_imageset_gids(ibs, imageset_text, unique=True, **kwargs):
imageset_id = ibs.get_imageset_imgsetids_from_text(imageset_text)
test_gid_list = ibs.get_imageset_gids(imageset_id)
if unique:
test_gid_list = list(set(test_gid_list))
return test_gid_list
[docs]def general_parse_gt_annots(
ibs, aid_list, include_parts=True, species_mapping={}, gt_species_mapping={}, **kwargs
):
gid_list = ibs.get_annot_gids(aid_list)
species_mapping_ = species_mapping.copy()
species_mapping_.update(gt_species_mapping)
species_set = set([])
gt_list = []
for gid, aid in zip(gid_list, aid_list):
width, height = ibs.get_image_sizes(gid)
bbox = ibs.get_annot_bboxes(aid)
theta = ibs.get_annot_thetas(aid)
# Transformation matrix
R = vt.rotation_around_bbox_mat3x3(theta, bbox)
# Get verticies of the annotation polygon
verts = vt.verts_from_bbox(bbox, close=True)
# Rotate and transform vertices
xyz_pts = vt.add_homogenous_coordinate(np.array(verts).T)
trans_pts = vt.remove_homogenous_coordinate(R.dot(xyz_pts))
new_verts = np.round(trans_pts).astype(np.int).T.tolist()
x_points = [pt[0] for pt in new_verts]
y_points = [pt[1] for pt in new_verts]
xtl = int(min(x_points))
xbr = int(max(x_points))
ytl = int(min(y_points))
ybr = int(max(y_points))
bbox = (xtl, ytl, xbr - xtl, ybr - ytl)
species = ibs.get_annot_species_texts(aid)
viewpoint = ibs.get_annot_viewpoints(aid)
interest = ibs.get_annot_interest(aid)
temp = {
'gid': gid,
'aid': aid,
'xtl': bbox[0] / width,
'ytl': bbox[1] / height,
'xbr': (bbox[0] + bbox[2]) / width,
'ybr': (bbox[1] + bbox[3]) / height,
'width': bbox[2] / width,
'height': bbox[3] / height,
'class': species_mapping_.get(species, species),
'viewpoint': viewpoint,
'interest': interest,
'confidence': 1.0,
}
species_set.add(temp['class'])
gt_list.append(temp)
part_rowid_list = ibs.get_annot_part_rowids(aid)
if include_parts:
for part_rowid in part_rowid_list:
bbox = ibs.get_part_bboxes(part_rowid)
theta = ibs.get_part_thetas(part_rowid)
# Transformation matrix
R = vt.rotation_around_bbox_mat3x3(theta, bbox)
# Get verticies of the annotation polygon
verts = vt.verts_from_bbox(bbox, close=True)
# Rotate and transform vertices
xyz_pts = vt.add_homogenous_coordinate(np.array(verts).T)
trans_pts = vt.remove_homogenous_coordinate(R.dot(xyz_pts))
new_verts = np.round(trans_pts).astype(np.int).T.tolist()
x_points = [pt[0] for pt in new_verts]
y_points = [pt[1] for pt in new_verts]
xtl = int(min(x_points))
xbr = int(max(x_points))
ytl = int(min(y_points))
ybr = int(max(y_points))
bbox = (xtl, ytl, xbr - xtl, ybr - ytl)
tag = ibs.get_part_tag_text(part_rowid)
if tag is None:
tag = species
else:
tag = '%s+%s' % (species, tag)
temp = {
'gid': gid,
'aid': aid,
'part_id': part_rowid,
'xtl': bbox[0] / width,
'ytl': bbox[1] / height,
'xbr': (bbox[0] + bbox[2]) / width,
'ybr': (bbox[1] + bbox[3]) / height,
'width': bbox[2] / width,
'height': bbox[3] / height,
'class': tag,
'viewpoint': viewpoint,
'interest': interest,
'confidence': 1.0,
}
species_set.add(temp['class'])
gt_list.append(temp)
return gt_list, species_set
[docs]def general_parse_gt(ibs, test_gid_list=None, **kwargs):
global GLOBAL_GT_CACHE_DICT
kwargs_str = ut.repr3(kwargs)
kwargs_cfgstr = ut.hash_data(kwargs_str)
if test_gid_list is None:
test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
uuid_list = ibs.get_image_uuids(test_gid_list)
gid_list = ibs.get_image_gids_from_uuid(uuid_list)
species_set = set([])
gt_dict = {}
for gid, uuid in zip(gid_list, uuid_list):
cache_tag = '%s_%s' % (
uuid,
kwargs_cfgstr,
)
if cache_tag not in GLOBAL_GT_CACHE_DICT:
aid_list = ibs.get_image_aids(gid)
gt_list, species_set = general_parse_gt_annots(ibs, aid_list, **kwargs)
species_set = species_set | species_set
GLOBAL_GT_CACHE_DICT[cache_tag] = gt_list
gt_dict[uuid] = GLOBAL_GT_CACHE_DICT[cache_tag]
# logger.info('General Parse GT species_set = %r' % (species_set, ))
return gt_dict
##########################################################################################
[docs]def localizer_parse_pred_dirty(ibs, test_gid_list, species_mapping_, **kwargs):
depc = ibs.depc_image
uuid_list = ibs.get_image_uuids(test_gid_list)
size_list = ibs.get_image_sizes(test_gid_list)
# Unsure, but we need to call this multiple times? Lazy loading bug?
bboxes_list = depc.get_property(
'localizations', test_gid_list, 'bboxes', config=kwargs
)
# Get actual data
bboxes_list = depc.get_property(
'localizations', test_gid_list, 'bboxes', config=kwargs
)
thetas_list = depc.get_property(
'localizations', test_gid_list, 'thetas', config=kwargs
)
confss_list = depc.get_property(
'localizations', test_gid_list, 'confs', config=kwargs
)
classs_list = depc.get_property(
'localizations', test_gid_list, 'classes', config=kwargs
)
length_list = [len(bbox_list) for bbox_list in bboxes_list]
# Establish primitives
test_gids_list = [
[test_gid] * length for test_gid, length in zip(test_gid_list, length_list)
]
sizes_list = [[size] * length for size, length in zip(size_list, length_list)]
keeps_list = [[True] * length for length in length_list]
features_list = [[None] * length for length in length_list]
features_lazy_list = [[None] * length for length in length_list]
viewpoints_list = [[None] * length for length in length_list]
interests_list = [[None] * length for length in length_list]
# Get features
if kwargs.get('features', False):
features_list = depc.get_property(
'localizations_features', test_gid_list, 'vector', config=kwargs
)
if kwargs.get('features_lazy', False):
from functools import partial
def features_lazy_func(gid, offset):
vector_list = depc.get_property(
'localizations_features', gid, 'vector', config=kwargs
)
vector = vector_list[offset]
return vector
features_lazy_list = [
[
partial(features_lazy_func, test_gid, test_offset)
for test_offset in range(length)
]
for test_gid, length in zip(test_gid_list, length_list)
]
# Get species and viewpoints labels
if kwargs.get('labels', False):
classs_list = depc.get_property(
'localizations_labeler', test_gid_list, 'species', config=kwargs
)
viewpoints_list = depc.get_property(
'localizations_labeler', test_gid_list, 'viewpoint', config=kwargs
)
# Get updated confidences for boxes
if kwargs.get('classify', False):
logger.info('Using alternate classifications')
# depc.delete_property('localizations_classifier', test_gid_list, config=kwargs)
confss_list = depc.get_property(
'localizations_classifier', test_gid_list, 'score', config=kwargs
)
# Get updated confidences for boxes
if kwargs.get('interest', False):
logger.info('Using alternate AoI interest flags')
interests_list = depc.get_property(
'localizations_classifier', test_gid_list, 'score', config=kwargs
)
# Reformat results for json
zipped_list_list = zip(
keeps_list,
test_gids_list,
sizes_list,
bboxes_list,
thetas_list,
confss_list,
classs_list,
viewpoints_list,
interests_list,
features_list,
features_lazy_list,
)
results_list = [
[
{
'gid': test_gid,
'xtl': bbox[0] / width,
'ytl': bbox[1] / height,
'xbr': (bbox[0] + bbox[2]) / width,
'ybr': (bbox[1] + bbox[3]) / height,
'width': bbox[2] / width,
'height': bbox[3] / height,
'theta': theta,
'confidence': conf,
'class': species_mapping_.get(class_, class_),
'viewpoint': viewpoint,
'interest': None if interest is None else interest >= 0.84,
'feature': feature,
'feature_lazy': feature_lazy,
}
for keep_, test_gid, (
width,
height,
), bbox, theta, conf, class_, viewpoint, interest, feature, feature_lazy in zip(
*zipped_list
)
if keep_
]
for zipped_list in zipped_list_list
]
pred_dict = {
uuid_: result_list for uuid_, result_list in zip(uuid_list, results_list)
}
return pred_dict
[docs]def localizer_parse_pred(
ibs, test_gid_list=None, species_mapping={}, pred_species_mapping={}, **kwargs
):
global GLOBAL_PRED_CACHE_DICT
if 'feature2_algo' not in kwargs:
kwargs['feature2_algo'] = 'resnet'
species_mapping_ = species_mapping.copy()
species_mapping_.update(pred_species_mapping)
kwargs_str = ut.repr3([species_mapping_, kwargs])
kwargs_cfgstr = ut.hash_data(kwargs_str)
if test_gid_list is None:
test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
uuid_list = ibs.get_image_uuids(test_gid_list)
gid_list = ibs.get_image_gids_from_uuid(uuid_list)
dirty_gids = []
for gid, uuid in zip(gid_list, uuid_list):
cache_tag = '%s_%s' % (
uuid,
kwargs_cfgstr,
)
if cache_tag not in GLOBAL_PRED_CACHE_DICT:
dirty_gids.append(gid)
pred_dict_dirty = localizer_parse_pred_dirty(
ibs, dirty_gids, species_mapping_=species_mapping_, **kwargs
)
for dirty_uuid in pred_dict_dirty:
cache_tag = '%s_%s' % (
dirty_uuid,
kwargs_cfgstr,
)
GLOBAL_PRED_CACHE_DICT[cache_tag] = pred_dict_dirty[dirty_uuid]
pred_dict = {}
for gid, uuid in zip(gid_list, uuid_list):
cache_tag = '%s_%s' % (
uuid,
kwargs_cfgstr,
)
assert cache_tag in GLOBAL_PRED_CACHE_DICT
pred_dict[uuid] = GLOBAL_PRED_CACHE_DICT[cache_tag]
return pred_dict
[docs]def localizer_precision_recall_algo(ibs, samples=SAMPLES, test_gid_list=None, **kwargs):
if test_gid_list is None:
test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
test_uuid_list = ibs.get_image_uuids(test_gid_list)
logger.info('\tGather Ground-Truth')
gt_dict = general_parse_gt(ibs, test_gid_list=test_gid_list, **kwargs)
logger.info('\tGather Predictions')
pred_dict = localizer_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)
species_set = kwargs.get('species_set', None)
if species_set is not None:
# filter out any prefix ! to denote interest only
species_set_ = set([species.lstrip('!') for species in species_set])
dict_list = [
(gt_dict, 'Ground-Truth'),
(pred_dict, 'Predictions'),
]
for dict_, dict_tag in dict_list:
for image_uuid in dict_:
dict_[image_uuid] = [
val
for val in dict_[image_uuid]
if val.get('class', None) in species_set_
]
values = localizer_tp_fp(test_uuid_list, gt_dict, pred_dict, **kwargs)
conf_list, tp_list, fp_list, total = values
conf_list_ = [-1.0, -1.0]
pr_list = [1.0, 0.0]
re_list = [0.0, 1.0]
for conf, tp, fp in zip(conf_list, tp_list, fp_list):
try:
pr = tp / (tp + fp)
re = tp / total
except ZeroDivisionError:
continue
conf_list_.append(conf)
pr_list.append(pr)
re_list.append(re)
return conf_list_, pr_list, re_list
[docs]def localizer_assign(gt_list, pred, min_overlap):
best_overlap = min_overlap
best_index = None
for index, gt in enumerate(gt_list):
if gt['class'] != pred['class']:
continue
overlap = general_intersection_over_union(gt, pred)
if overlap < best_overlap:
continue
best_overlap = overlap
best_index = index
if best_index is None:
best_overlap = None
return best_index, best_overlap
[docs]def localizer_assignments(pred_list, gt_list, gt_list_=[], min_overlap=0.5):
pred_list = sorted(pred_list, key=lambda pred: pred['confidence'], reverse=True)
index_list = list(range(len(gt_list)))
match_list = []
for pred in pred_list:
flag = False
match_index, best_overlap = localizer_assign(gt_list, pred, min_overlap)
match_index_, best_overlap_ = localizer_assign(gt_list_, pred, min_overlap)
if match_index is not None:
flag = True
del gt_list[match_index]
elif match_index_ is not None:
flag = None
match_index_external = (
None if match_index is None else index_list.pop(match_index)
)
if flag is not None:
match_list += [(pred['confidence'], flag, match_index_external, best_overlap)]
return match_list
[docs]def localizer_tp_fp(uuid_list, gt_dict, pred_dict, min_overlap=0.5, **kwargs):
total = 0.0
interest_species_set = set([])
species_set = kwargs.get('species_set', None)
if species_set is not None:
for species in species_set:
if species.startswith('!'):
species = species.lstrip('!')
interest_species_set.add(species)
match_list = []
for image_uuid in uuid_list:
gt_list = []
gt_list_ = []
pred_list = pred_dict[image_uuid]
for gt in gt_dict[image_uuid]:
species = gt['class']
interest = gt['interest']
if species in interest_species_set and not interest:
gt_list_.append(gt)
else:
gt_list.append(gt)
total += len(gt_list)
# Match predictions
match_list_ = localizer_assignments(pred_list, gt_list, gt_list_, min_overlap)
for match_ in match_list_:
match_list.append(match_)
# sort matches by confidence from high to low
match_list = sorted(match_list, key=lambda match: match[0], reverse=True)
conf_list = []
tp_list = []
fp_list = []
tp_counter = 0
fp_counter = 0
for conf, flag, index, overlap in match_list:
if flag:
tp_counter += 1
else:
fp_counter += 1
conf_list.append(conf)
tp_list.append(tp_counter)
fp_list.append(fp_counter)
# logger.info('\t tps [:10] : %r' % (tp_list[:10], ))
# logger.info('\t fps [:10] : %r' % (fp_list[:10], ))
# logger.info('\t con [:10] : %r' % (conf_list[:10], ))
# logger.info('\t tps [-10:] : %r' % (tp_list[-10:], ))
# logger.info('\t fps [-10:] : %r' % (fp_list[-10:], ))
# logger.info('\t con [-10:] : %r' % (conf_list[-10:], ))
# logger.info('\t num_annotations: %r' % (total, ))
return conf_list, tp_list, fp_list, total
[docs]def localizer_precision_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
logger.info('Processing Precision-Recall for: %r' % (label,))
conf_list, pr_list, re_list = localizer_precision_recall_algo(ibs, **kwargs)
return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
def _ignore_filter_identity_func(*args, **kwargs):
return False
[docs]def localizer_iou_recall_algo(
ibs, samples=100, test_gid_list=None, ignore_filter_func=None, **kwargs
):
assert 'min_overlap' not in kwargs
if test_gid_list is None:
test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
test_uuid_list = ibs.get_image_uuids(test_gid_list)
if ignore_filter_func is None:
ignore_filter_func = _ignore_filter_identity_func
logger.info('\tGather Ground-Truth')
gt_dict = general_parse_gt(ibs, test_gid_list=test_gid_list, **kwargs)
logger.info('\tGather Predictions')
pred_dict = localizer_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)
species_set = kwargs.get('species_set', None)
if species_set is not None:
# filter out any prefix ! to denote interest only
species_set_ = set([species.lstrip('!') for species in species_set])
dict_list = [
(gt_dict, 'Ground-Truth'),
(pred_dict, 'Predictions'),
]
for dict_, dict_tag in dict_list:
for image_uuid in dict_:
temp = []
for val in dict_[image_uuid]:
if val.get('class', None) not in species_set_:
continue
if ignore_filter_func(ibs, val):
continue
temp.append(val)
dict_[image_uuid] = temp
target = (1.0, 1.0)
iou_list = [_ / float(samples) for _ in range(0, int(samples) + 1)]
conf_list_ = []
iou_list_ = []
recall_list = []
for iou in tqdm.tqdm(iou_list):
values = localizer_tp_fp(
test_uuid_list, gt_dict, pred_dict, min_overlap=iou, **kwargs
)
conf_list, tp_list, fp_list, total = values
conf_list_ = []
pr_list = []
re_list = []
for conf, tp, fp in zip(conf_list, tp_list, fp_list):
try:
pr = tp / (tp + fp)
re = tp / total
except ZeroDivisionError:
continue
conf_list_.append(conf)
pr_list.append(pr)
re_list.append(re)
best_tup = general_identify_operating_point(
conf_list, re_list, pr_list, target=target
)
best_conf_list, best_re_list, best_pr_list, best_length = best_tup
if len(best_conf_list) > 1:
logger.info(
'WARNING: Multiple best operating points found %r' % (best_conf_list,)
)
assert len(best_conf_list) > 0
best_re_index = np.argmax(best_re_list)
best_re = best_re_list[best_re_index]
best_conf = best_conf_list[best_re_index]
conf_list_.append(best_conf)
iou_list_.append(iou)
recall_list.append(best_re)
return conf_list_, iou_list_, recall_list
[docs]def localizer_iou_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
logger.info('Processing IoU-Recall for: %r' % (label,))
conf_list, iou_list, recall_list = localizer_iou_recall_algo(ibs, **kwargs)
return general_area_best_conf(
conf_list, iou_list, recall_list, interpolate=False, **kwargs
)
# def localizer_iou_precision_algo_plot(ibs, **kwargs):
# label = kwargs['label']
# logger.info('Processing Precision-Recall for: %r' % (label, ))
# conf_list, iou_list, pr_list, re_list = localizer_iou_precision_recall_algo(ibs, **kwargs)
# return general_area_best_conf(conf_list, iou_list, re_list, **kwargs)
[docs]def localizer_confusion_matrix_algo_plot(
ibs, label=None, target_conf=None, test_gid_list=None, **kwargs
):
if test_gid_list is None:
test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
test_uuid_list = ibs.get_image_uuids(test_gid_list)
logger.info('\tGather Ground-Truth')
gt_dict = general_parse_gt(ibs, test_gid_list=test_gid_list, **kwargs)
logger.info('\tGather Predictions')
pred_dict = localizer_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)
species_set = kwargs.get('species_set', None)
if species_set is not None:
# filter out any prefix ! to denote interest only
species_set_ = set([species.lstrip('!') for species in species_set])
dict_list = [
(gt_dict, 'Ground-Truth'),
(pred_dict, 'Predictions'),
]
for dict_, dict_tag in dict_list:
for image_uuid in dict_:
dict_[image_uuid] = [
val
for val in dict_[image_uuid]
if val.get('class', None) in species_set_
]
values = localizer_tp_fp(test_uuid_list, gt_dict, pred_dict, **kwargs)
conf_list, tp_list, fp_list, total = values
best_conf = None
best_accuracy = None
best_args = None
for conf, tp, fp in sorted(zip(conf_list, tp_list, fp_list)):
fn = total - tp
accuracy = tp / (tp + fp + fn)
if target_conf is None:
if best_accuracy is None or accuracy > best_accuracy:
best_conf = conf
best_accuracy = accuracy
best_args = (tp, fp, fn)
else:
if target_conf <= conf:
best_conf = conf
best_accuracy = accuracy
best_args = (tp, fp, fn)
break
try:
assert None not in [best_conf, best_accuracy, best_args]
except AssertionError:
ut.embed()
return np.nan, (np.nan, None)
logger.info(
'Processing Confusion Matrix for: %r (Conf = %0.02f, Accuracy = %0.02f)'
% (label, best_conf, best_accuracy)
)
tp, fp, fn = best_args
label_list = []
prediction_list = []
for _ in range(int(tp)):
label_list.append('positive')
prediction_list.append('positive')
for _ in range(int(fp)):
label_list.append('negative')
prediction_list.append('positive')
for _ in range(int(fn)):
label_list.append('positive')
prediction_list.append('negative')
category_list = ['positive', 'negative']
category_mapping = {
'positive': 0,
'negative': 1,
}
values = general_confusion_matrix_algo(
label_list, prediction_list, category_list, category_mapping, size=20, **kwargs
)
return best_conf, values
[docs]@register_ibs_method
def localizer_precision_recall(
ibs, config_dict=None, output_path=None, test_gid_list=None, **kwargs
):
if config_dict is None:
if test_gid_list is not None:
logger.info('Using %d test gids' % (len(test_gid_list),))
# species_mapping = { # NOQA
# 'giraffe_masai' : 'giraffe',
# 'giraffe_reticulated' : 'giraffe',
# 'zebra_grevys' : 'zebra',
# 'zebra_plains' : 'zebra',
# }
config_dict = {
# 'seaturtle': (
# [
# {'label': 'Sea Turtle', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_green', 'turtle_hawksbill'])},
# {'label': 'Sea Turtle Heads', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_green+head', 'turtle_hawksbill+head'])},
# {'label': 'Green', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_green'])},
# {'label': 'Green Heads', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_green+head'])},
# {'label': 'Hawksbill', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill Heads', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_hawksbill+head'])},
# ],
# {'BEST_INDEX': 0},
# ),
# '!seaturtle': (
# [
# {'label': '! Sea Turtle', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_green', '!turtle_hawksbill'])},
# {'label': '! Sea Turtle Heads', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_green+head', '!turtle_hawksbill+head'])},
# {'label': '! Green', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_green'])},
# {'label': '! Green Heads', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_green+head'])},
# {'label': '! Hawksbill', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_hawksbill'])},
# {'label': '! Hawksbill Heads', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_hawksbill+head'])},
# ],
# {'BEST_INDEX': 0},
# ),
# 'hawksbills': (
# [
# {'label': 'Hawksbill NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['turtle_hawksbill'])},
# ],
# {},
# ),
# 'hawsbills+heads': (
# [
# {'label': 'Hawksbill Head NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['turtle_hawksbill+head'])},
# ],
# {},
# ),
# 'hammerhead': (
# [
# {'label': 'Hammerhead NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['shark_hammerhead'])},
# ],
# {},
# ),
# '!hammerhead': (
# [
# {'label': 'Hammerhead NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!shark_hammerhead'])},
# ],
# {'offset_color': 1},
# ),
# 'ggr2-giraffe-lightnet': (
# [
# {'label': 'Giraffe NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# ],
# {},
# ),
# 'ggr2-zebra-lightnet': (
# [
# {'label': 'Zebra NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# ],
# {},
# ),
# 'ggr2-!giraffe-lightnet': (
# [
# {'label': 'Giraffe ! NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# ],
# {},
# ),
# 'ggr2-!zebra-lightnet': (
# [
# {'label': 'Zebra ! NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# ],
# {},
# ),
# 'ggr2-giraffe-azure': (
# [
# {'label': 'Giraffe NMS 0%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 10%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 20%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 30%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 40%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 50%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 60%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 70%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 80%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 90%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 100%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# ],
# {},
# ),
# 'ggr2-zebra-azure': (
# [
# {'label': 'Zebra NMS 0%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 10%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 20%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 30%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 40%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 50%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 60%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 70%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 80%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 90%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 100%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# ],
# {},
# ),
# 'ggr2-!giraffe-azure': (
# [
# {'label': 'Giraffe ! NMS 0%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 10%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 20%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 30%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 40%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 50%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 60%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 70%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 80%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 90%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 100%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# ],
# {},
# ),
# 'ggr2-!zebra-azure': (
# [
# {'label': 'Zebra ! NMS 0%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 10%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 20%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 30%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 40%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 50%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 60%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 70%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 80%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 90%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 100%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# ],
# {},
# ),
# 'lynx': (
# [
# {'label': 'Lynx NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['lynx'])},
# ],
# {},
# ),
# 'jaguar': (
# [
# {'label': 'Jaguar NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['jaguar'])},
# ],
# {},
# ),
# '!jaguar': (
# [
# {'label': 'Jaguar NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!jaguar'])},
# ],
# {},
# ),
# 'manta': (
# [
# {'label': 'Manta NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['manta_ray_giant'])},
# ],
# {},
# ),
# '!manta': (
# [
# {'label': 'Manta NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!manta_ray_giant'])},
# ],
# {},
# ),
# 'giraffe': (
# [
# {'label': 'Giraffe NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
# {'label': 'Giraffe NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
# {'label': 'Giraffe NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
# {'label': 'Giraffe NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
# {'label': 'Giraffe NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
# {'label': 'Masai Giraffe NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['giraffe_masai'])},
# {'label': 'Masai Giraffe NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['giraffe_masai'])},
# {'label': 'Masai Giraffe NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['giraffe_masai'])},
# {'label': 'Masai Giraffe NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['giraffe_masai'])},
# {'label': 'Masai Giraffe NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['giraffe_masai'])},
# {'label': 'Reticulated Giraffe NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['giraffe_reticulated'])},
# {'label': 'Reticulated Giraffe NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['giraffe_reticulated'])},
# {'label': 'Reticulated Giraffe NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['giraffe_reticulated'])},
# {'label': 'Reticulated Giraffe NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['giraffe_reticulated'])},
# {'label': 'Reticulated Giraffe NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['giraffe_reticulated'])},
# ],
# {},
# ),
# 'spotted_skunk_v0': (
# [
# {'label': 'Spotted Skunk NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['skunk_spotted'])},
# ],
# {},
# ),
# '!spotted_skunk_v0': (
# [
# {'label': 'Spotted Skunk NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!skunk_spotted'])},
# ],
# {},
# ),
# 'nassau_grouper_v0': (
# [
# {'label': 'Nassau Grouper NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['grouper_nassau'])},
# ],
# {},
# ),
# '!nassau_grouper_v0': (
# [
# {'label': 'Nassau Grouper! NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!grouper_nassau'])},
# ],
# {},
# ),
# 'spotted_dolphin_v0': (
# [
# {'label': 'Spotted DolphinNMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['dolphin_spotted'])},
# ],
# {},
# ),
# '!spotted_dolphin_v0': (
# [
# {'label': 'Spotted Dolphin! NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!dolphin_spotted'])},
# ],
# {},
# ),
'seadragon_weedy_v1': (
[
{
'label': 'Weedy Body NMS 0%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.00,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 10%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.10,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 20%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.20,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 30%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.30,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 40%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.40,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 50%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.50,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 60%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.60,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 70%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.70,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 80%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.80,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 90%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.90,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 100%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 1.00,
'species_set': set(['seadragon_leafy']),
},
],
{},
),
'seadragon_leafy_v1': (
[
{
'label': 'Leafy Body NMS 0%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.00,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 10%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.10,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 20%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.20,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 30%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.30,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 40%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.40,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 50%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.50,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 60%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.60,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 70%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.70,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 80%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.80,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 90%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.90,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 100%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 1.00,
'species_set': set(['seadragon_weedy']),
},
],
{},
),
'seadragon_weedy_head_v1': (
[
{
'label': 'Weedy Head NMS 0%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.00,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 10%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.10,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 20%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.20,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 30%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.30,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 40%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.40,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 50%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.50,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 60%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.60,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 70%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.70,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 80%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.80,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 90%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.90,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 100%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 1.00,
'species_set': set(['seadragon_leafy+head']),
},
],
{},
),
'seadragon_leafy_head_v1': (
[
{
'label': 'Leafy Head NMS 0%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.00,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 10%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.10,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 20%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.20,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 30%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.30,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 40%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.40,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 50%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.50,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 60%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.60,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 70%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.70,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 80%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.80,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 90%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.90,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 100%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 1.00,
'species_set': set(['seadragon_weedy+head']),
},
],
{},
),
}
for config_key in config_dict:
config_list, config = config_dict[config_key]
for key in kwargs:
config[key] = kwargs[key]
# Backwards compatibility hack
if test_gid_list is not None:
for config_ in config_list:
if 'test_gid_list' not in config_:
config_['test_gid_list'] = test_gid_list
ibs.localizer_precision_recall_algo_display(
config_list, config_tag=config_key, output_path=output_path, **config
)
[docs]@register_ibs_method
def localizer_precision_recall_algo_display(
ibs,
config_list,
config_tag='',
min_overlap=0.5,
figsize=(40, 9),
target_recall=0.8,
BEST_INDEX=None,
offset_color=0,
write_images=False,
plot_point=True,
output_path=None,
plot_iou_recall=True,
**kwargs,
):
import matplotlib.pyplot as plt
import wbia.plottool as pt
if output_path is None:
output_path = abspath(expanduser(join('~', 'Desktop')))
color_list_ = []
for _ in range(offset_color):
color_list_ += [(0.2, 0.2, 0.2)]
color_list = pt.distinct_colors(len(config_list) - len(color_list_), randomize=False)
color_list = color_list_ + color_list
fig_ = plt.figure(figsize=figsize, dpi=400)
######################################################################################
axes_ = plt.subplot(141)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('Recall (Ground-Truth IOU >= %0.02f)' % (min_overlap,))
axes_.set_ylabel('Precision')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
localizer_precision_recall_algo_plot(
ibs,
color=color,
min_overlap=min_overlap,
plot_point=plot_point,
target_recall=target_recall,
**config,
)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
tup2_list = [ret[3] for ret in ret_list]
best_index = (
None if BEST_INDEX is None else BEST_INDEX
) # Match formatting of below, this is a silly conditional
best_y = 0.0
best_index_ = None
valid_best_index = []
for index, tup2 in enumerate(tup2_list):
if tup2 is None:
continue
conf_list, x_list, y_list, length = tup2
y = y_list[0]
if best_y < y:
valid_best_index.append(index)
best_index_ = index
best_y = y
# If user defined best_index is invalid, don't use it
if best_index is None:
best_index = best_index_
else:
if best_index not in valid_best_index:
best_index = None
if best_index is not None:
best_conf_list, best_x_list, best_y_list, best_length = tup2_list[best_index]
color = 'xkcd:gold'
marker = 'D'
plt.plot(best_x_list, best_y_list, color=color, marker=marker)
plt.title('Precision-Recall Curves', y=1.19)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
######################################################################################
if plot_iou_recall:
axes_ = plt.subplot(142)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('IOU (Intersection / Union)')
axes_.set_ylabel('Recall')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
localizer_iou_recall_algo_plot(ibs, color=color_, plot_point=False, **config_)
for color_, config_ in zip(color_list, config_list)
]
# area_list = [ ret[0] for ret in ret_list ]
# tup2_list = [ ret[3] for ret in ret_list ]
# best_index = None if BEST_INDEX is None else BEST_INDEX # Match formatting of below, this is a silly conditional
# best_y = 0.0
# best_index_ = None
# valid_best_index = []
# for index, tup2 in enumerate(tup2_list):
# if tup2 is None:
# continue
# conf_list, x_list, y_list, length = tup2
# y = y_list[0]
# if best_y < y:
# valid_best_index.append(index)
# best_index_ = index
# best_y = y
# # If user defined best_index is invalid, don't use it
# if best_index is None:
# best_index = best_index_
# else:
# if best_index not in valid_best_index:
# best_index = None
# if best_index is not None:
# best_conf_list, best_x_list, best_y_list, best_length = tup2_list[best_index]
# color = 'xkcd:gold'
# marker = 'D'
# plt.plot(best_x_list, best_y_list, color=color, marker=marker)
plt.title('Recall-IOU Curves', y=1.19)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
######################################################################################
# axes_ = plt.subplot(153)
# axes_.set_autoscalex_on(False)
# axes_.set_autoscaley_on(False)
# axes_.set_xlabel('IOU (Intersection / Union)')
# axes_.set_ylabel('Precision')
# axes_.set_xlim([0.0, 1.01])
# axes_.set_ylim([0.0, 1.01])
# ret_list = [
# localizer_iou_precision_algo_plot(ibs, color=color_, plot_point=False, **config_)
# for color_, config_ in zip(color_list, config_list)
# ]
# plt.title('Precision-IOU Curves', y=1.19)
# plt.legend(bbox_to_anchor=(0.0, 1.02, 1.0, .102), loc=3, ncol=2, mode="expand",
# borderaxespad=0.0)
######################################################################################
if best_index is not None:
axes_ = plt.subplot(144)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
target_conf = best_conf_list[0]
best_config = config_list[best_index]
best_label = config_list[best_index]['label']
best_area = area_list[best_index]
values = localizer_confusion_matrix_algo_plot(
ibs,
min_overlap=min_overlap,
fig_=fig_,
axes_=axes_,
target_conf=target_conf,
**best_config,
)
best_conf, (correct_rate, _) = values
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
args = (
target_recall,
best_label,
best_area,
best_conf,
)
plt.title(
'Confusion Matrix for Recall >= %0.02f\n(Algo: %s, mAP = %0.02f, OP = %0.02f)'
% args,
y=1.26,
)
######################################################################################
axes_ = plt.subplot(143)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
best_index = np.argmax(area_list) if BEST_INDEX is None else BEST_INDEX
best_config = config_list[best_index]
best_label = config_list[best_index]['label']
best_area = area_list[best_index]
values = localizer_confusion_matrix_algo_plot(
ibs, min_overlap=min_overlap, fig_=fig_, axes_=axes_, **best_config
)
best_conf, (correct_rate, _) = values
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
args = (
best_label,
best_area,
best_conf,
)
plt.title('Confusion Matrix\n(Algo: %s, mAP = %0.02f, OP = %0.02f)' % args, y=1.26)
######################################################################################
if len(config_tag) > 0:
config_tag = '%s-' % (config_tag,)
fig_filename = '%slocalizer-precision-recall-%0.2f.png' % (config_tag, min_overlap)
fig_path = join(output_path, fig_filename)
plt.savefig(fig_path, bbox_inches='tight')
return fig_path
[docs]@register_ibs_method
def localizer_precision_recall_algo_display_animate(ibs, config_list, **kwargs):
for value in range(10):
min_overlap = value / 10.0
logger.info('Processing: %r' % (min_overlap,))
ibs.localizer_precision_recall_algo_display(
config_list, min_overlap=min_overlap, **kwargs
)
# def localizer_classification_tp_tn_fp_fn(gt_list, pred_list, conf, min_overlap,
# check_species=False,
# check_viewpoint=False, **kwargs):
# overlap = general_overlap(gt_list, pred_list)
# num_gt, num_pred = overlap.shape
# # Get confidences
# conf_list = [pred['confidence'] for pred in pred_list]
# pred_flag_list = [conf <= conf_ for conf_ in conf_list]
# if num_gt == 0:
# tp_list = [False] * len(pred_list)
# tn_list = [not pred_flag for pred_flag in pred_flag_list]
# fp_list = [ pred_flag for pred_flag in pred_flag_list]
# fn_list = [False] * len(pred_list)
# elif num_pred == 0:
# tp_list = []
# tn_list = []
# fp_list = []
# fn_list = []
# else:
# max_overlap = np.max(overlap, axis=0)
# gt_flag_list = min_overlap < max_overlap
# status_list = []
# for gt_flag, pred_flag in zip(gt_flag_list, pred_flag_list):
# if gt_flag and pred_flag:
# status_list.append('tp')
# elif gt_flag and not pred_flag:
# status_list.append('fn')
# elif not gt_flag and pred_flag:
# status_list.append('fp')
# elif not gt_flag and not pred_flag:
# status_list.append('tn')
# else:
# raise ValueError
# tp_list = [status == 'tp' for status in status_list]
# tn_list = [status == 'tn' for status in status_list]
# fp_list = [status == 'fp' for status in status_list]
# fn_list = [status == 'fn' for status in status_list]
# return tp_list, tn_list, fp_list, fn_list
# def localizer_classification_confusion_matrix_algo_plot(ibs, color, conf,
# label=None,
# min_overlap=0.25,
# write_images=False,
# **kwargs):
# logger.info('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf, ))
# test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
# test_uuid_list = ibs.get_image_uuids(test_gid_list)
# logger.info('\tGather Ground-Truth')
# gt_dict = general_parse_gt(ibs, test_gid_list=test_gid_list, **kwargs)
# logger.info('\tGather Predictions')
# pred_dict = localizer_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)
# if write_images:
# output_folder = 'localizer-classification-confusion-matrix-%0.2f-%0.2f-images' % (min_overlap, conf, )
# output_path = abspath(expanduser(join('~', 'Desktop', output_folder)))
# ut.ensuredir(output_path)
# label_list = []
# prediction_list = []
# for index, (test_gid, test_uuid) in enumerate(zip(test_gid_list, test_uuid_list)):
# if test_uuid in pred_dict:
# gt_list = gt_dict[test_uuid]
# pred_list = pred_dict[test_uuid]
# values = localizer_classification_tp_tn_fp_fn(gt_list, pred_list, conf,
# min_overlap=min_overlap,
# **kwargs)
# tp_list, tn_list, fp_list, fn_list = values
# tp = tp_list.count(True)
# tn = tn_list.count(True)
# fp = fp_list.count(True)
# fn = fn_list.count(True)
# for _ in range(int(tp)):
# label_list.append('positive')
# prediction_list.append('positive')
# for _ in range(int(tn)):
# label_list.append('negative')
# prediction_list.append('negative')
# for _ in range(int(fp)):
# label_list.append('negative')
# prediction_list.append('positive')
# for _ in range(int(fn)):
# label_list.append('positive')
# prediction_list.append('negative')
# if write_images:
# test_image = ibs.get_images(test_gid)
# test_image = _resize(test_image, t_width=600, verbose=False)
# height_, width_, channels_ = test_image.shape
# for gt in gt_list:
# xtl = int(gt['xtl'] * width_)
# ytl = int(gt['ytl'] * height_)
# xbr = int(gt['xbr'] * width_)
# ybr = int(gt['ybr'] * height_)
# cv2.rectangle(test_image, (xtl, ytl), (xbr, ybr), (0, 0, 255))
# zipped = zip(pred_list, tp_list, tn_list, fp_list, fn_list)
# for pred, tp_, tn_, fp_, fn_ in zipped:
# if tp_:
# color = (0, 255, 0)
# elif fp_:
# continue
# # color = (255, 0, 0)
# elif fn_:
# color = (255, 0, 0)
# elif tn_:
# continue
# else:
# continue
# xtl = int(pred['xtl'] * width_)
# ytl = int(pred['ytl'] * height_)
# xbr = int(pred['xbr'] * width_)
# ybr = int(pred['ybr'] * height_)
# cv2.rectangle(test_image, (xtl, ytl), (xbr, ybr), color)
# status_str = 'success' if (fp + fn) == 0 else 'failure'
# status_val = tp - fp - fn
# args = (status_str, status_val, test_gid, tp, fp, fn, )
# output_filename = 'test_%s_%d_gid_%d_tp_%d_fp_%d_fn_%d.png' % args
# output_filepath = join(output_path, output_filename)
# cv2.imwrite(output_filepath, test_image)
# category_list = ['positive', 'negative']
# category_mapping = {
# 'positive': 0,
# 'negative': 1,
# }
# return general_confusion_matrix_algo(label_list, prediction_list, category_list,
# category_mapping, size=20, **kwargs)
# @register_ibs_method
# def localizer_classifications_confusion_matrix_algo_display(ibs, conf,
# min_overlap=0.25,
# figsize=(24, 7),
# write_images=False,
# target_recall=0.9,
# plot_point=True,
# masking=False,
# **kwargs):
# import matplotlib.pyplot as plt
# fig_ = plt.figure(figsize=figsize)
# config = {
# 'label' : 'WIC',
# 'algo' : '_COMBINED',
# 'species_set' : set(['zebra']),
# 'classify' : True,
# 'classifier_algo': 'svm',
# 'classifier_masking': masking,
# 'classifier_weight_filepath': '/home/jason/code/wbia/models-bootstrap/classifier.svm.image.zebra.pkl',
# }
# axes_ = plt.subplot(111)
# axes_.set_aspect(1)
# gca_ = plt.gca()
# gca_.grid(False)
# correct_rate, _ = localizer_classification_confusion_matrix_algo_plot(ibs, None, conf,
# min_overlap=min_overlap,
# write_images=write_images,
# fig_=fig_, axes_=axes_,
# **config)
# axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
# axes_.set_ylabel('Ground-Truth')
# args = (min_overlap, conf, )
# plt.title('Confusion Matrix (IoU %0.02f, Conf %0.02f)' % args, y=1.13)
# # plt.show()
# args = (min_overlap, conf, )
# fig_filename = 'localizer-classification-confusion-matrix-%0.2f-%0.2f.png' % args
# fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
# plt.savefig(fig_path, bbox_inches='tight')
# @register_ibs_method
# def localizer_classifications_confusion_matrix_algo_display_animate(ibs, total=10, **kwargs):
# for index in range(0, total + 1):
# conf = index / total
# ibs.localizer_classifications_confusion_matrix_algo_display(conf, **kwargs)
[docs]def classifier_cameratrap_precision_recall_algo(
ibs, positive_imageset_id, negative_imageset_id, **kwargs
):
depc = ibs.depc_image
test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_set_ = list(test_gid_set_)
positive_gid_set = set(ibs.get_imageset_gids(positive_imageset_id))
negative_gid_set = set(ibs.get_imageset_gids(negative_imageset_id))
test_gid_set = []
label_list = []
for gid in test_gid_set_:
if gid in positive_gid_set:
label = 'positive'
elif gid in negative_gid_set:
label = 'negative'
else:
# label = 'unknown'
continue
test_gid_set.append(gid)
label_list.append(label)
prediction_list = depc.get_property(
'classifier', test_gid_set, 'class', config=kwargs
)
confidence_list = depc.get_property(
'classifier', test_gid_set, 'score', config=kwargs
)
confidence_list = [
confidence if prediction == 'positive' else 1.0 - confidence
for prediction, confidence in zip(prediction_list, confidence_list)
]
return general_precision_recall_algo(ibs, label_list, confidence_list)
[docs]def classifier_cameratrap_precision_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
logger.info('Processing Precision-Recall for: %r' % (label,))
(
conf_list,
pr_list,
re_list,
tpr_list,
fpr_list,
) = classifier_cameratrap_precision_recall_algo(ibs, **kwargs)
return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
[docs]def classifier_cameratrap_roc_algo_plot(ibs, **kwargs):
label = kwargs['label']
logger.info('Processing ROC for: %r' % (label,))
(
conf_list,
pr_list,
re_list,
tpr_list,
fpr_list,
) = classifier_cameratrap_precision_recall_algo(ibs, **kwargs)
return general_area_best_conf(
conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
)
[docs]def classifier_cameratrap_confusion_matrix_algo_plot(
ibs,
label,
color,
conf,
positive_imageset_id,
negative_imageset_id,
output_cases=False,
**kwargs,
):
logger.info('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf))
depc = ibs.depc_image
test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_set_ = list(test_gid_set_)
positive_gid_set = set(ibs.get_imageset_gids(positive_imageset_id))
negative_gid_set = set(ibs.get_imageset_gids(negative_imageset_id))
test_gid_set = []
label_list = []
for gid in test_gid_set_:
if gid in positive_gid_set:
label = 'positive'
elif gid in negative_gid_set:
label = 'negative'
else:
# label = 'unknown'
continue
test_gid_set.append(gid)
label_list.append(label)
prediction_list = depc.get_property(
'classifier', test_gid_set, 'class', config=kwargs
)
confidence_list = depc.get_property(
'classifier', test_gid_set, 'score', config=kwargs
)
confidence_list = [
confidence if prediction == 'positive' else 1.0 - confidence
for prediction, confidence in zip(prediction_list, confidence_list)
]
prediction_list = [
'positive' if confidence >= conf else 'negative' for confidence in confidence_list
]
if output_cases:
output_path = 'cameratrap-confusion-incorrect'
output_path = abspath(expanduser(join('~', 'Desktop', output_path)))
positive_path = join(output_path, 'positive')
negative_path = join(output_path, 'negative')
ut.delete(output_path)
ut.ensuredir(output_path)
ut.ensuredir(positive_path)
ut.ensuredir(negative_path)
interpolation = cv2.INTER_LANCZOS4
warpkw = dict(interpolation=interpolation)
for gid, label, prediction in zip(test_gid_set, label_list, prediction_list):
if label == prediction:
continue
image = ibs.get_images(gid)
image = cv2.resize(image, (192, 192), **warpkw)
# Get path
image_path = positive_path if label == 'positive' else negative_path
image_filename = 'hardidx_%d_pred_%s_case_fail.jpg' % (gid, prediction)
image_filepath = join(image_path, image_filename)
# Save path
cv2.imwrite(image_filepath, image)
category_list = ['positive', 'negative']
category_mapping = {
'positive': 0,
'negative': 1,
}
return general_confusion_matrix_algo(
label_list, prediction_list, category_list, category_mapping, **kwargs
)
[docs]@register_ibs_method
def classifier_cameratrap_precision_recall_algo_display(
ibs, positive_imageset_id, negative_imageset_id, config_list=None, figsize=(20, 20)
):
import matplotlib.pyplot as plt
import wbia.plottool as pt
fig_ = plt.figure(figsize=figsize, dpi=400)
if config_list is None:
config_list = [
# {'label': 'Initial Model (5%) - IBEIS_CNN', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'ryan.wbia_cnn.v1'},
# {'label': 'Initial Model (5%) - DenseNet', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v1'},
# {'label': 'Initial Model (5%) - DenseNet 0', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v1:0'},
# {'label': 'Initial Model (5%) - DenseNet 1', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v1:1'},
# {'label': 'Initial Model (5%) - DenseNet 2', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v1:2'},
{
'label': 'Initial Model (10%) - DenseNet',
'classifier_algo': 'densenet',
'classifier_weight_filepath': 'ryan_densenet_v2',
},
# {'label': 'Initial Model (10%) - DenseNet 0', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v2:0'},
# {'label': 'Initial Model (10%) - DenseNet 1', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v2:1'},
# {'label': 'Initial Model (10%) - DenseNet 2', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v2:2'},
# {'label': 'Initial Model (0%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.1'},
# {'label': 'Retrained Model (1%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.2'},
# {'label': 'Retrained Model (2%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.3'},
# {'label': 'Retrained Model (3%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.4'},
# {'label': 'Retrained Model (4%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.5'},
# {'label': 'Retrained Model (5%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.6'},
# {'label': 'Initial Model (0%)', 'classifier_weight_filepath': 'megan1.1'},
# {'label': 'Retrained Model (1%)', 'classifier_weight_filepath': 'megan1.2'},
# {'label': 'Retrained Model (2%)', 'classifier_weight_filepath': 'megan1.3'},
# {'label': 'Retrained Model (3%)', 'classifier_weight_filepath': 'megan1.4'},
# {'label': 'Retrained Model (3.5%)', 'classifier_weight_filepath': 'megan1.5'},
# {'label': 'Retrained Model (5%)', 'classifier_weight_filepath': 'megan1.6'},
]
color_list = pt.distinct_colors(len(config_list), randomize=False)
axes_ = plt.subplot(221)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('Recall')
axes_.set_ylabel('Precision')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
classifier_cameratrap_precision_recall_algo_plot(
ibs,
color=color,
positive_imageset_id=positive_imageset_id,
negative_imageset_id=negative_imageset_id,
**config,
)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
conf_list = [ret[1] for ret in ret_list]
index = np.argmax(area_list)
# index = 0
best_label1 = config_list[index]['label']
best_config1 = config_list[index]
best_color1 = color_list[index]
best_area1 = area_list[index]
best_conf1 = conf_list[index]
plt.title(
'Precision-Recall Curve (Best: %s, AP = %0.02f)' % (best_label1, best_area1),
y=1.10,
)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(222)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('False-Positive Rate')
axes_.set_ylabel('True-Positive Rate')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
classifier_cameratrap_roc_algo_plot(
ibs,
color=color,
positive_imageset_id=positive_imageset_id,
negative_imageset_id=negative_imageset_id,
**config,
)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
conf_list = [ret[1] for ret in ret_list]
index = np.argmax(area_list)
# index = 0
best_label2 = config_list[index]['label']
best_config2 = config_list[index]
best_color2 = color_list[index]
best_area2 = area_list[index]
best_conf2 = conf_list[index]
plt.title('ROC Curve (Best: %s, AP = %0.02f)' % (best_label2, best_area2), y=1.10)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(223)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, _ = classifier_cameratrap_confusion_matrix_algo_plot(
ibs,
color=best_color1,
conf=best_conf1,
fig_=fig_,
axes_=axes_,
positive_imageset_id=positive_imageset_id,
negative_imageset_id=negative_imageset_id,
output_cases=True,
**best_config1,
)
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
plt.title('P-R Confusion Matrix (OP = %0.02f)' % (best_conf1,), y=1.12)
axes_ = plt.subplot(224)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, _ = classifier_cameratrap_confusion_matrix_algo_plot(
ibs,
color=best_color2,
conf=best_conf2,
fig_=fig_,
axes_=axes_,
positive_imageset_id=positive_imageset_id,
negative_imageset_id=negative_imageset_id,
**best_config2,
)
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
plt.title('ROC Confusion Matrix (OP = %0.02f)' % (best_conf2,), y=1.12)
fig_filename = 'classifier-cameratrap-precision-recall-roc.png'
fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
plt.savefig(fig_path, bbox_inches='tight')
# def classifier_binary_precision_recall_algo(ibs, category_set, **kwargs):
# depc = ibs.depc_image
# test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
# test_gid_set = list(test_gid_set)
# aids_list = ibs.get_image_aids(test_gid_set)
# species_set_list = [
# set(ibs.get_annot_species_texts(aid_list))
# for aid_list in aids_list
# ]
# label_list = [
# 'negative' if len(species_set & category_set) == 0 else 'positive'
# for species_set in species_set_list
# ]
# prediction_list = depc.get_property('classifier', test_gid_set, 'class', config=kwargs)
# confidence_list = depc.get_property('classifier', test_gid_set, 'score', config=kwargs)
# confidence_list = [
# confidence if prediction == 'positive' else 1.0 - confidence
# for prediction, confidence in zip(prediction_list, confidence_list)
# ]
# return general_precision_recall_algo(ibs, label_list, confidence_list)
# def classifier_binary_precision_recall_algo_plot(ibs, **kwargs):
# label = kwargs['label']
# logger.info('Processing Precision-Recall for: %r' % (label, ))
# conf_list, pr_list, re_list, tpr_list, fpr_list = classifier_binary_precision_recall_algo(ibs, **kwargs)
# return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
# def classifier_binary_roc_algo_plot(ibs, **kwargs):
# label = kwargs['label']
# logger.info('Processing ROC for: %r' % (label, ))
# conf_list, pr_list, re_list, tpr_list, fpr_list = classifier_binary_precision_recall_algo(ibs, **kwargs)
# return general_area_best_conf(conf_list, fpr_list, tpr_list, interpolate=False,
# target=(0.0, 1.0), **kwargs)
# def classifier_binary_confusion_matrix_algo_plot(ibs, label, color, conf, category_set, **kwargs):
# logger.info('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf, ))
# depc = ibs.depc_image
# test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
# test_gid_set = list(test_gid_set)
# aids_list = ibs.get_image_aids(test_gid_set)
# species_set_list = [
# set(ibs.get_annot_species_texts(aid_list))
# for aid_list in aids_list
# ]
# label_list = [
# 'negative' if len(species_set & category_set) == 0 else 'positive'
# for species_set in species_set_list
# ]
# prediction_list = depc.get_property('classifier', test_gid_set, 'class', config=kwargs)
# confidence_list = depc.get_property('classifier', test_gid_set, 'score', config=kwargs)
# confidence_list = [
# confidence if prediction == 'positive' else 1.0 - confidence
# for prediction, confidence in zip(prediction_list, confidence_list)
# ]
# prediction_list = [
# 'positive' if confidence >= conf else 'negative'
# for confidence in confidence_list
# ]
# category_list = ['positive', 'negative']
# category_mapping = {
# 'positive': 0,
# 'negative': 1,
# }
# return general_confusion_matrix_algo(label_list, prediction_list, category_list,
# category_mapping, **kwargs)
# @register_ibs_method
# def classifier_binary_precision_recall_algo_display(ibs, figsize=(16, 16), **kwargs):
# import matplotlib.pyplot as plt
# fig_ = plt.figure(figsize=figsize)
# # label = 'V1'
# # species_list = ['zebra']
# # kwargs['classifier_weight_filepath'] = 'coco_zebra'
# label = 'V3'
# species_list = ['zebra_plains', 'zebra_grevys']
# kwargs['classifier_weight_filepath'] = 'v3_zebra'
# category_set = set(species_list)
# axes_ = plt.subplot(221)
# axes_.set_autoscalex_on(False)
# axes_.set_autoscaley_on(False)
# axes_.set_xlabel('Recall')
# axes_.set_ylabel('Precision')
# axes_.set_xlim([0.0, 1.01])
# axes_.set_ylim([0.0, 1.01])
# area, best_conf1, _ = classifier_binary_precision_recall_algo_plot(ibs, label=label, color='r', category_set=category_set, **kwargs)
# plt.title('Precision-Recall Curve (AP = %0.02f)' % (area, ), y=1.10)
# plt.legend(bbox_to_anchor=(0.0, 1.02, 1.0, .102), loc=3, ncol=2, mode="expand",
# borderaxespad=0.0)
# axes_ = plt.subplot(222)
# axes_.set_autoscalex_on(False)
# axes_.set_autoscaley_on(False)
# axes_.set_xlabel('False-Positive Rate')
# axes_.set_ylabel('True-Positive Rate')
# axes_.set_xlim([0.0, 1.01])
# axes_.set_ylim([0.0, 1.01])
# area, best_conf2, _ = classifier_binary_roc_algo_plot(ibs, label=label, color='r', category_set=category_set, **kwargs)
# plt.title('ROC Curve (AP = %0.02f)' % (area, ), y=1.10)
# plt.legend(bbox_to_anchor=(0.0, 1.02, 1.0, .102), loc=3, ncol=2, mode="expand",
# borderaxespad=0.0)
# axes_ = plt.subplot(223)
# axes_.set_aspect(1)
# gca_ = plt.gca()
# gca_.grid(False)
# correct_rate, _ = classifier_binary_confusion_matrix_algo_plot(ibs, label, 'r', conf=best_conf1, fig_=fig_, axes_=axes_, category_set=category_set, **kwargs)
# axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
# axes_.set_ylabel('Ground-Truth')
# plt.title('P-R Confusion Matrix (OP = %0.02f)' % (best_conf1, ), y=1.12)
# axes_ = plt.subplot(224)
# axes_.set_aspect(1)
# gca_ = plt.gca()
# gca_.grid(False)
# correct_rate, _ = classifier_binary_confusion_matrix_algo_plot(ibs, label, 'r', conf=best_conf2, fig_=fig_, axes_=axes_, category_set=category_set, **kwargs)
# axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
# axes_.set_ylabel('Ground-Truth')
# plt.title('ROC Confusion Matrix (OP = %0.02f)' % (best_conf2, ), y=1.12)
# fig_filename = 'classifier-precision-recall-roc.png'
# fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
# plt.savefig(fig_path, bbox_inches='tight')
[docs]def classifier2_precision_recall_algo(
ibs,
category,
species_mapping={},
output_path=None,
test_gid_list=None,
test_label_list=None,
**kwargs,
):
depc = ibs.depc_image
if test_gid_list is None:
test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_list = list(test_gid_set)
if test_label_list is None:
aids_list = ibs.get_image_aids(test_gid_list)
species_list_list = list(map(ibs.get_annot_species_texts, aids_list))
species_set_list = []
for species_list in species_list_list:
species_set = set([])
for species in species_list:
species = species_mapping.get(species, species)
species_set.add(species)
species_set_list.append(species_set)
else:
species_set_list = [set([label]) for label in test_label_list]
label_list = [
'positive' if category in species_set_ else 'negative'
for species_set_ in species_set_list
]
confidence_dict_list = depc.get_property(
'classifier_two', test_gid_list, 'scores', config=kwargs
)
confidence_list = [
confidence_dict[category] for confidence_dict in confidence_dict_list
]
if output_path is not None:
ut.ensuredir(output_path)
config_ = {
'draw_annots': False,
'thumbsize': (192, 192),
}
thumbnail_list = depc.get_property(
'thumbnails', test_gid_list, 'img', config=config_
)
zipped = zip(
test_gid_list, thumbnail_list, species_set_list, confidence_dict_list
)
for index, (test_gid, thumbnail, species_set, confidence_dict) in enumerate(
zipped
):
logger.info(index)
x = ';'.join(species_set)
y = []
for key in confidence_dict:
y.append('%s-%0.04f' % (key, confidence_dict[key]))
y = ';'.join(y)
output_filename = 'image-index-%s-gid-%s-gt-%s-pred-%s.png' % (
index,
test_gid,
x,
y,
)
output_filepath = join(output_path, output_filename)
cv2.imwrite(output_filepath, thumbnail)
kwargs.pop('category', None)
return general_precision_recall_algo(ibs, label_list, confidence_list)
[docs]def classifier2_precision_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
logger.info('Processing Precision-Recall for: %r' % (label,))
conf_list, pr_list, re_list, tpr_list, fpr_list = classifier2_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
[docs]def classifier2_roc_algo_plot(ibs, **kwargs):
label = kwargs['label']
logger.info('Processing ROC for: %r' % (label,))
conf_list, pr_list, re_list, tpr_list, fpr_list = classifier2_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(
conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
)
[docs]@register_ibs_method
def classifier2_precision_recall_algo_display(
ibs,
species_list=None,
species_mapping={},
nice_mapping={},
test_gid_list=None,
test_label_list=None,
figsize=(20, 9),
**kwargs,
):
import matplotlib.pyplot as plt
import wbia.plottool as pt
depc = ibs.depc_image
fig_ = plt.figure(figsize=figsize, dpi=400) # NOQA
# kwargs['classifier_two_weight_filepath'] = 'v3'
# kwargs['classifier_two_weight_filepath'] = 'candidacy'
# kwargs['classifier_two_weight_filepath'] = 'ggr2'
is_labeled = test_label_list is not None
kwargs['classifier_two_algo'] = 'densenet'
kwargs['classifier_two_weight_filepath'] = 'flukebook_v1'
test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_list_ = list(test_gid_set) if test_gid_list is None else test_gid_list
test_label_list_ = test_label_list if is_labeled else [None] * len(test_gid_list_)
zipped = list(zip(test_gid_list_, test_label_list_))
test_gid_list_ = []
test_label_list_ = []
for test_gid_, test_label_ in zipped:
if test_gid_ in test_gid_set:
test_gid_list_.append(test_gid_)
test_label_list_.append(test_label_)
test_gid_list = test_gid_list_
test_label_list = test_label_list_ if is_labeled else None
# depc.delete_property('classifier_two', test_gid_list, config=kwargs)
if species_list is None:
test_gid = test_gid_list[0]
confidence_dict = depc.get_property(
'classifier_two', test_gid, 'scores', config=kwargs
)
species_list = confidence_dict.keys()
category_set = sorted(species_list)
config_list = []
for category in category_set:
category_nice = nice_mapping.get(category, category)
config_dict = {
'label': category_nice,
'category': category,
}
config_dict.update(kwargs)
config_list.append(config_dict)
color_list_ = []
color_list = pt.distinct_colors(len(config_list) - len(color_list_), randomize=False)
color_list = color_list_ + color_list
axes_ = plt.subplot(121)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('Recall')
axes_.set_ylabel('Precision')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
for color, config in zip(color_list, config_list):
classifier2_precision_recall_algo_plot(
ibs,
color=color,
test_gid_list=test_gid_list,
test_label_list=test_label_list,
species_mapping=species_mapping,
**config,
)
plt.title('Precision-Recall Curves', y=1.19)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(122)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('False-Positive Rate')
axes_.set_ylabel('True-Positive Rate')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
op_dict = {}
for color, config in zip(color_list, config_list):
values = classifier2_roc_algo_plot(
ibs,
color=color,
test_gid_list=test_gid_list,
test_label_list=test_label_list,
species_mapping=species_mapping,
**config,
)
ap, best_conf, tup1, tup2 = values
op_dict[config['category']] = best_conf
plt.title('ROC Curves', y=1.19)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
if is_labeled:
species_set_list = [set([label]) for label in test_label_list]
else:
aids_list = ibs.get_image_aids(test_gid_list)
species_list_list = list(map(ibs.get_annot_species_texts, aids_list))
species_set_list = []
for species_list in species_list_list:
species_set = set([])
for species in species_list:
species = species_mapping.get(species, species)
species_set.add(species)
species_set_list.append(species_set)
confidence_dict_list = depc.get_property(
'classifier_two', test_gid_list, 'scores', config=kwargs
)
correct = 0
for test_gid, confidence_dict, species_set in zip(
test_gid_list, confidence_dict_list, species_set_list
):
species_set_ = set([])
for key in confidence_dict:
if op_dict[key] <= confidence_dict[key]:
species_set_.add(key)
if len(species_set ^ species_set_) == 0:
correct += 1
else:
logger.info(test_gid, confidence_dict, species_set)
logger.info('Accuracy: %0.04f' % (100.0 * correct / len(test_gid_list)))
logger.info('\t using op_dict = %r' % (op_dict,))
fig_filename = 'classifier2-precision-recall-roc.png'
fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
plt.savefig(fig_path, bbox_inches='tight')
[docs]def labeler_tp_tn_fp_fn(
ibs,
category_list,
species_mapping={},
viewpoint_mapping={},
samples=SAMPLES,
test_gid_set=None,
**kwargs,
):
def errors(zipped, conf, category):
tp, tn, fp, fn = 0.0, 0.0, 0.0, 0.0
for index, (label, confidence) in enumerate(zipped):
if label == category:
if conf <= confidence:
tp += 1
else:
fn += 1
else:
if conf <= confidence:
fp += 1
else:
tn += 1
return tp, tn, fp, fn
depc = ibs.depc_annot
if test_gid_set is None:
test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_set = list(test_gid_set)
aids_list = ibs.get_image_aids(test_gid_set)
aid_list = ut.flatten(aids_list)
# Get annot species and viewpoints
species_list = ibs.get_annot_species_texts(aid_list)
viewpoint_list = ibs.get_annot_viewpoints(aid_list)
# Filter aids with species of interest and undefined viewpoints
species_list = [species_mapping.get(species, species) for species in species_list]
viewpoint_list = [
viewpoint_mapping.get(species, {}).get(viewpoint, viewpoint)
for species, viewpoint in zip(species_list, viewpoint_list)
]
flag_list = [
species in category_list and viewpoint is not None
for species, viewpoint in zip(species_list, viewpoint_list)
]
if False in flag_list:
aid_list = ut.compress(aid_list, flag_list)
species_list = ut.compress(species_list, flag_list)
viewpoint_list = ut.compress(viewpoint_list, flag_list)
# Make ground-truth
label_list = [
'%s:%s' % (species, viewpoint_)
for species, viewpoint_ in zip(species_list, viewpoint_list)
]
# Get predictions
# depc.delete_property('labeler', aid_list, config=kwargs)
probability_dict_list = depc.get_property('labeler', aid_list, 'probs', config=kwargs)
value1_list = set(label_list)
value2_list = set(probability_dict_list[0].keys())
print(value1_list)
print(value2_list)
assert len(value1_list - value2_list) == 0
assert len(value2_list - value1_list) == 0
conf_list = [_ / float(samples) for _ in range(0, int(samples) + 1)]
label_dict = {}
for key in value1_list:
logger.info('\t%r' % (key,))
conf_dict = {}
confidence_list = [
probability_dict[key] for probability_dict in probability_dict_list
]
zipped = list(zip(label_list, confidence_list))
for conf in conf_list:
conf_dict[conf] = errors(zipped, conf, key)
label_dict[key] = conf_dict
return label_dict
[docs]def labeler_precision_recall_algo(ibs, category_list, label_dict, **kwargs):
if category_list is None:
category_list_ = label_dict.keys()
else:
category_list_ = []
for category in category_list:
for key in label_dict:
if category in key or category is None:
category_list_.append(key)
global_conf_dict = {}
for category in category_list_:
conf_dict = label_dict[category]
for conf in conf_dict:
new_list = conf_dict[conf]
if conf not in global_conf_dict:
global_conf_dict[conf] = new_list
else:
cur_list = global_conf_dict[conf]
zipped_ = zip(cur_list, new_list)
global_conf_dict[conf] = [cur + new for cur, new in zipped_]
conf_list_ = [-1.0, -1.0]
pr_list = [1.0, 0.0]
re_list = [0.0, 1.0]
tpr_list = [0.0, 1.0]
fpr_list = [0.0, 1.0]
# conf_list_ = []
# pr_list = []
# re_list = []
# tpr_list = []
# fpr_list = []
for conf in sorted(global_conf_dict.keys(), reverse=True):
error_list = global_conf_dict[conf]
tp, tn, fp, fn = error_list
try:
pr = tp / (tp + fp)
re = tp / (tp + fn)
tpr = tp / (tp + fn)
fpr = fp / (fp + tn)
conf_list_.append(conf)
pr_list.append(pr)
re_list.append(re)
tpr_list.append(tpr)
fpr_list.append(fpr)
except ZeroDivisionError:
logger.info(
'Zero division error (%r) - tp: %r tn: %r fp: %r fn: %r'
% (conf, tp, tn, fp, fn)
)
return conf_list_, pr_list, re_list, tpr_list, fpr_list
[docs]def labeler_precision_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
category_list = kwargs['category_list']
logger.info(
'Processing Precision-Recall for: %r (category_list = %r)'
% (label, category_list)
)
conf_list, pr_list, re_list, tpr_list, fpr_list = labeler_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
[docs]def labeler_roc_algo_plot(ibs, **kwargs):
label = kwargs['label']
category_list = kwargs['category_list']
logger.info('Processing ROC for: %r (category_list = %r)' % (label, category_list))
conf_list, pr_list, re_list, tpr_list, fpr_list = labeler_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(
conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
)
[docs]def labeler_confusion_matrix_algo_plot(
ibs,
category_list,
species_mapping={},
viewpoint_mapping={},
category_mapping=None,
test_gid_set=None,
**kwargs,
):
logger.info('Processing Confusion Matrix')
depc = ibs.depc_annot
if test_gid_set is None:
test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_set = list(test_gid_set)
aids_list = ibs.get_image_aids(test_gid_set)
aid_list = ut.flatten(aids_list)
species_list = ibs.get_annot_species_texts(aid_list)
viewpoint_list = ibs.get_annot_viewpoints(aid_list)
label_list = [
'%s:%s'
% (
species_mapping.get(species, species),
viewpoint_mapping.get(species, {}).get(viewpoint, viewpoint),
)
for species, viewpoint in zip(species_list, viewpoint_list)
]
temp_list = [
(aid, label) for aid, label in zip(aid_list, label_list) if label in category_list
]
aid_list = [_[0] for _ in temp_list]
label_list = [_[1] for _ in temp_list]
conf_list = depc.get_property('labeler', aid_list, 'score', config=kwargs)
species_list = depc.get_property('labeler', aid_list, 'species', config=kwargs)
viewpoint_list = depc.get_property('labeler', aid_list, 'viewpoint', config=kwargs)
prediction_list = [
'%s:%s' % (species, viewpoint)
for species, viewpoint in zip(species_list, viewpoint_list)
]
category_list = list(map(simple_code, category_list))
label_list = list(map(simple_code, label_list))
prediction_list = list(map(simple_code, prediction_list))
if category_mapping is None:
category_mapping = {key: index for index, key in enumerate(category_list)}
category_mapping = {
simple_code(key): category_mapping[key] for key in category_mapping
}
return general_confusion_matrix_algo(
label_list,
prediction_list,
category_list,
category_mapping,
conf_list=conf_list,
size=8,
**kwargs,
)
[docs]@register_ibs_method
def labeler_precision_recall_algo_display(
ibs,
category_list=None,
species_mapping={},
viewpoint_mapping={},
category_mapping=None,
fuzzy_dict=None,
figsize=(30, 9),
test_gid_set=None,
use_axis_aligned_chips=False,
labeler_weight_filepath=None,
config_list=None,
**kwargs,
):
import matplotlib.pyplot as plt
from wbia import plottool as pt
if category_list is None:
if test_gid_set is None:
test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_set = list(test_gid_set)
aids_list = ibs.get_image_aids(test_gid_set)
aid_list = ut.flatten(aids_list)
species_list = ibs.get_annot_species_texts(aid_list)
species_list = [species_mapping.get(species, species) for species in species_list]
category_list = sorted(list(set(species_list)))
logger.info('Compiling raw numbers...')
kwargs['labeler_algo'] = 'densenet'
if labeler_weight_filepath is None:
# kwargs['labeler_weight_filepath'] = 'zebra_v1'
# kwargs['labeler_weight_filepath'] = 'seaturtle'
# kwargs['labeler_weight_filepath'] = 'giraffe_v1'
# kwargs['labeler_weight_filepath'] = 'lynx_v3'
# kwargs['labeler_weight_filepath'] = 'seaturtle_v3'
# kwargs['labeler_weight_filepath'] = 'jaguar_v3'
# kwargs['labeler_weight_filepath'] = 'hendrik_dorsal_v2'
# kwargs['labeler_weight_filepath'] = 'spotted_skunk_v0'
# kwargs['labeler_weight_filepath'] = 'nassau_grouper_v0'
# kwargs['labeler_weight_filepath'] = 'spotted_dolphin_v0'
# kwargs['labeler_weight_filepath'] = 'seadragon_v1'
kwargs['labeler_weight_filepath'] = 'seadragon_v2'
else:
kwargs['labeler_weight_filepath'] = labeler_weight_filepath
kwargs['labeler_axis_aligned'] = use_axis_aligned_chips
label_dict = labeler_tp_tn_fp_fn(
ibs,
category_list,
species_mapping=species_mapping,
viewpoint_mapping=viewpoint_mapping,
test_gid_set=test_gid_set,
**kwargs,
)
if config_list is None:
config_list = [
# {'label': 'Giraffe', 'category_list': None},
# {'label': 'Masai Giraffe', 'category_list': ['giraffe_masai']},
# {'label': 'Reticulated Giraffe', 'category_list': ['giraffe_reticulated']},
# {'label': 'Lynx', 'category_list': ['lynx_pardinus']},
# {'label': 'Sea Turtle', 'category_list': ['turtle_sea']},
# {'label': 'Sea Turtle Head', 'category_list': ['turtle_sea+head']},
# {'label': 'Manta', 'category_list': ['manta_ray_giant']},
# {'label': 'Jaguar', 'category_list': ['jaguar']},
# {'label': 'Dorsal Fin', 'category_list': ['dolphin_bottlenose_fin']},
# {'label': 'Reticulated Giraffe', 'category_list': ['giraffe_reticulated']},
# {'label': 'Sea Turtle', 'category_list': ['turtle_sea']},
# {'label': 'Whale Fluke', 'category_list': ['whale_fluke']},
# {'label': 'Grevy\'s Zebra', 'category_list': ['zebra_grevys']},
# {'label': 'Plains Zebra', 'category_list': ['zebra_plains']},
# {'label': 'Spotted Skunk', 'category_list': ['skunk_spotted']},
# {'label': 'Nassau Grouper', 'category_list': ['grouper_nassau']},
# {'label': 'Spotted Dolphin', 'category_list': ['dolphin_spotted']},
# {'label': 'Spotted Dolphin', 'category_list': ['dolphin_spotted']},
{'label': 'Weedy SD ', 'category_list': ['seadragon_weedy']},
{'label': 'Weedy Head', 'category_list': ['seadragon_weedy+head']},
{'label': 'Leafy SD ', 'category_list': ['seadragon_leafy']},
{'label': 'Leafy Head', 'category_list': ['seadragon_leafy+head']},
]
color_list = [(0.0, 0.0, 0.0)]
color_list += pt.distinct_colors(len(config_list) - len(color_list), randomize=False)
fig_ = plt.figure(figsize=figsize, dpi=400) # NOQA
axes_ = plt.subplot(131)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('Recall')
axes_.set_ylabel('Precision')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
area_list = []
for color, config in zip(color_list, config_list):
ret = labeler_precision_recall_algo_plot(
ibs, label_dict=label_dict, color=color, **config
)
area = ret[0]
area_list.append(area)
plt.title('Precision-Recall Curve', y=1.19)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(132)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('False-Positive Rate')
axes_.set_ylabel('True-Positive Rate')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
for color, config in zip(color_list, config_list):
labeler_roc_algo_plot(ibs, label_dict=label_dict, color=color, **config)
plt.title('ROC Curve', y=1.19)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
key_list = sorted(label_dict.keys())
fuzzy = fuzzy_dict is not None
if not fuzzy:
fuzzy_dict = {}
for index1, label1 in enumerate(key_list):
if label1 == 'ignore':
fuzzy_list = []
else:
species, viewpoint = label1.strip().split(':')
fuzzy_list = []
for index2, label2 in enumerate(key_list):
if species in label2:
fuzzy_list.append(index2)
fuzzy_dict[index1] = set(fuzzy_list)
axes_ = plt.subplot(133)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, fuzzy_rate = labeler_confusion_matrix_algo_plot(
ibs,
key_list,
species_mapping=species_mapping,
viewpoint_mapping=viewpoint_mapping,
category_mapping=category_mapping,
fig_=fig_,
axes_=axes_,
fuzzy_dict=fuzzy_dict,
test_gid_set=test_gid_set,
**kwargs,
)
if fuzzy:
axes_.set_xlabel(
'Predicted (Correct = %0.02f%%, Fuzzy = %0.02f%%)'
% (correct_rate * 100.0, fuzzy_rate * 100.0)
)
else:
axes_.set_xlabel(
'Predicted (Correct = %0.02f%%, Species = %0.02f%%)'
% (correct_rate * 100.0, fuzzy_rate * 100.0)
)
axes_.set_ylabel('Ground-Truth')
# area_list_ = area_list[1:]
area_list_ = area_list
mAP = sum(area_list_) / len(area_list_)
args = (mAP * 100.0,)
plt.title('Confusion Matrix\nmAP = %0.02f' % args, y=1.19)
fig_filename = 'labeler-precision-recall-roc.png'
fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
plt.savefig(fig_path, bbox_inches='tight')
[docs]def canonical_precision_recall_algo(ibs, species, **kwargs):
depc = ibs.depc_annot
test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_list_ = list(test_gid_set_)
test_aid_list_ = ut.flatten(ibs.get_image_aids(test_gid_list_))
test_aid_list_ = ibs.filter_annotation_set(test_aid_list_, species=species)
test_flag_list_ = ibs.get_annot_canonical(test_aid_list_)
test_aid_set = []
label_list = []
for aid, flag in zip(test_aid_list_, test_flag_list_):
if flag:
label = 'positive'
else:
label = 'negative'
test_aid_set.append(aid)
label_list.append(label)
prediction_list = depc.get_property(
'classifier', test_aid_set, 'class', config=kwargs
)
confidence_list = depc.get_property(
'classifier', test_aid_set, 'score', config=kwargs
)
confidence_list = [
confidence if prediction == 'positive' else 1.0 - confidence
for prediction, confidence in zip(prediction_list, confidence_list)
]
return general_precision_recall_algo(ibs, label_list, confidence_list)
[docs]def canonical_precision_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
logger.info('Processing Precision-Recall for: %r' % (label,))
conf_list, pr_list, re_list, tpr_list, fpr_list = canonical_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
[docs]def canonical_roc_algo_plot(ibs, **kwargs):
label = kwargs['label']
logger.info('Processing ROC for: %r' % (label,))
conf_list, pr_list, re_list, tpr_list, fpr_list = canonical_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(
conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
)
[docs]def canonical_confusion_matrix_algo_plot(
ibs, label, color, conf, species, output_cases=False, **kwargs
):
logger.info('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf))
depc = ibs.depc_annot
test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_list_ = list(test_gid_set_)
test_aid_list_ = ut.flatten(ibs.get_image_aids(test_gid_list_))
test_aid_list_ = ibs.filter_annotation_set(test_aid_list_, species=species)
test_flag_list_ = ibs.get_annot_canonical(test_aid_list_)
test_aid_set = []
label_list = []
for aid, flag in zip(test_aid_list_, test_flag_list_):
if flag:
label = 'positive'
else:
label = 'negative'
test_aid_set.append(aid)
label_list.append(label)
prediction_list = depc.get_property(
'classifier', test_aid_set, 'class', config=kwargs
)
confidence_list = depc.get_property(
'classifier', test_aid_set, 'score', config=kwargs
)
confidence_list = [
confidence if prediction == 'positive' else 1.0 - confidence
for prediction, confidence in zip(prediction_list, confidence_list)
]
prediction_list = [
'positive' if confidence >= conf else 'negative' for confidence in confidence_list
]
if output_cases:
output_path = 'canonical-confusion-incorrect'
output_path = abspath(expanduser(join('~', 'Desktop', output_path)))
positive_path = join(output_path, 'positive')
negative_path = join(output_path, 'negative')
ut.delete(output_path)
ut.ensuredir(output_path)
ut.ensuredir(positive_path)
ut.ensuredir(negative_path)
config = {
'dim_size': (192, 192),
'resize_dim': 'wh',
}
chip_list = ibs.depc_annot.get_property(
'chips', test_aid_set, 'img', config=config
)
zipped = zip(test_aid_set, chip_list, label_list, prediction_list)
for aid, chip, label, prediction in zipped:
if label == prediction:
continue
# Get path
image_path = positive_path if label == 'positive' else negative_path
image_filename = 'hardidx_%d_pred_%s_case_fail.jpg' % (aid, prediction)
image_filepath = join(image_path, image_filename)
# Save path
cv2.imwrite(image_filepath, chip)
category_list = ['positive', 'negative']
category_mapping = {
'positive': 0,
'negative': 1,
}
return general_confusion_matrix_algo(
label_list, prediction_list, category_list, category_mapping, **kwargs
)
[docs]@register_ibs_method
def canonical_precision_recall_algo_display(ibs, figsize=(20, 20)):
import matplotlib.pyplot as plt
from wbia import plottool as pt
fig_ = plt.figure(figsize=figsize, dpi=400)
config_list = [
{
'label': 'CA V1 Ensemble',
'classifier_algo': 'densenet',
'classifier_weight_filepath': 'canonical_zebra_grevys_v1',
'species': 'zebra_grevys',
}, # SMALLER DATASET
{
'label': 'CA V2 Ensemble',
'classifier_algo': 'densenet',
'classifier_weight_filepath': 'canonical_zebra_grevys_v2',
'species': 'zebra_grevys',
}, # BROKEN L/R AUGMENTATION
{
'label': 'CA V3 Ensemble',
'classifier_algo': 'densenet',
'classifier_weight_filepath': 'canonical_zebra_grevys_v3',
'species': 'zebra_grevys',
}, # LARGER DATASET, TOO HARSH AUGMENTATION
{
'label': 'CA V4 Ensemble',
'classifier_algo': 'densenet',
'classifier_weight_filepath': 'canonical_zebra_grevys_v4',
'species': 'zebra_grevys',
}, # BETTER AUGMENTATION
# {'label': 'CA V4 Model 0', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'canonical_zebra_grevys_v4:0', 'species': 'zebra_grevys'},
# {'label': 'CA V4 Model 1', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'canonical_zebra_grevys_v4:1', 'species': 'zebra_grevys'},
# {'label': 'CA V4 Model 2', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'canonical_zebra_grevys_v4:2', 'species': 'zebra_grevys'},
]
color_list = []
# color_list = [(0, 0, 0)]
color_list += pt.distinct_colors(len(config_list) - len(color_list), randomize=False)
axes_ = plt.subplot(221)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('Recall')
axes_.set_ylabel('Precision')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
canonical_precision_recall_algo_plot(ibs, color=color, **config)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
conf_list = [ret[1] for ret in ret_list]
# index = np.argmax(area_list)
index = -1
best_label1 = config_list[index]['label']
best_config1 = config_list[index]
best_color1 = color_list[index]
best_area1 = area_list[index]
best_conf1 = conf_list[index]
plt.title(
'Precision-Recall Curve (Best: %s, AP = %0.02f)' % (best_label1, best_area1),
y=1.10,
)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(222)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('False-Positive Rate')
axes_.set_ylabel('True-Positive Rate')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
canonical_roc_algo_plot(ibs, color=color, **config)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
conf_list = [ret[1] for ret in ret_list]
# index = np.argmax(area_list)
index = -1
best_label2 = config_list[index]['label']
best_config2 = config_list[index]
best_color2 = color_list[index]
best_area2 = area_list[index]
best_conf2 = conf_list[index]
plt.title('ROC Curve (Best: %s, AP = %0.02f)' % (best_label2, best_area2), y=1.10)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(223)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, _ = canonical_confusion_matrix_algo_plot(
ibs,
color=best_color1,
conf=best_conf1,
fig_=fig_,
axes_=axes_,
output_cases=True,
**best_config1,
)
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
plt.title('P-R Confusion Matrix (OP = %0.02f)' % (best_conf1,), y=1.12)
axes_ = plt.subplot(224)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, _ = canonical_confusion_matrix_algo_plot(
ibs, color=best_color2, conf=best_conf2, fig_=fig_, axes_=axes_, **best_config2
)
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
plt.title('ROC Confusion Matrix (OP = %0.02f)' % (best_conf2,), y=1.12)
fig_filename = 'canonical-precision-recall-roc.png'
fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
plt.savefig(fig_path, bbox_inches='tight')
def _canonical_get_boxes(ibs, gid_list, species):
from wbia.web.appfuncs import CANONICAL_PART_TYPE
aid_list = ut.flatten(ibs.get_image_aids(gid_list))
aid_list = ibs.filter_annotation_set(aid_list, species=species)
flag_list = ibs.get_annot_canonical(aid_list)
part_rowids_list = ibs.get_annot_part_rowids(aid_list)
part_types_list = list(map(ibs.get_part_types, part_rowids_list))
aid_set = []
bbox_set = []
zipped = zip(aid_list, flag_list, part_rowids_list, part_types_list)
for aid, flag, part_rowid_list, part_type_list in zipped:
part_rowid_ = None
if flag:
for part_rowid, part_type in zip(part_rowid_list, part_type_list):
if part_type == CANONICAL_PART_TYPE:
assert part_rowid_ is None, 'Cannot have multiple CA for one image'
part_rowid_ = part_rowid
if part_rowid_ is not None:
axtl, aytl, aw, ah = ibs.get_annot_bboxes(aid)
axbr, aybr = axtl + aw, aytl + ah
pxtl, pytl, pw, ph = ibs.get_part_bboxes(part_rowid_)
pxbr, pybr = pxtl + pw, pytl + ph
x0 = pxtl - axtl
y0 = pytl - aytl
x1 = axbr - pxbr
y1 = aybr - pybr
x0 = max(x0 / aw, 0.0)
y0 = max(y0 / ah, 0.0)
x1 = max(x1 / aw, 0.0)
y1 = max(y1 / ah, 0.0)
assert x0 + x1 < 0.99
assert y0 + y1 < 0.99
bbox = (x0, y0, x1, y1)
aid_set.append(aid)
bbox_set.append(bbox)
return aid_set, bbox_set
[docs]def canonical_localization_deviation_plot(
ibs, attribute, color, index, label=None, species=None, marker='o', **kwargs
):
import random
import matplotlib.pyplot as plt
assert None not in [label, species]
logger.info('Processing Deviation for: %r' % (label,))
depc = ibs.depc_annot
if attribute == 'x0':
take_index = 0
elif attribute == 'y0':
take_index = 1
elif attribute == 'x1':
take_index = 2
elif attribute == 'y1':
take_index = 3
else:
raise ValueError('attribute not valid')
test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_list_ = list(test_gid_set_)
test_aid_set, test_bbox_set = _canonical_get_boxes(ibs, test_gid_list_, species)
value_list = ut.take_column(test_bbox_set, take_index)
prediction_list = depc.get_property(
'canonical', test_aid_set, attribute, config=kwargs
)
x_list = []
y_list = []
overshoot = 0.0
for value, prediction in zip(value_list, prediction_list):
x = random.uniform(index, index + 1)
y = value - prediction
if y < 0:
overshoot += 1
x_list.append(x)
y_list.append(y)
mean = np.mean(y_list)
std = np.std(y_list)
overshoot /= len(y_list)
label = '%s (Over: %0.02f, %0.02f+/-%0.02f)' % (label, overshoot, mean, std)
plt.plot(
x_list,
y_list,
color=color,
linestyle='None',
marker=marker,
label=label,
alpha=0.5,
)
plt.plot(
[index, index + 1], [0.0, 0.0], color=(0.2, 0.2, 0.2), linestyle='-', alpha=0.3
)
if index % 4 == 3:
plt.plot(
[index + 1, index + 1],
[-1.0, 1.0],
color=(0.2, 0.2, 0.2),
linestyle='--',
alpha=0.1,
)
color = 'xkcd:gold'
marker = 'D'
plt.errorbar(
[index + 0.5],
[mean],
[std],
linestyle='None',
color=color,
marker=marker,
zorder=999,
barsabove=True,
)
# plt.plot([index + 0.5], [mean], color=color, marker=marker)
[docs]def canonical_localization_iou_plot(
ibs, color, index, label=None, species=None, marker='o', threshold=0.75, **kwargs
):
import random
import matplotlib.pyplot as plt
def _convert(bbox):
x0, y0, x1, y1 = bbox
retval = {
'xtl': x0,
'ytl': y0,
'xbr': 1.0 - x1,
'ybr': 1.0 - y1,
}
retval['width'] = retval['xbr'] - retval['xtl']
retval['height'] = retval['ybr'] - retval['ytl']
return retval
assert None not in [label, species]
logger.info('Processing IoU for: %r' % (label,))
depc = ibs.depc_annot
test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_list_ = list(test_gid_set_)
test_aid_set, test_bbox_set = _canonical_get_boxes(ibs, test_gid_list_, species)
prediction_list = depc.get_property('canonical', test_aid_set, None, config=kwargs)
gt_list = [_convert(test_bbox) for test_bbox in test_bbox_set]
pred_list = [_convert(prediction) for prediction in prediction_list]
correct = 0.0
x_list = []
y_list = []
for gt, pred in zip(gt_list, pred_list):
overlap = general_overlap([gt], [pred])
x = random.uniform(index, index + 1)
y = overlap[0][0]
if y >= threshold:
correct += 1.0
x_list.append(x)
y_list.append(y)
accuracy = correct / len(y_list)
mean = np.mean(y_list)
std = np.std(y_list)
label = '%s (Acc: %0.02f, %0.02f+/-%0.02f)' % (label, accuracy, mean, std)
plt.plot(
x_list,
y_list,
color=color,
linestyle='None',
marker=marker,
label=label,
alpha=0.5,
)
for y_value in [0.5, 0.75, 0.9]:
plt.plot(
[index, index + 1],
[y_value, y_value],
color=(0.2, 0.2, 0.2),
linestyle='-',
alpha=0.3,
)
if index % 4 == 3:
plt.plot(
[index + 1, index + 1],
[0.0, 1.0],
color=(0.2, 0.2, 0.2),
linestyle='--',
alpha=0.1,
)
color = 'xkcd:gold'
marker = 'D'
plt.errorbar(
[index + 0.5],
[mean],
[std],
linestyle='None',
color=color,
marker=marker,
zorder=999,
barsabove=True,
)
# plt.plot([index + 0.5], [mean], color=color, marker=marker)
return test_aid_set, test_bbox_set, prediction_list, y_list, accuracy
[docs]@register_ibs_method
def canonical_localization_iou_visualize(
ibs,
index,
test_aid_set,
test_bbox_set,
prediction_list,
overlap_list,
color_list,
label=None,
species=None,
**kwargs,
):
assert None not in [label, species]
assert len(color_list) == 4
logger.info('Processing Renderings for: %r' % (label,))
color_list_ = []
for color in color_list:
color_ = []
for value in color:
value_ = int(np.around(255.0 * value))
color_ = [value_] + color_
color_ = tuple(color_)
color_list_.append(color_)
color_list = color_list_
output_path = expanduser(join('~', 'Desktop', 'canonical-regression-%d' % (index,)))
ut.delete(output_path)
ut.ensuredir(output_path)
config = {
'dim_size': 600,
'resize_dim': 'maxwh',
}
chip_list = ibs.depc_annot.get_property('chips', test_aid_set, 'img', config=config)
zipped = list(
zip(test_aid_set, chip_list, test_bbox_set, prediction_list, overlap_list)
)
for test_aid, chip, test_bbox, prediction, overlap in zipped:
h, w = chip.shape[:2]
chipa = chip.copy()
chipb = chip.copy()
x0a, y0a, x1a, y1a = test_bbox
x0b, y0b, x1b, y1b = prediction
x0a = int(np.around(x0a * w))
y0a = int(np.around(y0a * h))
x1a = int(np.around(x1a * w))
y1a = int(np.around(y1a * h))
x0b = int(np.around(x0b * w))
y0b = int(np.around(y0b * h))
x1b = int(np.around(x1b * w))
y1b = int(np.around(y1b * h))
x1a = w - x1a
x1b = w - x1b
y1a = h - y1a
y1b = h - y1b
chipa = cv2.line(chipa, (x0a, y0a), (x0a, y1a), color_list[0], 3)
chipa = cv2.line(chipa, (x0a, y0a), (x1a, y0a), color_list[1], 3)
chipa = cv2.line(chipa, (x1a, y0a), (x1a, y1a), color_list[2], 3)
chipa = cv2.line(chipa, (x0a, y1a), (x1a, y1a), color_list[3], 3)
chipb = cv2.line(chipb, (x0b, y0b), (x0b, y1b), color_list[0], 3)
chipb = cv2.line(chipb, (x0b, y0b), (x1b, y0b), color_list[1], 3)
chipb = cv2.line(chipb, (x1b, y0b), (x1b, y1b), color_list[2], 3)
chipb = cv2.line(chipb, (x0b, y1b), (x1b, y1b), color_list[3], 3)
canvas = np.hstack((chipa, chipb))
canvas_filepath = join(
output_path,
'canonical-regression-iou-%0.02f-aid-%s.jpg' % (overlap, test_aid),
)
cv2.imwrite(canvas_filepath, canvas)
[docs]@register_ibs_method
def canonical_localization_precision_recall_algo_display(ibs, figsize=(20, 40)):
import matplotlib.pyplot as plt
from wbia import plottool as pt
fig_ = plt.figure(figsize=figsize, dpi=400) # NOQA
config_list = [
# {'label': 'CA V1 Ensemble', 'canonical_weight_filepath': 'canonical_zebra_grevys_v1', 'species': 'zebra_grevys'}, # OVER = 1.0, small dataset
# {'label': 'CA V1 Model 0', 'canonical_weight_filepath': 'canonical_zebra_grevys_v1:0', 'species': 'zebra_grevys'}, # OVER = 1.0, small dataset
# {'label': 'CA V1 Model 1', 'canonical_weight_filepath': 'canonical_zebra_grevys_v1:1', 'species': 'zebra_grevys'}, # OVER = 1.0, small dataset
# {'label': 'CA V1 Model 2', 'canonical_weight_filepath': 'canonical_zebra_grevys_v1:2', 'species': 'zebra_grevys'}, # OVER = 1.0, small dataset
# {'label': 'CA V2 Ensemble', 'canonical_weight_filepath': 'canonical_zebra_grevys_v2', 'species': 'zebra_grevys'}, # OVER = 1.0, large dataset
# {'label': 'CA V2 Model 0', 'canonical_weight_filepath': 'canonical_zebra_grevys_v2:0', 'species': 'zebra_grevys'}, # OVER = 1.0, large dataset
# {'label': 'CA V2 Model 1', 'canonical_weight_filepath': 'canonical_zebra_grevys_v2:1', 'species': 'zebra_grevys'}, # OVER = 1.0, large dataset
# {'label': 'CA V2 Model 2', 'canonical_weight_filepath': 'canonical_zebra_grevys_v2:2', 'species': 'zebra_grevys'}, # OVER = 1.0, large dataset
# {'label': 'CA V3 Ensemble', 'canonical_weight_filepath': 'canonical_zebra_grevys_v3', 'species': 'zebra_grevys'}, # OVER = 2.0
# {'label': 'CA V3 Model 0', 'canonical_weight_filepath': 'canonical_zebra_grevys_v3:0', 'species': 'zebra_grevys'}, # OVER = 2.0
# {'label': 'CA V3 Model 1', 'canonical_weight_filepath': 'canonical_zebra_grevys_v3:1', 'species': 'zebra_grevys'}, # OVER = 2.0
# {'label': 'CA V3 Model 2', 'canonical_weight_filepath': 'canonical_zebra_grevys_v3:2', 'species': 'zebra_grevys'}, # OVER = 2.0
{
'label': 'CA V5-1.0 Ens.',
'canonical_weight_filepath': 'canonical_zebra_grevys_v5',
'species': 'zebra_grevys',
}, # OVER = 1.0
{
'label': 'CA V5-1.0 M0',
'canonical_weight_filepath': 'canonical_zebra_grevys_v5:0',
'species': 'zebra_grevys',
}, # OVER = 1.0
{
'label': 'CA V5-1.0 M1',
'canonical_weight_filepath': 'canonical_zebra_grevys_v5:1',
'species': 'zebra_grevys',
}, # OVER = 1.0
{
'label': 'CA V5-1.0 M2',
'canonical_weight_filepath': 'canonical_zebra_grevys_v5:2',
'species': 'zebra_grevys',
}, # OVER = 1.0
{
'label': 'CA V6-2.0 Ens.',
'canonical_weight_filepath': 'canonical_zebra_grevys_v6',
'species': 'zebra_grevys',
}, # OVER = 2.0
{
'label': 'CA V6-2.0 M0',
'canonical_weight_filepath': 'canonical_zebra_grevys_v6:0',
'species': 'zebra_grevys',
}, # OVER = 2.0
{
'label': 'CA V6-2.0 M1',
'canonical_weight_filepath': 'canonical_zebra_grevys_v6:1',
'species': 'zebra_grevys',
}, # OVER = 2.0
{
'label': 'CA V6-2.0 M2',
'canonical_weight_filepath': 'canonical_zebra_grevys_v6:2',
'species': 'zebra_grevys',
}, # OVER = 2.0
{
'label': 'CA V4-4.0 Ens.',
'canonical_weight_filepath': 'canonical_zebra_grevys_v4',
'species': 'zebra_grevys',
}, # OVER = 4.0
{
'label': 'CA V4-4.0 M0',
'canonical_weight_filepath': 'canonical_zebra_grevys_v4:0',
'species': 'zebra_grevys',
}, # OVER = 4.0
{
'label': 'CA V4-4.0 M1',
'canonical_weight_filepath': 'canonical_zebra_grevys_v4:1',
'species': 'zebra_grevys',
}, # OVER = 4.0
{
'label': 'CA V4-4.0 M2',
'canonical_weight_filepath': 'canonical_zebra_grevys_v4:2',
'species': 'zebra_grevys',
}, # OVER = 4.0
]
color_list = []
# color_list = [(0, 0, 0)]
color_list += pt.distinct_colors(len(config_list) - len(color_list), randomize=False)
min_, max_ = -1.0, 1.0
axes_ = plt.subplot(321)
axes_.grid(True, which='major')
axes_.grid(False, which='minor')
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.get_xaxis().set_ticks([])
axes_.set_ylabel('GT - Pred Deviation (in percentages)')
axes_.set_xlim([0.0, len(config_list)])
axes_.set_ylim([min_, max_])
axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
for index, (color, config) in enumerate(zip(color_list, config_list)):
canonical_localization_deviation_plot(
ibs, 'x0', color=color, index=index, **config
)
plt.title('X0 Deviation Scatter Plot')
plt.legend(
bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(322)
axes_.grid(True, which='major')
axes_.grid(False, which='minor')
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.get_xaxis().set_ticks([])
axes_.set_ylabel('GT - Pred Deviation (in percentages)')
axes_.set_xlim([0.0, len(config_list)])
axes_.set_ylim([min_, max_])
axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
for index, (color, config) in enumerate(zip(color_list, config_list)):
canonical_localization_deviation_plot(
ibs, 'x1', color=color, index=index, **config
)
plt.title('Y0 Deviation Scatter Plot')
plt.legend(
bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(323)
axes_.grid(True, which='major')
axes_.grid(False, which='minor')
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.get_xaxis().set_ticks([])
axes_.set_ylabel('GT - Pred Deviation (in percentages)')
axes_.set_xlim([0.0, len(config_list)])
axes_.set_ylim([min_, max_])
axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
for index, (color, config) in enumerate(zip(color_list, config_list)):
canonical_localization_deviation_plot(
ibs, 'y0', color=color, index=index, **config
)
plt.title('X1 Deviation Scatter Plot')
plt.legend(
bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(324)
axes_.grid(True, which='major')
axes_.grid(False, which='minor')
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.get_xaxis().set_ticks([])
axes_.set_ylabel('GT - Pred Deviation (in percentages)')
axes_.set_xlim([0.0, len(config_list)])
axes_.set_ylim([min_, max_])
axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
for index, (color, config) in enumerate(zip(color_list, config_list)):
canonical_localization_deviation_plot(
ibs, 'y1', color=color, index=index, **config
)
plt.title('Y1 Deviation Scatter Plot')
plt.legend(
bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(325)
axes_.grid(True, which='major')
axes_.grid(False, which='minor')
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.get_xaxis().set_ticks([])
axes_.set_ylabel('GT - Pred Deviation (in percentages)')
axes_.set_xlim([0.0, len(config_list)])
axes_.set_ylim([min_, max_])
axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
assert len(config_list) % 4 == 0
rounds = len(config_list) // 4
colors = pt.distinct_colors(4, randomize=False)
attribute_list = []
color_list_ = []
for _ in range(rounds):
attribute_list += ['x0', 'y0', 'x1', 'y1']
color_list_ += colors
for index, (attribute, color_) in enumerate(zip(attribute_list, color_list_)):
index_ = (index // 4) * 4
config_ = config_list[index_].copy()
config_['label'] = '%s %s' % (config_['label'], attribute)
canonical_localization_deviation_plot(
ibs, attribute, color=color_, index=index, **config_
)
plt.title('Ensemble Deviation Scatter Plot')
plt.legend(
bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(326)
axes_.grid(True, which='major')
axes_.grid(False, which='minor')
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.get_xaxis().set_ticks([])
axes_.set_ylabel('IoU')
axes_.set_xlim([0.0, len(config_list)])
axes_.set_ylim([0.0, 1.0])
for index, (color, config) in enumerate(zip(color_list, config_list)):
values_ = canonical_localization_iou_plot(ibs, color=color, index=index, **config)
if index % 4 == 0:
config_ = config_list[index]
test_aid_set, test_bbox_set, prediction_list, y_list, accuracy = values_
ibs.canonical_localization_iou_visualize(
index,
test_aid_set,
test_bbox_set,
prediction_list,
y_list,
colors,
**config_,
)
plt.title('IoU Scatter Plot')
plt.legend(
bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
fig_filename = 'canonical-localization-deviance.png'
fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
plt.savefig(fig_path, bbox_inches='tight')
[docs]@register_ibs_method
def background_accuracy_display(ibs, category_list, test_gid_set=None, output_path=None):
if output_path is None:
output_path = abspath(expanduser(join('~', 'Desktop', 'background')))
ut.ensuredir(output_path)
if test_gid_set is None:
test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_set = list(test_gid_set)
aids_list = ibs.get_image_aids(test_gid_set)
aid_list = ut.flatten(aids_list)
species_list = ibs.get_annot_species_texts(aid_list)
aid_list = [
aid for aid, species in zip(aid_list, species_list) if species in category_list
]
species_list = ibs.get_annot_species_texts(aid_list)
gid_list = ibs.get_annot_gids(aid_list)
config2_ = {'fw_detector': 'cnn'}
hough_cpath_list = ibs.get_annot_probchip_fpath(aid_list, config2_=config2_)
image_list = [vt.imread(hough_cpath) for hough_cpath in hough_cpath_list]
chip_list = ibs.get_annot_chips(aid_list, config2_=config2_)
zipped = zip(aid_list, gid_list, species_list, image_list, chip_list)
for index, (aid, gid, species, image, chip) in enumerate(zipped):
logger.info(index)
mask = vt.resize_mask(image, chip)
blended = vt.blend_images_multiply(chip, mask)
blended *= 255.0
blended = np.around(blended)
blended[blended < 0] = 0
blended[blended > 255] = 255
blended = blended.astype(np.uint8)
canvas = np.hstack((chip, mask, blended))
output_filepath = join(
output_path, 'background.%s.%d.%d.png' % (species, gid, aid)
)
cv2.imwrite(output_filepath, canvas)
[docs]def aoi2_precision_recall_algo(ibs, category_list=None, test_gid_set_=None, **kwargs):
depc = ibs.depc_annot
if test_gid_set_ is None:
test_gid_set_ = general_get_imageset_gids(ibs, 'TEST_SET')
test_aid_list_ = list(set(ut.flatten(ibs.get_image_aids(test_gid_set_))))
species_list = ibs.get_annot_species_texts(test_aid_list_)
interest_list = ibs.get_annot_interest(test_aid_list_)
test_aid_list = []
label_list = []
for test_aid, species, interest in zip(test_aid_list_, species_list, interest_list):
if category_list is not None:
if species not in category_list:
continue
if interest is None:
continue
label = 'positive' if interest else 'negative'
test_aid_list.append(test_aid)
label_list.append(label)
prediction_list = depc.get_property('aoi_two', test_aid_list, 'class', config=kwargs)
confidence_list = depc.get_property('aoi_two', test_aid_list, 'score', config=kwargs)
confidence_list = [
confidence if prediction == 'positive' else 1.0 - confidence
for prediction, confidence in zip(prediction_list, confidence_list)
]
return general_precision_recall_algo(ibs, label_list, confidence_list, **kwargs)
[docs]def aoi2_precision_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
logger.info('Processing Precision-Recall for: %r' % (label,))
conf_list, pr_list, re_list, tpr_list, fpr_list = aoi2_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
[docs]def aoi2_roc_algo_plot(ibs, **kwargs):
label = kwargs['label']
logger.info('Processing ROC for: %r' % (label,))
conf_list, pr_list, re_list, tpr_list, fpr_list = aoi2_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(
conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
)
[docs]def aoi2_confusion_matrix_algo_plot(
ibs,
label,
color,
conf,
output_cases=False,
category_list=None,
test_gid_set_=None,
**kwargs,
):
logger.info('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf))
depc = ibs.depc_annot
if test_gid_set_ is None:
test_gid_set_ = general_get_imageset_gids(ibs, 'TEST_SET')
test_aid_list_ = list(set(ut.flatten(ibs.get_image_aids(test_gid_set_))))
species_list = ibs.get_annot_species_texts(test_aid_list_)
interest_list = ibs.get_annot_interest(test_aid_list_)
test_aid_list = []
label_list = []
for test_aid, species, interest in zip(test_aid_list_, species_list, interest_list):
if category_list is not None:
if species not in category_list:
continue
if interest is None:
continue
label = 'positive' if interest else 'negative'
test_aid_list.append(test_aid)
label_list.append(label)
prediction_list = depc.get_property('aoi_two', test_aid_list, 'class', config=kwargs)
confidence_list = depc.get_property('aoi_two', test_aid_list, 'score', config=kwargs)
confidence_list = [
confidence if prediction == 'positive' else 1.0 - confidence
for prediction, confidence in zip(prediction_list, confidence_list)
]
prediction_list = [
'positive' if confidence >= conf else 'negative' for confidence in confidence_list
]
if output_cases:
output_path = 'aoi2-confusion-incorrect'
output_path = abspath(expanduser(join('~', 'Desktop', output_path)))
ut.delete(output_path)
ut.ensuredir(output_path)
manifest_dict = {}
test_gid_list = ibs.get_annot_gids(test_aid_list)
zipped = zip(test_gid_list, test_aid_list, label_list, prediction_list)
for test_gid, test_aid, label, prediction in zipped:
if test_gid not in manifest_dict:
manifest_dict[test_gid] = {}
assert test_aid not in manifest_dict[test_gid]
manifest_dict[test_gid][test_aid] = (
label,
prediction,
)
for test_gid in manifest_dict:
image = ibs.get_images(test_gid)
w, h = ibs.get_image_sizes(test_gid)
image = _resize(image, t_width=600, verbose=False)
height_, width_, channels_ = image.shape
for test_aid in manifest_dict[test_gid]:
label, prediction = manifest_dict[test_gid][test_aid]
bbox = ibs.get_annot_bboxes(test_aid)
xtl, ytl, width, height = bbox
xbr = xtl + width
ybr = ytl + height
xtl = int(np.round((xtl / w) * width_))
ytl = int(np.round((ytl / h) * height_))
xbr = int(np.round((xbr / w) * width_))
ybr = int(np.round((ybr / h) * height_))
if label == 'positive':
color = (255, 99, 46)
else:
color = (127, 255, 127)
cv2.rectangle(image, (xtl, ytl), (xbr, ybr), color, 4)
if prediction == 'positive':
color = (255, 99, 46)
else:
color = (127, 255, 127)
cv2.rectangle(image, (xtl - 4, ytl - 4), (xbr + 4, ybr + 4), color, 4)
image_filename = 'image_%d.png' % (test_gid,)
image_filepath = join(output_path, image_filename)
cv2.imwrite(image_filepath, image)
category_list = ['positive', 'negative']
category_mapping = {
'positive': 0,
'negative': 1,
}
return general_confusion_matrix_algo(
label_list, prediction_list, category_list, category_mapping, size=20, **kwargs
)
[docs]@register_ibs_method
def aoi2_precision_recall_algo_display(
ibs, test_gid_list=None, output_cases=False, figsize=(20, 20)
):
import matplotlib.pyplot as plt
import wbia.plottool as pt
fig_ = plt.figure(figsize=figsize)
test_gid_set = None if test_gid_list is None else sorted(set(test_gid_list))
config_list = [
# {'label': 'All Species', 'aoi_two_weight_filepath': 'ggr2', 'category_list': None},
# {'label': 'Masai Giraffe', 'aoi_two_weight_filepath': 'ggr2', 'category_list': ['giraffe_masai']},
# {'label': 'Reticulated Giraffe', 'aoi_two_weight_filepath': 'ggr2', 'category_list': ['giraffe_reticulated']},
# {'label': 'Sea Turtle', 'aoi_two_weight_filepath': 'ggr2', 'category_list': ['turtle_sea']},
# {'label': 'Whale Fluke', 'aoi_two_weight_filepath': 'ggr2', 'category_list': ['whale_fluke']},
# {'label': 'Grevy\'s Zebra', 'aoi_two_weight_filepath': 'ggr2', 'category_list': ['zebra_grevys']},
# {'label': 'Plains Zebra', 'aoi_two_weight_filepath': 'ggr2', 'category_list': ['zebra_plains']},
# {'label': 'Hammerhead', 'aoi_two_weight_filepath': 'hammerhead', 'category_list': ['shark_hammerhead']},
{
'label': 'Jaguar',
'aoi_two_weight_filepath': 'jaguar',
'category_list': ['jaguar'],
},
]
color_list = [(0, 0, 0)]
color_list += pt.distinct_colors(len(config_list) - len(color_list), randomize=False)
axes_ = plt.subplot(221)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('Recall')
axes_.set_ylabel('Precision')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
aoi2_precision_recall_algo_plot(
ibs, color=color, test_gid_set_=test_gid_set, **config
)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
conf_list = [ret[1] for ret in ret_list]
# index = np.argmax(area_list)
index = 0
best_label1 = config_list[index]['label']
best_config1 = config_list[index]
best_color1 = color_list[index]
best_area1 = area_list[index]
best_conf1 = conf_list[index]
plt.title(
'Precision-Recall Curve (Best: %s, AP = %0.02f)' % (best_label1, best_area1),
y=1.10,
)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(222)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('False-Positive Rate')
axes_.set_ylabel('True-Positive Rate')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
aoi2_roc_algo_plot(ibs, color=color, **config)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
conf_list = [ret[1] for ret in ret_list]
# index = np.argmax(area_list)
index = 0
best_label2 = config_list[index]['label']
best_config2 = config_list[index]
best_color2 = color_list[index]
best_area2 = area_list[index]
best_conf2 = conf_list[index]
plt.title('ROC Curve (Best: %s, AP = %0.02f)' % (best_label2, best_area2), y=1.10)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
plt.plot([0.0, 1.0], [0.0, 1.0], color=(0.5, 0.5, 0.5), linestyle='--')
axes_ = plt.subplot(223)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, _ = aoi2_confusion_matrix_algo_plot(
ibs,
color=best_color1,
conf=best_conf1,
fig_=fig_,
axes_=axes_,
output_cases=output_cases,
test_gid_set_=test_gid_set,
**best_config1,
)
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
plt.title('P-R Confusion Matrix (OP = %0.02f)' % (best_conf1,), y=1.12)
axes_ = plt.subplot(224)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, _ = aoi2_confusion_matrix_algo_plot(
ibs,
color=best_color2,
conf=best_conf2,
fig_=fig_,
axes_=axes_,
test_gid_set_=test_gid_set,
**best_config2,
)
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
plt.title('ROC Confusion Matrix (OP = %0.02f)' % (best_conf2,), y=1.12)
fig_filename = 'aoi2-precision-recall-roc.png'
fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
plt.savefig(fig_path, bbox_inches='tight')
[docs]def detector_parse_gt(ibs, test_gid_list=None, **kwargs):
if test_gid_list is None:
test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
uuid_list = ibs.get_image_uuids(test_gid_list)
gid_list = ibs.get_image_gids_from_uuid(uuid_list)
gt_dict = {}
for gid, uuid in zip(gid_list, uuid_list):
width, height = ibs.get_image_sizes(gid)
aid_list = ibs.get_image_aids(gid)
gt_list = []
for aid in aid_list:
bbox = ibs.get_annot_bboxes(aid)
temp = {
'gid': gid,
'xtl': bbox[0] / width,
'ytl': bbox[1] / height,
'xbr': (bbox[0] + bbox[2]) / width,
'ybr': (bbox[1] + bbox[3]) / height,
'width': bbox[2] / width,
'height': bbox[3] / height,
'class': ibs.get_annot_species_texts(aid),
'viewpoint': ibs.get_annot_viewpoints(aid),
'confidence': 1.0,
}
gt_list.append(temp)
gt_dict[uuid] = gt_list
return gt_dict
# def detector_parse_pred(ibs, test_gid_list=None, **kwargs):
# depc = ibs.depc_image
# if test_gid_list is None:
# test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
# uuid_list = ibs.get_image_uuids(test_gid_list)
# # depc.delete_property('detections', test_gid_list, config=kwargs)
# results_list = depc.get_property('detections', test_gid_list, None, config=kwargs)
# size_list = ibs.get_image_sizes(test_gid_list)
# zipped_list = zip(results_list)
# # Reformat results for json
# results_list = [
# [
# {
# 'gid' : test_gid,
# 'xtl' : bbox[0] / width,
# 'ytl' : bbox[1] / height,
# 'width' : bbox[2] / width,
# 'height' : bbox[3] / height,
# 'theta' : theta, # round(theta, 4),
# 'confidence' : conf, # round(conf, 4),
# 'class' : class_,
# 'viewpoint' : viewpoint,
# }
# for bbox, theta, class_, viewpoint, conf in zip(*zipped[0][1:])
# ]
# for zipped, (width, height), test_gid in zip(zipped_list, size_list, test_gid_list)
# ]
# pred_dict = {
# uuid_ : result_list
# for uuid_, result_list in zip(uuid_list, results_list)
# }
# # logger.info(pred_dict)
# return pred_dict
# def detector_precision_recall_algo(ibs, samples=SAMPLES, force_serial=FORCE_SERIAL, **kwargs):
# test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
# uuid_list = ibs.get_image_uuids(test_gid_list)
# logger.info('\tGather Ground-Truth')
# gt_dict = detector_parse_gt(ibs, test_gid_list=test_gid_list)
# logger.info('\tGather Predictions')
# pred_dict = detector_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)
# logger.info('\tGenerate Curves...')
# conf_list = [ _ / float(samples) for _ in range(0, int(samples) + 1) ]
# conf_list = sorted(conf_list, reverse=True)
# uuid_list_list = [ uuid_list for _ in conf_list ]
# gt_dict_list = [ gt_dict for _ in conf_list ]
# pred_dict_list = [ pred_dict for _ in conf_list ]
# kwargs_list = [ kwargs for _ in conf_list ]
# arg_iter = zip(conf_list, uuid_list_list, gt_dict_list, pred_dict_list, kwargs_list)
# pr_re_gen = ut.generate2(detector_precision_recall_algo_worker, arg_iter,
# nTasks=len(conf_list), ordered=True,
# chunksize=CHUNK_SIZE, force_serial=force_serial)
# conf_list_ = [-1.0, -1.0]
# pr_list = [1.0, 0.0]
# re_list = [0.0, 1.0]
# # conf_list_ = []
# # pr_list = []
# # re_list = []
# for conf, pr, re in pr_re_gen:
# conf_list_.append(conf)
# pr_list.append(pr)
# re_list.append(re)
# logger.info('...complete')
# return conf_list_, pr_list, re_list
# def detector_precision_recall_algo_worker(conf, uuid_list, gt_dict, pred_dict,
# kwargs):
# tp, fp, fn = 0.0, 0.0, 0.0
# for index, uuid_ in enumerate(uuid_list):
# if uuid_ in pred_dict:
# pred_list = [
# pred
# for pred in pred_dict[uuid_]
# if pred['confidence'] >= conf
# ]
# tp_, fp_, fn_ = general_tp_fp_fn(gt_dict[uuid_], pred_list, **kwargs)
# tp += tp_
# fp += fp_
# fn += fn_
# pr = tp / (tp + fp)
# re = tp / (tp + fn)
# return (conf, pr, re)
# def detector_precision_recall_algo_plot(ibs, **kwargs):
# label = kwargs['label']
# logger.info('Processing Precision-Recall for: %r' % (label, ))
# conf_list, pr_list, re_list = detector_precision_recall_algo(ibs, **kwargs)
# return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
# def detector_confusion_matrix_algo_plot(ibs, label, color, conf, **kwargs):
# logger.info('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf, ))
# test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
# uuid_list = ibs.get_image_uuids(test_gid_list)
# logger.info('\tGather Ground-Truth')
# gt_dict = detector_parse_gt(ibs, test_gid_list=test_gid_list)
# logger.info('\tGather Predictions')
# pred_dict = detector_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)
# label_list = []
# prediction_list = []
# for index, uuid_ in enumerate(uuid_list):
# if uuid_ in pred_dict:
# gt_list = gt_dict[uuid_]
# pred_list = [
# pred
# for pred in pred_dict[uuid_]
# if pred['confidence'] >= conf
# ]
# tp, fp, fn = general_tp_fp_fn(gt_list, pred_list, **kwargs)
# for _ in range(int(tp)):
# label_list.append('positive')
# prediction_list.append('positive')
# for _ in range(int(fp)):
# label_list.append('negative')
# prediction_list.append('positive')
# for _ in range(int(fn)):
# label_list.append('positive')
# prediction_list.append('negative')
# category_list = ['positive', 'negative']
# category_mapping = {
# 'positive': 0,
# 'negative': 1,
# }
# return general_confusion_matrix_algo(label_list, prediction_list, category_list,
# category_mapping, **kwargs)
# @register_ibs_method
# def detector_precision_recall_algo_display(ibs, min_overlap=0.5, figsize=(24, 7), **kwargs):
# import matplotlib.pyplot as plt
# fig_ = plt.figure(figsize=figsize)
# axes_ = plt.subplot(131)
# axes_.set_autoscalex_on(False)
# axes_.set_autoscaley_on(False)
# axes_.set_xlabel('Recall (Ground-Truth IOU >= %0.02f)' % (min_overlap, ))
# axes_.set_ylabel('Precision')
# axes_.set_xlim([0.0, 1.01])
# axes_.set_ylim([0.0, 1.01])
# kwargs_list = [
# {
# 'min_overlap' : min_overlap,
# 'classifier_sensitivity' : 0.64,
# 'localizer_grid' : False,
# 'localizer_sensitivity' : 0.16,
# 'labeler_sensitivity' : 0.42,
# },
# {
# 'min_overlap' : min_overlap,
# 'classifier_sensitivity' : 0.64,
# 'localizer_grid' : False,
# 'localizer_sensitivity' : 0.16,
# 'labeler_sensitivity' : 0.42,
# 'check_species' : True,
# },
# {
# 'min_overlap' : min_overlap,
# 'classifier_sensitivity' : 0.64,
# 'localizer_grid' : False,
# 'localizer_sensitivity' : 0.16,
# 'labeler_sensitivity' : 0.42,
# 'check_viewpoint' : True,
# },
# {
# 'min_overlap' : min_overlap,
# 'classifier_sensitivity' : 0.04,
# 'localizer_grid' : True,
# 'localizer_sensitivity' : 0.05,
# 'labeler_sensitivity' : 0.39,
# },
# {
# 'min_overlap' : min_overlap,
# 'classifier_sensitivity' : 0.04,
# 'localizer_grid' : True,
# 'localizer_sensitivity' : 0.05,
# 'labeler_sensitivity' : 0.39,
# 'check_species' : True,
# },
# {
# 'min_overlap' : min_overlap,
# 'classifier_sensitivity' : 0.04,
# 'localizer_grid' : True,
# 'localizer_sensitivity' : 0.05,
# 'labeler_sensitivity' : 0.39,
# 'check_viewpoint' : True,
# },
# ]
# label_list = [
# 'Opt L',
# 'Opt L+S',
# 'Opt L+S+V',
# 'Rec L',
# 'Rec L+S',
# 'Rec L+S+V',
# ]
# color_list = [
# 'r',
# 'b',
# 'g',
# 'k',
# 'y',
# 'c',
# ]
# ret_list = [
# detector_precision_recall_algo_plot(ibs, label=label, color=color, **kwargs_)
# for label, color, kwargs_ in zip(label_list, color_list, kwargs_list)
# ]
# area_list = [ ret[0] for ret in ret_list ]
# conf_list = [ ret[1] for ret in ret_list ]
# index = np.argmax(area_list)
# best_label = label_list[index]
# best_kwargs = kwargs_list[index]
# best_area = area_list[index]
# best_conf = conf_list[index]
# plt.title('Precision-Recall Curve (Best: %s, AP = %0.02f)' % (best_label, best_area, ), y=1.20)
# # Display graph
# plt.legend(bbox_to_anchor=(0.0, 1.02, 1.0, .102), loc=3, ncol=2, mode="expand",
# borderaxespad=0.0)
# axes_ = plt.subplot(132)
# axes_.set_aspect(1)
# gca_ = plt.gca()
# gca_.grid(False)
# correct_rate, _ = detector_confusion_matrix_algo_plot(ibs, 'V1', 'r', conf=best_conf, fig_=fig_, axes_=axes_, **best_kwargs)
# axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
# axes_.set_ylabel('Ground-Truth')
# plt.title('P-R Confusion Matrix (Algo: %s, OP = %0.02f)' % (best_label, best_conf, ), y=1.26)
# best_index = None
# best_conf = None
# best_pr = 0.0
# best_re = 0.0
# tup_list = [ ret[2] for ret in ret_list ]
# for index, tup in enumerate(tup_list):
# for conf, re, pr in zip(*tup):
# if pr > best_pr:
# best_index = index
# best_conf = conf
# best_pr = pr
# best_re = re
# if best_index is not None:
# axes_ = plt.subplot(131)
# plt.plot([best_re], [best_pr], 'yo')
# best_label = label_list[best_index]
# best_kwargs = kwargs_list[best_index]
# axes_ = plt.subplot(133)
# axes_.set_aspect(1)
# gca_ = plt.gca()
# gca_.grid(False)
# correct_rate, _ = detector_confusion_matrix_algo_plot(ibs, 'V1', 'r', conf=best_conf, fig_=fig_, axes_=axes_, **best_kwargs)
# axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
# axes_.set_ylabel('Ground-Truth')
# plt.title('P-R Confusion Matrix (Algo: %s, OP = %0.02f)' % (best_label, best_conf, ), y=1.26)
# # plt.show()
# fig_filename = 'detector-precision-recall-%0.2f.png' % (min_overlap, )
# fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
# plt.savefig(fig_path, bbox_inches='tight')
# @register_ibs_method
# def detector_metric_graphs(ibs, species_list=[]):
# ibs.classifier_precision_recall_algo_display(species_list)
# ibs.localizer_precision_recall_algo_display()
# ibs.labeler_precision_recall_algo_display()
# ibs.detector_precision_recall_algo_display()