# -*- coding: utf-8 -*-
"""
Hotspotter pipeline module
Module Notation and Concepts:
PREFIXES:
qaid2_XXX - prefix mapping query chip index to
qfx2_XXX - prefix mapping query chip feature index to
* nns - a (qfx2_idx, qfx2_dist) tuple
* idx - the index into the nnindexers descriptors
* qfx - query feature index wrt the query chip
* dfx - query feature index wrt the database chip
* dist - the distance to a corresponding feature
* fm - a list of feature match pairs / correspondences (qfx, dfx)
* fsv - a score vector of a corresponding feature
* valid - a valid bit for a corresponding feature
PIPELINE_VARS:
nns_list - maping from query chip index to nns
* qfx2_idx - ranked list of query feature indexes to database feature indexes
* qfx2_dist - ranked list of query feature indexes to database feature indexes
* qaid2_norm_weight - mapping from qaid to (qfx2_normweight, qfx2_selnorm)
= qaid2_nnfiltagg[qaid]
CommandLine:
To see the ouput of a complete pipeline run use
# Set to whichever database you like
python main.py --db PZ_MTEST --setdb
python main.py --db NAUT_test --setdb
python main.py --db testdb1 --setdb
# Then run whichever configuration you like
python main.py --query 1 --yes --noqcache -t default:codename=vsmany
python main.py --query 1 --yes --noqcache -t default:codename=vsmany_nsum
TODO:
* Don't preload the nn-indexer in case the nearest neighbors have already
been computed?
"""
import logging
import numpy as np
import vtool as vt
from wbia.algo.hots import hstypes
from wbia.algo.hots import chip_match
from wbia.algo.hots import nn_weights
from wbia.algo.hots import scoring
from wbia.algo.hots import _pipeline_helpers as plh # NOQA
from collections import namedtuple
import utool as ut
print, rrr, profile = ut.inject2(__name__)
logger = logging.getLogger('wbia')
# =================
# Globals
# =================
TAU = 2 * np.pi # References: tauday.com
NOT_QUIET = ut.NOT_QUIET and not ut.get_argflag('--quiet-query')
DEBUG_PIPELINE = ut.get_argflag(('--debug-pipeline', '--debug-pipe'))
VERB_PIPELINE = NOT_QUIET and (
ut.VERBOSE or ut.get_argflag(('--verbose-pipeline', '--verb-pipe'))
)
VERYVERBOSE_PIPELINE = ut.get_argflag(('--very-verbose-pipeline', '--very-verb-pipe'))
USE_HOTSPOTTER_CACHE = not ut.get_argflag('--nocache-hs') and ut.USE_CACHE
USE_NN_MID_CACHE = (
(True and ut.is_developer())
and not ut.get_argflag('--nocache-nnmid')
and USE_HOTSPOTTER_CACHE
)
USE_NN_MID_CACHE = False
NN_LBL = 'Assign NN: '
FILT_LBL = 'Filter NN: '
WEIGHT_LBL = 'Weight NN: '
BUILDCM_LBL = 'Build Chipmatch: '
SVER_LVL = 'SVER: '
PROGKW = dict(freq=1, time_thresh=30.0, adjust=True)
# Internal tuples denoting return types
WeightRet_ = namedtuple(
'weight_ret',
('filtkey_list', 'filtweights_list', 'filtvalids_list', 'filtnormks_list'),
)
[docs]class Neighbors(ut.NiceRepr):
__slots__ = ['qaid', 'qfx_list', 'neighb_idxs', 'neighb_dists']
# TODO: replace with named tuple?
def __init__(self, qaid, idxs, dists, qfxs):
self.qaid = qaid
self.qfx_list = qfxs
self.neighb_idxs = idxs
self.neighb_dists = dists
@property
def num_query_feats(self):
if self.qfx_list is None:
return len(self.neighb_idxs)
else:
return len(self.qfx_list)
def __iter__(self):
return iter([self.neighb_idxs, self.neighb_dists])
def __getitem__(self, index):
return (self.neighb_idxs, self.neighb_dists)[index]
def __nice__(self):
return '(qaid=%r,nQfxs=%r,nNbs=%r)' % (
self.qaid,
self.num_query_feats,
self.neighb_idxs.shape[1],
)
def __getstate__(self):
return self.__dict__
def __setstate__(self, state):
return self.__dict__.update(**state)
# @profile
[docs]def request_wbia_query_L0(ibs, qreq_, verbose=VERB_PIPELINE):
r"""Driver logic of query pipeline
Note:
Make sure _pipeline_helpres.testrun_pipeline_upto reflects what happens
in this function.
Args:
ibs (wbia.IBEISController): IBEIS database object to be queried.
technically this object already lives inside of qreq_.
qreq_ (wbia.QueryRequest): hyper-parameters. use
``ibs.new_query_request`` to create one
Returns:
list: cm_list containing ``wbia.ChipMatch`` objects
CommandLine:
python -m wbia.algo.hots.pipeline --test-request_wbia_query_L0:0 --show
python -m wbia.algo.hots.pipeline --test-request_wbia_query_L0:1 --show
python -m wbia.algo.hots.pipeline --test-request_wbia_query_L0:0 --db testdb1 --qaid 325
python -m wbia.algo.hots.pipeline --test-request_wbia_query_L0:0 --db testdb3 --qaid 325
# background match
python -m wbia.algo.hots.pipeline --test-request_wbia_query_L0:0 --db NNP_Master3 --qaid 12838
python -m wbia.algo.hots.pipeline --test-request_wbia_query_L0:0
python -m wbia.algo.hots.pipeline --test-request_wbia_query_L0:0 --db PZ_MTEST -a timectrl:qindex=0:256
python -m wbia.algo.hots.pipeline --test-request_wbia_query_L0:0 --db PZ_Master1 -a timectrl:qindex=0:256
utprof.py -m wbia.algo.hots.pipeline --test-request_wbia_query_L0:0 --db PZ_Master1 -a timectrl:qindex=0:256
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> import wbia
>>> qreq_ = wbia.init.main_helpers.testdata_qreq_(a=['default:qindex=0:2,dindex=0:10'])
>>> ibs = qreq_.ibs
>>> print(qreq_.qparams.query_cfgstr)
>>> verbose = True
>>> cm_list = request_wbia_query_L0(ibs, qreq_, verbose=verbose)
>>> cm = cm_list[0]
>>> ut.quit_if_noshow()
>>> cm.ishow_analysis(qreq_, fnum=0, make_figtitle=True)
>>> ut.show_if_requested()
"""
# Load data for nearest neighbors
if verbose:
assert ibs is qreq_.ibs
logger.info('\n\n[hs] +--- STARTING HOTSPOTTER PIPELINE ---')
logger.info(ut.indent(qreq_.get_infostr(), '[hs] '))
ibs.assert_valid_aids(qreq_.get_internal_qaids(), msg='pipeline qaids')
ibs.assert_valid_aids(qreq_.get_internal_daids(), msg='pipeline daids')
if qreq_.qparams.pipeline_root == 'smk':
from wbia.algo.hots.smk import smk_match
# Alternative to naive bayes matching:
# Selective match kernel
qaid2_scores, qaid2_chipmatch_FILT_ = smk_match.execute_smk_L5(qreq_)
elif qreq_.qparams.pipeline_root in ['vsone', 'vsmany']:
assert qreq_.qparams.pipeline_root != 'vsone', 'pipeline no longer supports vsone'
if qreq_.prog_hook is not None:
qreq_.prog_hook.initialize_subhooks(5)
# qreq_.lazy_load(verbose=(verbose and ut.NOT_QUIET))
qreq_.lazy_preload(verbose=(verbose and ut.NOT_QUIET))
impossible_daids_list, Kpad_list = build_impossible_daids_list(qreq_)
# Nearest neighbors (nns_list)
# a nns object is a tuple(ndarray, ndarray) - (qfx2_dx, qfx2_dist)
# * query descriptors assigned to database descriptors
# * FLANN used here
nns_list = nearest_neighbors(
qreq_, Kpad_list, impossible_daids_list, verbose=verbose
)
# Remove Impossible Votes
# a nnfilt object is an ndarray qfx2_valid
# * marks matches to the same image as invalid
nnvalid0_list = baseline_neighbor_filter(
qreq_, nns_list, impossible_daids_list, verbose=verbose
)
# Nearest neighbors weighting / scoring (filtweights_list)
# filtweights_list maps qaid to filtweights which is a dict
# that maps a filter name to that query's weights for that filter
weight_ret = weight_neighbors(qreq_, nns_list, nnvalid0_list, verbose=verbose)
filtkey_list, filtweights_list, filtvalids_list, filtnormks_list = weight_ret
# Nearest neighbors to chip matches (cm_list)
# * Initial scoring occurs
# * vsone un-swapping occurs here
cm_list_FILT = build_chipmatches(
qreq_,
nns_list,
nnvalid0_list,
filtkey_list,
filtweights_list,
filtvalids_list,
filtnormks_list,
verbose=verbose,
)
else:
logger.info('invalid pipeline root %r' % (qreq_.qparams.pipeline_root))
# Spatial verification (cm_list) (TODO: cython)
# * prunes chip results and feature matches
# TODO: allow for reweighting of feature matches to happen.
cm_list_SVER = spatial_verification(qreq_, cm_list_FILT, verbose=verbose)
if cm_list_FILT[0].filtnorm_aids is not None:
pass
# assert cm_list_SVER[0].filtnorm_aids is not None
cm_list = cm_list_SVER
# Final Scoring
score_method = qreq_.qparams.score_method
scoring.score_chipmatch_list(qreq_, cm_list, score_method)
if VERB_PIPELINE:
logger.info('[hs] L___ FINISHED HOTSPOTTER PIPELINE ___')
return cm_list
# ============================
# 0) Nearest Neighbors
# ============================
[docs]def build_impossible_daids_list(qreq_, verbose=VERB_PIPELINE):
r"""
Args:
qreq_ (QueryRequest): query request object with hyper-parameters
CommandLine:
python -m wbia.algo.hots.pipeline --test-build_impossible_daids_list
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> import wbia
>>> qreq_ = wbia.testdata_qreq_(
>>> defaultdb='testdb1',
>>> a='default:species=zebra_plains,qhackerrors=True',
>>> p='default:use_k_padding=True,can_match_sameimg=False,can_match_samename=False')
>>> impossible_daids_list, Kpad_list = build_impossible_daids_list(qreq_)
>>> impossible_daids_list = [x.tolist() for x in impossible_daids_list]
>>> vals = ut.dict_subset(locals(), ['impossible_daids_list', 'Kpad_list'])
>>> result = ut.repr2(vals, nl=1, explicit=True, nobr=True, strvals=True)
>>> print(result)
>>> assert np.all(qreq_.qaids == [1, 4, 5, 6])
>>> assert np.all(qreq_.daids == [1, 2, 3, 4, 5, 6])
...
impossible_daids_list=[[1], [4], [5, 6], [5, 6]],
Kpad_list=[1, 1, 2, 2],
"""
if verbose:
logger.info('[hs] Step 0) Build impossible matches')
can_match_sameimg = qreq_.qparams.can_match_sameimg
can_match_samename = qreq_.qparams.can_match_samename
use_k_padding = qreq_.qparams.use_k_padding
can_match_self = False
internal_qaids = qreq_.get_internal_qaids()
internal_daids = qreq_.get_internal_daids()
internal_data_nids = qreq_.get_qreq_annot_nids(internal_daids)
_impossible_daid_lists = []
if not can_match_self:
if can_match_sameimg and can_match_samename:
# we can skip this if sameimg or samename is specified.
# it will cover this case for us
_impossible_daid_lists.append([[qaid] for qaid in internal_qaids])
if not can_match_sameimg:
# slow way of getting contact_aids (now incorporates faster way)
contact_aids_list = qreq_.ibs.get_annot_contact_aids(
internal_qaids, daid_list=internal_daids
)
_impossible_daid_lists.append(contact_aids_list)
EXTEND_TO_OTHER_CONTACT_GT = False
# TODO: flag overlapping keypoints with another annot as likely to
# cause photobombs.
# Also cannot match any aids with a name of an annotation in this image
if EXTEND_TO_OTHER_CONTACT_GT:
# TODO: need a test set that can accomidate testing this case
# testdb1 might cut it if we spruced it up
nonself_contact_aids = [
np.setdiff1d(aids, qaid)
for aids, qaid in zip(contact_aids_list, internal_qaids)
]
nonself_contact_nids = qreq_.ibs.unflat_map(
qreq_.get_qreq_annot_nids, nonself_contact_aids
)
contact_aids_gt_list = [
internal_daids.compress(vt.get_covered_mask(internal_data_nids, nids))
for nids in nonself_contact_nids
]
_impossible_daid_lists.append(contact_aids_gt_list)
if not can_match_samename:
internal_data_nids = qreq_.get_qreq_annot_nids(internal_daids)
internal_query_nids = qreq_.get_qreq_annot_nids(internal_qaids)
gt_aids = [
internal_daids.compress(internal_data_nids == nid)
for nid in internal_query_nids
]
_impossible_daid_lists.append(gt_aids)
# TODO: add explicit not a match case in here
_impossible_daids_list = list(map(ut.flatten, zip(*_impossible_daid_lists)))
impossible_daids_list = [
np.unique(impossible_daids) for impossible_daids in _impossible_daids_list
]
# TODO: we need to pad K for each bad annotation
if use_k_padding:
Kpad_list = list(map(len, impossible_daids_list))
else:
# always at least pad K for self queries
Kpad_list = [1 if qaid in internal_daids else 0 for qaid in internal_qaids]
return impossible_daids_list, Kpad_list
# ============================
# 1) Nearest Neighbors
# ============================
[docs]@profile
def nearest_neighbor_cacheid2(qreq_, Kpad_list):
r"""
Returns a hacky cacheid for neighbor configs.
DEPRICATE: This will be replaced by dtool caching
Args:
qreq_ (QueryRequest): query request object with hyper-parameters
Kpad_list (list):
Returns:
tuple: (nn_mid_cacheid_list, nn_cachedir)
CommandLine:
python -m wbia.algo.hots.pipeline --exec-nearest_neighbor_cacheid2
python -m wbia.algo.hots.pipeline --exec-nearest_neighbor_cacheid2 --superstrict
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> import wbia
>>> verbose = True
>>> cfgdict = dict(K=4, Knorm=1, checks=800, use_k_padding=False)
>>> # test 1
>>> p = 'default' + ut.get_cfg_lbl(cfgdict)
>>> qreq_ = wbia.testdata_qreq_(
>>> defaultdb='testdb1', p=[p], qaid_override=[1, 2],
>>> daid_override=[1, 2, 3, 4, 5])
>>> locals_ = plh.testrun_pipeline_upto(qreq_, 'nearest_neighbors')
>>> Kpad_list, = ut.dict_take(locals_, ['Kpad_list'])
>>> tup = nearest_neighbor_cacheid2(qreq_, Kpad_list)
>>> (nn_cachedir, nn_mid_cacheid_list) = tup
>>> result1 = 'nn_mid_cacheid_list1 = ' + ut.repr2(nn_mid_cacheid_list, nl=1)
>>> # test 2
>>> cfgdict2 = dict(K=2, Knorm=3, use_k_padding=True)
>>> p2 = 'default' + ut.get_cfg_lbl(cfgdict)
>>> ibs = qreq_.ibs
>>> qreq_ = wbia.testdata_qreq_(defaultdb='testdb1', p=[p2], qaid_override=[1, 2], daid_override=[1, 2, 3, 4, 5])
>>> locals_ = plh.testrun_pipeline_upto(qreq_, 'nearest_neighbors')
>>> Kpad_list, = ut.dict_take(locals_, ['Kpad_list'])
>>> tup = nearest_neighbor_cacheid2(qreq_, Kpad_list)
>>> (nn_cachedir, nn_mid_cacheid_list) = tup
>>> result2 = 'nn_mid_cacheid_list2 = ' + ut.repr2(nn_mid_cacheid_list, nl=1)
>>> result = result1 + '\n' + result2
>>> print(result)
nn_mid_cacheid_list1 = [
'nnobj_8687dcb6-1f1f-fdd3-8b72-8f36f9f41905_DVUUIDS((5)oavtblnlrtocnrpm)_NN(single,cks800)_Chip(sz700,maxwh)_Feat(hesaff+sift)_FLANN(8_kdtrees)_truek6',
'nnobj_a2aef668-20c1-1897-d8f3-09a47a73f26a_DVUUIDS((5)oavtblnlrtocnrpm)_NN(single,cks800)_Chip(sz700,maxwh)_Feat(hesaff+sift)_FLANN(8_kdtrees)_truek6',
]
nn_mid_cacheid_list2 = [
'nnobj_8687dcb6-1f1f-fdd3-8b72-8f36f9f41905_DVUUIDS((5)oavtblnlrtocnrpm)_NN(single,cks800)_Chip(sz700,maxwh)_Feat(hesaff+sift)_FLANN(8_kdtrees)_truek6',
'nnobj_a2aef668-20c1-1897-d8f3-09a47a73f26a_DVUUIDS((5)oavtblnlrtocnrpm)_NN(single,cks800)_Chip(sz700,maxwh)_Feat(hesaff+sift)_FLANN(8_kdtrees)_truek6',
]
"""
from wbia.algo import Config
chip_cfgstr = qreq_.qparams.chip_cfgstr
feat_cfgstr = qreq_.qparams.feat_cfgstr
flann_cfgstr = qreq_.qparams.flann_cfgstr
requery = qreq_.qparams.requery
# assert requery is False, 'can not be on yet'
internal_daids = qreq_.get_internal_daids()
internal_qaids = qreq_.get_internal_qaids()
if requery:
assert qreq_.qparams.vsmany
data_hashid = qreq_.get_data_hashid()
else:
data_hashid = qreq_.ibs.get_annot_hashid_visual_uuid(internal_daids, prefix='D')
if requery:
query_hashid_list = qreq_.get_qreq_pcc_uuids(internal_qaids)
else:
# TODO: get attribute from qreq_, not wbia
query_hashid_list = qreq_.get_qreq_annot_visual_uuids(internal_qaids)
HACK_KCFG = True
if HACK_KCFG:
# hack config so we consolidate different k values
# (ie, K=2,Knorm=1 == K=1,Knorm=2)
nn_cfgstr = Config.NNConfig(**qreq_.qparams).get_cfgstr(
ignore_keys={'K', 'Knorm', 'use_k_padding'}
)
else:
nn_cfgstr = qreq_.qparams.nn_cfgstr
aug_cfgstr = 'aug_quryside' if qreq_.qparams.query_rotation_heuristic else ''
nn_mid_cacheid = ''.join(
[data_hashid, nn_cfgstr, chip_cfgstr, feat_cfgstr, flann_cfgstr, aug_cfgstr]
)
logger.info('nn_mid_cacheid = %r' % (nn_mid_cacheid,))
if HACK_KCFG:
kbase = qreq_.qparams.K + int(qreq_.qparams.Knorm)
nn_mid_cacheid_list = [
'nnobj_' + str(query_hashid) + nn_mid_cacheid + '_truek' + str(kbase + Kpad)
for query_hashid, Kpad in zip(query_hashid_list, Kpad_list)
]
else:
nn_mid_cacheid_list = [
'nnobj_' + str(query_hashid) + nn_mid_cacheid + '_' + str(Kpad)
for query_hashid, Kpad in zip(query_hashid_list, Kpad_list)
]
nn_cachedir = qreq_.ibs.get_neighbor_cachedir()
# ut.unixjoin(qreq_.ibs.get_cachedir(), 'neighborcache2')
ut.ensuredir(nn_cachedir)
if ut.VERBOSE:
logger.info('nn_mid_cacheid = %r' % (nn_mid_cacheid,))
pass
return nn_cachedir, nn_mid_cacheid_list
[docs]@profile
def cachemiss_nn_compute_fn(
flags_list, qreq_, Kpad_list, impossible_daids_list, K, Knorm, requery, verbose
):
"""
Logic for computing neighbors if there is a cache miss
>>> flags_list = [True] * len(Kpad_list)
>>> flags_list = [True, False, True]
"""
# Cant do this here because of get_nn_aids. bleh
# Could make this slightly more efficient
# qreq_.load_indexer(verbose=verbose)
# internal_qaids = qreq_.get_internal_qaids()
# internal_qaids = internal_qaids.compress(flags_list)
# Get only the data that needs to be computed
internal_qannots = qreq_.internal_qannots
internal_qannots = internal_qannots.compress(flags_list)
Kpad_list = ut.compress(Kpad_list, flags_list)
# do computation
if not requery:
num_neighbors_list = [K + Kpad + Knorm for Kpad in Kpad_list]
else:
num_neighbors_list = [K + Knorm] * len(Kpad_list)
Kpad_list = 2 * np.array(Kpad_list)
config2_ = qreq_.get_internal_query_config2()
qvecs_list = internal_qannots.vecs
qfxs_list = [np.arange(len(qvecs)) for qvecs in qvecs_list]
if config2_.minscale_thresh is not None or config2_.maxscale_thresh is not None:
min_ = -np.inf if config2_.minscale_thresh is None else config2_.minscale_thresh
max_ = np.inf if config2_.maxscale_thresh is None else config2_.maxscale_thresh
# qkpts_list = qreq_.ibs.get_annot_kpts(internal_qaids, config2_=config2_)
qkpts_list = internal_qannots.kpts
qkpts_list = vt.ziptake(qkpts_list, qfxs_list, axis=0)
# kpts_list = vt.ziptake(kpts_list, fxs_list, axis=0) # not needed for first filter
scales_list = [vt.get_scales(kpts) for kpts in qkpts_list]
# Remove data under the threshold
flags_list1 = [
np.logical_and(scales >= min_, scales <= max_) for scales in scales_list
]
qvecs_list = vt.zipcompress(qvecs_list, flags_list1, axis=0)
qfxs_list = vt.zipcompress(qfxs_list, flags_list1, axis=0)
if config2_.fgw_thresh is not None:
# qfgw_list = qreq_.ibs.get_annot_fgweights(
# internal_qaids, config2_=config2_)
qfgw_list = internal_qannots.fgweights
qfgw_list = vt.ziptake(qfgw_list, qfxs_list, axis=0)
fgw_thresh = config2_.fgw_thresh
flags_list2 = [fgws >= fgw_thresh for fgws in qfgw_list]
qfxs_list = vt.zipcompress(qfxs_list, flags_list2, axis=0)
qvecs_list = vt.zipcompress(qvecs_list, flags_list2, axis=0)
if verbose:
if len(qvecs_list) == 1:
logger.info('[hs] depth(qvecs_list) = %r' % (ut.depth_profile(qvecs_list),))
# Mark progress ane execute nearest indexer nearest neighbor code
prog_hook = None if qreq_.prog_hook is None else qreq_.prog_hook.next_subhook()
if requery:
# assert False, (
# 'need to implement part where matches with the same name are not considered'
# )
qvec_iter = ut.ProgressIter(qvecs_list, lbl=NN_LBL, prog_hook=prog_hook, **PROGKW)
"""
# Maybe some query vector stacking would help here
qvecs_stack = np.vstack(qvecs_list)
# Nope, really doesn't help that much
# For 100 annotations
%timeit np.vstack(qvecs_list)
# 100 loops, best of 3: 5.15 ms per loop
%timeit qreq_.indexer.knn(qvecs_stack, K)
# 1 loop, best of 3: 18.1 s per loop
%timeit [qreq_.indexer.knn(qfx2_vec, K) for qfx2_vec in qvecs_list]
# 1 loop, best of 3: 19.4 s per loop
"""
idx_dist_list = [
qreq_.indexer.requery_knn(qfx2_vec, K, pad, impossible_daids)
for qfx2_vec, K, pad, impossible_daids in zip(
qvec_iter, num_neighbors_list, Kpad_list, impossible_daids_list
)
]
else:
qvec_iter = ut.ProgressIter(qvecs_list, lbl=NN_LBL, prog_hook=prog_hook, **PROGKW)
idx_dist_list = [
qreq_.indexer.knn(qfx2_vec, num_neighbors)
for qfx2_vec, num_neighbors in zip(qvec_iter, num_neighbors_list)
]
# Move into new object structure
nns_list = [
Neighbors(qaid, idxs, dists, qfxs)
for qaid, qfxs, (idxs, dists) in zip(
internal_qannots.aid, qfxs_list, idx_dist_list
)
]
return nns_list
[docs]@profile
def nearest_neighbors(
qreq_, Kpad_list, impossible_daids_list=None, verbose=VERB_PIPELINE
):
"""
Plain Nearest Neighbors
Tries to load nearest neighbors from a cache instead of recomputing them.
CommandLine:
python -m wbia.algo.hots.pipeline --test-nearest_neighbors
python -m wbia.algo.hots.pipeline --test-nearest_neighbors --db PZ_MTEST --qaids=1:100
utprof.py -m wbia.algo.hots.pipeline --test-nearest_neighbors --db PZ_MTEST --qaids=1:100
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> import wbia
>>> verbose = True
>>> qreq_ = wbia.testdata_qreq_(defaultdb='testdb1', qaid_override=[1])
>>> locals_ = plh.testrun_pipeline_upto(qreq_, 'nearest_neighbors')
>>> Kpad_list, impossible_daids_list = ut.dict_take(
>>> locals_, ['Kpad_list', 'impossible_daids_list'])
>>> nns_list = nearest_neighbors(qreq_, Kpad_list, impossible_daids_list,
>>> verbose=verbose)
>>> qaid = qreq_.internal_qaids[0]
>>> nn = nns_list[0]
>>> (qfx2_idx, qfx2_dist) = nn
>>> num_neighbors = Kpad_list[0] + qreq_.qparams.K + qreq_.qparams.Knorm
>>> # Assert nns tuple is valid
>>> ut.assert_eq(qfx2_idx.shape, qfx2_dist.shape)
>>> ut.assert_eq(qfx2_idx.shape[1], num_neighbors)
>>> ut.assert_inbounds(qfx2_idx.shape[0], 1000, 3000)
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> import wbia
>>> verbose = True
>>> qreq_ = wbia.testdata_qreq_(defaultdb='testdb1', qaid_override=[1])
>>> locals_ = plh.testrun_pipeline_upto(qreq_, 'nearest_neighbors')
>>> Kpad_list, impossible_daids_list = ut.dict_take(
>>> locals_, ['Kpad_list', 'impossible_daids_list'])
>>> nns_list = nearest_neighbors(qreq_, Kpad_list, impossible_daids_list,
>>> verbose=verbose)
>>> qaid = qreq_.internal_qaids[0]
>>> nn = nns_list[0]
>>> (qfx2_idx, qfx2_dist) = nn
>>> num_neighbors = Kpad_list[0] + qreq_.qparams.K + qreq_.qparams.Knorm
>>> # Assert nns tuple is valid
>>> ut.assert_eq(qfx2_idx.shape, qfx2_dist.shape)
>>> ut.assert_eq(qfx2_idx.shape[1], num_neighbors)
>>> ut.assert_inbounds(qfx2_idx.shape[0], 1000, 3000)
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> import wbia
>>> verbose = True
>>> custom_nid_lookup = {a: a for a in range(14)}
>>> qreq1_ = wbia.testdata_qreq_(
>>> defaultdb='testdb1', t=['default:K=2,requery=True,can_match_samename=False'],
>>> daid_override=[2, 3, 4, 5, 6, 7, 8],
>>> qaid_override=[2, 5, 1], custom_nid_lookup=custom_nid_lookup)
>>> locals_ = plh.testrun_pipeline_upto(qreq1_, 'nearest_neighbors')
>>> Kpad_list, impossible_daids_list = ut.dict_take(
>>> locals_, ['Kpad_list', 'impossible_daids_list'])
>>> nns_list1 = nearest_neighbors(qreq1_, Kpad_list, impossible_daids_list,
>>> verbose=verbose)
>>> nn1 = nns_list1[0]
>>> nnvalid0_list1 = baseline_neighbor_filter(qreq1_, nns_list1,
>>> impossible_daids_list)
>>> assert np.all(nnvalid0_list1[0]), (
>>> 'requery should never produce impossible results')
>>> # Compare versus not using requery
>>> qreq2_ = wbia.testdata_qreq_(
>>> defaultdb='testdb1', t=['default:K=2,requery=False'],
>>> daid_override=[1, 2, 3, 4, 5, 6, 7, 8],
>>> qaid_override=[2, 5, 1])
>>> locals_ = plh.testrun_pipeline_upto(qreq2_, 'nearest_neighbors')
>>> Kpad_list, impossible_daids_list = ut.dict_take(
>>> locals_, ['Kpad_list', 'impossible_daids_list'])
>>> nns_list2 = nearest_neighbors(qreq2_, Kpad_list, impossible_daids_list,
>>> verbose=verbose)
>>> nn2 = nns_list2[0]
>>> nn1.neighb_dists
>>> nn2.neighb_dists
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> import wbia
>>> verbose = True
>>> qreq1_ = wbia.testdata_qreq_(
>>> defaultdb='testdb1', t=['default:K=5,requery=True,can_match_samename=False'],
>>> daid_override=[2, 3, 4, 5, 6, 7, 8],
>>> qaid_override=[2, 5, 1])
>>> locals_ = plh.testrun_pipeline_upto(qreq1_, 'nearest_neighbors')
>>> Kpad_list, impossible_daids_list = ut.dict_take(
>>> locals_, ['Kpad_list', 'impossible_daids_list'])
>>> nns_list1 = nearest_neighbors(qreq1_, Kpad_list, impossible_daids_list,
>>> verbose=verbose)
>>> nn1 = nns_list1[0]
>>> nnvalid0_list1 = baseline_neighbor_filter(qreq1_, nns_list1,
>>> impossible_daids_list)
>>> assert np.all(nnvalid0_list1[0]), 'should always be valid'
"""
K = qreq_.qparams.K
Knorm = qreq_.qparams.Knorm
requery = qreq_.qparams.requery
# checks = qreq_.qparams.checks
# Get both match neighbors (including padding) and normalizing neighbors
if verbose:
logger.info(
'[hs] Step 1) Assign nearest neighbors: %s' % (qreq_.qparams.nn_cfgstr,)
)
prog_hook = None if qreq_.prog_hook is None else qreq_.prog_hook.next_subhook()
qreq_.load_indexer(verbose=verbose, prog_hook=prog_hook)
# For each internal query annotation
# Find the nearest neighbors of each descriptor vector
# USE_NN_MID_CACHE = ut.is_developer()
use_cache = USE_NN_MID_CACHE
if use_cache:
nn_cachedir, nn_mid_cacheid_list = nearest_neighbor_cacheid2(qreq_, Kpad_list)
else:
# hacks
nn_mid_cacheid_list = [None] * len(qreq_.get_internal_qaids())
nn_cachedir = None
nns_list = ut.tryload_cache_list_with_compute(
use_cache,
nn_cachedir,
'neighbs4',
nn_mid_cacheid_list,
cachemiss_nn_compute_fn,
qreq_,
Kpad_list,
impossible_daids_list,
K,
Knorm,
requery,
verbose,
)
return nns_list
# ============================
# 2) Remove Impossible Weights
# ============================
[docs]@profile
def baseline_neighbor_filter(
qreq_, nns_list, impossible_daids_list, verbose=VERB_PIPELINE
):
"""
Removes matches to self, the same image, or the same name.
CommandLine:
python -m wbia.algo.hots.pipeline --test-baseline_neighbor_filter
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> qreq_, args = plh.testdata_pre(
>>> 'baseline_neighbor_filter', defaultdb='testdb1',
>>> qaid_override=[1, 2, 3, 4],
>>> daid_override=list(range(1, 11)),
>>> p=['default:QRH=False,requery=False,can_match_samename=False'],
>>> verbose=True)
>>> nns_list, impossible_daids_list = args
>>> nnvalid0_list = baseline_neighbor_filter(qreq_, nns_list,
>>> impossible_daids_list)
>>> ut.assert_eq(len(nnvalid0_list), len(qreq_.qaids))
>>> assert not np.any(nnvalid0_list[0][:, 0]), (
... 'first col should be all invalid because of self match')
>>> assert not np.all(nnvalid0_list[0][:, 1]), (
... 'second col should have some good matches')
>>> ut.assert_inbounds(nnvalid0_list[0].sum(), 1000, 10000)
"""
if verbose:
logger.info('[hs] Step 2) Baseline neighbor filter')
Knorm = qreq_.qparams.Knorm
# Find which annotations each query matched against
neighb_aids_iter = (
qreq_.indexer.get_nn_aids(nn.neighb_idxs.T[0 : nn.neighb_idxs.shape[1] - Knorm].T)
for nn in nns_list
)
filter_iter_ = zip(neighb_aids_iter, impossible_daids_list)
prog_hook = None if qreq_.prog_hook is None else qreq_.prog_hook.next_subhook()
filter_iter = ut.ProgressIter(
filter_iter_,
length=len(nns_list),
enabled=False,
lbl=FILT_LBL,
prog_hook=prog_hook,
**PROGKW,
)
# Check to be sure that none of the matched annotations are in the impossible set
nnvalid0_list = [
vt.get_uncovered_mask(neighb_aids, impossible_daids)
for neighb_aids, impossible_daids in filter_iter
]
return nnvalid0_list
# ============================
# 3) Nearest Neighbor weights
# ============================
[docs]@profile
def weight_neighbors(qreq_, nns_list, nnvalid0_list, verbose=VERB_PIPELINE):
"""
pipeline step 3 -
assigns weights to feature matches based on the active filter list
CommandLine:
python -m wbia.algo.hots.pipeline --test-weight_neighbors
python -m wbia.algo.hots.pipeline --test-weight_neighbors:0 --verbose --verbtd --ainfo --nocache --veryverbose
python -m wbia.algo.hots.pipeline --test-weight_neighbors:0 --show
python -m wbia.algo.hots.pipeline --test-weight_neighbors:1 --show
python -m wbia.algo.hots.pipeline --test-weight_neighbors:0 --show -t default:lnbnn_normer=lnbnn_fg_0.9__featscore,lnbnn_norm_thresh=.9
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> qreq_, args = plh.testdata_pre(
>>> 'weight_neighbors', defaultdb='testdb1',
>>> a=['default:qindex=0:3,dindex=0:5,hackerrors=False'],
>>> p=['default:codename=vsmany,bar_l2_on=True,fg_on=False'], verbose=True)
>>> nns_list, nnvalid0_list = args
>>> verbose = True
>>> weight_ret = weight_neighbors(qreq_, nns_list, nnvalid0_list, verbose)
>>> filtkey_list, filtweights_list, filtvalids_list, filtnormks_list = weight_ret
>>> import wbia.plottool as pt
>>> verbose = True
>>> cm_list = build_chipmatches(
>>> qreq_, nns_list, nnvalid0_list, filtkey_list, filtweights_list,
>>> filtvalids_list, filtnormks_list, verbose=verbose)
>>> ut.quit_if_noshow()
>>> cm = cm_list[0]
>>> cm.score_name_nsum(qreq_)
>>> cm.ishow_analysis(qreq_)
>>> ut.show_if_requested()
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> qreq_, args = plh.testdata_pre(
>>> 'weight_neighbors', defaultdb='testdb1',
>>> a=['default:qindex=0:3,dindex=0:5,hackerrors=False'],
>>> p=['default:codename=vsmany,bar_l2_on=True,fg_on=False'], verbose=True)
>>> nns_list, nnvalid0_list = args
>>> verbose = True
>>> weight_ret = weight_neighbors(qreq_, nns_list, nnvalid0_list, verbose)
>>> filtkey_list, filtweights_list, filtvalids_list, filtnormks_list = weight_ret
>>> nInternAids = len(qreq_.get_internal_qaids())
>>> nFiltKeys = len(filtkey_list)
>>> filtweight_depth = ut.depth_profile(filtweights_list)
>>> filtvalid_depth = ut.depth_profile(filtvalids_list)
>>> ut.assert_eq(nInternAids, len(filtweights_list))
>>> ut.assert_eq(nInternAids, len(filtvalids_list))
>>> ut.assert_eq(ut.get_list_column(filtweight_depth, 0), [nFiltKeys] * nInternAids)
>>> ut.assert_eq(filtvalid_depth, (nInternAids, nFiltKeys))
>>> ut.assert_eq(filtvalids_list, [[None, None], [None, None], [None, None]])
>>> ut.assert_eq(filtkey_list, [hstypes.FiltKeys.LNBNN, hstypes.FiltKeys.BARL2])
>>> ut.quit_if_noshow()
>>> import wbia.plottool as pt
>>> verbose = True
>>> cm_list = build_chipmatches(
>>> qreq_, nns_list, nnvalid0_list, filtkey_list, filtweights_list,
>>> filtvalids_list, filtnormks_list, verbose=verbose)
>>> cm = cm_list[0]
>>> cm.score_name_nsum(qreq_)
>>> cm.ishow_analysis(qreq_)
>>> ut.show_if_requested()
"""
if verbose:
logger.info('[hs] Step 3) Weight neighbors: ' + qreq_.qparams.nnweight_cfgstr)
if len(nns_list) == 1:
logger.info('[hs] depth(nns_list) ' + str(ut.depth_profile(nns_list)))
# logger.info(WEIGHT_LBL)
# intern_qaid_iter = ut.ProgressIter(internal_qaids, lbl=BUILDCM_LBL,
# **PROGKW)
# Build weights for each active filter
filtkey_list = []
_filtweight_list = []
_filtvalid_list = []
_filtnormk_list = []
config2_ = qreq_.extern_data_config2
if not config2_.sqrd_dist_on:
# Take the square root of the squared distances
for nns in nns_list:
nns.neighb_dists = np.sqrt(nns.neighb_dists.astype(np.float64))
# nns_list_ = [(neighb_idx, np.sqrt(neighb_dist.astype(np.float64)))
# for neighb_idx, neighb_dist in nns_list]
# nns_list = nns_list_
if config2_.lnbnn_on:
filtname = 'lnbnn'
lnbnn_weight_list, normk_list = nn_weights.NN_WEIGHT_FUNC_DICT[filtname](
nns_list, nnvalid0_list, qreq_
)
if config2_.lnbnn_normer is not None:
logger.info('[hs] normalizing feat scores')
if qreq_.lnbnn_normer is None:
qreq_.lnbnn_normer = vt.ScoreNormalizer()
# qreq_.lnbnn_normer.load(cfgstr=config2_.lnbnn_normer)
qreq_.lnbnn_normer.fuzzyload(partial_cfgstr=config2_.lnbnn_normer)
lnbnn_weight_list = [
qreq_.lnbnn_normer.normalize_scores(s.ravel()).reshape(s.shape)
for s in lnbnn_weight_list
]
# Thresholding like a champ!
lnbnn_norm_thresh = config2_.lnbnn_norm_thresh
# lnbnn_weight_list = [
# s * [s > lnbnn_norm_thresh] for s in lnbnn_weight_list
# ]
lnbnn_isvalid = [s > lnbnn_norm_thresh for s in lnbnn_weight_list]
# Softmaxing
# from scipy.special import expit
# y = expit(x * 6)
# lnbnn_weight_list = [
# vt.logistic_01(s)
# for s in lnbnn_weight_list
# ]
filtname += '_norm'
_filtvalid_list.append(lnbnn_isvalid) # None means all valid
else:
_filtvalid_list.append(None) # None means all valid
_filtweight_list.append(lnbnn_weight_list)
_filtnormk_list.append(normk_list)
filtkey_list.append(filtname)
if config2_.normonly_on:
filtname = 'normonly'
normonly_weight_list, normk_list = nn_weights.NN_WEIGHT_FUNC_DICT[filtname](
nns_list, nnvalid0_list, qreq_
)
_filtweight_list.append(normonly_weight_list)
_filtvalid_list.append(None) # None means all valid
_filtnormk_list.append(normk_list)
filtkey_list.append(filtname)
if config2_.bar_l2_on:
filtname = 'bar_l2'
bar_l2_weight_list, normk_list = nn_weights.NN_WEIGHT_FUNC_DICT[filtname](
nns_list, nnvalid0_list, qreq_
)
_filtweight_list.append(bar_l2_weight_list)
_filtvalid_list.append(None) # None means all valid
_filtnormk_list.append(None)
filtkey_list.append(filtname)
if config2_.ratio_thresh:
filtname = 'ratio'
ratio_weight_list, normk_list = nn_weights.NN_WEIGHT_FUNC_DICT[filtname](
nns_list, nnvalid0_list, qreq_
)
ratio_isvalid = [
neighb_ratio <= qreq_.qparams.ratio_thresh
for neighb_ratio in ratio_weight_list
]
# HACK TO GET 1 - RATIO AS SCORE
ratioscore_list = [
np.subtract(1, neighb_ratio) for neighb_ratio in ratio_weight_list
]
_filtweight_list.append(ratioscore_list)
_filtvalid_list.append(ratio_isvalid)
_filtnormk_list.append(normk_list)
filtkey_list.append(filtname)
# --simple weighted implm
if config2_.const_on:
filtname = 'const'
constvote_weight_list = nn_weights.NN_WEIGHT_FUNC_DICT[filtname](
nns_list, nnvalid0_list, qreq_
)
_filtweight_list.append(constvote_weight_list)
_filtvalid_list.append(None) # None means all valid
_filtnormk_list.append(None)
filtkey_list.append(filtname)
if config2_.fg_on:
filtname = 'fg'
fgvote_weight_list = nn_weights.NN_WEIGHT_FUNC_DICT[filtname](
nns_list, nnvalid0_list, qreq_
)
_filtweight_list.append(fgvote_weight_list)
_filtvalid_list.append(None) # None means all valid
_filtnormk_list.append(None)
filtkey_list.append(filtname)
# Switch nested list structure from [filt, qaid] to [qaid, filt]
nInternAids = len(nns_list)
filtweights_list = [
ut.get_list_column(_filtweight_list, index) for index in range(nInternAids)
]
filtvalids_list = [
[None if filtvalid is None else filtvalid[index] for filtvalid in _filtvalid_list]
for index in range(nInternAids)
]
filtnormks_list = [
[None if normk is None else normk[index] for normk in _filtnormk_list]
for index in range(nInternAids)
]
assert len(filtkey_list) > 0, 'no feature correspondece filter keys were specified'
weight_ret = WeightRet_(
filtkey_list, filtweights_list, filtvalids_list, filtnormks_list
)
return weight_ret
# ============================
# 4) Conversion from featurematches to chipmatches neighb -> aid2
# ============================
# @profile
[docs]def build_chipmatches(
qreq_,
nns_list,
nnvalid0_list,
filtkey_list,
filtweights_list,
filtvalids_list,
filtnormks_list,
verbose=VERB_PIPELINE,
):
"""
pipeline step 4 - builds sparse chipmatches
Takes the dense feature matches from query feature to (what could be any)
database features and builds sparse matching pairs for each annotation to
annotation match.
CommandLine:
python -m wbia build_chipmatches
python -m wbia build_chipmatches:0 --show
python -m wbia build_chipmatches:1 --show
python -m wbia build_chipmatches:2 --show
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> qreq_, args = plh.testdata_pre(
>>> 'build_chipmatches', p=['default:codename=vsmany'])
>>> (nns_list, nnvalid0_list, filtkey_list, filtweights_list,
>>> filtvalids_list, filtnormks_list) = args
>>> verbose = True
>>> cm_list = build_chipmatches(qreq_, *args, verbose=verbose)
>>> # verify results
>>> [cm.assert_self(qreq_) for cm in cm_list]
>>> cm = cm_list[0]
>>> fm = cm.fm_list[cm.daid2_idx[2]]
>>> num_matches = len(fm)
>>> print('vsmany num_matches = %r' % num_matches)
>>> ut.assert_inbounds(num_matches, 500, 2000, 'vsmany nmatches out of bounds')
>>> ut.quit_if_noshow()
>>> cm.score_annot_csum(qreq_)
>>> cm_list[0].ishow_single_annotmatch(qreq_)
>>> ut.show_if_requested()
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> # Test to make sure filtering by feature weights works
>>> qreq_, args = plh.testdata_pre(
>>> 'build_chipmatches',
>>> p=['default:codename=vsmany,fgw_thresh=.9'])
>>> (nns_list, nnvalid0_list, filtkey_list, filtweights_list,
>>> filtvalids_list, filtnormks_list) = args
>>> verbose = True
>>> cm_list = build_chipmatches(qreq_, *args, verbose=verbose)
>>> # verify results
>>> [cm.assert_self(qreq_) for cm in cm_list]
>>> cm = cm_list[0]
>>> fm = cm.fm_list[cm.daid2_idx[2]]
>>> num_matches = len(fm)
>>> print('num_matches = %r' % num_matches)
>>> ut.assert_inbounds(num_matches, 100, 410, 'vsmany nmatches out of bounds')
>>> ut.quit_if_noshow()
>>> cm.score_annot_csum(qreq_)
>>> cm_list[0].ishow_single_annotmatch(qreq_)
>>> ut.show_if_requested()
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> qreq_, args = plh.testdata_pre(
>>> 'build_chipmatches', p=['default:requery=True'], a='default')
>>> (nns_list, nnvalid0_list, filtkey_list, filtweights_list,
>>> filtvalids_list, filtnormks_list) = args
>>> verbose = True
>>> cm_list = build_chipmatches(qreq_, *args, verbose=verbose)
>>> # verify results
>>> [cm.assert_self(qreq_) for cm in cm_list]
>>> scoring.score_chipmatch_list(qreq_, cm_list, 'csum')
>>> cm = cm_list[0]
>>> for cm in cm_list:
>>> # should be positive for LNBNN
>>> assert np.all(cm.score_list[np.isfinite(cm.score_list)] >= 0)
"""
assert not qreq_.qparams.vsone, 'can no longer do vsone in pipeline'
Knorm = qreq_.qparams.Knorm
if verbose:
pipeline_root = qreq_.qparams.pipeline_root
logger.info('[hs] Step 4) Building chipmatches %s' % (pipeline_root,))
# Iterate over INTERNAL query annotation ids
prog_hook = None if qreq_.prog_hook is None else qreq_.prog_hook.next_subhook()
nns_iter = ut.ProgressIter(
nns_list, lbl=BUILDCM_LBL, enabled=False, prog_hook=prog_hook, **PROGKW
)
cm_list = [
get_sparse_matchinfo_nonagg(
qreq_,
nns,
neighb_valid0,
neighb_score_list,
neighb_valid_list,
neighb_normk_list,
Knorm,
fsv_col_lbls=filtkey_list,
)
for nns, neighb_valid0, neighb_score_list, neighb_valid_list, neighb_normk_list in zip(
nns_iter, nnvalid0_list, filtweights_list, filtvalids_list, filtnormks_list
)
]
return cm_list
# @profile
[docs]def get_sparse_matchinfo_nonagg(
qreq_,
nns,
neighb_valid0,
neighb_score_list,
neighb_valid_list,
neighb_normk_list,
Knorm,
fsv_col_lbls,
):
"""
builds sparse iterator that generates feature match pairs, scores, and ranks
Returns:
ValidMatchTup_ : vmt a tuple of corresponding lists. Each item in the
list corresponds to a daid, dfx, scorevec, rank, norm_aid, norm_fx...
CommandLine:
python -m wbia.algo.hots.pipeline --test-get_sparse_matchinfo_nonagg --show
python -m wbia.algo.hots.pipeline --test-get_sparse_matchinfo_nonagg:1 --show
utprof.py -m wbia.algo.hots.pipeline --test-get_sparse_matchinfo_nonagg
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> verbose = True
>>> qreq_, qaid, daid, args = plh.testdata_sparse_matchinfo_nonagg(
>>> defaultdb='PZ_MTEST', p=['default:Knorm=3,normalizer_rule=name,const_on=True,ratio_thresh=.2,sqrd_dist_on=True'])
>>> nns, neighb_valid0, neighb_score_list, neighb_valid_list, neighb_normk_list, Knorm, fsv_col_lbls = args
>>> cm = get_sparse_matchinfo_nonagg(qreq_, *args)
>>> qannot = qreq_.ibs.annots([qaid], config=qreq_.qparams)
>>> dannot = qreq_.ibs.annots(cm.daid_list, config=qreq_.qparams)
>>> cm.assert_self(verbose=False)
>>> ut.quit_if_noshow()
>>> cm.score_annot_csum(qreq_)
>>> cm.show_single_annotmatch(qreq_)
>>> ut.show_if_requested()
"""
# Unpack neighbor ids, indices, filter scores, and flags
indexer = qreq_.indexer
neighb_idx = nns.neighb_idxs
neighb_nnidx = neighb_idx.T[:-Knorm].T
qfx_list = nns.qfx_list
K = neighb_nnidx.T.shape[0]
neighb_daid = indexer.get_nn_aids(neighb_nnidx)
neighb_dfx = indexer.get_nn_featxs(neighb_nnidx)
# Determine matches that are valid using all measurements
neighb_valid_list_ = [neighb_valid0] + ut.filter_Nones(neighb_valid_list)
neighb_valid_agg = np.logical_and.reduce(neighb_valid_list_)
# We fill filter each relavant matrix by aggregate validity
flat_validx = np.flatnonzero(neighb_valid_agg)
# Infer the valid internal query feature indexes and ranks
valid_x = np.floor_divide(flat_validx, K, dtype=hstypes.INDEX_TYPE)
valid_qfx = qfx_list.take(valid_x)
valid_rank = np.mod(flat_validx, K, dtype=hstypes.FK_DTYPE)
# TODO: valid_qfx, valid_rank = np.unravel_index(flat_validx, (neighb_nnidx.shape[0], K))?
# Then take the valid indices from internal database
# annot_rowids, feature indexes, and all scores
valid_daid = neighb_daid.take(flat_validx, axis=None)
valid_dfx = neighb_dfx.take(flat_validx, axis=None)
valid_scorevec = np.concatenate(
[neighb_score.take(flat_validx)[:, None] for neighb_score in neighb_score_list],
axis=1,
)
# Incorporate Normalizers
# Normalizers for each weight filter that used a normalizer
# Determine which feature per annot was used as the normalizer for each filter
# Each non-None sub list is still in neighb_ format
num_filts = len(neighb_normk_list)
K = len(neighb_idx.T) - Knorm
norm_filtxs = ut.where_not_None(neighb_normk_list)
num_normed_filts = len(norm_filtxs)
if num_normed_filts > 0:
_normks = ut.take(neighb_normk_list, norm_filtxs)
# Offset index to get flat normalizer positions
_offset = np.arange(0, neighb_idx.size, neighb_idx.shape[1])
flat_normxs = [_offset + neighb_normk for neighb_normk in _normks]
flat_normidxs = [neighb_idx.take(ks) for ks in flat_normxs]
flat_norm_aids = [indexer.get_nn_aids(idx) for idx in flat_normidxs]
flat_norm_fxs = [indexer.get_nn_featxs(idx) for idx in flat_normidxs]
# Take the valid indicies
_valid_norm_aids = [aids.take(valid_x) for aids in flat_norm_aids]
_valid_norm_fxs = [fxs.take(valid_x) for fxs in flat_norm_fxs]
else:
_valid_norm_aids = []
_valid_norm_fxs = []
valid_norm_aids = ut.ungroup([_valid_norm_aids], [norm_filtxs], num_filts - 1)
valid_norm_fxs = ut.ungroup([_valid_norm_fxs], [norm_filtxs], num_filts - 1)
# ValidMatchTup_ = namedtuple('vmt', ( # valid_match_tup
# 'daid', 'qfx', 'dfx', 'scorevec', 'rank', 'norm_aids', 'norm_fxs'))
# vmt = ValidMatchTup_(valid_daid, valid_qfx, valid_dfx, valid_scorevec,
# valid_rank, valid_norm_aids, valid_norm_fxs)
# NOTE: CONTIGUOUS ARRAYS MAKE A HUGE DIFFERENCE
valid_fm = np.concatenate((valid_qfx[:, None], valid_dfx[:, None]), axis=1)
assert valid_fm.flags.c_contiguous, 'non-contiguous'
# valid_fm = np.ascontiguousarray(valid_fm)
daid_list, daid_groupxs = vt.group_indices(valid_daid)
fm_list = vt.apply_grouping(valid_fm, daid_groupxs)
fsv_list = vt.apply_grouping(valid_scorevec, daid_groupxs)
fk_list = vt.apply_grouping(valid_rank, daid_groupxs)
filtnorm_aids = [
None # [None] * len(daid_groupxs)
if aids is None
else vt.apply_grouping(aids, daid_groupxs)
for aids in valid_norm_aids
]
filtnorm_fxs = [
None # [None] * len(daid_groupxs)
if fxs is None
else vt.apply_grouping(fxs, daid_groupxs)
for fxs in valid_norm_fxs
]
assert len(filtnorm_aids) == len(fsv_col_lbls), 'bad normer'
assert len(filtnorm_fxs) == len(fsv_col_lbls), 'bad normer'
cm = chip_match.ChipMatch(
nns.qaid,
daid_list,
fm_list,
fsv_list,
fk_list,
fsv_col_lbls=fsv_col_lbls,
filtnorm_aids=filtnorm_aids,
filtnorm_fxs=filtnorm_fxs,
)
return cm
# ============================
# 5) Spatial Verification
# ============================
[docs]def spatial_verification(qreq_, cm_list_FILT, verbose=VERB_PIPELINE):
r"""
pipeline step 5 - spatially verify feature matches
Returns:
list: cm_listSVER - new list of spatially verified chipmatches
CommandLine:
python -m wbia.algo.hots.pipeline --test-spatial_verification --show
python -m wbia.algo.hots.pipeline --test-spatial_verification --show --qaid 1
python -m wbia.algo.hots.pipeline --test-spatial_verification:0
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> ibs, qreq_, cm_list = plh.testdata_pre_sver('PZ_MTEST', qaid_list=[18])
>>> scoring.score_chipmatch_list(qreq_, cm_list, qreq_.qparams.prescore_method) # HACK
>>> cm = cm_list[0]
>>> top_nids = cm.get_top_nids(6)
>>> verbose = True
>>> cm_list_SVER = spatial_verification(qreq_, cm_list)
>>> # Test Results
>>> cmSV = cm_list_SVER[0]
>>> scoring.score_chipmatch_list(qreq_, cm_list_SVER, qreq_.qparams.score_method) # HACK
>>> top_nids_SV = cmSV.get_top_nids(6)
>>> cm.print_csv(sort=True)
>>> cmSV.print_csv(sort=False)
>>> gt_daids = np.intersect1d(cm.get_groundtruth_daids(), cmSV.get_groundtruth_daids())
>>> fm_list = cm.get_annot_fm(gt_daids)
>>> fmSV_list = cmSV.get_annot_fm(gt_daids)
>>> maplen = lambda list_: np.array(list(map(len, list_)))
>>> assert len(gt_daids) > 0, 'ground truth did not survive'
>>> ut.assert_lessthan(maplen(fmSV_list), maplen(fm_list)), 'feature matches were not filtered'
>>> ut.quit_if_noshow()
>>> cmSV.show_daids_matches(qreq_, gt_daids)
>>> import wbia.plottool as pt
>>> #homog_tup = (refined_inliers, H)
>>> #aff_tup = (aff_inliers, Aff)
>>> #pt.draw_sv.show_sv(rchip1, rchip2, kpts1, kpts2, fm, aff_tup=aff_tup, homog_tup=homog_tup, refine_method=refine_method)
>>> ut.show_if_requested()
"""
cm_list = cm_list_FILT
if not qreq_.qparams.sv_on or qreq_.qparams.xy_thresh is None:
if verbose:
logger.info('[hs] Step 5) Spatial verification: off')
return cm_list
else:
cm_list_SVER = _spatial_verification(qreq_, cm_list, verbose=verbose)
return cm_list_SVER
# @profile
def _spatial_verification(qreq_, cm_list, verbose=VERB_PIPELINE):
"""
make only spatially valid features survive
>>> from wbia.algo.hots.pipeline import * # NOQA
"""
if verbose:
logger.info('[hs] Step 5) Spatial verification: ' + qreq_.qparams.sv_cfgstr)
# dbg info (can remove if there is a speed issue)
score_method = qreq_.qparams.score_method
prescore_method = qreq_.qparams.prescore_method
nNameShortList = qreq_.qparams.nNameShortlistSVER
nAnnotPerName = qreq_.qparams.nAnnotPerNameSVER
scoring.score_chipmatch_list(qreq_, cm_list, prescore_method)
cm_shortlist = scoring.make_chipmatch_shortlists(
qreq_, cm_list, nNameShortList, nAnnotPerName, score_method
)
prog_hook = None if qreq_.prog_hook is None else qreq_.prog_hook.next_subhook()
cm_progiter = ut.ProgressIter(
cm_shortlist,
length=len(cm_shortlist),
prog_hook=prog_hook,
lbl=SVER_LVL,
**PROGKW,
)
cm_list_SVER = [sver_single_chipmatch(qreq_, cm) for cm in cm_progiter]
# rescore after verification?
return cm_list_SVER
# @profile
[docs]def sver_single_chipmatch(qreq_, cm, verbose=False):
r"""
Spatially verifies a shortlist of a single chipmatch
TODO: move to chip match?
loops over a shortlist of results for a specific query annotation
Args:
qreq_ (QueryRequest): query request object with hyper-parameters
cm (ChipMatch):
Returns:
wbia.ChipMatch: cmSV
CommandLine:
python -m wbia draw_rank_cmc --db PZ_Master1 --show \
-t best:refine_method=[homog,affine,cv2-homog,cv2-ransac-homog,cv2-lmeds-homog] \
-a timectrlhard ---acfginfo --veryverbtd
python -m wbia draw_rank_cmc --db PZ_Master1 --show \
-t best:refine_method=[homog,cv2-lmeds-homog],full_homog_checks=[True,False] \
-a timectrlhard ---acfginfo --veryverbtd
python -m wbia sver_single_chipmatch --show \
-t default:full_homog_checks=True -a default --qaid 18
python -m wbia sver_single_chipmatch --show \
-t default:refine_method=affine -a default --qaid 18
python -m wbia sver_single_chipmatch --show \
-t default:refine_method=cv2-homog -a default --qaid 18
python -m wbia sver_single_chipmatch --show \
-t default:refine_method=cv2-homog,full_homog_checks=True -a default --qaid 18
python -m wbia sver_single_chipmatch --show \
-t default:refine_method=cv2-homog,full_homog_checks=False -a default --qaid 18
python -m wbia sver_single_chipmatch --show \
-t default:refine_method=cv2-lmeds-homog,full_homog_checks=False -a default --qaid 18
python -m wbia sver_single_chipmatch --show \
-t default:refine_method=cv2-ransac-homog,full_homog_checks=False -a default --qaid 18
python -m wbia sver_single_chipmatch --show \
-t default:full_homog_checks=False -a default --qaid 18
python -m wbia sver_single_chipmatch --show --qaid=18 --y=0
python -m wbia sver_single_chipmatch --show --qaid=18 --y=1
Example:
>>> # DISABLE_DOCTEST
>>> # Visualization
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> qreq_, args = plh.testdata_pre('spatial_verification', defaultdb='PZ_MTEST') #, qaid_list=[18])
>>> cm_list = args.cm_list_FILT
>>> ibs = qreq_.ibs
>>> cm = cm_list[0]
>>> scoring.score_chipmatch_list(qreq_, cm_list, qreq_.qparams.prescore_method) # HACK
>>> #locals_ = ut.exec_func_src(sver_single_chipmatch, key_list=['svtup_list'], sentinal='# <SENTINAL>')
>>> #svtup_list1, = locals_
>>> verbose = True
>>> source = ut.get_func_sourcecode(sver_single_chipmatch, stripdef=True, strip_docstr=True)
>>> source = ut.replace_between_tags(source, '', '# <SENTINAL>', '# </SENTINAL>')
>>> globals_ = globals().copy()
>>> exec(source, globals_)
>>> svtup_list = globals_['svtup_list']
>>> gt_daids = cm.get_groundtruth_daids()
>>> x = ut.get_argval('--y', type_=int, default=0)
>>> #print('x = %r' % (x,))
>>> #daid = daids[x % len(daids)]
>>> notnone_list = ut.not_list(ut.flag_None_items(svtup_list))
>>> valid_idxs = np.where(notnone_list)
>>> valid_daids = cm.daid_list[valid_idxs]
>>> assert len(valid_daids) > 0, 'cannot spatially verify'
>>> valid_gt_daids = np.intersect1d(gt_daids, valid_daids)
>>> #assert len(valid_gt_daids) == 0, 'no sver groundtruth'
>>> daid = valid_gt_daids[x] if len(valid_gt_daids) > 0 else valid_daids[x]
>>> idx = cm.daid2_idx[daid]
>>> svtup = svtup_list[idx]
>>> assert svtup is not None, 'SV TUP IS NONE'
>>> refined_inliers, refined_errors, H = svtup[0:3]
>>> aff_inliers, aff_errors, Aff = svtup[3:6]
>>> homog_tup = (refined_inliers, H)
>>> aff_tup = (aff_inliers, Aff)
>>> fm = cm.fm_list[idx]
>>> aid1 = cm.qaid
>>> aid2 = daid
>>> rchip1, = ibs.get_annot_chips([aid1], config2_=qreq_.extern_query_config2)
>>> kpts1, = ibs.get_annot_kpts([aid1], config2_=qreq_.extern_query_config2)
>>> rchip2, = ibs.get_annot_chips([aid2], config2_=qreq_.extern_data_config2)
>>> kpts2, = ibs.get_annot_kpts([aid2], config2_=qreq_.extern_data_config2)
>>> import wbia.plottool as pt
>>> import matplotlib as mpl
>>> from wbia.scripts.thesis import TMP_RC
>>> mpl.rcParams.update(TMP_RC)
>>> show_aff = not ut.get_argflag('--noaff')
>>> refine_method = qreq_.qparams.refine_method if not ut.get_argflag('--norefinelbl') else ''
>>> pt.draw_sv.show_sv(rchip1, rchip2, kpts1, kpts2, fm, aff_tup=aff_tup,
>>> homog_tup=homog_tup, show_aff=show_aff,
>>> refine_method=refine_method)
>>> ut.show_if_requested()
"""
qaid = cm.qaid
use_chip_extent = qreq_.qparams.use_chip_extent
xy_thresh = qreq_.qparams.xy_thresh
scale_thresh = qreq_.qparams.scale_thresh
ori_thresh = qreq_.qparams.ori_thresh
min_nInliers = qreq_.qparams.min_nInliers
full_homog_checks = qreq_.qparams.full_homog_checks
refine_method = qreq_.qparams.refine_method
sver_output_weighting = qreq_.qparams.sver_output_weighting
# Precompute sver cmtup_old
kpts1 = qreq_.get_qreq_qannot_kpts(qaid).astype(np.float64)
kpts2_list = qreq_.get_qreq_dannot_kpts(cm.daid_list)
if use_chip_extent:
top_dlen_sqrd_list = qreq_.ibs.get_annot_chip_dlensqrd(
cm.daid_list, config2_=qreq_.extern_data_config2
)
else:
top_dlen_sqrd_list = compute_matching_dlen_extent(qreq_, cm.fm_list, kpts2_list)
config2_ = qreq_.extern_query_config2
if qreq_.qparams.weight_inliers:
# Weights for inlier scoring
if config2_.get('fg_on'):
qweights = qreq_.ibs.get_annot_fgweights(
[qaid], ensure=True, config2_=config2_
)[0].astype(np.float64)
else:
num = qreq_.ibs.get_annot_num_feats([qaid], config2_=config2_)[0]
qweights = np.ones(num, np.float64)
match_weight_list = [qweights.take(fm.T[0]) for fm in cm.fm_list]
else:
match_weight_list = [np.ones(len(fm), dtype=np.float64) for fm in cm.fm_list]
# Make an svtup for every daid in the shortlist
_iter1 = zip(
cm.daid_list,
cm.fm_list,
cm.fsv_list,
kpts2_list,
top_dlen_sqrd_list,
match_weight_list,
)
if verbose:
_iter1 = ut.ProgIter(
_iter1, length=len(cm.daid_list), lbl='sver shortlist', freq=1
)
svtup_list = []
for daid, fm, fsv, kpts2, dlen_sqrd2, match_weights in _iter1:
if len(fm) == 0:
# skip results without any matches
sv_tup = None
else:
# sver_testdata = dict(
# kpts1=kpts1,
# kpts2=kpts2,
# fm=fm,
# xy_thresh=xy_thresh,
# scale_thresh=scale_thresh,
# ori_thresh=ori_thresh,
# dlen_sqrd2=dlen_sqrd2,
# min_nInliers=min_nInliers,
# match_weights=match_weights,
# full_homog_checks=full_homog_checks,
# refine_method=refine_method
# )
# locals().update(ut.load_data('sver_testdata.pkl'))
try:
# Compute homography from chip2 to chip1 returned homography
# maps image1 space into image2 space image1 is a query chip
# and image2 is a database chip
sv_tup = vt.spatially_verify_kpts(
kpts1,
kpts2,
fm,
xy_thresh,
scale_thresh,
ori_thresh,
dlen_sqrd2,
min_nInliers,
match_weights=match_weights,
full_homog_checks=full_homog_checks,
refine_method=refine_method,
returnAff=True,
)
except Exception as ex:
ut.printex(
ex,
'Unknown error in spatial verification.',
keys=[
'kpts1',
'kpts2',
'fm',
'xy_thresh',
'scale_thresh',
'dlen_sqrd2',
'min_nInliers',
],
)
sv_tup = None
svtup_list.append(sv_tup)
# <SENTINAL>
# New way
inliers_list = []
for sv_tup in svtup_list:
if sv_tup is None:
inliers_list.append(None)
else:
(homog_inliers, homog_errors, H, aff_inliers, aff_errors, Aff) = sv_tup
inliers_list.append(homog_inliers)
indicies_list = inliers_list
cmSV = cm.take_feature_matches(indicies_list, keepscores=False)
# NOTE: It is not very clear explicitly, but the way H_list and
# homog_err_weight_list are built will correspond with the daid_list in
# cmSV returned by cm.take_feature_matches
svtup_list_ = ut.filter_Nones(svtup_list)
H_list_SV = ut.get_list_column(svtup_list_, 2)
cmSV.H_list = H_list_SV
if sver_output_weighting:
homog_err_weight_list = []
xy_thresh_sqrd = dlen_sqrd2 * xy_thresh
for sv_tup in svtup_list_:
(homog_inliers, homog_errors) = sv_tup[0:2]
homog_xy_errors = homog_errors[0].take(homog_inliers, axis=0)
homog_err_weight = 1.0 - np.sqrt(homog_xy_errors / xy_thresh_sqrd)
homog_err_weight_list.append(homog_err_weight)
# Rescore based on homography errors
filtkey = hstypes.FiltKeys.HOMOGERR
filtweight_list = homog_err_weight_list
cmSV.append_featscore_column(filtkey, filtweight_list)
return cmSV
[docs]def compute_matching_dlen_extent(qreq_, fm_list, kpts_list):
r"""
helper for spatial verification, computes the squared diagonal length of
matching chips
CommandLine:
python -m wbia.algo.hots.pipeline --test-compute_matching_dlen_extent
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.pipeline import * # NOQA
>>> ibs, qreq_, cm_list = plh.testdata_pre_sver('PZ_MTEST')
>>> verbose = True
>>> cm = cm_list[0]
>>> cm.set_cannonical_annot_score(cm.get_num_matches_list())
>>> cm.sortself()
>>> fm_list = cm.fm_list
>>> kpts_list = qreq_.get_qreq_dannot_kpts(cm.daid_list.tolist())
>>> topx2_dlen_sqrd = compute_matching_dlen_extent(qreq_, fm_list, kpts_list)
>>> ut.assert_inbounds(np.sqrt(topx2_dlen_sqrd)[0:5], 600, 1500)
"""
# Use extent of matching keypoints
# first get matching keypoints
fx2_list = [fm.T[1] for fm in fm_list]
kpts2_m_list = vt.ziptake(kpts_list, fx2_list, axis=0)
# [kpts.take(fx2, axis=0) for (kpts, fx2) in zip(kpts_list, fx2_list)]
dlen_sqrd_list = [vt.get_kpts_dlen_sqrd(kpts2_m) for kpts2_m in kpts2_m_list]
return dlen_sqrd_list