mirror of
https://gitcode.com/gh_mirrors/ope/OpenFace.git
synced 2026-05-16 12:17:57 +00:00
478 lines
16 KiB
C++
478 lines
16 KiB
C++
// Copyright (C) 2008 Davis E. King (davis@dlib.net)
|
|
// License: Boost Software License See LICENSE.txt for the full license.
|
|
#ifndef DLIB_KERNEL_FEATURE_RANKINg_H_
|
|
#define DLIB_KERNEL_FEATURE_RANKINg_H_
|
|
|
|
#include <vector>
|
|
#include <limits>
|
|
|
|
#include "feature_ranking_abstract.h"
|
|
#include "kcentroid.h"
|
|
#include "../optimization.h"
|
|
#include "../statistics.h"
|
|
#include <iostream>
|
|
|
|
namespace dlib
|
|
{
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename kernel_type,
|
|
typename sample_matrix_type,
|
|
typename label_matrix_type
|
|
>
|
|
matrix<typename kernel_type::scalar_type,0,2,typename kernel_type::mem_manager_type> rank_features_impl (
|
|
const kcentroid<kernel_type>& kc,
|
|
const sample_matrix_type& samples,
|
|
const label_matrix_type& labels
|
|
)
|
|
{
|
|
/*
|
|
This function ranks features by doing recursive feature elimination
|
|
|
|
*/
|
|
typedef typename kernel_type::scalar_type scalar_type;
|
|
typedef typename kernel_type::mem_manager_type mm;
|
|
|
|
|
|
// make sure requires clause is not broken
|
|
DLIB_ASSERT(is_binary_classification_problem(samples, labels) == true,
|
|
"\tmatrix rank_features()"
|
|
<< "\n\t you have given invalid arguments to this function"
|
|
);
|
|
|
|
matrix<scalar_type,0,2,mm> results(samples(0).nr(), 2);
|
|
matrix<scalar_type,sample_matrix_type::type::NR,1,mm> mask(samples(0).nr());
|
|
set_all_elements(mask,1);
|
|
|
|
// figure out what the separation is between the two centroids when all the features are
|
|
// present.
|
|
scalar_type first_separation;
|
|
{
|
|
kcentroid<kernel_type> c1(kc);
|
|
kcentroid<kernel_type> c2(kc);
|
|
// find the centers of each class
|
|
for (long s = 0; s < samples.size(); ++s)
|
|
{
|
|
if (labels(s) < 0)
|
|
{
|
|
c1.train(samples(s));
|
|
}
|
|
else
|
|
{
|
|
c2.train(samples(s));
|
|
}
|
|
|
|
}
|
|
first_separation = c1(c2);
|
|
}
|
|
|
|
|
|
using namespace std;
|
|
|
|
for (long i = results.nr()-1; i >= 0; --i)
|
|
{
|
|
long worst_feature_idx = 0;
|
|
scalar_type worst_feature_score = -std::numeric_limits<scalar_type>::infinity();
|
|
|
|
// figure out which feature to remove next
|
|
for (long j = 0; j < mask.size(); ++j)
|
|
{
|
|
// skip features we have already removed
|
|
if (mask(j) == 0)
|
|
continue;
|
|
|
|
kcentroid<kernel_type> c1(kc);
|
|
kcentroid<kernel_type> c2(kc);
|
|
|
|
// temporarily remove this feature from the working set of features
|
|
mask(j) = 0;
|
|
|
|
// find the centers of each class
|
|
for (long s = 0; s < samples.size(); ++s)
|
|
{
|
|
if (labels(s) < 0)
|
|
{
|
|
c1.train(pointwise_multiply(samples(s),mask));
|
|
}
|
|
else
|
|
{
|
|
c2.train(pointwise_multiply(samples(s),mask));
|
|
}
|
|
|
|
}
|
|
|
|
// find the distance between the two centroids and use that
|
|
// as the score
|
|
const double score = c1(c2);
|
|
|
|
if (score > worst_feature_score)
|
|
{
|
|
worst_feature_score = score;
|
|
worst_feature_idx = j;
|
|
}
|
|
|
|
// add this feature back to the working set of features
|
|
mask(j) = 1;
|
|
|
|
}
|
|
|
|
// now that we know what the next worst feature is record it
|
|
mask(worst_feature_idx) = 0;
|
|
results(i,0) = worst_feature_idx;
|
|
results(i,1) = worst_feature_score;
|
|
}
|
|
|
|
// now normalize the results
|
|
const scalar_type max_separation = std::max(max(colm(results,1)), first_separation);
|
|
set_colm(results,1) = colm(results,1)/max_separation;
|
|
for (long r = 0; r < results.nr()-1; ++r)
|
|
{
|
|
results(r,1) = results(r+1,1);
|
|
}
|
|
results(results.nr()-1,1) = first_separation/max_separation;
|
|
|
|
return results;
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename kernel_type,
|
|
typename sample_matrix_type,
|
|
typename label_matrix_type
|
|
>
|
|
matrix<typename kernel_type::scalar_type,0,2,typename kernel_type::mem_manager_type> rank_features (
|
|
const kcentroid<kernel_type>& kc,
|
|
const sample_matrix_type& samples,
|
|
const label_matrix_type& labels
|
|
)
|
|
{
|
|
return rank_features_impl(kc, mat(samples), mat(labels));
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename kernel_type,
|
|
typename sample_matrix_type,
|
|
typename label_matrix_type
|
|
>
|
|
matrix<typename kernel_type::scalar_type,0,2,typename kernel_type::mem_manager_type> rank_features_impl (
|
|
const kcentroid<kernel_type>& kc,
|
|
const sample_matrix_type& samples,
|
|
const label_matrix_type& labels,
|
|
const long num_features
|
|
)
|
|
{
|
|
/*
|
|
This function ranks features by doing recursive feature addition
|
|
|
|
*/
|
|
typedef typename kernel_type::scalar_type scalar_type;
|
|
typedef typename kernel_type::mem_manager_type mm;
|
|
|
|
// make sure requires clause is not broken
|
|
DLIB_ASSERT(is_binary_classification_problem(samples, labels) == true,
|
|
"\tmatrix rank_features()"
|
|
<< "\n\t you have given invalid arguments to this function"
|
|
);
|
|
DLIB_ASSERT(0 < num_features && num_features <= samples(0).nr(),
|
|
"\tmatrix rank_features()"
|
|
<< "\n\t you have given invalid arguments to this function"
|
|
<< "\n\t num_features: " << num_features
|
|
<< "\n\t samples(0).nr(): " << samples(0).nr()
|
|
);
|
|
|
|
matrix<scalar_type,0,2,mm> results(num_features, 2);
|
|
matrix<scalar_type,sample_matrix_type::type::NR,1,mm> mask(samples(0).nr());
|
|
set_all_elements(mask,0);
|
|
|
|
using namespace std;
|
|
|
|
for (long i = 0; i < results.nr(); ++i)
|
|
{
|
|
long best_feature_idx = 0;
|
|
scalar_type best_feature_score = -std::numeric_limits<scalar_type>::infinity();
|
|
|
|
// figure out which feature to add next
|
|
for (long j = 0; j < mask.size(); ++j)
|
|
{
|
|
// skip features we have already added
|
|
if (mask(j) == 1)
|
|
continue;
|
|
|
|
kcentroid<kernel_type> c1(kc);
|
|
kcentroid<kernel_type> c2(kc);
|
|
|
|
// temporarily add this feature to the working set of features
|
|
mask(j) = 1;
|
|
|
|
// find the centers of each class
|
|
for (long s = 0; s < samples.size(); ++s)
|
|
{
|
|
if (labels(s) < 0)
|
|
{
|
|
c1.train(pointwise_multiply(samples(s),mask));
|
|
}
|
|
else
|
|
{
|
|
c2.train(pointwise_multiply(samples(s),mask));
|
|
}
|
|
|
|
}
|
|
|
|
// find the distance between the two centroids and use that
|
|
// as the score
|
|
const double score = c1(c2);
|
|
|
|
if (score > best_feature_score)
|
|
{
|
|
best_feature_score = score;
|
|
best_feature_idx = j;
|
|
}
|
|
|
|
// take this feature back out of the working set of features
|
|
mask(j) = 0;
|
|
|
|
}
|
|
|
|
// now that we know what the next best feature is record it
|
|
mask(best_feature_idx) = 1;
|
|
results(i,0) = best_feature_idx;
|
|
results(i,1) = best_feature_score;
|
|
}
|
|
|
|
// now normalize the results
|
|
set_colm(results,1) = colm(results,1)/max(colm(results,1));
|
|
|
|
return results;
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename kernel_type,
|
|
typename sample_matrix_type,
|
|
typename label_matrix_type
|
|
>
|
|
matrix<typename kernel_type::scalar_type,0,2,typename kernel_type::mem_manager_type> rank_features (
|
|
const kcentroid<kernel_type>& kc,
|
|
const sample_matrix_type& samples,
|
|
const label_matrix_type& labels,
|
|
const long num_features
|
|
)
|
|
{
|
|
if (mat(samples).nr() > 0 && num_features == mat(samples)(0).nr())
|
|
{
|
|
// if we are going to rank them all then might as well do the recursive feature elimination version
|
|
return rank_features_impl(kc, mat(samples), mat(labels));
|
|
}
|
|
else
|
|
{
|
|
return rank_features_impl(kc, mat(samples), mat(labels), num_features);
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
// ----------------------------------------------------------------------------------------
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
namespace rank_features_helpers
|
|
{
|
|
template <
|
|
typename K,
|
|
typename sample_matrix_type,
|
|
typename label_matrix_type
|
|
>
|
|
typename K::scalar_type centroid_gap (
|
|
const kcentroid<K>& kc,
|
|
const sample_matrix_type& samples,
|
|
const label_matrix_type& labels
|
|
)
|
|
{
|
|
kcentroid<K> kc1(kc);
|
|
kcentroid<K> kc2(kc);
|
|
|
|
// toss all the samples into our kcentroids
|
|
for (long i = 0; i < samples.size(); ++i)
|
|
{
|
|
if (labels(i) > 0)
|
|
kc1.train(samples(i));
|
|
else
|
|
kc2.train(samples(i));
|
|
}
|
|
|
|
// now return the separation between the mean of these two centroids
|
|
return kc1(kc2);
|
|
}
|
|
|
|
template <
|
|
typename sample_matrix_type,
|
|
typename label_matrix_type
|
|
>
|
|
class test
|
|
{
|
|
typedef typename sample_matrix_type::type sample_type;
|
|
typedef typename sample_type::type scalar_type;
|
|
typedef typename sample_type::mem_manager_type mem_manager_type;
|
|
|
|
public:
|
|
test (
|
|
const sample_matrix_type& samples_,
|
|
const label_matrix_type& labels_,
|
|
unsigned long num_sv_,
|
|
bool verbose_
|
|
) : samples(samples_), labels(labels_), num_sv(num_sv_), verbose(verbose_)
|
|
{
|
|
}
|
|
|
|
double operator() (
|
|
double gamma
|
|
) const
|
|
{
|
|
using namespace std;
|
|
|
|
// we are doing the optimization in log space so don't forget to convert back to normal space
|
|
gamma = std::exp(gamma);
|
|
|
|
typedef radial_basis_kernel<sample_type> kernel_type;
|
|
// Make a kcentroid and find out what the gap is at the current gamma. Try to pick a reasonable
|
|
// tolerance.
|
|
const double tolerance = std::min(gamma*0.01, 0.01);
|
|
const kernel_type kern(gamma);
|
|
kcentroid<kernel_type> kc(kern, tolerance, num_sv);
|
|
scalar_type temp = centroid_gap(kc, samples, labels);
|
|
|
|
if (verbose)
|
|
{
|
|
cout << "\rChecking goodness of gamma = " << gamma << ". Goodness = "
|
|
<< temp << " " << flush;
|
|
}
|
|
return temp;
|
|
}
|
|
|
|
const sample_matrix_type& samples;
|
|
const label_matrix_type& labels;
|
|
unsigned long num_sv;
|
|
bool verbose;
|
|
|
|
};
|
|
|
|
template <
|
|
typename sample_matrix_type,
|
|
typename label_matrix_type
|
|
>
|
|
double find_gamma_with_big_centroid_gap_impl (
|
|
const sample_matrix_type& samples,
|
|
const label_matrix_type& labels,
|
|
double initial_gamma,
|
|
unsigned long num_sv,
|
|
bool verbose
|
|
)
|
|
{
|
|
using namespace std;
|
|
|
|
if (verbose)
|
|
{
|
|
cout << endl;
|
|
}
|
|
|
|
test<sample_matrix_type, label_matrix_type> funct(samples, labels, num_sv, verbose);
|
|
double best_gamma = std::log(initial_gamma);
|
|
double goodness = find_max_single_variable(funct, best_gamma, -15, 15, 1e-3, 100);
|
|
|
|
if (verbose)
|
|
{
|
|
cout << "\rBest gamma = " << std::exp(best_gamma) << ". Goodness = "
|
|
<< goodness << " " << endl;
|
|
}
|
|
|
|
return std::exp(best_gamma);
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename sample_matrix_type,
|
|
typename label_matrix_type
|
|
>
|
|
double find_gamma_with_big_centroid_gap (
|
|
const sample_matrix_type& samples,
|
|
const label_matrix_type& labels,
|
|
double initial_gamma = 0.1,
|
|
unsigned long num_sv = 40
|
|
)
|
|
{
|
|
DLIB_ASSERT(initial_gamma > 0 && num_sv > 0 && is_binary_classification_problem(samples, labels),
|
|
"\t double find_gamma_with_big_centroid_gap()"
|
|
<< "\n\t initial_gamma: " << initial_gamma
|
|
<< "\n\t num_sv: " << num_sv
|
|
<< "\n\t is_binary_classification_problem(): " << is_binary_classification_problem(samples, labels)
|
|
);
|
|
|
|
return rank_features_helpers::find_gamma_with_big_centroid_gap_impl(mat(samples),
|
|
mat(labels),
|
|
initial_gamma,
|
|
num_sv,
|
|
false);
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename sample_matrix_type,
|
|
typename label_matrix_type
|
|
>
|
|
double verbose_find_gamma_with_big_centroid_gap (
|
|
const sample_matrix_type& samples,
|
|
const label_matrix_type& labels,
|
|
double initial_gamma = 0.1,
|
|
unsigned long num_sv = 40
|
|
)
|
|
{
|
|
DLIB_ASSERT(initial_gamma > 0 && num_sv > 0 && is_binary_classification_problem(samples, labels),
|
|
"\t double verbose_find_gamma_with_big_centroid_gap()"
|
|
<< "\n\t initial_gamma: " << initial_gamma
|
|
<< "\n\t num_sv: " << num_sv
|
|
<< "\n\t is_binary_classification_problem(): " << is_binary_classification_problem(samples, labels)
|
|
);
|
|
|
|
return rank_features_helpers::find_gamma_with_big_centroid_gap_impl(mat(samples),
|
|
mat(labels),
|
|
initial_gamma,
|
|
num_sv,
|
|
true);
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename vector_type
|
|
>
|
|
double compute_mean_squared_distance (
|
|
const vector_type& samples
|
|
)
|
|
{
|
|
running_stats<double> rs;
|
|
for (unsigned long i = 0; i < samples.size(); ++i)
|
|
{
|
|
for (unsigned long j = i+1; j < samples.size(); ++j)
|
|
{
|
|
rs.add(length_squared(samples[i] - samples[j]));
|
|
}
|
|
}
|
|
|
|
return rs.mean();
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
}
|
|
|
|
#endif // DLIB_KERNEL_FEATURE_RANKINg_H_
|
|
|
|
|