I developed my own Gaussian Bernoulli RBM in scikit style; It consists of a real-valued visible layer and a binary hidden-layer.
I followed the guidelines of a gaussian rbm explained here :
pylearn2 Gaussian RBM
Problem is, that this gaussian RBM horribly degraded classification performance for image recognition. In running a test case, a gaussian RBM extracts hidden features from a digits dataset. Applying Logistic Regression on those features gets 0.419 score, whereas on Raw pixels gets 0.947 score.
The code below outlines the file contents. You can download the complete file using this public, dropbox link,
gaussian_rbm.py
You can run the file to execute the test case that uses the digits dataset. It compares between the score achieved using raw pixels with that of hidden RBM features.
Thanks!
class GaussianBernoulliRBM(BaseEstimator, TransformerMixin):
def __init__(self, n_components=256, learning_rate=0.1,
n_iter=10, sigma=1,random_state=None):
self.n_components = n_components
self.learning_rate = learning_rate
self.n_iter = n_iter
self.random_state = random_state
self.sigma=sigma
def transform(self, X):
"""returns the hidden neurons as features"
X, = check_arrays(X, sparse_format='csc', dtype=np.float)
return self._mean_hiddens_given_visibles(X)
def _mean_hiddens_given_visibles(self, v):
"""Computes the probabilities P(h|v).
P(h|v) = sigmoid( (v^T Wh + bias_hid) / sigma^2 )
"""
return logistic_sigmoid(safe_sparse_dot(v, self.components_.T)
+ self.intercept_hidden_) / self.sigma**2
def _mean_visibles_given_hiddens(self, h):
"""Computes the probabilities P(v|h).
P(v|h) = N( Wh + bias_vis, sigma^2)
"""
return np.dot(h, self.components_) + self.intercept_visible_
def _free_energy(self, v):
""" Computes the free energy F(v)
F(v) = ( (1/2) v.T v - bias_vis.T v) / sigma^2 - sum_i softplus( ( v^T W + c) / sigma^2 )_i
"""
sq_term = 0.5 *np.square(v).sum(axis=1)
bias_term = v.dot(self.intercept_visible_)
softplus_term = _softplus(safe_sparse_dot(v, self.components_.T) + self.intercept_hidden_).sum(axis=1)
return (sq_term-bias_term)/self.sigma -softplus_term
def reconstruct(self, v):
"""reconstruct by computing positive phase
followed by the negative phase
"""
h_ = self._mean_hiddens_given_visibles(v)
v_ = self._mean_visibles_given_hiddens(h_)
return v_
def _fit(self, v_pos, rng):
"""trains gaussian RBM"""
h_pos = self._mean_hiddens_given_visibles(v_pos)
v_neg = self._mean_visibles_given_hiddens(self.h_samples_)
h_neg = self._mean_hiddens_given_visibles(v_neg)
lr = float(self.learning_rate) / v_pos.shape[0]
update = safe_sparse_dot(v_pos.T, h_pos, dense_output=True).T
update -= np.dot(v_neg.T, h_neg).T
self.components_ += lr * update
self.intercept_hidden_ += lr * (h_pos.sum(axis=0) - h_neg.sum(axis=0))
self.intercept_visible_ += lr * (np.asarray(
v_pos.sum(axis=0)).squeeze() -
v_neg.sum(axis=0))
h_neg[rng.uniform(size=h_neg.shape) < h_neg] = 1.0 # sample binomial
self.h_samples_ = np.floor(h_neg, h_neg)
return self.scores(v_pos)
def scores(self, v):
"""returns reconstruction score
scores(v) = -( v - bias_vis - sigmoid( (v^T W + bias_hid) / sigma^2 ) W^T )/sigma^2
"""
return -((v-self.reconstruct(v))/self.sigma**2)
def fit(self, X, y=None):
"""initializes parameters for training"""
n_samples = X.shape[0]
rng = check_random_state(self.random_state)
self.components_ = np.asarray(
rng.normal(0, 0.01, (self.n_components, X.shape[1])),
order='fortran')
self.intercept_hidden_ = np.zeros(self.n_components, )
self.intercept_visible_ = np.zeros(X.shape[1], )
self.h_samples_ = np.zeros((n_samples, self.n_components))
for iteration in xrange(self.n_iter):
pl = self._fit(X, rng).sum()
pl /= n_samples
print("Iteration %d, score = %.2f"
% (iteration, pl))
return self
