diff options
| author | ziejd2 | 2017-09-28 15:04:40 -0500 |
|---|---|---|
| committer | ziejd2 | 2017-09-28 15:04:40 -0500 |
| commit | 8070dc963753142bb86c4ed698d91fd623ed28e7 (patch) | |
| tree | d0f6dd8fc46a49b819aa55c1a90faa14d8448883 /sourcecodes/bnt-master/netlab3.3 | |
| parent | 7cc31810d53176e805532b2789955f4eedbce6bb (diff) | |
| download | BNW-8070dc963753142bb86c4ed698d91fd623ed28e7.tar.gz | |
BNW using Octave instead of Matlab.
This version of BNW should perform the same as the original version. The only difference is that it uses Octave instead of Matlab when running BayesNet Toolbox during parameter learning. I am calling this BNW_1.02. It can be accessed at: compbio.uthsc.edu/BNW_1.02
Diffstat (limited to 'sourcecodes/bnt-master/netlab3.3')
182 files changed, 16107 insertions, 0 deletions
diff --git a/sourcecodes/bnt-master/netlab3.3/CVS/Entries b/sourcecodes/bnt-master/netlab3.3/CVS/Entries new file mode 100644 index 00000000..5a92ce95 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/CVS/Entries @@ -0,0 +1,180 @@ +/Contents.m/1.1.1.1/Wed Apr 27 17:59:04 2005// +/LICENSE/1.1.1.1/Wed Apr 27 17:59:04 2005// +/conffig.m/1.1.1.1/Wed Apr 27 17:59:04 2005// +/confmat.m/1.1.1.1/Wed Apr 27 17:59:04 2005// +/conjgrad.m/1.1.1.1/Wed Apr 27 17:59:04 2005// +/consist.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/convertoldnet.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/datread.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/datwrite.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/dem2ddat.m/1.1.1.1/Tue May 10 01:32:18 2005// +/demard.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demev1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demev2.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demev3.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgauss.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demglm1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demglm2.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgmm1.m/1.1.1.1/Sun May 8 17:51:14 2005// +/demgmm2.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgmm3.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgmm4.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgmm5.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgp.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgpard.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgpot.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgtm1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgtm2.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demhint.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demhmc1.m/1.1.1.1/Tue May 24 00:03:34 2005// +/demhmc2.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demhmc3.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demkmn1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demknn1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demmdn1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demmet1.m/1.1.1.1/Mon May 23 06:23:08 2005// +/demmlp1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demmlp2.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demnlab.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demns1.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demolgd1.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demopt1.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/dempot.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demprgp.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demprior.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demrbf1.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demsom1.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demtrain.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/dist2.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/eigdec.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/errbayes.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/evidence.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/fevbayes.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gauss.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gbayes.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glm.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmderiv.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmerr.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmevfwd.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmfwd.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmgrad.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmhess.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glminit.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmpak.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmtrain.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmunpak.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gmm.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gmmactiv.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gmmem.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gmminit.m/1.1.1.1/Mon May 23 23:44:46 2005// +/gmmpak.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gmmpost.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gmmprob.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gmmsamp.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gmmunpak.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gp.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpcovar.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpcovarf.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpcovarp.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gperr.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpfwd.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpgrad.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpinit.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gppak.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpunpak.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gradchek.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/graddesc.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gsamp.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtm.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmem.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmfwd.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtminit.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmlmean.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmlmode.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmmag.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmpost.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmprob.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/hbayes.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/hesschek.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/hintmat.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/hinton.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/histp.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/hmc.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/kmeansNetlab.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/knn.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/knnfwd.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/linef.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/linemin.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/maxitmess.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdn.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdn2gmm.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdndist2.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnerr.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnfwd.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdngrad.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdninit.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnnet.mat/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnpak.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnpost.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnprob.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnunpak.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/metrop.m/1.1.1.1/Mon May 23 23:07:40 2005// +/minbrack.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlp.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpbkp.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpderiv.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlperr.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpevfwd.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpfwd.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpgrad.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlphdotv.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlphess.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlphint.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpinit.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlppak.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpprior.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlptrain.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpunpak.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/netderiv.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/neterr.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netevfwd.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netgrad.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/nethess.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netinit.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netlab3.3.zip/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netlogo.mat/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netopt.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netpak.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netunpak.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/oilTrn.dat/1.1.1.1/Wed Apr 27 17:59:14 2005// +/oilTst.dat/1.1.1.1/Wed Apr 27 17:59:14 2005// +/olgd.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/pca.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/plotmat.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/ppca.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/quasinew.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbf.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfbkp.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfderiv.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbferr.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfevfwd.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbffwd.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfgrad.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfhess.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfjacob.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfpak.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfprior.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfsetbf.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfsetfw.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbftrain.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/rbfunpak.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/rosegrad.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/rosen.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/scg.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/som.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/somfwd.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/sompak.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/somtrain.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/somunpak.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/xor.dat/1.1.1.1/Wed Apr 27 17:59:16 2005// +D diff --git a/sourcecodes/bnt-master/netlab3.3/CVS/Repository b/sourcecodes/bnt-master/netlab3.3/CVS/Repository new file mode 100644 index 00000000..297cfa8a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/CVS/Repository @@ -0,0 +1 @@ +FullBNT/netlab3.3 diff --git a/sourcecodes/bnt-master/netlab3.3/CVS/Root b/sourcecodes/bnt-master/netlab3.3/CVS/Root new file mode 100644 index 00000000..f3bd14a6 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/CVS/Root @@ -0,0 +1 @@ +:ext:nsaunier@bnt.cvs.sourceforge.net:/cvsroot/bnt diff --git a/sourcecodes/bnt-master/netlab3.3/Contents.m b/sourcecodes/bnt-master/netlab3.3/Contents.m new file mode 100644 index 00000000..aaf32e1f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/Contents.m @@ -0,0 +1,176 @@ +% Netlab Toolbox +% Version 3.3.1 18-Jun-2004 +% +% conffig - Display a confusion matrix. +% confmat - Compute a confusion matrix. +% conjgrad - Conjugate gradients optimization. +% consist - Check that arguments are consistent. +% convertoldnet- Convert pre-2.3 release MLP and MDN nets to new format +% datread - Read data from an ascii file. +% datwrite - Write data to ascii file. +% dem2ddat - Generates two dimensional data for demos. +% demard - Automatic relevance determination using the MLP. +% demev1 - Demonstrate Bayesian regression for the MLP. +% demev2 - Demonstrate Bayesian classification for the MLP. +% demev3 - Demonstrate Bayesian regression for the RBF. +% demgauss - Demonstrate sampling from Gaussian distributions. +% demglm1 - Demonstrate simple classification using a generalized linear model. +% demglm2 - Demonstrate simple classification using a generalized linear model. +% demgmm1 - Demonstrate density modelling with a Gaussian mixture model. +% demgmm3 - Demonstrate density modelling with a Gaussian mixture model. +% demgmm4 - Demonstrate density modelling with a Gaussian mixture model. +% demgmm5 - Demonstrate density modelling with a PPCA mixture model. +% demgp - Demonstrate simple regression using a Gaussian Process. +% demgpard - Demonstrate ARD using a Gaussian Process. +% demgpot - Computes the gradient of the negative log likelihood for a mixture model. +% demgtm1 - Demonstrate EM for GTM. +% demgtm2 - Demonstrate GTM for visualisation. +% demhint - Demonstration of Hinton diagram for 2-layer feed-forward network. +% demhmc1 - Demonstrate Hybrid Monte Carlo sampling on mixture of two Gaussians. +% demhmc2 - Demonstrate Bayesian regression with Hybrid Monte Carlo sampling. +% demhmc3 - Demonstrate Bayesian regression with Hybrid Monte Carlo sampling. +% demkmean - Demonstrate simple clustering model trained with K-means. +% demknn1 - Demonstrate nearest neighbour classifier. +% demmdn1 - Demonstrate fitting a multi-valued function using a Mixture Density Network. +% demmet1 - Demonstrate Markov Chain Monte Carlo sampling on a Gaussian. +% demmlp1 - Demonstrate simple regression using a multi-layer perceptron +% demmlp2 - Demonstrate simple classification using a multi-layer perceptron +% demnlab - A front-end Graphical User Interface to the demos +% demns1 - Demonstrate Neuroscale for visualisation. +% demolgd1 - Demonstrate simple MLP optimisation with on-line gradient descent +% demopt1 - Demonstrate different optimisers on Rosenbrock's function. +% dempot - Computes the negative log likelihood for a mixture model. +% demprgp - Demonstrate sampling from a Gaussian Process prior. +% demprior - Demonstrate sampling from a multi-parameter Gaussian prior. +% demrbf1 - Demonstrate simple regression using a radial basis function network. +% demsom1 - Demonstrate SOM for visualisation. +% demtrain - Demonstrate training of MLP network. +% dist2 - Calculates squared distance between two sets of points. +% eigdec - Sorted eigendecomposition +% errbayes - Evaluate Bayesian error function for network. +% evidence - Re-estimate hyperparameters using evidence approximation. +% fevbayes - Evaluate Bayesian regularisation for network forward propagation. +% gauss - Evaluate a Gaussian distribution. +% gbayes - Evaluate gradient of Bayesian error function for network. +% glm - Create a generalized linear model. +% glmderiv - Evaluate derivatives of GLM outputs with respect to weights. +% glmerr - Evaluate error function for generalized linear model. +% glmevfwd - Forward propagation with evidence for GLM +% glmfwd - Forward propagation through generalized linear model. +% glmgrad - Evaluate gradient of error function for generalized linear model. +% glmhess - Evaluate the Hessian matrix for a generalised linear model. +% glminit - Initialise the weights in a generalized linear model. +% glmpak - Combines weights and biases into one weights vector. +% glmtrain - Specialised training of generalized linear model +% glmunpak - Separates weights vector into weight and bias matrices. +% gmm - Creates a Gaussian mixture model with specified architecture. +% gmmactiv - Computes the activations of a Gaussian mixture model. +% gmmem - EM algorithm for Gaussian mixture model. +% gmminit - Initialises Gaussian mixture model from data +% gmmpak - Combines all the parameters in a Gaussian mixture model into one vector. +% gmmpost - Computes the class posterior probabilities of a Gaussian mixture model. +% gmmprob - Computes the data probability for a Gaussian mixture model. +% gmmsamp - Sample from a Gaussian mixture distribution. +% gmmunpak - Separates a vector of Gaussian mixture model parameters into its components. +% gp - Create a Gaussian Process. +% gpcovar - Calculate the covariance for a Gaussian Process. +% gpcovarf - Calculate the covariance function for a Gaussian Process. +% gpcovarp - Calculate the prior covariance for a Gaussian Process. +% gperr - Evaluate error function for Gaussian Process. +% gpfwd - Forward propagation through Gaussian Process. +% gpgrad - Evaluate error gradient for Gaussian Process. +% gpinit - Initialise Gaussian Process model. +% gppak - Combines GP hyperparameters into one vector. +% gpunpak - Separates hyperparameter vector into components. +% gradchek - Checks a user-defined gradient function using finite differences. +% graddesc - Gradient descent optimization. +% gsamp - Sample from a Gaussian distribution. +% gtm - Create a Generative Topographic Map. +% gtmem - EM algorithm for Generative Topographic Mapping. +% gtmfwd - Forward propagation through GTM. +% gtminit - Initialise the weights and latent sample in a GTM. +% gtmlmean - Mean responsibility for data in a GTM. +% gtmlmode - Mode responsibility for data in a GTM. +% gtmmag - Magnification factors for a GTM +% gtmpost - Latent space responsibility for data in a GTM. +% gtmprob - Probability for data under a GTM. +% hbayes - Evaluate Hessian of Bayesian error function for network. +% hesschek - Use central differences to confirm correct evaluation of Hessian matrix. +% hintmat - Evaluates the coordinates of the patches for a Hinton diagram. +% hinton - Plot Hinton diagram for a weight matrix. +% histp - Histogram estimate of 1-dimensional probability distribution. +% hmc - Hybrid Monte Carlo sampling. +% kmeans - Trains a k means cluster model. +% knn - Creates a K-nearest-neighbour classifier. +% knnfwd - Forward propagation through a K-nearest-neighbour classifier. +% linef - Calculate function value along a line. +% linemin - One dimensional minimization. +% maxitmess- Create a standard error message when training reaches max. iterations. +% mdn - Creates a Mixture Density Network with specified architecture. +% mdn2gmm - Converts an MDN mixture data structure to array of GMMs. +% mdndist2 - Calculates squared distance between centres of Gaussian kernels and data +% mdnerr - Evaluate error function for Mixture Density Network. +% mdnfwd - Forward propagation through Mixture Density Network. +% mdngrad - Evaluate gradient of error function for Mixture Density Network. +% mdninit - Initialise the weights in a Mixture Density Network. +% mdnpak - Combines weights and biases into one weights vector. +% mdnpost - Computes the posterior probability for each MDN mixture component. +% mdnprob - Computes the data probability likelihood for an MDN mixture structure. +% mdnunpak - Separates weights vector into weight and bias matrices. +% metrop - Markov Chain Monte Carlo sampling with Metropolis algorithm. +% minbrack - Bracket a minimum of a function of one variable. +% mlp - Create a 2-layer feedforward network. +% mlpbkp - Backpropagate gradient of error function for 2-layer network. +% mlpderiv - Evaluate derivatives of network outputs with respect to weights. +% mlperr - Evaluate error function for 2-layer network. +% mlpevfwd - Forward propagation with evidence for MLP +% mlpfwd - Forward propagation through 2-layer network. +% mlpgrad - Evaluate gradient of error function for 2-layer network. +% mlphdotv - Evaluate the product of the data Hessian with a vector. +% mlphess - Evaluate the Hessian matrix for a multi-layer perceptron network. +% mlphint - Plot Hinton diagram for 2-layer feed-forward network. +% mlpinit - Initialise the weights in a 2-layer feedforward network. +% mlppak - Combines weights and biases into one weights vector. +% mlpprior - Create Gaussian prior for mlp. +% mlptrain - Utility to train an MLP network for demtrain +% mlpunpak - Separates weights vector into weight and bias matrices. +% netderiv - Evaluate derivatives of network outputs by weights generically. +% neterr - Evaluate network error function for generic optimizers +% netevfwd - Generic forward propagation with evidence for network +% netgrad - Evaluate network error gradient for generic optimizers +% nethess - Evaluate network Hessian +% netinit - Initialise the weights in a network. +% netopt - Optimize the weights in a network model. +% netpak - Combines weights and biases into one weights vector. +% netunpak - Separates weights vector into weight and bias matrices. +% olgd - On-line gradient descent optimization. +% pca - Principal Components Analysis +% plotmat - Display a matrix. +% ppca - Probabilistic Principal Components Analysis +% quasinew - Quasi-Newton optimization. +% rbf - Creates an RBF network with specified architecture +% rbfbkp - Backpropagate gradient of error function for RBF network. +% rbfderiv - Evaluate derivatives of RBF network outputs with respect to weights. +% rbferr - Evaluate error function for RBF network. +% rbfevfwd - Forward propagation with evidence for RBF +% rbffwd - Forward propagation through RBF network with linear outputs. +% rbfgrad - Evaluate gradient of error function for RBF network. +% rbfhess - Evaluate the Hessian matrix for RBF network. +% rbfjacob - Evaluate derivatives of RBF network outputs with respect to inputs. +% rbfpak - Combines all the parameters in an RBF network into one weights vector. +% rbfprior - Create Gaussian prior and output layer mask for RBF. +% rbfsetbf - Set basis functions of RBF from data. +% rbfsetfw - Set basis function widths of RBF. +% rbftrain - Two stage training of RBF network. +% rbfunpak - Separates a vector of RBF weights into its components. +% rosegrad - Calculate gradient of Rosenbrock's function. +% rosen - Calculate Rosenbrock's function. +% scg - Scaled conjugate gradient optimization. +% som - Creates a Self-Organising Map. +% somfwd - Forward propagation through a Self-Organising Map. +% sompak - Combines node weights into one weights matrix. +% somtrain - Kohonen training algorithm for SOM. +% somunpak - Replaces node weights in SOM. +% +% Copyright (c) Ian T Nabney (1996-2001) +% diff --git a/sourcecodes/bnt-master/netlab3.3/LICENSE b/sourcecodes/bnt-master/netlab3.3/LICENSE new file mode 100644 index 00000000..0dd4a362 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/LICENSE @@ -0,0 +1,42 @@ + Copyright (c) 1996-2001, Ian T. Nabney + All rights reserved. + + Redistribution and use in source and binary + forms, with or without modification, are + permitted provided that the following + conditions are met: + + * Redistributions of source code must + retain the above copyright notice, this + list of conditions and the following + disclaimer. + * Redistributions in binary form must + reproduce the above copyright notice, + this list of conditions and the + following disclaimer in the + documentation and/or other materials + provided with the distribution. + * Neither the name of the Aston University, Birmingham, U.K. + nor the names of its contributors may be + used to endorse or promote products + derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT + HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT + NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT + OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. diff --git a/sourcecodes/bnt-master/netlab3.3/conffig.m b/sourcecodes/bnt-master/netlab3.3/conffig.m new file mode 100644 index 00000000..23d774e0 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/conffig.m @@ -0,0 +1,29 @@ +function fh=conffig(y, t) +%CONFFIG Display a confusion matrix. +% +% Description +% CONFFIG(Y, T) displays the confusion matrix and classification +% performance for the predictions mat{y} compared with the targets T. +% The data is assumed to be in a 1-of-N encoding, unless there is just +% one column, when it is assumed to be a 2 class problem with a 0-1 +% encoding. Each row of Y and T corresponds to a single example. +% +% In the confusion matrix, the rows represent the true classes and the +% columns the predicted classes. +% +% FH = CONFFIG(Y, T) also returns the figure handle FH which can be +% used, for instance, to delete the figure when it is no longer needed. +% +% See also +% CONFMAT, DEMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[C, rate] = confmat(y, t); + +fh = figure('Name', 'Confusion matrix', ... + 'NumberTitle', 'off'); + +plotmat(C, 'k', 'k', 14); +title(['Classification rate: ' num2str(rate(1)) '%'], 'FontSize', 14); diff --git a/sourcecodes/bnt-master/netlab3.3/confmat.m b/sourcecodes/bnt-master/netlab3.3/confmat.m new file mode 100644 index 00000000..e03315bb --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/confmat.m @@ -0,0 +1,56 @@ +function [C,rate]=confmat(Y,T) +%CONFMAT Compute a confusion matrix. +% +% Description +% [C, RATE] = CONFMAT(Y, T) computes the confusion matrix C and +% classification performance RATE for the predictions mat{y} compared +% with the targets T. The data is assumed to be in a 1-of-N encoding, +% unless there is just one column, when it is assumed to be a 2 class +% problem with a 0-1 encoding. Each row of Y and T corresponds to a +% single example. +% +% In the confusion matrix, the rows represent the true classes and the +% columns the predicted classes. The vector RATE has two entries: the +% percentage of correct classifications and the total number of correct +% classifications. +% +% See also +% CONFFIG, DEMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[n c]=size(Y); +[n2 c2]=size(T); + +if n~=n2 | c~=c2 + error('Outputs and targets are different sizes') +end + +if c > 1 + % Find the winning class assuming 1-of-N encoding + [maximum Yclass] = max(Y', [], 1); + + TL=[1:c]*T'; +else + % Assume two classes with 0-1 encoding + c = 2; + class2 = find(T > 0.5); + TL = ones(n, 1); + TL(class2) = 2; + class2 = find(Y > 0.5); + Yclass = ones(n, 1); + Yclass(class2) = 2; +end + +% Compute +correct = (Yclass==TL); +total=sum(sum(correct)); +rate=[total*100/n total]; + +C=zeros(c,c); +for i=1:c + for j=1:c + C(i,j) = sum((Yclass==j).*(TL==i)); + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/conjgrad.m b/sourcecodes/bnt-master/netlab3.3/conjgrad.m new file mode 100644 index 00000000..d459af2d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/conjgrad.m @@ -0,0 +1,168 @@ +function [x, options, flog, pointlog] = conjgrad(f, x, options, gradf, ... + varargin) +%CONJGRAD Conjugate gradients optimization. +% +% Description +% [X, OPTIONS, FLOG, POINTLOG] = CONJGRAD(F, X, OPTIONS, GRADF) uses a +% conjugate gradients algorithm to find the minimum of the function +% F(X) whose gradient is given by GRADF(X). Here X is a row vector and +% F returns a scalar value. The point at which F has a local minimum +% is returned as X. The function value at that point is returned in +% OPTIONS(8). A log of the function values after each cycle is +% (optionally) returned in FLOG, and a log of the points visited is +% (optionally) returned in POINTLOG. +% +% CONJGRAD(F, X, OPTIONS, GRADF, P1, P2, ...) allows additional +% arguments to be passed to F() and GRADF(). +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG, and the points visited in the +% return argument POINTSLOG. If OPTIONS(1) is set to 0, then only +% warning messages are displayed. If OPTIONS(1) is -1, then nothing is +% displayed. +% +% OPTIONS(2) is a measure of the absolute precision required for the +% value of X at the solution. If the absolute difference between the +% values of X between two successive steps is less than OPTIONS(2), +% then this condition is satisfied. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. If the absolute difference between the +% objective function values between two successive steps is less than +% OPTIONS(3), then this condition is satisfied. Both this and the +% previous condition must be satisfied for termination. +% +% OPTIONS(9) is set to 1 to check the user defined gradient function. +% +% OPTIONS(10) returns the total number of function evaluations +% (including those in any line searches). +% +% OPTIONS(11) returns the total number of gradient evaluations. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% OPTIONS(15) is the precision in parameter space of the line search; +% default 1E-4. +% +% See also +% GRADDESC, LINEMIN, MINBRACK, QUASINEW, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set up the options. +if length(options) < 18 + error('Options vector too short') +end + +if(options(14)) + niters = options(14); +else + niters = 100; +end + +% Set up options for line search +line_options = foptions; +% Need a precise line search for success +if options(15) > 0 + line_options(2) = options(15); +else + line_options(2) = 1e-4; +end + +display = options(1); + +% Next two lines allow conjgrad to work with expression strings +f = fcnchk(f, length(varargin)); +gradf = fcnchk(gradf, length(varargin)); + +% Check gradients +if (options(9)) + feval('gradchek', x, f, gradf, varargin{:}); +end + +options(10) = 0; +options(11) = 0; +nparams = length(x); +fnew = feval(f, x, varargin{:}); +options(10) = options(10) + 1; +gradnew = feval(gradf, x, varargin{:}); +options(11) = options(11) + 1; +d = -gradnew; % Initial search direction +br_min = 0; +br_max = 1.0; % Initial value for maximum distance to search along +tol = sqrt(eps); + +j = 1; +if nargout >= 3 + flog(j, :) = fnew; + if nargout == 4 + pointlog(j, :) = x; + end +end + +while (j <= niters) + + xold = x; + fold = fnew; + gradold = gradnew; + + gg = gradold*gradold'; + if (gg == 0.0) + % If the gradient is zero then we are done. + options(8) = fnew; + return; + end + + % This shouldn't occur, but rest of code depends on d being downhill + if (gradnew*d' > 0) + d = -d; + if options(1) >= 0 + warning('search direction uphill in conjgrad'); + end + end + + line_sd = d./norm(d); + [lmin, line_options] = feval('linemin', f, xold, line_sd, fold, ... + line_options, varargin{:}); + options(10) = options(10) + line_options(10); + options(11) = options(11) + line_options(11); + % Set x and fnew to be the actual search point we have found + x = xold + lmin * line_sd; + fnew = line_options(8); + + % Check for termination + if (max(abs(x - xold)) < options(2) & max(abs(fnew - fold)) < options(3)) + options(8) = fnew; + return; + end + + gradnew = feval(gradf, x, varargin{:}); + options(11) = options(11) + 1; + + % Use Polak-Ribiere formula to update search direction + gamma = ((gradnew - gradold)*(gradnew)')/gg; + d = (d .* gamma) - gradnew; + + if (display > 0) + fprintf(1, 'Cycle %4d Function %11.6f\n', j, line_options(8)); + end + + j = j + 1; + if nargout >= 3 + flog(j, :) = fnew; + if nargout == 4 + pointlog(j, :) = x; + end + end +end + +% If we get here, then we haven't terminated in the given number of +% iterations. + +options(8) = fold; +if (options(1) >= 0) + disp(maxitmess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/consist.m b/sourcecodes/bnt-master/netlab3.3/consist.m new file mode 100644 index 00000000..9305b265 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/consist.m @@ -0,0 +1,87 @@ +function errstring = consist(model, type, inputs, outputs) +%CONSIST Check that arguments are consistent. +% +% Description +% +% ERRSTRING = CONSIST(NET, TYPE, INPUTS) takes a network data structure +% NET together with a string TYPE containing the correct network type, +% a matrix INPUTS of input vectors and checks that the data structure +% is consistent with the other arguments. An empty string is returned +% if there is no error, otherwise the string contains the relevant +% error message. If the TYPE string is empty, then any type of network +% is allowed. +% +% ERRSTRING = CONSIST(NET, TYPE) takes a network data structure NET +% together with a string TYPE containing the correct network type, and +% checks that the two types match. +% +% ERRSTRING = CONSIST(NET, TYPE, INPUTS, OUTPUTS) also checks that the +% network has the correct number of outputs, and that the number of +% patterns in the INPUTS and OUTPUTS is the same. The fields in NET +% that are used are +% type +% nin +% nout +% +% See also +% MLPFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Assume that all is OK as default +errstring = ''; + +% If type string is not empty +if ~isempty(type) + % First check that model has type field + if ~isfield(model, 'type') + errstring = 'Data structure does not contain type field'; + return + end + % Check that model has the correct type + s = model.type; + if ~strcmp(s, type) + errstring = ['Model type ''', s, ''' does not match expected type ''',... + type, '''']; + return + end +end + +% If inputs are present, check that they have correct dimension +if nargin > 2 + if ~isfield(model, 'nin') + errstring = 'Data structure does not contain nin field'; + return + end + + data_nin = size(inputs, 2); + if model.nin ~= data_nin + errstring = ['Dimension of inputs ', num2str(data_nin), ... + ' does not match number of model inputs ', num2str(model.nin)]; + return + end +end + +% If outputs are present, check that they have correct dimension +if nargin > 3 + if ~isfield(model, 'nout') + errstring = 'Data structure does not conatin nout field'; + return + end + data_nout = size(outputs, 2); + if model.nout ~= data_nout + errstring = ['Dimension of outputs ', num2str(data_nout), ... + ' does not match number of model outputs ', num2str(model.nout)]; + return + end + +% Also check that number of data points in inputs and outputs is the same + num_in = size(inputs, 1); + num_out = size(outputs, 1); + if num_in ~= num_out + errstring = ['Number of input patterns ', num2str(num_in), ... + ' does not match number of output patterns ', num2str(num_out)]; + return + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/convertoldnet.m b/sourcecodes/bnt-master/netlab3.3/convertoldnet.m new file mode 100644 index 00000000..bf281c83 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/convertoldnet.m @@ -0,0 +1,26 @@ +function net = convertoldnet(net) +%CONVERTOLDNET Convert pre-2.3 release MLP and MDN nets to new format +% +% Description +% NET = CONVERTOLDNET(NET) takes a network NET and, if appropriate, +% converts it from pre-2.3 to the current format. The difference is +% simply that in MLPs and the MLP sub-net of MDNs the field ACTFN has +% been renamed OUTFN to make it consistent with GLM and RBF networks. +% If the network is not old-format or an MLP or MDN it is left +% unchanged. +% +% See also +% MLP, MDN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +switch net.type + case 'mlp' + if (isfield(net, 'actfn')) + net.outfn = net.actfn; + net = rmfield(net, 'actfn'); + end + case 'mdn' + net.mlp = convertoldnet(net.mlp); +end diff --git a/sourcecodes/bnt-master/netlab3.3/datread.m b/sourcecodes/bnt-master/netlab3.3/datread.m new file mode 100644 index 00000000..f729ef53 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/datread.m @@ -0,0 +1,99 @@ +function [x, t, nin, nout, ndata] = datread(filename) +%DATREAD Read data from an ascii file. +% +% Description +% +% [X, T, NIN, NOUT, NDATA] = DATREAD(FILENAME) reads from the file +% FILENAME and returns a matrix X of input vectors, a matrix T of +% target vectors, and integers NIN, NOUT and NDATA specifying the +% number of inputs, the number of outputs and the number of data points +% respectively. +% +% The format of the data file is as follows: the first row contains the +% string NIN followed by the number of inputs, the second row contains +% the string NOUT followed by the number of outputs, and the third row +% contains the string NDATA followed by the number of data vectors. +% Subsequent lines each contain one input vector followed by one output +% vector, with individual values separated by spaces. +% +% See also +% nin 2 nout 1 ndata 4 0.000000e+00 0.000000e+00 +% 1.000000e+00 0.000000e+00 1.000000e+00 0.000000e+00 +% 1.000000e+00 0.000000e+00 0.000000e+00 1.000000e+00 +% 1.000000e+00 1.000000e+00 See Also +% DATWRITE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +fid = fopen(filename, 'rt'); +if fid == -1 + error('Failed to open file.') +end + +% Read number of inputs +s1 = fscanf(fid, '%s', 1); +if ~strcmp(s1, 'nin') + fclose(fid); + error('String ''nin'' not found') +end +nin = fscanf(fid, '%d\n', 1); +if ~isnumeric(nin) + fclose(fid); + error('No number for nin') +end +if nin < 0 | round(nin) ~= nin + fclose(fid); + error('nin must be a non-negative integer') +end + +% Read number of outputs +s2 = fscanf(fid, '%s', 1); +if ~strcmp(s2, 'nout') + fclose(fid); + error('String ''nout'' not found') +end +nout = fscanf(fid, '%d\n', 1); +if ~isnumeric(nout) + fclose(fid); + error('No number for nout') +end +if nout < 0 | round(nout) ~= nout + fclose(fid); + error('nout must be a non-negative integer') +end + +% Read number of data values +s3 = fscanf(fid, '%s', 1); +if ~strcmp(s3, 'ndata') + fclose(fid); + error('String ''ndata'' not found') +end +ndata = fscanf(fid, '%d\n', 1); +if ~isnumeric(ndata) + fclose(fid); + error('No number for ndata') +end +if ndata < 0 | round(ndata) ~= ndata + fclose(fid); + error('ndata must be a non-negative integer') +end + +% The following line reads all of the remaining data to the end of file. +temp = fscanf(fid, '%f', inf); + +% Check that size of temp is correct +if size(temp, 1) * size(temp,2) ~= (nin+nout) * ndata + fclose(fid); + error('Incorrect number of elements in file') +end + +temp = reshape(temp, nin + nout, ndata)'; +x = temp(:, 1:nin); +t = temp(:, nin + 1 : nin + nout); + +flag = fclose(fid); +if flag == -1 + error('Failed to close file.') +end + diff --git a/sourcecodes/bnt-master/netlab3.3/datwrite.m b/sourcecodes/bnt-master/netlab3.3/datwrite.m new file mode 100644 index 00000000..75df14d1 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/datwrite.m @@ -0,0 +1,44 @@ +function datwrite(filename, x, t) +%DATWRITE Write data to ascii file. +% +% Description +% +% DATWRITE(FILENAME, X, T) takes a matrix X of input vectors and a +% matrix T of target vectors and writes them to an ascii file named +% FILENAME. The file format is as follows: the first row contains the +% string NIN followed by the number of inputs, the second row contains +% the string NOUT followed by the number of outputs, and the third row +% contains the string NDATA followed by the number of data vectors. +% Subsequent lines each contain one input vector followed by one output +% vector, with individual values separated by spaces. +% +% See also +% DATREAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +nin = size(x, 2); +nout = size(t, 2); +ndata = size(x, 1); + +fid = fopen(filename, 'wt'); +if fid == -1 + error('Failed to open file.') +end + +if size(t, 1) ~= ndata + error('x and t must have same number of rows.'); +end + +fprintf(fid, ' nin %d\n nout %d\n ndata %d\n', nin , nout, ndata); +for i = 1 : ndata + fprintf(fid, '%13e ', x(i,:), t(i,:)); + fprintf(fid, '\n'); +end + +flag = fclose(fid); +if flag == -1 + error('Failed to close file.') +end + diff --git a/sourcecodes/bnt-master/netlab3.3/dem2ddat.m b/sourcecodes/bnt-master/netlab3.3/dem2ddat.m new file mode 100644 index 00000000..abdb7cc7 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/dem2ddat.m @@ -0,0 +1,48 @@ +function [data, c, prior, sd] = dem2ddat(ndata) +%DEM2DDAT Generates two dimensional data for demos. +% +% Description +% The data is drawn from three spherical Gaussian distributions with +% priors 0.3, 0.5 and 0.2; centres (2, 3.5), (0, 0) and (0,2); and +% standard deviations 0.2, 0.5 and 1.0. DATA = DEM2DDAT(NDATA) +% generates NDATA points. +% +% [DATA, C] = DEM2DDAT(NDATA) also returns a matrix containing the +% centres of the Gaussian distributions. +% +% See also +% DEMGMM1, DEMKMEAN, DEMKNN1 +% + +% Copyright (c) Ian T Nabney (1996-2001) + +input_dim = 2; + +% Fix seed for reproducible results +randn('state', 42); + +% Generate mixture of three Gaussians in two dimensional space +data = randn(ndata, input_dim); + +% Priors for the three clusters +prior(1) = 0.3; +prior(2) = 0.5; +prior(3) = 0.2; + +% Cluster centres +c = [2.0, 3.5; 0.0, 0.0; 0.0, 2.0]; + +% Cluster standard deviations +sd = [0.2 0.5 1.0]; + +% Put first cluster at (2, 3.5) +data(1:prior(1)*ndata, 1) = data(1:prior(1)*ndata, 1) * 0.2 + c(1,1); +data(1:prior(1)*ndata, 2) = data(1:prior(1)*ndata, 2) * 0.2 + c(1,2); + +% Leave second cluster at (0,0) +data((prior(1)*ndata + 1):(prior(2)+prior(1))*ndata, :) = ... + data((prior(1)*ndata + 1):(prior(2)+prior(1))*ndata, :) * 0.5; + +% Put third cluster at (0,2) +data((prior(1)+prior(2))*ndata +1:ndata, 2) = ... + data((prior(1)+prior(2))*ndata+1:ndata, 2) + c(3, 2); diff --git a/sourcecodes/bnt-master/netlab3.3/demard.m b/sourcecodes/bnt-master/netlab3.3/demard.m new file mode 100644 index 00000000..83478305 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demard.m @@ -0,0 +1,157 @@ +%DEMARD Automatic relevance determination using the MLP. +% +% Description +% This script demonstrates the technique of automatic relevance +% determination (ARD) using a synthetic problem having three input +% variables: X1 is sampled uniformly from the range (0,1) and has a low +% level of added Gaussian noise, X2 is a copy of X1 with a higher level +% of added noise, and X3 is sampled randomly from a Gaussian +% distribution. The single target variable is determined by +% SIN(2*PI*X1) with additive Gaussian noise. Thus X1 is very relevant +% for determining the target value, X2 is of some relevance, while X3 +% is irrelevant. The prior over weights is given by the ARD Gaussian +% prior with a separate hyper-parameter for the group of weights +% associated with each input. A multi-layer perceptron is trained on +% this data, with re-estimation of the hyper-parameters using EVIDENCE. +% The final values for the hyper-parameters reflect the relative +% importance of the three inputs. +% +% See also +% DEMMLP1, DEMEV1, MLP, EVIDENCE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc; +disp('This demonstration illustrates the technique of automatic relevance') +disp('determination (ARD) using a multi-layer perceptron.') +disp(' '); +disp('First, we set up a synthetic data set involving three input variables:') +disp('x1 is sampled uniformly from the range (0,1) and has a low level of') +disp('added Gaussian noise, x2 is a copy of x1 with a higher level of added') +disp('noise, and x3 is sampled randomly from a Gaussian distribution. The') +disp('single target variable is given by t = sin(2*pi*x1) with additive') +disp('Gaussian noise. Thus x1 is very relevant for determining the target') +disp('value, x2 is of some relevance, while x3 should in principle be') +disp('irrelevant.') +disp(' '); +disp('Press any key to see a plot of t against x1.') +pause; + +% Generate the data set. +randn('state', 0); +rand('state', 0); +ndata = 100; +noise = 0.05; +x1 = rand(ndata, 1) + 0.002*randn(ndata, 1); +x2 = x1 + 0.02*randn(ndata, 1); +x3 = 0.5 + 0.2*randn(ndata, 1); +x = [x1, x2, x3]; +t = sin(2*pi*x1) + noise*randn(ndata, 1); + +% Plot the data and the original function. +h = figure; +plotvals = linspace(0, 1, 200)'; +plot(x1, t, 'ob') +hold on +axis([0 1 -1.5 1.5]) +[fx, fy] = fplot('sin(2*pi*x)', [0 1]); +plot(fx, fy, '-g', 'LineWidth', 2); +legend('data', 'function'); + +disp(' '); +disp('Press any key to continue') +pause; clc; + +disp('The prior over weights is given by the ARD Gaussian prior with a') +disp('separate hyper-parameter for the group of weights associated with each') +disp('input. This prior is set up using the utility MLPPRIOR. The network is') +disp('trained by error minimization using scaled conjugate gradient function') +disp('SCG. There are two cycles of training, and at the end of each cycle') +disp('the hyper-parameters are re-estimated using EVIDENCE.') +disp(' '); +disp('Press any key to create and train the network.') +disp(' '); +pause; + +% Set up network parameters. +nin = 3; % Number of inputs. +nhidden = 2; % Number of hidden units. +nout = 1; % Number of outputs. +aw1 = 0.01*ones(1, nin); % First-layer ARD hyperparameters. +ab1 = 0.01; % Hyperparameter for hidden unit biases. +aw2 = 0.01; % Hyperparameter for second-layer weights. +ab2 = 0.01; % Hyperparameter for output unit biases. +beta = 50.0; % Coefficient of data error. + +% Create and initialize network. +prior = mlpprior(nin, nhidden, nout, aw1, ab1, aw2, ab2); +net = mlp(nin, nhidden, nout, 'linear', prior, beta); + +% Set up vector of options for the optimiser. +nouter = 2; % Number of outer loops +ninner = 10; % Number of inner loops +options = zeros(1,18); % Default options vector. +options(1) = 1; % This provides display of error values. +options(2) = 1.0e-7; % This ensures that convergence must occur +options(3) = 1.0e-7; +options(14) = 300; % Number of training cycles in inner loop. + +% Train using scaled conjugate gradients, re-estimating alpha and beta. +for k = 1:nouter + net = netopt(net, options, x, t, 'scg'); + [net, gamma] = evidence(net, x, t, ninner); + fprintf(1, '\n\nRe-estimation cycle %d:\n', k); + disp('The first three alphas are the hyperparameters for the corresponding'); + disp('input to hidden unit weights. The remainder are the hyperparameters'); + disp('for the hidden unit biases, second layer weights and output unit') + disp('biases, respectively.') + fprintf(1, ' alpha = %8.5f\n', net.alpha); + fprintf(1, ' beta = %8.5f\n', net.beta); + fprintf(1, ' gamma = %8.5f\n\n', gamma); + disp(' ') + disp('Press any key to continue.') + pause +end + +% Plot the function corresponding to the trained network. +figure(h); hold on; +[y, z] = mlpfwd(net, plotvals*ones(1,3)); +plot(plotvals, y, '-r', 'LineWidth', 2) +legend('data', 'function', 'network'); + +disp('Press any key to continue.'); +pause; clc; + +disp('We can now read off the hyperparameter values corresponding to the') +disp('three inputs x1, x2 and x3:') +disp(' '); +fprintf(1, ' alpha1: %8.5f\n', net.alpha(1)); +fprintf(1, ' alpha2: %8.5f\n', net.alpha(2)); +fprintf(1, ' alpha3: %8.5f\n', net.alpha(3)); +disp(' '); +disp('Since each alpha corresponds to an inverse variance, we see that the') +disp('posterior variance for weights associated with input x1 is large, that') +disp('of x2 has an intermediate value and the variance of weights associated') +disp('with x3 is small.') +disp(' ') +disp('Press any key to continue.') +disp(' ') +pause +disp('This is confirmed by looking at the corresponding weight values:') +disp(' '); +fprintf(1, ' %8.5f %8.5f\n', net.w1'); +disp(' '); +disp('where the three rows correspond to weights asssociated with x1, x2 and') +disp('x3 respectively. We see that the network is giving greatest emphasis') +disp('to x1 and least emphasis to x3, with intermediate emphasis on') +disp('x2. Since the target t is statistically independent of x3 we might') +disp('expect the weights associated with this input would go to') +disp('zero. However, for any finite data set there may be some chance') +disp('correlation between x3 and t, and so the corresponding alpha remains') +disp('finite.') + +disp(' '); +disp('Press any key to end.') +pause; clc; close(h); clear all + diff --git a/sourcecodes/bnt-master/netlab3.3/demev1.m b/sourcecodes/bnt-master/netlab3.3/demev1.m new file mode 100644 index 00000000..b8fd7464 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demev1.m @@ -0,0 +1,146 @@ +%DEMEV1 Demonstrate Bayesian regression for the MLP. +% +% Description +% The problem consists an input variable X which sampled from a +% Gaussian distribution, and a target variable T generated by computing +% SIN(2*PI*X) and adding Gaussian noise. A 2-layer network with linear +% outputs is trained by minimizing a sum-of-squares error function with +% isotropic Gaussian regularizer, using the scaled conjugate gradient +% optimizer. The hyperparameters ALPHA and BETA are re-estimated using +% the function EVIDENCE. A graph is plotted of the original function, +% the training data, the trained network function, and the error bars. +% +% See also +% EVIDENCE, MLP, SCG, DEMARD, DEMMLP1 +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc; +disp('This demonstration illustrates the application of Bayesian') +disp('re-estimation to determine the hyperparameters in a simple regression') +disp('problem. It is based on a local quadratic approximation to a mode of') +disp('the posterior distribution and the evidence maximization framework of') +disp('MacKay.') +disp(' ') +disp('First, we generate a synthetic data set consisting of a single input') +disp('variable x sampled from a Gaussian distribution, and a target variable') +disp('t obtained by evaluating sin(2*pi*x) and adding Gaussian noise.') +disp(' ') +disp('Press any key to see a plot of the data together with the sine function.') +pause; + +% Generate the matrix of inputs x and targets t. + +ndata = 16; % Number of data points. +noise = 0.1; % Standard deviation of noise distribution. +randn('state', 0); +x = 0.25 + 0.07*randn(ndata, 1); +t = sin(2*pi*x) + noise*randn(size(x)); + +% Plot the data and the original sine function. +h = figure; +nplot = 200; +plotvals = linspace(0, 1, nplot)'; +plot(x, t, 'ok') +xlabel('Input') +ylabel('Target') +hold on +axis([0 1 -1.5 1.5]) +fplot('sin(2*pi*x)', [0 1], '-g') +legend('data', 'function'); + +disp(' ') +disp('Press any key to continue') +pause; clc; + +disp('Next we create a two-layer MLP network having 3 hidden units and one') +disp('linear output. The model assumes Gaussian target noise governed by an') +disp('inverse variance hyperparmeter beta, and uses a simple Gaussian prior') +disp('distribution governed by an inverse variance hyperparameter alpha.') +disp(' '); +disp('The network weights and the hyperparameters are initialised and then') +disp('the weights are optimized with the scaled conjugate gradient') +disp('algorithm using the SCG function, with the hyperparameters kept') +disp('fixed. After a maximum of 500 iterations, the hyperparameters are') +disp('re-estimated using the EVIDENCE function. The process of optimizing') +disp('the weights with fixed hyperparameters and then re-estimating the') +disp('hyperparameters is repeated for a total of 3 cycles.') +disp(' ') +disp('Press any key to train the network and determine the hyperparameters.') +pause; + +% Set up network parameters. +nin = 1; % Number of inputs. +nhidden = 3; % Number of hidden units. +nout = 1; % Number of outputs. +alpha = 0.01; % Initial prior hyperparameter. +beta_init = 50.0; % Initial noise hyperparameter. + +% Create and initialize network weight vector. +net = mlp(nin, nhidden, nout, 'linear', alpha, beta_init); + +% Set up vector of options for the optimiser. +nouter = 3; % Number of outer loops. +ninner = 1; % Number of innter loops. +options = zeros(1,18); % Default options vector. +options(1) = 1; % This provides display of error values. +options(2) = 1.0e-7; % Absolute precision for weights. +options(3) = 1.0e-7; % Precision for objective function. +options(14) = 500; % Number of training cycles in inner loop. + +% Train using scaled conjugate gradients, re-estimating alpha and beta. +for k = 1:nouter + net = netopt(net, options, x, t, 'scg'); + [net, gamma] = evidence(net, x, t, ninner); + fprintf(1, '\nRe-estimation cycle %d:\n', k); + fprintf(1, ' alpha = %8.5f\n', net.alpha); + fprintf(1, ' beta = %8.5f\n', net.beta); + fprintf(1, ' gamma = %8.5f\n\n', gamma); + disp(' ') + disp('Press any key to continue.') + pause; +end + +fprintf(1, 'true beta: %f\n', 1/(noise*noise)); + +disp(' ') +disp('Network training and hyperparameter re-estimation are now complete.') +disp('Compare the final value for the hyperparameter beta with the true') +disp('value.') +disp(' ') +disp('Notice that the final error value is close to the number of data') +disp(['points (', num2str(ndata),') divided by two.']) +disp(' ') +disp('Press any key to continue.') +pause; clc; +disp('We can now plot the function represented by the trained network. This') +disp('corresponds to the mean of the predictive distribution. We can also') +disp('plot ''error bars'' representing one standard deviation of the') +disp('predictive distribution around the mean.') +disp(' ') +disp('Press any key to add the network function and error bars to the plot.') +pause; + +% Evaluate error bars. +[y, sig2] = netevfwd(mlppak(net), net, x, t, plotvals); +sig = sqrt(sig2); + +% Plot the data, the original function, and the trained network function. +[y, z] = mlpfwd(net, plotvals); +figure(h); hold on; +plot(plotvals, y, '-r') +xlabel('Input') +ylabel('Target') +plot(plotvals, y + sig, '-b'); +plot(plotvals, y - sig, '-b'); +legend('data', 'function', 'network', 'error bars'); + +disp(' ') +disp('Notice how the confidence interval spanned by the ''error bars'' is') +disp('smaller in the region of input space where the data density is high,') +disp('and becomes larger in regions away from the data.') +disp(' ') +disp('Press any key to end.') +pause; clc; close(h); +%clear all diff --git a/sourcecodes/bnt-master/netlab3.3/demev2.m b/sourcecodes/bnt-master/netlab3.3/demev2.m new file mode 100644 index 00000000..b479d88f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demev2.m @@ -0,0 +1,248 @@ +%DEMEV2 Demonstrate Bayesian classification for the MLP. +% +% Description +% A synthetic two class two-dimensional dataset X is sampled from a +% mixture of four Gaussians. Each class is associated with two of the +% Gaussians so that the optimal decision boundary is non-linear. A 2- +% layer network with logistic outputs is trained by minimizing the +% cross-entropy error function with isotroipc Gaussian regularizer (one +% hyperparameter for each of the four standard weight groups), using +% the scaled conjugate gradient optimizer. The hyperparameter vectors +% ALPHA and BETA are re-estimated using the function EVIDENCE. A graph +% is plotted of the optimal, regularised, and unregularised decision +% boundaries. A further plot of the moderated versus unmoderated +% contours is generated. +% +% See also +% EVIDENCE, MLP, SCG, DEMARD, DEMMLP2 +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +clc; + +disp('This program demonstrates the use of the evidence procedure on') +disp('a two-class problem. It also shows the improved generalisation') +disp('performance that can be achieved with moderated outputs; that is') +disp('predictions where an approximate integration over the true') +disp('posterior distribution is carried out.') +disp(' ') +disp('First we generate a synthetic dataset with two-dimensional input') +disp('sampled from a mixture of four Gaussians. Each class is') +disp('associated with two of the Gaussians so that the optimal decision') +disp('boundary is non-linear.') +disp(' ') +disp('Press any key to see a plot of the data.') +pause; + +% Generate the matrix of inputs x and targets t. + +rand('state', 423); +randn('state', 423); + +ClassSymbol1 = 'r.'; +ClassSymbol2 = 'y.'; +PointSize = 12; +titleSize = 10; + +fh1 = figure; +set(fh1, 'Name', 'True Data Distribution'); +whitebg(fh1, 'k'); + +% +% Generate the data +% +n=200; + +% Set up mixture model: 2d data with four centres +% Class 1 is first two centres, class 2 from the other two +mix = gmm(2, 4, 'full'); +mix.priors = [0.25 0.25 0.25 0.25]; +mix.centres = [0 -0.1; 1.5 0; 1 1; 1 -1]; +mix.covars(:,:,1) = [0.625 -0.2165; -0.2165 0.875]; +mix.covars(:,:,2) = [0.25 0; 0 0.25]; +mix.covars(:,:,3) = [0.2241 -0.1368; -0.1368 0.9759]; +mix.covars(:,:,4) = [0.2375 0.1516; 0.1516 0.4125]; + +[data, label] = gmmsamp(mix, n); + +% +% Calculate some useful axis limits +% +x0 = min(data(:,1)); +x1 = max(data(:,1)); +y0 = min(data(:,2)); +y1 = max(data(:,2)); +dx = x1-x0; +dy = y1-y0; +expand = 5/100; % Add on 5 percent each way +x0 = x0 - dx*expand; +x1 = x1 + dx*expand; +y0 = y0 - dy*expand; +y1 = y1 + dy*expand; +resolution = 100; +step = dx/resolution; +xrange = [x0:step:x1]; +yrange = [y0:step:y1]; +% +% Generate the grid +% +[X Y]=meshgrid([x0:step:x1],[y0:step:y1]); +% +% Calculate the class conditional densities, the unconditional densities and +% the posterior probabilities +% +px_j = gmmactiv(mix, [X(:) Y(:)]); +px = reshape(px_j*(mix.priors)',size(X)); +post = gmmpost(mix, [X(:) Y(:)]); +p1_x = reshape(post(:, 1) + post(:, 2), size(X)); +p2_x = reshape(post(:, 3) + post(:, 4), size(X)); + +plot(data((label<=2),1),data(label<=2,2),ClassSymbol1, 'MarkerSize', ... +PointSize) +hold on +axis([x0 x1 y0 y1]) +plot(data((label>2),1),data(label>2,2),ClassSymbol2, 'MarkerSize', ... + PointSize) + +% Convert targets to 0-1 encoding +target=[label<=2]; +disp(' ') +disp('Press any key to continue') +pause; clc; + +disp('Next we create a two-layer MLP network with 6 hidden units and') +disp('one logistic output. We use a separate inverse variance') +disp('hyperparameter for each group of weights (inputs, input bias,') +disp('outputs, output bias) and the weights are optimised with the') +disp('scaled conjugate gradient algorithm. After each 100 iterations') +disp('the hyperparameters are re-estimated twice. There are eight') +disp('cycles of the whole algorithm.') +disp(' ') +disp('Press any key to train the network and determine the hyperparameters.') +pause; + +% Set up network parameters. +nin = 2; % Number of inputs. +nhidden = 6; % Number of hidden units. +nout = 1; % Number of outputs. +alpha = 0.01; % Initial prior hyperparameter. +aw1 = 0.01; +ab1 = 0.01; +aw2 = 0.01; +ab2 = 0.01; + +% Create and initialize network weight vector. +prior = mlpprior(nin, nhidden, nout, aw1, ab1, aw2, ab2); +net = mlp(nin, nhidden, nout, 'logistic', prior); + +% Set up vector of options for the optimiser. +nouter = 8; % Number of outer loops. +ninner = 2; % Number of innter loops. +options = foptions; % Default options vector. +options(1) = 1; % This provides display of error values. +options(2) = 1.0e-5; % Absolute precision for weights. +options(3) = 1.0e-5; % Precision for objective function. +options(14) = 100; % Number of training cycles in inner loop. + +% Train using scaled conjugate gradients, re-estimating alpha and beta. +for k = 1:nouter + net = netopt(net, options, data, target, 'scg'); + [net, gamma] = evidence(net, data, target, ninner); + fprintf(1, '\nRe-estimation cycle %d:\n', k); + disp([' alpha = ', num2str(net.alpha')]); + fprintf(1, ' gamma = %8.5f\n\n', gamma); + disp(' ') + disp('Press any key to continue.') + pause; +end + +disp(' ') +disp('Network training and hyperparameter re-estimation are now complete.') +disp('Notice that the final error value is close to the number of data') +disp(['points (', num2str(n), ') divided by two.']) +disp('Also, the hyperparameter values differ, which suggests that a single') +disp('hyperparameter would not be so effective.') +disp(' ') +disp('First we train an MLP without Bayesian regularisation on the') +disp('same dataset using 400 iterations of scaled conjugate gradient') +disp(' ') +disp('Press any key to train the network by maximum likelihood.') +pause; +% Train standard network +net2 = mlp(nin, nhidden, nout, 'logistic'); +options(14) = 400; +net2 = netopt(net2, options, data, target, 'scg'); +y2g = mlpfwd(net2, [X(:), Y(:)]); +y2g = reshape(y2g(:, 1), size(X)); + +disp(' ') +disp('We can now plot the function represented by the trained networks.') +disp('We show the decision boundaries (output = 0.5) and the optimal') +disp('decision boundary given by applying Bayes'' theorem to the true') +disp('data model.') +disp(' ') +disp('Press any key to add the boundaries to the plot.') +pause; + +% Evaluate predictions. +[yg, ymodg] = mlpevfwd(net, data, target, [X(:) Y(:)]); +yg = reshape(yg(:,1),size(X)); +ymodg = reshape(ymodg(:,1),size(X)); + +% Bayesian decision boundary +[cB, hB] = contour(xrange,yrange,p1_x,[0.5 0.5],'b-'); +[cNb, hNb] = contour(xrange,yrange,yg,[0.5 0.5],'r-'); +[cN, hN] = contour(xrange,yrange,y2g,[0.5 0.5],'g-'); +set(hB, 'LineWidth', 2); +set(hNb, 'LineWidth', 2); +set(hN, 'LineWidth', 2); +Chandles = [hB(1) hNb(1) hN(1)]; +legend(Chandles, 'Bayes', ... + 'Reg. Network', 'Network', 3); + +disp(' ') +disp('Note how the regularised network predictions are closer to the') +disp('optimal decision boundary, while the unregularised network is') +disp('overtrained.') + +disp(' ') +disp('We will now compare moderated and unmoderated outputs for the'); +disp('regularised network by showing the contour plot of the posterior') +disp('probability estimates.') +disp(' ') +disp('The first plot shows the regularised (moderated) predictions') +disp('and the second shows the standard predictions from the same network.') +disp('These agree at the level 0.5.') +disp('Press any key to continue') +pause +levels = 0:0.1:1; +fh4 = figure; +set(fh4, 'Name', 'Moderated outputs'); +hold on +plot(data((label<=2),1),data(label<=2,2),'r.', 'MarkerSize', PointSize) +plot(data((label>2),1),data(label>2,2),'y.', 'MarkerSize', PointSize) + +[cNby, hNby] = contour(xrange, yrange, ymodg, levels, 'k-'); +set(hNby, 'LineWidth', 1); + +fh5 = figure; +set(fh5, 'Name', 'Unmoderated outputs'); +hold on +plot(data((label<=2),1),data(label<=2,2),'r.', 'MarkerSize', PointSize) +plot(data((label>2),1),data(label>2,2),'y.', 'MarkerSize', PointSize) + +[cNbm, hNbm] = contour(xrange, yrange, yg, levels, 'k-'); +set(hNbm, 'LineWidth', 1); + +disp(' ') +disp('Note how the moderated contours are more widely spaced. This shows') +disp('that there is a larger region where the outputs are close to 0.5') +disp('and a smaller region where the outputs are close to 0 or 1.') +disp(' ') +disp('Press any key to exit') +pause +close(fh1); +close(fh4); +close(fh5); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demev3.m b/sourcecodes/bnt-master/netlab3.3/demev3.m new file mode 100644 index 00000000..d288e037 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demev3.m @@ -0,0 +1,154 @@ +%DEMEV3 Demonstrate Bayesian regression for the RBF. +% +% Description +% The problem consists an input variable X which sampled from a +% Gaussian distribution, and a target variable T generated by computing +% SIN(2*PI*X) and adding Gaussian noise. An RBF network with linear +% outputs is trained by minimizing a sum-of-squares error function with +% isotropic Gaussian regularizer, using the scaled conjugate gradient +% optimizer. The hyperparameters ALPHA and BETA are re-estimated using +% the function EVIDENCE. A graph is plotted of the original function, +% the training data, the trained network function, and the error bars. +% +% See also +% DEMEV1, EVIDENCE, RBF, SCG, NETEVFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc; +disp('This demonstration illustrates the application of Bayesian') +disp('re-estimation to determine the hyperparameters in a simple regression') +disp('problem using an RBF netowk. It is based on a the fact that the') +disp('posterior distribution for the output weights of an RBF is Gaussian') +disp('and uses the evidence maximization framework of MacKay.') +disp(' ') +disp('First, we generate a synthetic data set consisting of a single input') +disp('variable x sampled from a Gaussian distribution, and a target variable') +disp('t obtained by evaluating sin(2*pi*x) and adding Gaussian noise.') +disp(' ') +disp('Press any key to see a plot of the data together with the sine function.') +pause; + +% Generate the matrix of inputs x and targets t. + +ndata = 16; % Number of data points. +noise = 0.1; % Standard deviation of noise distribution. +randn('state', 0); +rand('state', 0); +x = 0.25 + 0.07*randn(ndata, 1); +t = sin(2*pi*x) + noise*randn(size(x)); + +% Plot the data and the original sine function. +h = figure; +nplot = 200; +plotvals = linspace(0, 1, nplot)'; +plot(x, t, 'ok') +xlabel('Input') +ylabel('Target') +hold on +axis([0 1 -1.5 1.5]) +fplot('sin(2*pi*x)', [0 1], '-g') +legend('data', 'function'); + +disp(' ') +disp('Press any key to continue') +pause; clc; + +disp('Next we create a two-layer MLP network having 3 hidden units and one') +disp('linear output. The model assumes Gaussian target noise governed by an') +disp('inverse variance hyperparmeter beta, and uses a simple Gaussian prior') +disp('distribution governed by an inverse variance hyperparameter alpha.') +disp(' '); +disp('The network weights and the hyperparameters are initialised and then') +disp('the output layer weights are optimized with the scaled conjugate gradient') +disp('algorithm using the SCG function, with the hyperparameters kept') +disp('fixed. After a maximum of 50 iterations, the hyperparameters are') +disp('re-estimated using the EVIDENCE function. The process of optimizing') +disp('the weights with fixed hyperparameters and then re-estimating the') +disp('hyperparameters is repeated for a total of 3 cycles.') +disp(' ') +disp('Press any key to train the network and determine the hyperparameters.') +pause; + +% Set up network parameters. +nin = 1; % Number of inputs. +nhidden = 3; % Number of hidden units. +nout = 1; % Number of outputs. +alpha = 0.01; % Initial prior hyperparameter. +beta_init = 50.0; % Initial noise hyperparameter. + +% Create and initialize network weight vector. +net = rbf(nin, nhidden, nout, 'tps', 'linear', alpha, beta_init); +[net.mask, prior] = rbfprior('tps', nin, nhidden, nout, alpha, alpha); +net = netinit(net, prior); + +options = foptions; +options(14) = 5; % At most 5 EM iterations for basis functions +options(1) = -1; % Turn off all messages +net = rbfsetbf(net, options, x); % Initialise the basis functions + +% Now train the network +nouter = 5; +ninner = 2; +options = foptions; +options(1) = 1; +options(2) = 1.0e-5; % Absolute precision for weights. +options(3) = 1.0e-5; % Precision for objective function. +options(14) = 50; % Number of training cycles in inner loop. + +% Train using scaled conjugate gradients, re-estimating alpha and beta. +for k = 1:nouter + net = netopt(net, options, x, t, 'scg'); + [net, gamma] = evidence(net, x, t, ninner); + fprintf(1, '\nRe-estimation cycle %d:\n', k); + fprintf(1, ' alpha = %8.5f\n', net.alpha); + fprintf(1, ' beta = %8.5f\n', net.beta); + fprintf(1, ' gamma = %8.5f\n\n', gamma); + disp(' ') + disp('Press any key to continue.') + pause; +end + +fprintf(1, 'true beta: %f\n', 1/(noise*noise)); + +disp(' ') +disp('Network training and hyperparameter re-estimation are now complete.') +disp('Compare the final value for the hyperparameter beta with the true') +disp('value.') +disp(' ') +disp('Notice that the final error value is close to the number of data') +disp(['points (', num2str(ndata),') divided by two.']) +disp(' ') +disp('Press any key to continue.') +pause; clc; +disp('We can now plot the function represented by the trained network. This') +disp('corresponds to the mean of the predictive distribution. We can also') +disp('plot ''error bars'' representing one standard deviation of the') +disp('predictive distribution around the mean.') +disp(' ') +disp('Press any key to add the network function and error bars to the plot.') +pause; + +% Evaluate error bars. +[y, sig2] = netevfwd(netpak(net), net, x, t, plotvals); +sig = sqrt(sig2); + +% Plot the data, the original function, and the trained network function. +[y, z] = rbffwd(net, plotvals); +figure(h); hold on; +plot(plotvals, y, '-r') +xlabel('Input') +ylabel('Target') +plot(plotvals, y + sig, '-b'); +plot(plotvals, y - sig, '-b'); +legend('data', 'function', 'network', 'error bars'); + +disp(' ') +disp('Notice how the confidence interval spanned by the ''error bars'' is') +disp('smaller in the region of input space where the data density is high,') +disp('and becomes larger in regions away from the data.') +disp(' ') +disp('Press any key to end.') +pause; clc; close(h); + diff --git a/sourcecodes/bnt-master/netlab3.3/demgauss.m b/sourcecodes/bnt-master/netlab3.3/demgauss.m new file mode 100644 index 00000000..c9751ae9 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgauss.m @@ -0,0 +1,80 @@ +%DEMGAUSS Demonstrate sampling from Gaussian distributions. +% +% Description +% +% DEMGAUSS provides a simple illustration of the generation of data +% from Gaussian distributions. It first samples from a one-dimensional +% distribution using RANDN, and then plots a normalized histogram +% estimate of the distribution using HISTP together with the true +% density calculated using GAUSS. +% +% DEMGAUSS then demonstrates sampling from a Gaussian distribution in +% two dimensions. It creates a mean vector and a covariance matrix, and +% then plots contours of constant density using the function GAUSS. A +% sample of points drawn from this distribution, obtained using the +% function GSAMP, is then superimposed on the contours. +% +% See also +% GAUSS, GSAMP, HISTP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc +mean = 2; var = 5; nsamp = 3000; +xmin = -10; xmax = 10; nbins = 30; +disp('Demonstration of sampling from a uni-variate Gaussian with mean') +dstring = [num2str(mean), ' and variance ', num2str(var), '. ', ... + num2str(nsamp), ' samples are taken.']; +disp(dstring); +x = mean + sqrt(var)*randn(nsamp, 1); +fh1 = figure; +histp(x, xmin, xmax, nbins); +hold on; +axis([xmin xmax 0 0.2]); +plotvals = linspace(xmin, xmax, 200)'; +probs = gauss(mean, var, plotvals); +plot(plotvals, probs, '-r'); +xlabel('X') +ylabel('Density') + +disp(' ') +disp('Press any key to continue') +pause; +mu = [3 2]; +lam1 = 0.5; +lam2 = 5.0; +Sigma = lam1*[1,1]'*[1,1] + lam2*[1,-1]'*[1,-1]; +disp(' ') +disp('Demonstration of sampling from a bi-variate Gaussian. The mean is') +dstring = ['[', num2str(mu(1)), ', ', num2str(mu(2)), ... + '] and the covariance matrix is']; +disp(dstring) +disp(Sigma); +ngrid = 40; +cmin = -5; cmax = 10; +cvals = linspace(cmin, cmax, ngrid); +[X1, X2] = meshgrid(cvals, cvals); +XX = [X1(:), X2(:)]; +probs = gauss(mu, Sigma, XX); +probs = reshape(probs, ngrid, ngrid); + +fh2 = figure; +contour(X1, X2, probs, 'b'); +hold on + +nsamp = 300; +dstring = [num2str(nsamp), ' samples are generated.']; +disp('The plot shows the sampled data points with a contour plot of their density.') +samples = gsamp(mu, Sigma, nsamp); +plot(samples(:,1), samples(:,2), 'or'); +xlabel('X1') +ylabel('X2') +grid off; + +disp(' ') +disp('Press any key to end') +pause; +close(fh1); +close(fh2); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demglm1.m b/sourcecodes/bnt-master/netlab3.3/demglm1.m new file mode 100644 index 00000000..d6c54bac --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demglm1.m @@ -0,0 +1,95 @@ +%DEMGLM1 Demonstrate simple classification using a generalized linear model. +% +% Description +% The problem consists of a two dimensional input matrix DATA and a +% vector of classifications T. The data is generated from two +% Gaussian clusters, and a generalized linear model with logistic +% output is trained using iterative reweighted least squares. A plot of +% the data together with the 0.1, 0.5 and 0.9 contour lines of the +% conditional probability is generated. +% +% See also +% DEMGLM2, GLM, GLMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate data from two classes in 2d +input_dim = 2; + +% Fix seeds for reproducible results +randn('state', 42); +rand('state', 42); + +ndata = 100; +% Generate mixture of two Gaussians in two dimensional space +mix = gmm(2, 2, 'spherical'); +mix.priors = [0.4 0.6]; % Cluster priors +mix.centres = [2.0, 2.0; 0.0, 0.0]; % Cluster centres +mix.covars = [0.5, 1.0]; + +[data, label] = gmmsamp(mix, ndata); +targets = label - ones(ndata, 1); + +% Plot the result + +clc +disp('This demonstration illustrates the use of a generalized linear model') +disp('to classify data from two classes in a two-dimensional space. We') +disp('begin by generating and plotting the data.') +disp(' ') +disp('Press any key to continue.') +pause + +fh1 = figure; +plot(data(label==1,1), data(label==1,2), 'bo'); +hold on +axis([-4 5 -4 5]) +set(gca, 'box', 'on') +plot(data(label==2,1), data(label==2,2), 'rx') +title('Data') + +clc +disp('Now we fit a model consisting of a logistic sigmoid function of') +disp('a linear combination of the input variables.') +disp(' ') +disp('The model is trained using the IRLS algorithm for 5 iterations') +disp(' ') +disp('Press any key to continue.') +pause + +net = glm(input_dim, 1, 'logistic'); +options = foptions; +options(1) = 1; +options(14) = 5; +net = glmtrain(net, options, data, targets); + +disp(' ') +disp('We now plot some density contours given by this model.') +disp('The contour labelled 0.5 is the decision boundary.') +disp(' ') +disp('Press any key to continue.') +pause +x = -4.0:0.2:5.0; +y = -4.0:0.2:5.0; +[X, Y] = meshgrid(x,y); +X = X(:); +Y = Y(:); +grid = [X Y]; +Z = glmfwd(net, grid); +Z = reshape(Z, length(x), length(y)); +v = [0.1 0.5 0.9]; +[c, h] = contour(x, y, Z, v); +title('Generalized Linear Model') +set(h, 'linewidth', 3) +clabel(c, h); + +clc +disp('Note that the contours of constant density are straight lines.') +disp(' ') +disp('Press any key to end.') +pause +close(fh1); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demglm2.m b/sourcecodes/bnt-master/netlab3.3/demglm2.m new file mode 100644 index 00000000..ea3d6bde --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demglm2.m @@ -0,0 +1,103 @@ +%DEMGLM2 Demonstrate simple classification using a generalized linear model. +% +% Description +% The problem consists of a two dimensional input matrix DATA and a +% vector of classifications T. The data is generated from three +% Gaussian clusters, and a generalized linear model with softmax output +% is trained using iterative reweighted least squares. A plot of the +% data together with regions shaded by the classification given by the +% network is generated. +% +% See also +% DEMGLM1, GLM, GLMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate data from three classes in 2d +input_dim = 2; + +% Fix seeds for reproducible results +randn('state', 42); +rand('state', 42); + +ndata = 100; +% Generate mixture of three Gaussians in two dimensional space +mix = gmm(2, 3, 'spherical'); +mix.priors = [0.4 0.3 0.3]; % Cluster priors +mix.centres = [2, 2; 0.0, 0.0; 1, -1]; % Cluster centres +mix.covars = [0.5 1.0 0.6]; + +[data, label] = gmmsamp(mix, ndata); +id = eye(3); +targets = id(label,:); + +% Plot the result + +clc +disp('This demonstration illustrates the use of a generalized linear model') +disp('to classify data from three classes in a two-dimensional space. We') +disp('begin by generating and plotting the data.') +disp(' ') +disp('Press any key to continue.') +pause + +fh1 = figure; +plot(data(label==1,1), data(label==1,2), 'bo'); +hold on +axis([-4 5 -4 5]); +set(gca, 'Box', 'on') +plot(data(label==2,1), data(label==2,2), 'rx') +plot(data(label==3, 1), data(label==3, 2), 'go') +title('Data') + +clc +disp('Now we fit a model consisting of a softmax function of') +disp('a linear combination of the input variables.') +disp(' ') +disp('The model is trained using the IRLS algorithm for up to 10 iterations') +disp(' ') +disp('Press any key to continue.') +pause + +net = glm(input_dim, size(targets, 2), 'softmax'); +options = foptions; +options(1) = 1; +options(14) = 10; +net = glmtrain(net, options, data, targets); + +disp(' ') +disp('We now plot the decision regions given by this model.') +disp(' ') +disp('Press any key to continue.') +pause + +x = -4.0:0.2:5.0; +y = -4.0:0.2:5.0; +[X, Y] = meshgrid(x,y); +X = X(:); +Y = Y(:); +grid = [X Y]; +Z = glmfwd(net, grid); +[foo , class] = max(Z'); +class = class'; +colors = ['b.'; 'r.'; 'g.']; +for i = 1:3 + thisX = X(class == i); + thisY = Y(class == i); + h = plot(thisX, thisY, colors(i,:)); + set(h, 'MarkerSize', 8); +end +title('Plot of Decision regions') + +hold off + +clc +disp('Note that the boundaries of decision regions are straight lines.') +disp(' ') +disp('Press any key to end.') +pause +close(fh1); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demgmm1.m b/sourcecodes/bnt-master/netlab3.3/demgmm1.m new file mode 100644 index 00000000..e2d1ef4c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgmm1.m @@ -0,0 +1,173 @@ +%DEMGMM1 Demonstrate EM for Gaussian mixtures. +% +% Description +% This script demonstrates the use of the EM algorithm to fit a mixture +% of Gaussians to a set of data using maximum likelihood. A colour +% coding scheme is used to illustrate the evaluation of the posterior +% probabilities in the E-step of the EM algorithm. +% +% See also +% DEMGMM2, DEMGMM3, DEMGMM4, GMM, GMMEM, GMMPOST +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc; +disp('This demonstration illustrates the use of the EM (expectation-') +disp('maximization) algorithm for fitting of a mixture of Gaussians to a') +disp('data set by maximum likelihood.') +disp(' ') +disp('The data set consists of 40 data points in a 2-dimensional') +disp('space, generated by sampling from a mixture of 2 Gaussian') +disp('distributions.') +disp(' ') +disp('Press any key to see a plot of the data.') +pause; + +% Generate the data +randn('state', 0); rand('state', 0); +gmix = gmm(2, 2, 'spherical'); +ndat1 = 20; ndat2 = 20; ndata = ndat1+ndat2; +gmix.centres = [0.3 0.3; 0.7 0.7]; +gmix.covars = [0.01 0.01]; +x = gmmsamp(gmix, ndata); + +h = figure; +hd = plot(x(:, 1), x(:, 2), '.g', 'markersize', 30); +hold on; axis([0 1 0 1]); axis square; set(gca, 'box', 'on'); +ht = text(0.5, 1.05, 'Data', 'horizontalalignment', 'center'); +disp(' '); +disp('Press any key to continue.') +pause; clc; + +disp('We next create and initialize a mixture model consisting of a mixture') +disp('of 2 Gaussians having ''spherical'' covariance matrices, using the') +disp('function GMM. The Gaussian components can be displayed on the same') +disp('plot as the data by drawing a contour of constant probability density') +disp('for each component having radius equal to the corresponding standard') +disp('deviation. Component 1 is coloured red and component 2 is coloured') +disp('blue.') +disp(' ') +disp('Note that a particulary poor choice of initial parameters has been') +disp('made in order to illustrate more effectively the operation of the') +disp('EM algorithm.') +disp(' ') +disp('Press any key to see the initial configuration of the mixture model.') +pause; + +% Set up mixture model +ncentres = 2; input_dim = 2; +mix = gmm(input_dim, ncentres, 'spherical'); + +% Initialise the mixture model +mix.centres = [0.2 0.8; 0.8, 0.2]; +mix.covars = [0.01 0.01]; + +% Plot the initial model +ncirc = 30; theta = linspace(0, 2*pi, ncirc); +xs = cos(theta); ys = sin(theta); +xvals = mix.centres(:, 1)*ones(1,ncirc) + sqrt(mix.covars')*xs; +yvals = mix.centres(:, 2)*ones(1,ncirc) + sqrt(mix.covars')*ys; +hc(1)=line(xvals(1,:), yvals(1,:), 'color', 'r'); +hc(2)=line(xvals(2,:), yvals(2,:), 'color', 'b'); +set(ht, 'string', 'Initial Configuration'); +figure(h); +disp(' ') +disp('Press any key to continue'); +pause; clc; + +disp('Now we adapt the parameters of the mixture model iteratively using the') +disp('EM algorithm. Each cycle of the EM algorithm consists of an E-step') +disp('followed by an M-step. We start with the E-step, which involves the') +disp('evaluation of the posterior probabilities (responsibilities) which the') +disp('two components have for each of the data points.') +disp(' ') +disp('Since we have labelled the two components using the colours red and') +disp('blue, a convenient way to indicate the value of a posterior') +disp('probability for a given data point is to colour the point using a') +disp('scale ranging from pure red (corresponding to a posterior probability') +disp('of 1.0 for the red component and 0.0 for the blue component) through') +disp('to pure blue.') +disp(' ') +disp('Press any key to see the result of applying the first E-step.') +pause; + +% Initial E-step. +set(ht, 'string', 'E-step'); +post = gmmpost(mix, x); +dcols = [post(:,1), zeros(ndata, 1), post(:,2)]; +delete(hd); +for i = 1 : ndata + hd(i) = plot(x(i, 1), x(i, 2), 'color', dcols(i,:), ... + 'marker', '.', 'markersize', 30); +end +figure(h); + +disp(' '); +disp('Press any key to continue') +pause; clc; + +disp('Next we perform the corresponding M-step. This involves replacing the') +disp('centres of the component Gaussians by the corresponding weighted means') +disp('of the data. Thus the centre of the red component is replaced by the') +disp('mean of the data set, in which each data point is weighted according to') +disp('the amount of red ink (corresponding to the responsibility of') +disp('component 1 for explaining that data point). The variances and mixing') +disp('proportions of the two components are similarly re-estimated.') +disp(' ') +disp('Press any key to see the result of applying the first M-step.') +pause; + +% M-step. +set(ht, 'string', 'M-step'); +options = foptions; +options(14) = 1; % A single iteration +options(1) = -1; % Switch off all messages, including warning +mix = gmmem(mix, x, options); +delete(hc); +xvals = mix.centres(:, 1)*ones(1,ncirc) + sqrt(mix.covars')*xs; +yvals = mix.centres(:, 2)*ones(1,ncirc) + sqrt(mix.covars')*ys; +hc(1)=line(xvals(1,:), yvals(1,:), 'color', 'r'); +hc(2)=line(xvals(2,:), yvals(2,:), 'color', 'b'); +figure(h); +disp(' ') +disp('Press any key to continue') +pause; clc; + +disp('We can continue making alternate E and M steps until the changes in') +disp('the log likelihood at each cycle become sufficiently small.') +disp(' ') +disp('Press any key to see an animation of a further 9 EM cycles.') +pause; +figure(h); + +% Loop over EM iterations. +numiters = 9; +for n = 1 : numiters + + set(ht, 'string', 'E-step'); + post = gmmpost(mix, x); + dcols = [post(:,1), zeros(ndata, 1), post(:,2)]; + delete(hd); + for i = 1 : ndata + hd(i) = plot(x(i, 1), x(i, 2), 'color', dcols(i,:), ... + 'marker', '.', 'markersize', 30); + end + pause(1) + + set(ht, 'string', 'M-step'); + [mix, options] = gmmem(mix, x, options); + fprintf(1, 'Cycle %4d Error %11.6f\n', n, options(8)); + delete(hc); + xvals = mix.centres(:, 1)*ones(1,ncirc) + sqrt(mix.covars')*xs; + yvals = mix.centres(:, 2)*ones(1,ncirc) + sqrt(mix.covars')*ys; + hc(1)=line(xvals(1,:), yvals(1,:), 'color', 'r'); + hc(2)=line(xvals(2,:), yvals(2,:), 'color', 'b'); + pause(1) + +end + +disp(' '); +disp('Press any key to end.') +pause; clc; close(h); clear all + diff --git a/sourcecodes/bnt-master/netlab3.3/demgmm2.m b/sourcecodes/bnt-master/netlab3.3/demgmm2.m new file mode 100644 index 00000000..ae0e04fa --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgmm2.m @@ -0,0 +1,152 @@ +%DEMGMM1 Demonstrate density modelling with a Gaussian mixture model. +% +% Description +% The problem consists of modelling data generated by a mixture of +% three Gaussians in 2 dimensions. The priors are 0.3, 0.5 and 0.2; +% the centres are (2, 3.5), (0, 0) and (0,2); the variances are 0.2, +% 0.5 and 1.0. The first figure contains a scatter plot of the data. +% +% A Gaussian mixture model with three components is trained using EM. +% The parameter vector is printed before training and after training. +% The user should press any key to continue at these points. The +% parameter vector consists of priors (the column), centres (given as +% (x, y) pairs as the next two columns), and variances (the last +% column). +% +% The second figure is a 3 dimensional view of the density function, +% while the third shows the 1-standard deviation circles for the three +% components of the mixture model. +% +% See also +% GMM, GMMINIT, GMMEM, GMMPROB, GMMUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Generate the data +% Fix seeds for reproducible results +randn('state', 42); +rand('state', 42); + +ndata = 500; +[data, datac, datap, datasd] = dem2ddat(ndata); + +clc +disp('This demonstration illustrates the use of a Gaussian mixture model') +disp('to approximate the unconditional probability density of data in') +disp('a two-dimensional space. We begin by generating the data from') +disp('a mixture of three Gaussians and plotting it.') +disp(' ') +disp('Press any key to continue') +pause + +fh1 = figure; +plot(data(:, 1), data(:, 2), 'o') +set(gca, 'Box', 'on') +% Set up mixture model +ncentres = 3; +input_dim = 2; +mix = gmm(input_dim, ncentres, 'spherical'); + +options = foptions; +options(14) = 5; % Just use 5 iterations of k-means in initialisation +% Initialise the model parameters from the data +mix = gmminit(mix, data, options); + +clc +disp('The data is drawn from a mixture with parameters') +disp(' Priors Centres Variances') +disp([datap' datac (datasd.^2)']) +disp(' ') +disp('The mixture model has three components and spherical covariance') +disp('matrices. The model parameters after initialisation using the') +disp('k-means algorithm are as follows') +% Print out model +disp(' Priors Centres Variances') +disp([mix.priors' mix.centres mix.covars']) +disp('Press any key to continue') +pause + +% Set up vector of options for EM trainer +options = zeros(1, 18); +options(1) = 1; % Prints out error values. +options(14) = 10; % Max. Number of iterations. + +disp('We now train the model using the EM algorithm for 10 iterations') +disp(' ') +disp('Press any key to continue') +pause +[mix, options, errlog] = gmmem(mix, data, options); + +% Print out model +disp(' ') +disp('The trained model has parameters ') +disp(' Priors Centres Variances') +disp([mix.priors' mix.centres mix.covars']) +disp('Note the close correspondence between these parameters and those') +disp('of the distribution used to generate the data, which are repeated here.') +disp(' Priors Centres Variances') +disp([datap' datac (datasd.^2)']) +disp(' ') +disp('Press any key to continue') +pause + +clc +disp('We now plot the density given by the mixture model as a surface plot') +disp(' ') +disp('Press any key to continue') +pause +% Plot the result +x = -4.0:0.2:5.0; +y = -4.0:0.2:5.0; +[X, Y] = meshgrid(x,y); +X = X(:); +Y = Y(:); +grid = [X Y]; +Z = gmmprob(mix, grid); +Z = reshape(Z, length(x), length(y)); +c = mesh(x, y, Z); +hold on +title('Surface plot of probability density') +hold off + +clc +disp('The final plot shows the centres and widths, given by one standard') +disp('deviation, of the three components of the mixture model.') +disp(' ') +disp('Press any key to continue.') +pause +% Try to calculate a sensible position for the second figure, below the first +fig1_pos = get(fh1, 'Position'); +fig2_pos = fig1_pos; +fig2_pos(2) = fig2_pos(2) - fig1_pos(4); +fh2 = figure; +set(fh2, 'Position', fig2_pos) + +hp1 = plot(data(:, 1), data(:, 2), 'bo'); +axis('equal'); +hold on +hp2 = plot(mix.centres(:, 1), mix.centres(:,2), 'g+'); +set(hp2, 'MarkerSize', 10); +set(hp2, 'LineWidth', 3); + +title('Plot of data and mixture centres') +angles = 0:pi/30:2*pi; +for i = 1 : mix.ncentres + x_circle = mix.centres(i,1)*ones(1, length(angles)) + ... + sqrt(mix.covars(i))*cos(angles); + y_circle = mix.centres(i,2)*ones(1, length(angles)) + ... + sqrt(mix.covars(i))*sin(angles); + plot(x_circle, y_circle, 'r') +end +hold off +disp('Note how the data cluster positions and widths are captured by') +disp('the mixture model.') +disp(' ') +disp('Press any key to end.') +pause + +close(fh1); +close(fh2); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demgmm3.m b/sourcecodes/bnt-master/netlab3.3/demgmm3.m new file mode 100644 index 00000000..4e4210d1 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgmm3.m @@ -0,0 +1,192 @@ +%DEMGMM3 Demonstrate density modelling with a Gaussian mixture model. +% +% Description +% The problem consists of modelling data generated by a mixture of +% three Gaussians in 2 dimensions with a mixture model using diagonal +% covariance matrices. The priors are 0.3, 0.5 and 0.2; the centres +% are (2, 3.5), (0, 0) and (0,2); the covariances are all axis aligned +% (0.16, 0.64), (0.25, 1) and the identity matrix. The first figure +% contains a scatter plot of the data. +% +% A Gaussian mixture model with three components is trained using EM. +% The parameter vector is printed before training and after training. +% The user should press any key to continue at these points. The +% parameter vector consists of priors (the column), and centres (given +% as (x, y) pairs as the next two columns). The diagonal entries of +% the covariance matrices are printed separately. +% +% The second figure is a 3 dimensional view of the density function, +% while the third shows the axes of the 1-standard deviation circles +% for the three components of the mixture model. +% +% See also +% GMM, GMMINIT, GMMEM, GMMPROB, GMMUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Generate the data +ndata = 500; + +% Fix the seeds for reproducible results +randn('state', 42); +rand('state', 42); +data = randn(ndata, 2); +prior = [0.3 0.5 0.2]; +% Mixture model swaps clusters 1 and 3 +datap = [0.2 0.5 0.3]; +datac = [0 2; 0 0; 2 3.5]; +datacov = [1 1;1 0.25; 0.4*0.4 0.8*0.8]; +data1 = data(1:prior(1)*ndata,:); +data2 = data(prior(1)*ndata+1:(prior(2)+prior(1))*ndata, :); +data3 = data((prior(1)+prior(2))*ndata +1:ndata, :); + +% First cluster has axis aligned variance and centre (2, 3.5) +data1(:, 1) = data1(:, 1)*0.4 + 2.0; +data1(:, 2) = data1(:, 2)*0.8 + 3.5; + +% Second cluster has axis aligned variance and centre (0, 0) +data2(:,2) = data2(:, 2)*0.5; + +% Third cluster is at (0,2) with identity matrix for covariance +data3 = data3 + repmat([0 2], prior(3)*ndata, 1); + +% Put the dataset together again +data = [data1; data2; data3]; + +clc +disp('This demonstration illustrates the use of a Gaussian mixture model') +disp('with diagonal covariance matrices to approximate the unconditional') +disp('probability density of data in a two-dimensional space.') +disp('We begin by generating the data from a mixture of three Gaussians') +disp('with axis aligned covariance structure and plotting it.') +disp(' ') +disp('The first cluster has centre (0, 2).') +disp('The second cluster has centre (0, 0).') +disp('The third cluster has centre (2, 3.5).') +disp(' ') +disp('Press any key to continue') +pause + +fh1 = figure; +plot(data(:, 1), data(:, 2), 'o') +set(gca, 'Box', 'on') + +% Set up mixture model +ncentres = 3; +input_dim = 2; +mix = gmm(input_dim, ncentres, 'diag'); + +options = foptions; +options(14) = 5; % Just use 5 iterations of k-means in initialisation +% Initialise the model parameters from the data +mix = gmminit(mix, data, options); + +% Print out model +disp('The mixture model has three components and diagonal covariance') +disp('matrices. The model parameters after initialisation using the') +disp('k-means algorithm are as follows') +disp(' Priors Centres') +disp([mix.priors' mix.centres]) +disp('Covariance diagonals are') +disp(mix.covars) +disp('Press any key to continue.') +pause + +% Set up vector of options for EM trainer +options = zeros(1, 18); +options(1) = 1; % Prints out error values. +options(14) = 20; % Number of iterations. + +disp('We now train the model using the EM algorithm for 20 iterations.') +disp(' ') +disp('Press any key to continue.') +pause + +[mix, options, errlog] = gmmem(mix, data, options); + +% Print out model +disp(' ') +disp('The trained model has priors and centres:') +disp(' Priors Centres') +disp([mix.priors' mix.centres]) +disp('The data generator has priors and centres') +disp(' Priors Centres') +disp([datap' datac]) +disp('Model covariance diagonals are') +disp(mix.covars) +disp('Data generator covariance diagonals are') +disp(datacov) +disp('Note the close correspondence between these parameters and those') +disp('of the distribution used to generate the data.') +disp(' ') +disp('Press any key to continue.') +pause + +clc +disp('We now plot the density given by the mixture model as a surface plot.') +disp(' ') +disp('Press any key to continue.') +pause + +% Plot the result +x = -4.0:0.2:5.0; +y = -4.0:0.2:5.0; +[X, Y] = meshgrid(x,y); +X = X(:); +Y = Y(:); +grid = [X Y]; +Z = gmmprob(mix, grid); +Z = reshape(Z, length(x), length(y)); +c = mesh(x, y, Z); +hold on +title('Surface plot of probability density') +hold off +drawnow + +clc +disp('The final plot shows the centres and widths, given by one standard') +disp('deviation, of the three components of the mixture model. The axes') +disp('of the ellipses of constant density are shown.') +disp(' ') +disp('Press any key to continue.') +pause + +% Try to calculate a sensible position for the second figure, below the first +fig1_pos = get(fh1, 'Position'); +fig2_pos = fig1_pos; +fig2_pos(2) = fig2_pos(2) - fig1_pos(4); +fh2 = figure('Position', fig2_pos); + +h = plot(data(:, 1), data(:, 2), 'bo'); +hold on +axis('equal'); +title('Plot of data and covariances') +for i = 1:ncentres + v = [1 0]; + for j = 1:2 + start=mix.centres(i,:)-sqrt(mix.covars(i,:).*v); + endpt=mix.centres(i,:)+sqrt(mix.covars(i,:).*v); + linex = [start(1) endpt(1)]; + liney = [start(2) endpt(2)]; + line(linex, liney, 'Color', 'k', 'LineWidth', 3) + v = [0 1]; + end + % Plot ellipses of one standard deviation + theta = 0:0.02:2*pi; + x = sqrt(mix.covars(i,1))*cos(theta) + mix.centres(i,1); + y = sqrt(mix.covars(i,2))*sin(theta) + mix.centres(i,2); + plot(x, y, 'r-'); +end +hold off + +disp('Note how the data cluster positions and widths are captured by') +disp('the mixture model.') +disp(' ') +disp('Press any key to end.') +pause + +close(fh1); +close(fh2); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demgmm4.m b/sourcecodes/bnt-master/netlab3.3/demgmm4.m new file mode 100644 index 00000000..12426787 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgmm4.m @@ -0,0 +1,210 @@ +%DEMGMM4 Demonstrate density modelling with a Gaussian mixture model. +% +% Description +% The problem consists of modelling data generated by a mixture of +% three Gaussians in 2 dimensions with a mixture model using full +% covariance matrices. The priors are 0.3, 0.5 and 0.2; the centres +% are (2, 3.5), (0, 0) and (0,2); the variances are (0.16, 0.64) axis +% aligned, (0.25, 1) rotated by 30 degrees and the identity matrix. The +% first figure contains a scatter plot of the data. +% +% A Gaussian mixture model with three components is trained using EM. +% The parameter vector is printed before training and after training. +% The user should press any key to continue at these points. The +% parameter vector consists of priors (the column), and centres (given +% as (x, y) pairs as the next two columns). The covariance matrices +% are printed separately. +% +% The second figure is a 3 dimensional view of the density function, +% while the third shows the axes of the 1-standard deviation ellipses +% for the three components of the mixture model. +% +% See also +% GMM, GMMINIT, GMMEM, GMMPROB, GMMUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the data + +ndata = 500; + +% Fix the seeds for reproducible results +randn('state', 42); +rand('state', 42); +data = randn(ndata, 2); +prior = [0.3 0.5 0.2]; +% Mixture model swaps clusters 1 and 3 +datap = [0.2 0.5 0.3]; +datac = [0 2; 0 0; 2 3.5]; +datacov = repmat(eye(2), [1 1 3]); +data1 = data(1:prior(1)*ndata,:); +data2 = data(prior(1)*ndata+1:(prior(2)+prior(1))*ndata, :); +data3 = data((prior(1)+prior(2))*ndata +1:ndata, :); + +% First cluster has axis aligned variance and centre (2, 3.5) +data1(:, 1) = data1(:, 1)*0.4 + 2.0; +data1(:, 2) = data1(:, 2)*0.8 + 3.5; +datacov(:, :, 3) = [0.4*0.4 0; 0 0.8*0.8]; + +% Second cluster has variance axes rotated by 30 degrees and centre (0, 0) +rotn = [cos(pi/6) -sin(pi/6); sin(pi/6) cos(pi/6)]; +data2(:,1) = data2(:, 1)*0.5; +data2 = data2*rotn; +datacov(:, :, 2) = rotn' * [0.25 0; 0 1] * rotn; + +% Third cluster is at (0,2) +data3 = data3 + repmat([0 2], prior(3)*ndata, 1); + +% Put the dataset together again +data = [data1; data2; data3]; + +clc +disp('This demonstration illustrates the use of a Gaussian mixture model') +disp('with full covariance matrices to approximate the unconditional ') +disp('probability density of data in a two-dimensional space.') +disp('We begin by generating the data from a mixture of three Gaussians and') +disp('plotting it.') +disp(' ') +disp('The first cluster has axis aligned variance and centre (0, 2).') +disp('The second cluster has variance axes rotated by 30 degrees') +disp('and centre (0, 0). The third cluster has unit variance and centre') +disp('(2, 3.5).') +disp(' ') +disp('Press any key to continue.') +pause + +fh1 = figure; +plot(data(:, 1), data(:, 2), 'o') +set(gca, 'Box', 'on') + +% Set up mixture model +ncentres = 3; +input_dim = 2; +mix = gmm(input_dim, ncentres, 'full'); + +% Initialise the model parameters from the data +options = foptions; +options(14) = 5; % Just use 5 iterations of k-means in initialisation +mix = gmminit(mix, data, options); + +% Print out model +clc +disp('The mixture model has three components and full covariance') +disp('matrices. The model parameters after initialisation using the') +disp('k-means algorithm are as follows') +disp(' Priors Centres') +disp([mix.priors' mix.centres]) +disp('Covariance matrices are') +disp(mix.covars) +disp('Press any key to continue.') +pause + +% Set up vector of options for EM trainer +options = zeros(1, 18); +options(1) = 1; % Prints out error values. +options(14) = 50; % Number of iterations. + +disp('We now train the model using the EM algorithm for 50 iterations.') +disp(' ') +disp('Press any key to continue.') +pause +[mix, options, errlog] = gmmem(mix, data, options); + +% Print out model +disp(' ') +disp('The trained model has priors and centres:') +disp(' Priors Centres') +disp([mix.priors' mix.centres]) +disp('The data generator has priors and centres') +disp(' Priors Centres') +disp([datap' datac]) +disp('Model covariance matrices are') +disp(mix.covars(:, :, 1)) +disp(mix.covars(:, :, 2)) +disp(mix.covars(:, :, 3)) +disp('Data generator covariance matrices are') +disp(datacov(:, :, 1)) +disp(datacov(:, :, 2)) +disp(datacov(:, :, 3)) +disp('Note the close correspondence between these parameters and those') +disp('of the distribution used to generate the data. The match for') +disp('covariance matrices is not that close, but would be improved with') +disp('more iterations of the training algorithm.') +disp(' ') +disp('Press any key to continue.') +pause + +clc +disp('We now plot the density given by the mixture model as a surface plot.') +disp(' ') +disp('Press any key to continue.') +pause + +% Plot the result +x = -4.0:0.2:5.0; +y = -4.0:0.2:5.0; +[X, Y] = meshgrid(x,y); +X = X(:); +Y = Y(:); +grid = [X Y]; +Z = gmmprob(mix, grid); +Z = reshape(Z, length(x), length(y)); +c = mesh(x, y, Z); +hold on +title('Surface plot of probability density') +hold off +drawnow + +clc +disp('The final plot shows the centres and widths, given by one standard') +disp('deviation, of the three components of the mixture model. The axes') +disp('of the ellipses of constant density are shown.') +disp(' ') +disp('Press any key to continue.') +pause + +% Try to calculate a sensible position for the second figure, below the first +fig1_pos = get(fh1, 'Position'); +fig2_pos = fig1_pos; +fig2_pos(2) = fig2_pos(2) - fig1_pos(4) - 30; +fh2 = figure('Position', fig2_pos); + +h3 = plot(data(:, 1), data(:, 2), 'bo'); +axis equal; +hold on +title('Plot of data and covariances') +for i = 1:ncentres + [v,d] = eig(mix.covars(:,:,i)); + for j = 1:2 + % Ensure that eigenvector has unit length + v(:,j) = v(:,j)/norm(v(:,j)); + start=mix.centres(i,:)-sqrt(d(j,j))*(v(:,j)'); + endpt=mix.centres(i,:)+sqrt(d(j,j))*(v(:,j)'); + linex = [start(1) endpt(1)]; + liney = [start(2) endpt(2)]; + line(linex, liney, 'Color', 'k', 'LineWidth', 3) + end + % Plot ellipses of one standard deviation + theta = 0:0.02:2*pi; + x = sqrt(d(1,1))*cos(theta); + y = sqrt(d(2,2))*sin(theta); + % Rotate ellipse axes + ellipse = (v*([x; y]))'; + % Adjust centre + ellipse = ellipse + ones(length(theta), 1)*mix.centres(i,:); + plot(ellipse(:,1), ellipse(:,2), 'r-'); +end +hold off + +disp('Note how the data cluster positions and widths are captured by') +disp('the mixture model.') +disp(' ') +disp('Press any key to end.') +pause + +close(fh1); +close(fh2); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demgmm5.m b/sourcecodes/bnt-master/netlab3.3/demgmm5.m new file mode 100644 index 00000000..f265b35d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgmm5.m @@ -0,0 +1,147 @@ +%DEMGMM5 Demonstrate density modelling with a PPCA mixture model. +% +% Description +% The problem consists of modelling data generated by a mixture of +% three Gaussians in 2 dimensions with a mixture model using full +% covariance matrices. The priors are 0.3, 0.5 and 0.2; the centres +% are (2, 3.5), (0, 0) and (0,2); the variances are (0.16, 0.64) axis +% aligned, (0.25, 1) rotated by 30 degrees and the identity matrix. The +% first figure contains a scatter plot of the data. +% +% A mixture model with three one-dimensional PPCA components is trained +% using EM. The parameter vector is printed before training and after +% training. The parameter vector consists of priors (the column), and +% centres (given as (x, y) pairs as the next two columns). +% +% The second figure is a 3 dimensional view of the density function, +% while the third shows the axes of the 1-standard deviation ellipses +% for the three components of the mixture model together with the one +% standard deviation along the principal component of each mixture +% model component. +% +% See also +% GMM, GMMINIT, GMMEM, GMMPROB, PPCA +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +ndata = 500; +data = randn(ndata, 2); +prior = [0.3 0.5 0.2]; +% Mixture model swaps clusters 1 and 3 +datap = [0.2 0.5 0.3]; +datac = [0 2; 0 0; 2 3.5]; +datacov = repmat(eye(2), [1 1 3]); +data1 = data(1:prior(1)*ndata,:); +data2 = data(prior(1)*ndata+1:(prior(2)+prior(1))*ndata, :); +data3 = data((prior(1)+prior(2))*ndata +1:ndata, :); + +% First cluster has axis aligned variance and centre (2, 3.5) +data1(:, 1) = data1(:, 1)*0.1 + 2.0; +data1(:, 2) = data1(:, 2)*0.8 + 3.5; +datacov(:, :, 3) = [0.1*0.1 0; 0 0.8*0.8]; + +% Second cluster has variance axes rotated by 30 degrees and centre (0, 0) +rotn = [cos(pi/6) -sin(pi/6); sin(pi/6) cos(pi/6)]; +data2(:,1) = data2(:, 1)*0.2; +data2 = data2*rotn; +datacov(:, :, 2) = rotn' * [0.04 0; 0 1] * rotn; + +% Third cluster is at (0,2) +data3(:, 2) = data3(:, 2)*0.1; +data3 = data3 + repmat([0 2], prior(3)*ndata, 1); + +% Put the dataset together again +data = [data1; data2; data3]; + +ndata = 100; % Number of data points. +noise = 0.2; % Standard deviation of noise distribution. +x = [0:1/(2*(ndata - 1)):0.5]'; +randn('state', 1); +rand('state', 1); +t = sin(2*pi*x) + noise*randn(ndata, 1); + +% Fit three one-dimensional PPCA models +ncentres = 3; +ppca_dim = 1; + +clc +disp('This demonstration illustrates the use of a Gaussian mixture model') +disp('with a probabilistic PCA covariance structure to approximate the') +disp('unconditional probability density of data in a two-dimensional space.') +disp('We begin by generating the data from a mixture of three Gaussians and') +disp('plotting it.') +disp(' ') +disp('The first cluster has axis aligned variance and centre (0, 2).') +disp('The variance parallel to the x-axis is significantly greater') +disp('than that parallel to the y-axis.') +disp('The second cluster has variance axes rotated by 30 degrees') +disp('and centre (0, 0). The third cluster has significant variance') +disp('parallel to the y-axis and centre (2, 3.5).') +disp(' ') +disp('Press any key to continue.') +pause + +fh1 = figure; +plot(data(:, 1), data(:, 2), 'o') +set(gca, 'Box', 'on') +axis equal +hold on + +mix = gmm(2, ncentres, 'ppca', ppca_dim); +options = foptions; +options(14) = 10; +options(1) = -1; % Switch off all warnings + +% Just use 10 iterations of k-means in initialisation +% Initialise the model parameters from the data +mix = gmminit(mix, data, options); +disp('The mixture model has three components with 1-dimensional') +disp('PPCA subspaces. The model parameters after initialisation using') +disp('the k-means algorithm are as follows') +disp(' Priors Centres') +disp([mix.priors' mix.centres]) +disp(' ') +disp('Press any key to continue') +pause + +options(1) = 1; % Prints out error values. +options(14) = 30; % Number of iterations. + +disp('We now train the model using the EM algorithm for up to 30 iterations.') +disp(' ') +disp('Press any key to continue.') +pause + +[mix, options, errlog] = gmmem(mix, data, options); +disp('The trained model has priors and centres:') +disp(' Priors Centres') +disp([mix.priors' mix.centres]) + +% Now plot the result +for i = 1:ncentres + % Plot the PC vectors + v = mix.U(:,:,i); + start=mix.centres(i,:)-sqrt(mix.lambda(i))*(v'); + endpt=mix.centres(i,:)+sqrt(mix.lambda(i))*(v'); + linex = [start(1) endpt(1)]; + liney = [start(2) endpt(2)]; + line(linex, liney, 'Color', 'k', 'LineWidth', 3) + % Plot ellipses of one standard deviation + theta = 0:0.02:2*pi; + x = sqrt(mix.lambda(i))*cos(theta); + y = sqrt(mix.covars(i))*sin(theta); + % Rotate ellipse axes + rot_matrix = [v(1) -v(2); v(2) v(1)]; + ellipse = (rot_matrix*([x; y]))'; + % Adjust centre + ellipse = ellipse + ones(length(theta), 1)*mix.centres(i,:); + plot(ellipse(:,1), ellipse(:,2), 'r-') +end + +disp(' ') +disp('Press any key to exit') +pause +close (fh1); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demgp.m b/sourcecodes/bnt-master/netlab3.3/demgp.m new file mode 100644 index 00000000..65c587a3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgp.m @@ -0,0 +1,160 @@ +%DEMGP Demonstrate simple regression using a Gaussian Process. +% +% Description +% The problem consists of one input variable X and one target variable +% T. The values in X are chosen in two separated clusters and the +% target data is generated by computing SIN(2*PI*X) and adding Gaussian +% noise. Two Gaussian Processes, each with different covariance +% functions are trained by optimising the hyperparameters using the +% scaled conjugate gradient algorithm. The final predictions are +% plotted together with 2 standard deviation error bars. +% +% See also +% GP, GPERR, GPFWD, GPGRAD, GPINIT, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Find out if flops is available (i.e. pre-version 6 Matlab) +v = version; +if (str2num(strtok(v, '.')) >= 6) + flops_works = logical(0); +else + flops_works = logical(1); +end + +randn('state', 42); +x = [0.1 0.15 0.2 0.25 0.65 0.7 0.75 0.8 0.85 0.9]'; +ndata = length(x); +t = sin(2*pi*x) + 0.05*randn(ndata, 1); + +xtest = linspace(0, 1, 50)'; + +clc +disp('This demonstration illustrates the use of a Gaussian Process') +disp('model for regression problems. The data is generated from a noisy') +disp('sine function.') +disp(' ') +disp('Press any key to continue.') +pause + +flops(0); +% Initialise the parameters. +net = gp(1, 'sqexp'); +prior.pr_mean = 0; +prior.pr_var = 1; +net = gpinit(net, x, t, prior); + +clc +disp('The first GP uses the squared exponential covariance function.') +disp('The hyperparameters are initialised by sampling from a Gaussian with a') +disp(['mean of ', num2str(prior.pr_mean), ' and variance ', ... + num2str(prior.pr_var), '.']) +disp('After initializing the network, we train it using the scaled conjugate') +disp('gradients algorithm for 20 cycles.') +disp(' ') +disp('Press any key to continue') +pause + +% Now train to find the hyperparameters. +options = foptions; +options(1) = 1; % Display training error values +options(14) = 20; +flops(0) +[net, options] = netopt(net, options, x, t, 'scg'); +if flops_works + sflops = flops; +end + +disp('The second GP uses the rational quadratic covariance function.') +disp('The hyperparameters are initialised by sampling from a Gaussian with a') +disp(['mean of ', num2str(prior.pr_mean), ' and variance ', num2str(prior.pr_var)]) +disp('After initializing the network, we train it using the scaled conjugate') +disp('gradients algorithm for 20 cycles.') +disp(' ') +disp('Press any key to continue') +pause +flops(0) +net2 = gp(1, 'ratquad'); +net2 = gpinit(net2, x, t, prior); +flops(0) +[net2, options] = netopt(net2, options, x, t, 'scg'); +if flops_works + rflops = flops; +end + +disp(' ') +disp('Press any key to continue') +disp(' ') +pause +clc + +fprintf(1, 'For squared exponential covariance function,'); +if flops_works + fprintf(1, 'flops = %d', sflops); +end +fprintf(1, '\nfinal hyperparameters:\n') +format_string = strcat(' bias:\t\t\t%10.6f\n noise:\t\t%10.6f\n', ... + ' inverse lengthscale:\t%10.6f\n vertical scale:\t%10.6f\n'); +fprintf(1, format_string, ... + exp(net.bias), exp(net.noise), exp(net.inweights(1)), exp(net.fpar(1))); +fprintf(1, '\n\nFor rational quadratic covariance function,'); +if flops_works + fprintf(1, 'flops = %d', rflops); +end +fprintf(1, '\nfinal hyperparameters:\n') +format_string = [format_string ' cov decay order:\t%10.6f\n']; +fprintf(1, format_string, ... + exp(net2.bias), exp(net2.noise), exp(net2.inweights(1)), ... + exp(net2.fpar(1)), exp(net2.fpar(2))); +disp(' ') +disp('Press any key to continue') +pause + +disp(' ') +disp('Now we plot the data, underlying function, model outputs and two') +disp('standard deviation error bars on a single graph to compare the results.') +disp(' ') +disp('Press any key to continue.') +pause +cn = gpcovar(net, x); +cninv = inv(cn); +[ytest, sigsq] = gpfwd(net, xtest, cninv); +sig = sqrt(sigsq); + +fh1 = figure; +hold on +plot(x, t, 'ok'); +xlabel('Input') +ylabel('Target') +fplot('sin(2*pi*x)', [0 1], '--m'); +plot(xtest, ytest, '-k'); +plot(xtest, ytest+(2*sig), '-b', xtest, ytest-(2*sig), '-b'); +axis([0 1 -1.5 1.5]); +title('Squared exponential covariance function') +legend('data', 'function', 'GP', 'error bars'); +hold off + +cninv2 = inv(gpcovar(net2, x)); +[ytest2, sigsq2] = gpfwd(net2, xtest, cninv2); +sig2 = sqrt(sigsq2); +fh2 = figure; +hold on +plot(x, t, 'ok'); +xlabel('Input') +ylabel('Target') +fplot('sin(2*pi*x)', [0 1], '--m'); +plot(xtest, ytest2, '-k'); +plot(xtest, ytest2+(2*sig2), '-b', xtest, ytest2-(2*sig2), '-b'); +axis([0 1 -1.5 1.5]); +title('Rational quadratic covariance function') +legend('data', 'function', 'GP', 'error bars'); +hold off + +disp(' ') +disp('Press any key to end.') +pause +close(fh1); +close(fh2); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demgpard.m b/sourcecodes/bnt-master/netlab3.3/demgpard.m new file mode 100644 index 00000000..82e8d2a1 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgpard.m @@ -0,0 +1,126 @@ +%DEMGPARD Demonstrate ARD using a Gaussian Process. +% +% Description +% The data consists of three input variables X1, X2 and X3, and one +% target variable T. The target data is generated by computing +% SIN(2*PI*X1) and adding Gaussian noise, x2 is a copy of x1 with a +% higher level of added noise, and x3 is sampled randomly from a +% Gaussian distribution. A Gaussian Process, is trained by optimising +% the hyperparameters using the scaled conjugate gradient algorithm. +% The final values of the hyperparameters show that the model +% successfully identifies the importance of each input. +% +% See also +% DEMGP, GP, GPERR, GPFWD, GPGRAD, GPINIT, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc; +randn('state', 1729); +rand('state', 1729); +disp('This demonstration illustrates the technique of automatic relevance') +disp('determination (ARD) using a Gaussian Process.') +disp(' '); +disp('First, we set up a synthetic data set involving three input variables:') +disp('x1 is sampled uniformly from the range (0,1) and has a low level of') +disp('added Gaussian noise, x2 is a copy of x1 with a higher level of added') +disp('noise, and x3 is sampled randomly from a Gaussian distribution. The') +disp('single target variable is given by t = sin(2*pi*x1) with additive') +disp('Gaussian noise. Thus x1 is very relevant for determining the target') +disp('value, x2 is of some relevance, while x3 should in principle be') +disp('irrelevant.') +disp(' '); +disp('Press any key to see a plot of t against x1.') +pause; + +ndata = 100; +x1 = rand(ndata, 1); +x2 = x1 + 0.05*randn(ndata, 1); +x3 = 0.5 + 0.5*randn(ndata, 1); +x = [x1, x2, x3]; +t = sin(2*pi*x1) + 0.1*randn(ndata, 1); + +% Plot the data and the original function. +h = figure; +plotvals = linspace(0, 1, 200)'; +plot(x1, t, 'ob') +hold on +xlabel('Input x1') +ylabel('Target') +axis([0 1 -1.5 1.5]) +[fx, fy] = fplot('sin(2*pi*x)', [0 1]); +plot(fx, fy, '-g', 'LineWidth', 2); +legend('data', 'function'); + +disp(' '); +disp('Press any key to continue') +pause; clc; + +disp('The Gaussian Process has a separate hyperparameter for each input.') +disp('The hyperparameters are trained by error minimisation using the scaled.') +disp('conjugate gradient optimiser.') +disp(' '); +disp('Press any key to create and train the model.') +disp(' '); +pause; + +net = gp(3, 'sqexp'); +% Initialise the parameters. +prior.pr_mean = 0; +prior.pr_var = 0.1; +net = gpinit(net, x, t, prior); + +% Now train to find the hyperparameters. +options = foptions; +options(1) = 1; +options(14) = 30; + +[net, options] = netopt(net, options, x, t, 'scg'); + +rel = exp(net.inweights); + +fprintf(1, ... + '\nFinal hyperparameters:\n\n bias:\t\t%10.6f\n noise:\t%10.6f\n', ... + exp(net.bias), exp(net.noise)); +fprintf(1, ' Vertical scale: %8.6f\n', exp(net.fpar(1))); +fprintf(1, ' Input 1:\t%10.6f\n Input 2:\t%10.6f\n', ... + rel(1), rel(2)); +fprintf(1, ' Input 3:\t%10.6f\n\n', rel(3)); +disp(' '); +disp('We see that the inverse lengthscale associated with') +disp('input x1 is large, that of x2 has an intermediate value and the variance') +disp('of weights associated with x3 is small.') +disp(' '); +disp('This implies that the Gaussian Process is giving greatest emphasis') +disp('to x1 and least emphasis to x3, with intermediate emphasis on') +disp('x2 in the covariance function.') +disp(' ') +disp('Since the target t is statistically independent of x3 we might') +disp('expect the weights associated with this input would go to') +disp('zero. However, for any finite data set there may be some chance') +disp('correlation between x3 and t, and so the corresponding hyperparameter remains') +disp('finite.') +disp('Press any key to continue.') +pause + +disp('Finally, we plot the output of the Gaussian Process along the line') +disp('x1 = x2 = x3, together with the true underlying function.') +xt = linspace(0, 1, 50); +xtest = [xt', xt', xt']; + +cn = gpcovar(net, x); +cninv = inv(cn); +[ytest, sigsq] = gpfwd(net, xtest, cninv); +sig = sqrt(sigsq); + +figure(h); hold on; +plot(xt, ytest, '-k'); +plot(xt, ytest+(2*sig), '-b', xt, ytest-(2*sig), '-b'); +axis([0 1 -1.5 1.5]); +fplot('sin(2*pi*x)', [0 1], '--m'); + +disp(' '); +disp('Press any key to end.') +pause; clc; close(h); clear all + diff --git a/sourcecodes/bnt-master/netlab3.3/demgpot.m b/sourcecodes/bnt-master/netlab3.3/demgpot.m new file mode 100644 index 00000000..c859fd0b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgpot.m @@ -0,0 +1,29 @@ +function g = demgpot(x, mix) +%DEMGPOT Computes the gradient of the negative log likelihood for a mixture model. +% +% Description +% This function computes the gradient of the negative log of the +% unconditional data density P(X) with respect to the coefficients of +% the data vector X for a Gaussian mixture model. The data structure +% MIX defines the mixture model, while the matrix X contains the data +% vector as a row vector. Note the unusual order of the arguments: this +% is so that the function can be used in DEMHMC1 directly for sampling +% from the distribution P(X). +% +% See also +% DEMHMC1, DEMMET1, DEMPOT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Computes the potential gradient + +temp = (ones(mix.ncentres,1)*x)-mix.centres; +temp = temp.*(gmmactiv(mix,x)'*ones(1, mix.nin)); +% Assume spherical covariance structure +if ~strcmp(mix.covar_type, 'spherical') + error('Spherical covariance only.') +end +temp = temp./(mix.covars'*ones(1, mix.nin)); +temp = temp.*(mix.priors'*ones(1, mix.nin)); +g = sum(temp, 1)/gmmprob(mix, x); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demgtm1.m b/sourcecodes/bnt-master/netlab3.3/demgtm1.m new file mode 100644 index 00000000..a127617f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgtm1.m @@ -0,0 +1,147 @@ +%DEMGTM1 Demonstrate EM for GTM. +% +% Description +% This script demonstrates the use of the EM algorithm to fit a one- +% dimensional GTM to a two-dimensional set of data using maximum +% likelihood. The location and spread of the Gaussian kernels in the +% data space is shown during training. +% +% See also +% DEMGTM2, GTM, GTMEM, GTMPOST +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Demonstrates the GTM with a 2D target space and a 1D latent space. +% +% This script generates a simple data set in 2 dimensions, +% with an intrinsic dimensionality of 1, and trains a GTM +% with a 1-dimensional latent variable to model this data +% set, visually illustrating the training process +% +% Synopsis: gtm_demo + +% Generate and plot a 2D data set + +data_min = 0.15; +data_max = 3.05; +T = [data_min:0.05:data_max]'; +T = [T (T + 1.25*sin(2*T))]; +fh1 = figure; +plot(T(:,1), T(:,2), 'ro'); +axis([data_min-0.05 data_max+0.05 data_min-0.05 data_max+0.05]); +clc; +disp('This demonstration shows in detail how the EM algorithm works') +disp('for training a GTM with a one dimensional latent space.') +disp(' ') +fprintf([... +'The figure shows data generated by feeding a 1D uniform distribution\n', ... +'(on the X-axis) through a non-linear function (y = x + 1.25*sin(2*x))\n', ... +'\nPress any key to continue ...\n\n']); +pause; + +% Generate a unit circle figure, to be used for plotting +src = [0:(2*pi)/(20-1):2*pi]'; +unitC = [sin(src) cos(src)]; + +% Generate and plot (along with the data) an initial GTM model + +clc; +num_latent_points = 20; +num_rbf_centres = 5; + +net = gtm(1, num_latent_points, 2, num_rbf_centres, 'gaussian'); + +options = zeros(1, 18); +options(7) = 1; +net = gtminit(net, options, T, 'regular', num_latent_points, ... + num_rbf_centres); + +mix = gtmfwd(net); +% Replot the figure +hold off; +plot(mix.centres(:,1), mix.centres(:,2), 'g'); +hold on; +for i=1:num_latent_points + c = 2*unitC*sqrt(mix.covars(1)) + [ones(20,1)*mix.centres(i,1) ... + ones(num_latent_points,1)*mix.centres(i,2)]; + fill(c(:,1), c(:,2), [0.8 1 0.8]); +end +plot(T(:,1), T(:,2), 'ro'); +plot(mix.centres(:,1), mix.centres(:,2), 'g+'); +plot(mix.centres(:,1), mix.centres(:,2), 'g'); +axis([data_min-0.05 data_max+0.05 data_min-0.05 data_max+0.05]); +drawnow; +title('Initial configuration'); +disp(' ') +fprintf([... +'The figure shows the starting point for the GTM, before the training.\n', ... +'A discrete latent variable distribution of %d points in 1 dimension \n', ... +'is mapped to the 1st principal component of the target data by an RBF.\n', ... +'with %d basis functions. Each of the %d points defines the centre of\n', ... +'a Gaussian in a Gaussian mixture, marked by the green ''+''-signs. The\n', ... +'mixture components all have equal variance, illustrated by the filled\n', ... +'circle around each ''+''-sign, the radii corresponding to 2 standard\n', ... +'deviations. The ''+''-signs are connected with a line according to their\n', ... +'corresponding ordering in latent space.\n\n', ... +'Press any key to begin training ...\n\n'], num_latent_points, ... +num_rbf_centres, num_latent_points); +pause; + +figure(fh1); +%%%% Train the GTM and plot it (along with the data) as training proceeds %%%% +options = foptions; +options(1) = -1; % Turn off all warning messages +options(14) = 1; +for j = 1:15 + [net, options] = gtmem(net, T, options); + hold off; + mix = gtmfwd(net); + plot(mix.centres(:,1), mix.centres(:,2), 'g'); + hold on; + for i=1:20 + c = 2*unitC*sqrt(mix.covars(1)) + [ones(20,1)*mix.centres(i,1) ... + ones(20,1)*mix.centres(i,2)]; + fill(c(:,1), c(:,2), [0.8 1.0 0.8]); + end + plot(T(:,1), T(:,2), 'ro'); + plot(mix.centres(:,1), mix.centres(:,2), 'g+'); + plot(mix.centres(:,1), mix.centres(:,2), 'g'); + axis([0 3.5 0 3.5]); + title(['After ', int2str(j),' iterations of training.']); + drawnow; + if (j == 4) + fprintf([... +'The GTM initially adapts relatively quickly - already after \n', ... +'4 iterations of training, a rough fit is attained.\n\n', ... +'Press any key to continue training ...\n\n']); +pause; +figure(fh1); + elseif (j == 8) + fprintf([... +'After another 4 iterations of training: from now on further \n', ... +'training only makes small changes to the mapping, which combined with \n', ... +'decrements of the Gaussian mixture variance, optimize the fit in \n', ... +'terms of likelihood.\n\n', ... +'Press any key to continue training ...\n\n']); +pause; +figure(fh1); + else + pause(1); + end +end + +clc; +fprintf([... +'After 15 iterations of training the GTM can be regarded as converged. \n', ... +'Is has been adapted to fit the target data distribution as well \n', ... +'as possible, given prior smoothness constraints on the mapping. It \n', ... +'captures the fact that the probabilty density is higher at the two \n', ... +'bends of the curve, and lower towards its end points.\n\n']); +disp(' '); +disp('Press any key to exit.'); +pause; + +close(fh1); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demgtm2.m b/sourcecodes/bnt-master/netlab3.3/demgtm2.m new file mode 100644 index 00000000..2cfa907c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgtm2.m @@ -0,0 +1,194 @@ +%DEMGTM2 Demonstrate GTM for visualisation. +% +% Description +% This script demonstrates the use of a GTM with a two-dimensional +% latent space to visualise data in a higher dimensional space. This is +% done through the use of the mean responsibility and magnification +% factors. +% +% See also +% DEMGTM1, GTM, GTMEM, GTMPOST +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Fix seeds for reproducible results +rand('state', 420); +randn('state', 420); + +ndata = 300 +clc; +disp('This demonstration shows how a Generative Topographic Mapping') +disp('can be used to model and visualise high dimensional data. The') +disp('data is generated from a mixture of two spherical Gaussians in') +dstring = ['four dimensional space. ', num2str(ndata), ... + ' data points are generated.']; +disp(dstring); +disp(' '); +disp('Press any key to continue.') +pause +% Create data +data_dim = 4; +latent_dim = 2; +mix = gmm(data_dim, 2, 'spherical'); +mix.centres = [1 1 1 1; 0 0 0 0]; +mix.priors = [0.5 0.5]; +mix.covars = [0.1 0.1]; + +[data, labels] = gmmsamp(mix, ndata); + +latent_shape = [15 15]; % Number of latent points in each dimension +nlatent = prod(latent_shape); % Number of latent points +num_rbf_centres = 16; + +clc; +dstring = ['Next we generate and initialise the GTM. There are ',... + num2str(nlatent), ' latent points']; +disp(dstring); +dstring = ['arranged in a square of ', num2str(latent_shape(1)), ... + ' points on a side. There are ', num2str(num_rbf_centres), ... + ' centres in the']; +disp(dstring); +disp('RBF model, which has Gaussian activation functions.') +disp(' ') +disp('Once the model is created, the latent data sample') +disp('and RBF centres are placed uniformly in the square [-1 1 -1 1].') +disp('The output weights of the RBF are computed to map the latent'); +disp('space to the two dimensional PCA subspace of the data.'); +disp(' ') +disp('Press any key to continue.'); +pause; + +% Create and initialise GTM model +net = gtm(latent_dim, nlatent, data_dim, num_rbf_centres, ... + 'gaussian', 0.1); + +options = foptions; +options(1) = -1; +options(7) = 1; % Set width factor of RBF +net = gtminit(net, options, data, 'regular', latent_shape, [4 4]); + +options = foptions; +options(14) = 30; +options(1) = 1; + +clc; +dstring = ['We now train the model with ', num2str(options(14)), ... + ' iterations of']; +disp(dstring) +disp('the EM algorithm for the GTM.') +disp(' ') +disp('Press any key to continue.') +pause; + +[net, options] = gtmem(net, data, options); + +disp(' ') +disp('Press any key to continue.') +pause; + +clc; +disp('We now visualise the data by plotting, for each data point,'); +disp('the posterior mean and mode (in latent space). These give'); +disp('a summary of the entire posterior distribution in latent space.') +disp('The corresponding values are joined by a line to aid the') +disp('interpretation.') +disp(' ') +disp('Press any key to continue.'); +pause; +% Plot posterior means +means = gtmlmean(net, data); +modes = gtmlmode(net, data); +PointSize = 12; +ClassSymbol1 = 'r.'; +ClassSymbol2 = 'b.'; +fh1 = figure; +hold on; +title('Visualisation in latent space') +plot(means((labels==1),1), means(labels==1,2), ... + ClassSymbol1, 'MarkerSize', PointSize) +plot(means((labels>1),1),means(labels>1,2),... + ClassSymbol2, 'MarkerSize', PointSize) + +ClassSymbol1 = 'ro'; +ClassSymbol2 = 'bo'; +plot(modes(labels==1,1), modes(labels==1,2), ... + ClassSymbol1) +plot(modes(labels>1,1),modes(labels>1,2),... + ClassSymbol2) + +% Join up means and modes +for n = 1:ndata + plot([means(n,1); modes(n,1)], [means(n,2); modes(n,2)], 'g-') +end +% Place legend outside data plot +legend('Mean (class 1)', 'Mean (class 2)', 'Mode (class 1)',... + 'Mode (class 2)', -1); + +% Display posterior for a data point +% Choose an interesting one with a large distance between mean and +% mode +[distance, point] = max(sum((means-modes).^2, 2)); +resp = gtmpost(net, data(point, :)); + +disp(' ') +disp('For more detailed information, the full posterior distribution') +disp('(or responsibility) can be plotted in latent space for a') +disp('single data point. This point has been chosen as the one') +disp('with the largest distance between mean and mode.') +disp(' ') +disp('Press any key to continue.'); +pause; + +R = reshape(resp, fliplr(latent_shape)); +XL = reshape(net.X(:,1), fliplr(latent_shape)); +YL = reshape(net.X(:,2), fliplr(latent_shape)); + +fh2 = figure; +imagesc(net.X(:, 1), net.X(:,2), R); +hold on; +tstr = ['Responsibility for point ', num2str(point)]; +title(tstr); +set(gca,'YDir','normal') +colormap(hot); +colorbar +disp(' '); +disp('Press any key to continue.') +pause + +clc +disp('Finally, we visualise the data with the posterior means in') +disp('latent space as before, but superimpose the magnification') +disp('factors to highlight the separation between clusters.') +disp(' ') +disp('Note the large magnitude factors down the centre of the') +disp('graph, showing that the manifold is stretched more in') +disp('this region than within each of the two clusters.') +ClassSymbol1 = 'g.'; +ClassSymbol2 = 'b.'; + +fh3 = figure; +mags = gtmmag(net, net.X); +% Reshape into grid form +Mags = reshape(mags, fliplr(latent_shape)); +imagesc(net.X(:, 1), net.X(:,2), Mags); +hold on +title('Dataset visualisation with magnification factors') +set(gca,'YDir','normal') +colormap(hot); +colorbar +hold on; % Else the magnification plot disappears +plot(means(labels==1,1), means(labels==1,2), ... + ClassSymbol1, 'MarkerSize', PointSize) +plot(means(labels>1,1), means(labels>1,2), ... + ClassSymbol2, 'MarkerSize', PointSize) + +disp(' ') +disp('Press any key to exit.') +pause + +close(fh1); +close(fh2); +close(fh3); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demhint.m b/sourcecodes/bnt-master/netlab3.3/demhint.m new file mode 100644 index 00000000..bd2e5025 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demhint.m @@ -0,0 +1,49 @@ +function demhint(nin, nhidden, nout) +%DEMHINT Demonstration of Hinton diagram for 2-layer feed-forward network. +% +% Description +% +% DEMHINT plots a Hinton diagram for a 2-layer feedforward network with +% 5 inputs, 4 hidden units and 3 outputs. The weight vector is chosen +% from a Gaussian distribution as described under MLP. +% +% DEMHINT(NIN, NHIDDEN, NOUT) allows the user to specify the number of +% inputs, hidden units and outputs. +% +% See also +% HINTON, HINTMAT, MLP, MLPPAK, MLPUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargin < 1 nin = 5; end +if nargin < 2 nhidden = 7; end +if nargin < 3 nout = 3; end + +% Fix the seed for reproducible results +randn('state', 42); +clc +disp('This demonstration illustrates the plotting of Hinton diagrams') +disp('for Multi-Layer Perceptron networks.') +disp(' ') +disp('Press any key to continue.') +pause +net = mlp(nin, nhidden, nout, 'linear'); + +[h1, h2] = mlphint(net); +clc +disp('The MLP has been created with') +disp([' ' int2str(nin) ' inputs']) +disp([' ' int2str(nhidden) ' hidden units']) +disp([' ' int2str(nout) ' outputs']) +disp(' ') +disp('One figure is produced for each layer of weights.') +disp('For each layer the fan-in weights are arranged in rows for each unit.') +disp('The bias weight is separated from the rest by a red vertical line.') +disp('The area of each box is proportional to the weight value: positive') +disp('values are white, and negative are black.') +disp(' ') +disp('Press any key to exit.'); +pause; +delete(h1); +delete(h2); diff --git a/sourcecodes/bnt-master/netlab3.3/demhmc1.m b/sourcecodes/bnt-master/netlab3.3/demhmc1.m new file mode 100644 index 00000000..69102f9e --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demhmc1.m @@ -0,0 +1,145 @@ +%DEMHMC1 Demonstrate Hybrid Monte Carlo sampling on mixture of two Gaussians. +% +% Description +% The problem consists of generating data from a mixture of two +% Gaussians in two dimensions using a hybrid Monte Carlo algorithm with +% persistence. A mixture model is then fitted to the sample to compare +% it with the true underlying generator. +% +% See also +% DEMHMC3, HMC, DEMPOT, DEMGPOT +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +dim = 2; % Data dimension +ncentres = 2; % Number of centres in mixture model + +seed = 42; % Seed for random weight initialization. +randn('state', seed); +rand('state', seed); + +clc +disp('This demonstration illustrates the use of the hybrid Monte Carlo') +disp('algorithm to sample from a mixture of two Gaussians.') +disp('The means of the two components are [0 0] and [2 2].') +disp(' ') +disp('First we set up the parameters of the mixture model we are sampling') +disp('from.') +disp(' ') +disp('Press any key to continue.') +pause + +% Set up mixture model to sample from +mix = gmm(dim, ncentres, 'spherical'); +mix.centres(1, :) = [0 0]; +mix.centres(2, :) = [2 2]; +x = [0 1]; % Start vector + +% Set up vector of options for hybrid Monte Carlo. + +nsamples = 160; % Number of retained samples. + +options = foptions; % Default options vector. +options(1) = 1; % Switch on diagnostics. +options(5) = 1; % Use persistence +options(7) = 50; % Number of steps in trajectory. +options(14) = nsamples; % Number of Monte Carlo samples returned. +options(15) = 30; % Number of samples omitted at start of chain. +options(18) = 0.02; + +clc +disp(['Next we take ', num2str(nsamples),' samples from the distribution.'... + , 'The first ', num2str(options(15))]) +disp('samples at the start of the chain are omitted. As persistence') +disp('is used, the momentum has a small random component added at each step.') +disp([num2str(options(7)), ... + ' iterations are used at each step and the step size is ',... + num2str(options(18))]) +disp('Sampling starts at the point [0 1].') +disp('The new state is accepted if the threshold value is greater than') +disp('a random number between 0 and 1.') +disp(' ') +disp('Negative step numbers indicate samples discarded from the start of the') +disp('chain.') +disp(' ') +disp('Press any key to continue.') +pause + +[samples, energies] = hmc('dempot', x, options, 'demgpot', mix); + +disp(' ') +disp('Press any key to continue.') +pause +clc +disp('The plot shows the samples generated by the HMC function.') +disp('The different colours are used to show how the samples move from') +disp('one component to the other over time.') +disp(' ') +disp('Press any key to continue.') +pause +probs = exp(-energies); +fh1 = figure; +% Plot data in 4 groups +ngroups = 4; +g1end = floor(nsamples/ngroups); +g2end = floor(2*nsamples/ngroups); +g3end = floor(3*nsamples/ngroups); +p1 = plot(samples(1:g1end,1), samples(1:g1end,2), 'k.', 'MarkerSize', 12); +hold on +lstrings = char(['Samples 1-' int2str(g1end)], ... + ['Samples ' int2str(g1end+1) '-' int2str(g2end)], ... + ['Samples ' int2str(g2end+1) '-' int2str(g3end)], ... + ['Samples ' int2str(g3end+1) '-' int2str(nsamples)]); +p2 = plot(samples(g1end+1:g2end,1), samples(g1end+1:g2end,2), ... + 'r.', 'MarkerSize', 12); +p3 = plot(samples(g2end+1:g3end,1), samples(g2end+1:g3end,2), ... + 'g.', 'MarkerSize', 12); +p4 = plot(samples(g3end+1:nsamples,1), samples(g3end+1:nsamples,2), ... + 'b.', 'MarkerSize', 12); +legend([p1 p2 p3 p4], lstrings, 2); + +clc +disp('We now fit a Gaussian mixture model to the sampled data.') +disp('The model has spherical covariance structure and the correct') +disp('number of components.') +disp(' ') +disp('Press any key to continue.') +pause +% Fit a mixture model to the sample +newmix = gmm(dim, ncentres, 'spherical'); +options = foptions; +options(1) = -1; % Switch off all diagnostics +options(14) = 5; % Just use 5 iterations of k-means in initialisation +% Initialise the model parameters from the samples +newmix = gmminit(newmix, samples, options); + +% Set up vector of options for EM trainer +options = zeros(1, 18); +options(1) = 1; % Prints out error values. +options(14) = 15; % Max. Number of iterations. + +disp('We now train the model using the EM algorithm for 15 iterations') +disp(' ') +disp('Press any key to continue') +pause +[newmix, options, errlog] = gmmem(newmix, samples, options); + +% Print out model +disp(' ') +disp('The trained model has parameters ') +disp(' Priors Centres Variances') +disp([newmix.priors' newmix.centres newmix.covars']) +disp('Note the close correspondence between these parameters and those') +disp('of the distribution used to generate the data') +disp(' ') +disp(' Priors Centres Variances') +disp([mix.priors' mix.centres mix.covars']) +disp(' ') +disp('Press any key to exit') +pause + +close(fh1); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demhmc2.m b/sourcecodes/bnt-master/netlab3.3/demhmc2.m new file mode 100644 index 00000000..d3df79d8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demhmc2.m @@ -0,0 +1,132 @@ +%DEMHMC2 Demonstrate Bayesian regression with Hybrid Monte Carlo sampling. +% +% Description +% The problem consists of one input variable X and one target variable +% T with data generated by sampling X at equal intervals and then +% generating target data by computing SIN(2*PI*X) and adding Gaussian +% noise. The model is a 2-layer network with linear outputs, and the +% hybrid Monte Carlo algorithm (without persistence) is used to sample +% from the posterior distribution of the weights. The graph shows the +% underlying function, 100 samples from the function given by the +% posterior distribution of the weights, and the average prediction +% (weighted by the posterior probabilities). +% +% See also +% DEMHMC3, HMC, MLP, MLPERR, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the matrix of inputs x and targets t. +ndata = 20; % Number of data points. +noise = 0.1; % Standard deviation of noise distribution. +nin = 1; % Number of inputs. +nout = 1; % Number of outputs. + +seed = 42; % Seed for random weight initialization. +randn('state', seed); +rand('state', seed); + +x = 0.25 + 0.1*randn(ndata, nin); +t = sin(2*pi*x) + noise*randn(size(x)); + +clc +disp('This demonstration illustrates the use of the hybrid Monte Carlo') +disp('algorithm to sample from the posterior weight distribution of a') +disp('multi-layer perceptron.') +disp(' ') +disp('A regression problem is used, with the one-dimensional data drawn') +disp('from a noisy sine function. The x values are sampled from a normal') +disp('distribution with mean 0.25 and variance 0.01.') +disp(' ') +disp('First we initialise the network.') +disp(' ') +disp('Press any key to continue.') +pause + +% Set up network parameters. +nhidden = 5; % Number of hidden units. +alpha = 0.001; % Coefficient of weight-decay prior. +beta = 100.0; % Coefficient of data error. + +% Create and initialize network model. +% Initialise weights reasonably close to 0 +net = mlp(nin, nhidden, nout, 'linear', alpha, beta); +net = mlpinit(net, 10); + +clc +disp('Next we take 100 samples from the posterior distribution. The first') +disp('200 samples at the start of the chain are omitted. As persistence') +disp('is not used, the momentum is randomised at each step. 100 iterations') +disp('are used at each step. The new state is accepted if the threshold') +disp('value is greater than a random number between 0 and 1.') +disp(' ') +disp('Negative step numbers indicate samples discarded from the start of the') +disp('chain.') +disp(' ') +disp('Press any key to continue.') +pause +% Set up vector of options for hybrid Monte Carlo. +nsamples = 100; % Number of retained samples. + +options = foptions; % Default options vector. +options(1) = 1; % Switch on diagnostics. +options(7) = 100; % Number of steps in trajectory. +options(14) = nsamples; % Number of Monte Carlo samples returned. +options(15) = 200; % Number of samples omitted at start of chain. +options(18) = 0.002; % Step size. + +w = mlppak(net); +% Initialise HMC +hmc('state', 42); +[samples, energies] = hmc('neterr', w, options, 'netgrad', net, x, t); + +clc +disp('The plot shows the underlying noise free function, the 100 samples') +disp('produced from the MLP, and their average as a Monte Carlo estimate') +disp('of the true posterior average.') +disp(' ') +disp('Press any key to continue.') +pause +nplot = 300; +plotvals = [0 : 1/(nplot - 1) : 1]'; +pred = zeros(size(plotvals)); +fh = figure; +for k = 1:nsamples + w2 = samples(k,:); + net2 = mlpunpak(net, w2); + y = mlpfwd(net2, plotvals); + % Average sample predictions as Monte Carlo estimate of true integral + pred = pred + y; + h4 = plot(plotvals, y, '-r', 'LineWidth', 1); + if k == 1 + hold on + end +end +pred = pred./nsamples; + +% Plot data +h1 = plot(x, t, 'ob', 'LineWidth', 2, 'MarkerFaceColor', 'blue'); +axis([0 1 -3 3]) + +% Plot function +[fx, fy] = fplot('sin(2*pi*x)', [0 1], '--g'); +h2 = plot(fx, fy, '--g', 'LineWidth', 2); +set(gca, 'box', 'on'); + +% Plot averaged prediction +h3 = plot(plotvals, pred, '-c', 'LineWidth', 2); +hold off + +lstrings = char('Data', 'Function', 'Prediction', 'Samples'); +legend([h1 h2 h3 h4], lstrings, 3); + +disp('Note how the predictions become much further from the true function') +disp('away from the region of high data density.') +disp(' ') +disp('Press any key to exit.') +pause +close(fh); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demhmc3.m b/sourcecodes/bnt-master/netlab3.3/demhmc3.m new file mode 100644 index 00000000..f2956ae5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demhmc3.m @@ -0,0 +1,135 @@ +%DEMHMC3 Demonstrate Bayesian regression with Hybrid Monte Carlo sampling. +% +% Description +% The problem consists of one input variable X and one target variable +% T with data generated by sampling X at equal intervals and then +% generating target data by computing SIN(2*PI*X) and adding Gaussian +% noise. The model is a 2-layer network with linear outputs, and the +% hybrid Monte Carlo algorithm (with persistence) is used to sample +% from the posterior distribution of the weights. The graph shows the +% underlying function, 300 samples from the function given by the +% posterior distribution of the weights, and the average prediction +% (weighted by the posterior probabilities). +% +% See also +% DEMHMC2, HMC, MLP, MLPERR, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the matrix of inputs x and targets t. +ndata = 20; % Number of data points. +noise = 0.1; % Standard deviation of noise distribution. +nin = 1; % Number of inputs. +nout = 1; % Number of outputs. + +seed = 42; % Seed for random number generators. +randn('state', seed); +rand('state', seed); + +x = 0.25 + 0.1*randn(ndata, nin); +t = sin(2*pi*x) + noise*randn(size(x)); + +clc +disp('This demonstration illustrates the use of the hybrid Monte Carlo') +disp('algorithm to sample from the posterior weight distribution of a') +disp('multi-layer perceptron.') +disp(' ') +disp('A regression problem is used, with the one-dimensional data drawn') +disp('from a noisy sine function. The x values are sampled from a normal') +disp('distribution with mean 0.25 and variance 0.01.') +disp(' ') +disp('First we initialise the network.') +disp(' ') +disp('Press any key to continue.') +pause + +% Set up network parameters. +nhidden = 5; % Number of hidden units. +alpha = 0.001; % Coefficient of weight-decay prior. +beta = 100.0; % Coefficient of data error. + +% Create and initialize network model. + +% Initialise weights reasonably close to 0 +net = mlp(nin, nhidden, nout, 'linear', alpha, beta); +net = mlpinit(net, 10); + +clc +disp('Next we take 100 samples from the posterior distribution. The first') +disp('300 samples at the start of the chain are omitted. As persistence') +disp('is used, the momentum has a small random component added at each step.') +disp('10 iterations are used at each step (compared with 100 in demhmc2).') +disp('The step size is 0.005 (compared with 0.002).') +disp('The new state is accepted if the threshold') +disp('value is greater than a random number between 0 and 1.') +disp(' ') +disp('Negative step numbers indicate samples discarded from the start of the') +disp('chain.') +disp(' ') +disp('Press any key to continue.') +pause + +% Set up vector of options for hybrid Monte Carlo. +nsamples = 100; % Number of retained samples. + +options = foptions; % Default options vector. +options(1) = 1; % Switch on diagnostics. +options(5) = 1; % Use persistence +options(7) = 10; % Number of steps in trajectory. +options(14) = nsamples; % Number of Monte Carlo samples returned. +options(15) = 300; % Number of samples omitted at start of chain. +options(17) = 0.95; % Alpha value in persistence +options(18) = 0.005; % Step size. + +w = mlppak(net); +% Initialise HMC +hmc('state', 42); +[samples, energies] = hmc('neterr', w, options, 'netgrad', net, x, t); + +clc +disp('The plot shows the underlying noise free function, the 100 samples') +disp('produced from the MLP, and their average as a Monte Carlo estimate') +disp('of the true posterior average.') +disp(' ') +disp('Press any key to continue.') +pause + +nplot = 300; +plotvals = [0 : 1/(nplot - 1) : 1]'; +pred = zeros(size(plotvals)); +fh1 = figure; +hold on +for k = 1:nsamples + w2 = samples(k,:); + net2 = mlpunpak(net, w2); + y = mlpfwd(net2, plotvals); + % Sum predictions + pred = pred + y; + h4 = plot(plotvals, y, '-r', 'LineWidth', 1); +end +pred = pred./nsamples; +% Plot data +h1 = plot(x, t, 'ob', 'LineWidth', 2, 'MarkerFaceColor', 'blue'); +axis([0 1 -3 3]) + +% Plot function +[fx, fy] = fplot('sin(2*pi*x)', [0 1], '--g'); +h2 = plot(fx, fy, '--g', 'LineWidth', 2); +set(gca, 'box', 'on'); + +% Plot averaged prediction +h3 = plot(plotvals, pred, '-c', 'LineWidth', 2); + +lstrings = char('Data', 'Function', 'Prediction', 'Samples'); +legend([h1 h2 h3 h4], lstrings, 3); +hold off + +disp('Note how the predictions become much further from the true function') +disp('away from the region of high data density.') +disp(' ') +disp('Press any key to exit.') +pause +close(fh1); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demkmn1.m b/sourcecodes/bnt-master/netlab3.3/demkmn1.m new file mode 100644 index 00000000..d9a7c6e2 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demkmn1.m @@ -0,0 +1,111 @@ +%DEMKMEAN Demonstrate simple clustering model trained with K-means. +% +% Description +% The problem consists of data in a two-dimensional space. The data is +% drawn from three spherical Gaussian distributions with priors 0.3, +% 0.5 and 0.2; centres (2, 3.5), (0, 0) and (0,2); and standard +% deviations 0.2, 0.5 and 1.0. The first figure contains a scatter plot +% of the data. The data is the same as in DEMGMM1. +% +% A cluster model with three components is trained using the batch K- +% means algorithm. The matrix of centres is printed after training. The +% second figure shows the data labelled with a colour derived from the +% corresponding cluster +% +% See also +% DEM2DDAT, DEMGMM1, KNN1, KMEANS +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Generate the data, fixing seeds for reproducible results +ndata = 250; +randn('state', 42); +rand('state', 42); +data = dem2ddat(ndata); + +% Randomise data order +data = data(randperm(ndata),:); + +clc +disp('This demonstration illustrates the use of a cluster model to') +disp('find centres that reflect the distribution of data points.') +disp('We begin by generating the data from a mixture of three Gaussians') +disp('in two-dimensional space and plotting it.') +disp(' ') +disp('Press any key to continue.') +pause + +fh1 = figure; +plot(data(:, 1), data(:, 2), 'o') +set(gca, 'Box', 'on') +title('Data') + +% Set up cluster model +ncentres = 3; +centres = zeros(ncentres, 2); + +% Set up vector of options for kmeans trainer +options = foptions; +options(1) = 1; % Prints out error values. +options(5) = 1; +options(14) = 10; % Number of iterations. + +clc +disp('The model is chosen to have three centres, which are initialised') +disp('at randomly selected data points. We now train the model using') +disp('the batch K-means algorithm with a maximum of 10 iterations and') +disp('stopping tolerance of 1e-4.') +disp(' ') +disp('Press any key to continue.') +pause + +% Train the centres from the data +[centres, options, post] = kmeans(centres, data, options); + +% Print out model +disp(' ') +disp('Note that training has terminated before 10 iterations as there') +disp('has been no change in the centres or error function.') +disp(' ') +disp('The trained model has centres:') +disp(centres); +disp('Press any key to continue.') +pause + +clc +disp('We now plot each data point coloured according to its classification') +disp('given by the nearest cluster centre. The cluster centres are denoted') +disp('by black crosses.') + +% Plot the result +fh2 = figure; + +hold on +colours = ['b.'; 'r.'; 'g.']; + +[tempi, tempj] = find(post); +hold on +for i = 1:3 + % Select data points closest to ith centre + thisX = data(tempi(tempj == i), 1); + thisY = data(tempi(tempj == i), 2); + hp(i) = plot(thisX, thisY, colours(i,:)); + set(hp(i), 'MarkerSize', 12); +end +set(gca, 'Box', 'on') +legend('Class 1', 'Class 2', 'Class 3', 2) +hold on +plot(centres(:, 1), centres(:,2), 'k+', 'LineWidth', 2, ... + 'MarkerSize', 8) +title('Centres and data labels') +hold off + +disp(' ') +disp('Press any key to end.') +pause + +close(fh1); +close(fh2); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demknn1.m b/sourcecodes/bnt-master/netlab3.3/demknn1.m new file mode 100644 index 00000000..6e3ec60a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demknn1.m @@ -0,0 +1,103 @@ +%DEMKNN1 Demonstrate nearest neighbour classifier. +% +% Description +% The problem consists of data in a two-dimensional space. The data is +% drawn from three spherical Gaussian distributions with priors 0.3, +% 0.5 and 0.2; centres (2, 3.5), (0, 0) and (0,2); and standard +% deviations 0.2, 0.5 and 1.0. The first figure contains a scatter plot +% of the data. The data is the same as in DEMGMM1. +% +% The second figure shows the data labelled with the corresponding +% class given by the classifier. +% +% See also +% DEM2DDAT, DEMGMM1, KNN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc +disp('This program demonstrates the use of the K nearest neighbour algorithm.') +disp(' ') +disp('Press any key to continue.') +pause +% Generate the test data +ndata = 250; +randn('state', 42); +rand('state', 42); + +[data, c] = dem2ddat(ndata); + +% Randomise data order +data = data(randperm(ndata),:); + +clc +disp('We generate the data in two-dimensional space from a mixture of') +disp('three spherical Gaussians. The centres are shown as black crosses') +disp('in the plot.') +disp(' ') +disp('Press any key to continue.') +pause +fh1 = figure; +plot(data(:, 1), data(:, 2), 'o') +set(gca, 'Box', 'on') +hold on +title('Data') +hp1 = plot(c(:, 1), c(:,2), 'k+') +% Increase size of crosses +set(hp1, 'MarkerSize', 8); +set(hp1, 'LineWidth', 2); +hold off + +clc +disp('We next use the centres as training examplars for the K nearest') +disp('neighbour algorithm.') +disp(' ') +disp('Press any key to continue.') +pause + +% Use centres as training data +train_labels = [1, 0, 0; 0, 1, 0; 0, 0, 1]; + +% Label the test data up to kmax neighbours +kmax = 1; +net = knn(2, 3, kmax, c, train_labels); +[y, l] = knnfwd(net, data); + +clc +disp('We now plot each data point coloured according to its classification.') +disp(' ') +disp('Press any key to continue.') +pause +% Plot the result +fh2 = figure; +colors = ['b.'; 'r.'; 'g.']; +for i = 1:3 + thisX = data(l == i,1); + thisY = data(l == i,2); + hp(i) = plot(thisX, thisY, colors(i,:)); + set(hp(i), 'MarkerSize', 12); + if i == 1 + hold on + end +end +set(gca, 'Box', 'on'); +legend('Class 1', 'Class 2', 'Class 3', 2) +hold on +labels = ['1', '2', '3']; +hp2 = plot(c(:, 1), c(:,2), 'k+'); +% Increase size of crosses +set(hp2, 'MarkerSize', 8); +set(hp2, 'LineWidth', 2); + +test_labels = labels(l(:,1)); + +title('Training data and data labels') +hold off + +disp('The demonstration is now complete: press any key to exit.') +pause +close(fh1); +close(fh2); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demmdn1.m b/sourcecodes/bnt-master/netlab3.3/demmdn1.m new file mode 100644 index 00000000..43c2d98b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demmdn1.m @@ -0,0 +1,211 @@ +%DEMMDN1 Demonstrate fitting a multi-valued function using a Mixture Density Network. +% +% Description +% The problem consists of one input variable X and one target variable +% T with data generated by sampling T at equal intervals and then +% generating target data by computing T + 0.3*SIN(2*PI*T) and adding +% Gaussian noise. A Mixture Density Network with 3 centres in the +% mixture model is trained by minimizing a negative log likelihood +% error function using the scaled conjugate gradient optimizer. +% +% The conditional means, mixing coefficients and variances are plotted +% as a function of X, and a contour plot of the full conditional +% density is also generated. +% +% See also +% MDN, MDNERR, MDNGRAD, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the matrix of inputs x and targets t. +seedn = 42; +seed = 42; +randn('state', seedn); +rand('state', seed); +ndata = 300; % Number of data points. +noise = 0.2; % Range of noise distribution. +t = [0:1/(ndata - 1):1]'; +x = t + 0.3*sin(2*pi*t) + noise*rand(ndata, 1) - noise/2; +axis_limits = [-0.2 1.2 -0.2 1.2]; + +clc +disp('This demonstration illustrates the use of a Mixture Density Network') +disp('to model multi-valued functions. The data is generated from the') +disp('mapping x = t + 0.3 sin(2 pi t) + e, where e is a noise term.') +disp('We begin by plotting the data.') +disp(' ') +disp('Press any key to continue') +pause +% Plot the data +fh1 = figure; +p1 = plot(x, t, 'ob'); +axis(axis_limits); +hold on +disp('Note that for x in the range 0.35 to 0.65, there are three possible') +disp('branches of the function.') +disp(' ') +disp('Press any key to continue') +pause + +% Set up network parameters. +nin = 1; % Number of inputs. +nhidden = 5; % Number of hidden units. +ncentres = 3; % Number of mixture components. +dim_target = 1; % Dimension of target space +mdntype = '0'; % Currently unused: reserved for future use +alpha = 100; % Inverse variance for weight initialisation + % Make variance small for good starting point + +% Create and initialize network weight vector. +net = mdn(nin, nhidden, ncentres, dim_target, mdntype); +init_options = zeros(1, 18); +init_options(1) = -1; % Suppress all messages +init_options(14) = 10; % 10 iterations of K means in gmminit +net = mdninit(net, alpha, t, init_options); + +% Set up vector of options for the optimiser. +options = foptions; +options(1) = 1; % This provides display of error values. +options(14) = 200; % Number of training cycles. + +clc +disp('We initialise the neural network model, which is an MLP with a') +disp('Gaussian mixture model with three components and spherical variance') +disp('as the error function. This enables us to model the complete') +disp('conditional density function.') +disp(' ') +disp('Next we train the model for 200 epochs using a scaled conjugate gradient') +disp('optimizer. The error function is the negative log likelihood of the') +disp('training data.') +disp(' ') +disp('Press any key to continue.') +pause + +% Train using scaled conjugate gradients. +[net, options] = netopt(net, options, x, t, 'scg'); + +disp(' ') +disp('Press any key to continue.') +pause + +clc +disp('We can also train a conventional MLP with sum of squares error function.') +disp('This will approximate the conditional mean, which is not always a') +disp('good representation of the data. Note that the error function is the') +disp('sum of squares error on the training data, which accounts for the') +disp('different values from training the MDN.') +disp(' ') +disp('We train the network with the quasi-Newton optimizer for 80 epochs.') +disp(' ') +disp('Press any key to continue.') +pause +mlp_nhidden = 8; +net2 = mlp(nin, mlp_nhidden, dim_target, 'linear'); +options(14) = 80; +[net2, options] = netopt(net2, options, x, t, 'quasinew'); +disp(' ') +disp('Press any key to continue.') +pause + +clc +disp('Now we plot the underlying function, the MDN prediction,') +disp('represented by the mode of the conditional distribution, and the') +disp('prediction of the conventional MLP.') +disp(' ') +disp('Press any key to continue.') +pause + +% Plot the original function, and the trained network function. +plotvals = [0:0.01:1]'; +mixes = mdn2gmm(mdnfwd(net, plotvals)); +axis(axis_limits); +yplot = t+0.3*sin(2*pi*t); +p2 = plot(yplot, t, '--y'); + +% Use the mode to represent the function +y = zeros(1, length(plotvals)); +priors = zeros(length(plotvals), ncentres); +c = zeros(length(plotvals), 3); +widths = zeros(length(plotvals), ncentres); +for i = 1:length(plotvals) + [m, j] = max(mixes(i).priors); + y(i) = mixes(i).centres(j,:); + c(i,:) = mixes(i).centres'; +end +p3 = plot(plotvals, y, '*r'); +p4 = plot(plotvals, mlpfwd(net2, plotvals), 'g'); +set(p4, 'LineWidth', 2); +legend([p1 p2 p3 p4], 'data', 'function', 'MDN mode', 'MLP mean', 4); +hold off + +clc +disp('We can also plot how the mixture model parameters depend on x.') +disp('First we plot the mixture centres, then the priors and finally') +disp('the variances.') +disp(' ') +disp('Press any key to continue.') +pause +fh2 = figure; +subplot(3, 1, 1) +plot(plotvals, c) +hold on +title('Mixture centres') +legend('centre 1', 'centre 2', 'centre 3') +hold off + +priors = reshape([mixes.priors], mixes(1).ncentres, size(mixes, 2))'; +%%fh3 = figure; +subplot(3, 1, 2) +plot(plotvals, priors) +hold on +title('Mixture priors') +legend('centre 1', 'centre 2', 'centre 3') +hold off + +variances = reshape([mixes.covars], mixes(1).ncentres, size(mixes, 2))'; +%%fh4 = figure; +subplot(3, 1, 3) +plot(plotvals, variances) +hold on +title('Mixture variances') +legend('centre 1', 'centre 2', 'centre 3') +hold off + +disp('The last figure is a contour plot of the conditional probability') +disp('density generated by the Mixture Density Network. Note how it') +disp('is well matched to the regions of high data density.') +disp(' ') +disp('Press any key to continue.') +pause +% Contour plot for MDN. +i = 0:0.01:1.0; +j = 0:0.01:1.0; + +[I, J] = meshgrid(i,j); +I = I(:); +J = J(:); +li = length(i); +lj = length(j); +Z = zeros(li, lj); +for k = 1:li; + Z(:,k) = gmmprob(mixes(k), j'); +end +fh5 = figure; +% Set up levels by hand to make a good figure +v = [2 2.5 3 3.5 5:3:18]; +contour(i, j, Z, v) +hold on +title('Contour plot of conditional density') +hold off + +disp(' ') +disp('Press any key to exit.') +pause +close(fh1); +close(fh2); +%%close(fh3); +%%close(fh4); +close(fh5); +%%clear all; diff --git a/sourcecodes/bnt-master/netlab3.3/demmet1.m b/sourcecodes/bnt-master/netlab3.3/demmet1.m new file mode 100644 index 00000000..804622a4 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demmet1.m @@ -0,0 +1,103 @@ +function samples=demmet1(plot_wait) +%DEMMET1 Demonstrate Markov Chain Monte Carlo sampling on a Gaussian. +% +% Description +% The problem consists of generating data from a Gaussian in two +% dimensions using a Markov Chain Monte Carlo algorithm. The points are +% plotted one after another to show the path taken by the chain. +% +% DEMMET1(PLOTWAIT) allows the user to set the time (in a whole number +% of seconds) between the plotting of points. This is passed to PAUSE +% +% See also +% DEMHMC1, METROP, GMM, DEMPOT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargin == 0 | plot_wait < 0 + plot_wait = 0; % No wait if not specified or incorrect +end +dim = 2; % Data dimension +ncentres = 1; % Number of centres in mixture model + +seed = 42; % Seed for random weight initialization. +randn('state', seed); +rand('state', seed); + +clc +disp('This demonstration illustrates the use of the Markov chain Monte Carlo') +disp('algorithm to sample from a Gaussian distribution.') +disp('The mean is at [0 0].') +disp(' ') +disp('First we set up the parameters of the mixture model we are sampling') +disp('from.') +disp(' ') +disp('Press any key to continue.') +pause + +% Set up mixture model to sample from +mix = gmm(dim, ncentres, 'spherical'); +mix.centres(1, :) = [0 0]; +x = [0 4]; % Start vector + +% Set up vector of options for hybrid Monte Carlo. + +nsamples = 150; % Number of retained samples. + +options = foptions; % Default options vector. +options(1) = 0; % Switch off diagnostics. +options(14) = nsamples; % Number of Monte Carlo samples returned. +options(18) = 0.1; + +clc +disp('Next we take 150 samples from the distribution.') +disp('Sampling starts at the point [0 4].') +disp('The new state is accepted if the threshold value is greater than') +disp('a random number between 0 and 1.') +disp(' ') +disp('Press any key to continue.') +pause + +[samples, energies] = metrop('dempot', x, options, '', mix); + +clc +disp('The plot shows the samples generated by the MCMC function in order') +disp('as an animation to show the path taken by the Markov chain.') +disp('The different colours are used to show that the first few samples') +disp('should be discarded as they lie too far from the mean.') +disp(' ') +disp('Press any key to continue.') +pause +probs = exp(-energies); +fh1 = figure; +g1end = floor(nsamples/4); + +for n = 1:nsamples + + if n < g1end + Marker = 'k.'; + p1 = plot(samples(n,1), samples(n,2), Marker, ... + 'EraseMode', 'none', 'MarkerSize', 12); + if n == 1 + axis([-3 5 -2 5]) + end + else + Marker = 'r.'; + p2 = plot(samples(n,1), samples(n,2), Marker, ... + 'EraseMode', 'none', 'MarkerSize', 12); + end + hold on + drawnow; % Force drawing immediately + pause(plot_wait); +end +lstrings = char(['Samples 1-' int2str(g1end)], ... + ['Samples ' int2str(g1end+1) '-' int2str(nsamples)]); +legend([p1 p2], lstrings, 1); + +disp(' ') +disp('Press any key to exit.') +pause +close(fh1); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demmlp1.m b/sourcecodes/bnt-master/netlab3.3/demmlp1.m new file mode 100644 index 00000000..4f102b73 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demmlp1.m @@ -0,0 +1,88 @@ +%DEMMLP1 Demonstrate simple regression using a multi-layer perceptron +% +% Description +% The problem consists of one input variable X and one target variable +% T with data generated by sampling X at equal intervals and then +% generating target data by computing SIN(2*PI*X) and adding Gaussian +% noise. A 2-layer network with linear outputs is trained by minimizing +% a sum-of-squares error function using the scaled conjugate gradient +% optimizer. +% +% See also +% MLP, MLPERR, MLPGRAD, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the matrix of inputs x and targets t. + +ndata = 20; % Number of data points. +noise = 0.2; % Standard deviation of noise distribution. +x = [0:1/(ndata - 1):1]'; +randn('state', 1); +t = sin(2*pi*x) + noise*randn(ndata, 1); + +clc +disp('This demonstration illustrates the use of a Multi-Layer Perceptron') +disp('network for regression problems. The data is generated from a noisy') +disp('sine function.') +disp(' ') +disp('Press any key to continue.') +pause + +% Set up network parameters. +nin = 1; % Number of inputs. +nhidden = 3; % Number of hidden units. +nout = 1; % Number of outputs. +alpha = 0.01; % Coefficient of weight-decay prior. + +% Create and initialize network weight vector. + +net = mlp(nin, nhidden, nout, 'linear', alpha); + +% Set up vector of options for the optimiser. + +options = zeros(1,18); +options(1) = 1; % This provides display of error values. +options(14) = 100; % Number of training cycles. + +clc +disp(['The network has ', num2str(nhidden), ' hidden units and a weight decay']) +disp(['coefficient of ', num2str(alpha), '.']) +disp(' ') +disp('After initializing the network, we train it use the scaled conjugate') +disp('gradients algorithm for 100 cycles.') +disp(' ') +disp('Press any key to continue') +pause + +% Train using scaled conjugate gradients. +[net, options] = netopt(net, options, x, t, 'scg'); + +disp(' ') +disp('Now we plot the data, underlying function, and network outputs') +disp('on a single graph to compare the results.') +disp(' ') +disp('Press any key to continue.') +pause + +% Plot the data, the original function, and the trained network function. +plotvals = [0:0.01:1]'; +y = mlpfwd(net, plotvals); +fh1 = figure; +plot(x, t, 'ob') +hold on +xlabel('Input') +ylabel('Target') +axis([0 1 -1.5 1.5]) +[fx, fy] = fplot('sin(2*pi*x)', [0 1]); +plot(fx, fy, '-r', 'LineWidth', 2) +plot(plotvals, y, '-k', 'LineWidth', 2) +legend('data', 'function', 'network'); + +disp(' ') +disp('Press any key to end.') +pause +close(fh1); +clear all; diff --git a/sourcecodes/bnt-master/netlab3.3/demmlp2.m b/sourcecodes/bnt-master/netlab3.3/demmlp2.m new file mode 100644 index 00000000..20a86cf4 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demmlp2.m @@ -0,0 +1,337 @@ +%DEMMLP2 Demonstrate simple classification using a multi-layer perceptron +% +% Description +% The problem consists of input data in two dimensions drawn from a +% mixture of three Gaussians: two of which are assigned to a single +% class. An MLP with logistic outputs trained with a quasi-Newton +% optimisation algorithm is compared with the optimal Bayesian decision +% rule. +% +% See also +% MLP, MLPFWD, NETERR, QUASINEW +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Set up some figure parameters +AxisShift = 0.05; +ClassSymbol1 = 'r.'; +ClassSymbol2 = 'y.'; +PointSize = 12; +titleSize = 10; + +% Fix the seeds +rand('state', 423); +randn('state', 423); + +clc +disp('This demonstration shows how an MLP with logistic outputs and') +disp('and cross entropy error function can be trained to model the') +disp('posterior class probabilities in a classification problem.') +disp('The results are compared with the optimal Bayes rule classifier,') +disp('which can be computed exactly as we know the form of the generating') +disp('distribution.') +disp(' ') +disp('Press any key to continue.') +pause + +fh1 = figure; +set(fh1, 'Name', 'True Data Distribution'); +whitebg(fh1, 'k'); + +% +% Generate the data +% +n=200; + +% Set up mixture model: 2d data with three centres +% Class 1 is first centre, class 2 from the other two +mix = gmm(2, 3, 'full'); +mix.priors = [0.5 0.25 0.25]; +mix.centres = [0 -0.1; 1 1; 1 -1]; +mix.covars(:,:,1) = [0.625 -0.2165; -0.2165 0.875]; +mix.covars(:,:,2) = [0.2241 -0.1368; -0.1368 0.9759]; +mix.covars(:,:,3) = [0.2375 0.1516; 0.1516 0.4125]; + +[data, label] = gmmsamp(mix, n); + +% +% Calculate some useful axis limits +% +x0 = min(data(:,1)); +x1 = max(data(:,1)); +y0 = min(data(:,2)); +y1 = max(data(:,2)); +dx = x1-x0; +dy = y1-y0; +expand = 5/100; % Add on 5 percent each way +x0 = x0 - dx*expand; +x1 = x1 + dx*expand; +y0 = y0 - dy*expand; +y1 = y1 + dy*expand; +resolution = 100; +step = dx/resolution; +xrange = [x0:step:x1]; +yrange = [y0:step:y1]; +% +% Generate the grid +% +[X Y]=meshgrid([x0:step:x1],[y0:step:y1]); +% +% Calculate the class conditional densities, the unconditional densities and +% the posterior probabilities +% +px_j = gmmactiv(mix, [X(:) Y(:)]); +px = reshape(px_j*(mix.priors)',size(X)); +post = gmmpost(mix, [X(:) Y(:)]); +p1_x = reshape(post(:, 1), size(X)); +p2_x = reshape(post(:, 2) + post(:, 3), size(X)); + +% +% Generate some pretty pictures !! +% +colormap(hot) +colorbar +subplot(1,2,1) +hold on +plot(data((label==1),1),data(label==1,2),ClassSymbol1, 'MarkerSize', PointSize) +plot(data((label>1),1),data(label>1,2),ClassSymbol2, 'MarkerSize', PointSize) +contour(xrange,yrange,p1_x,[0.5 0.5],'w-'); +axis([x0 x1 y0 y1]) +set(gca,'Box','On') +title('The Sampled Data'); +rect=get(gca,'Position'); +rect(1)=rect(1)-AxisShift; +rect(3)=rect(3)+AxisShift; +set(gca,'Position',rect) +hold off + +subplot(1,2,2) +imagesc(X(:),Y(:),px); +hold on +[cB, hB] = contour(xrange,yrange,p1_x,[0.5 0.5],'w:'); +set(hB,'LineWidth', 2); +axis([x0 x1 y0 y1]) +set(gca,'YDir','normal') +title('Probability Density p(x)') +hold off + +drawnow; +clc; +disp('The first figure shows the data sampled from a mixture of three') +disp('Gaussians, the first of which (whose centre is near the origin) is') +disp('labelled red and the other two are labelled yellow. The second plot') +disp('shows the unconditional density of the data with the optimal Bayesian') +disp('decision boundary superimposed.') +disp(' ') +disp('Press any key to continue.') +pause +fh2 = figure; +set(fh2, 'Name', 'Class-conditional Densities and Posterior Probabilities'); +whitebg(fh2, 'w'); + +subplot(2,2,1) +p1=reshape(px_j(:,1),size(X)); +imagesc(X(:),Y(:),p1); +colormap hot +colorbar +axis(axis) +set(gca,'YDir','normal') +hold on +plot(mix.centres(:,1),mix.centres(:,2),'b+','MarkerSize',8,'LineWidth',2) +title('Density p(x|red)') +hold off + +subplot(2,2,2) +p2=reshape((px_j(:,2)+px_j(:,3)),size(X)); +imagesc(X(:),Y(:),p2); +colorbar +set(gca,'YDir','normal') +hold on +plot(mix.centres(:,1),mix.centres(:,2),'b+','MarkerSize',8,'LineWidth',2) +title('Density p(x|yellow)') +hold off + +subplot(2,2,3) +imagesc(X(:),Y(:),p1_x); +set(gca,'YDir','normal') +colorbar +title('Posterior Probability p(red|x)') +hold on +plot(mix.centres(:,1),mix.centres(:,2),'b+','MarkerSize',8,'LineWidth',2) +hold off + +subplot(2,2,4) +imagesc(X(:),Y(:),p2_x); +set(gca,'YDir','normal') +colorbar +title('Posterior Probability p(yellow|x)') +hold on +plot(mix.centres(:,1),mix.centres(:,2),'b+','MarkerSize',8,'LineWidth',2) +hold off + +% Now set up and train the MLP +nhidden=6; +nout=1; +alpha = 0.2; % Weight decay +ncycles = 60; % Number of training cycles. +% Set up MLP network +net = mlp(2, nhidden, nout, 'logistic', alpha); +options = zeros(1,18); +options(1) = 1; % Print out error values +options(14) = ncycles; + +mlpstring = ['We now set up an MLP with ', num2str(nhidden), ... + ' hidden units, logistic output and cross']; +trainstring = ['entropy error function, and train it for ', ... + num2str(ncycles), ' cycles using the']; +wdstring = ['quasi-Newton optimisation algorithm with weight decay of ', ... + num2str(alpha), '.']; + +% Force out the figure before training the MLP +drawnow; +disp(' ') +disp('The second figure shows the class conditional densities and posterior') +disp('probabilities for each class. The blue crosses mark the centres of') +disp('the three Gaussians.') +disp(' ') +disp(mlpstring) +disp(trainstring) +disp(wdstring) +disp(' ') +disp('Press any key to continue.') +pause + +% Convert targets to 0-1 encoding +target=[label==1]; + +% Train using quasi-Newton. +[net] = netopt(net, options, data, target, 'quasinew'); +y = mlpfwd(net, data); +yg = mlpfwd(net, [X(:) Y(:)]); +yg = reshape(yg(:,1),size(X)); + +fh3 = figure; +set(fh3, 'Name', 'Network Output'); +whitebg(fh3, 'k') +subplot(1, 2, 1) +hold on +plot(data((label==1),1),data(label==1,2),'r.', 'MarkerSize', PointSize) +plot(data((label>1),1),data(label>1,2),'y.', 'MarkerSize', PointSize) +% Bayesian decision boundary +[cB, hB] = contour(xrange,yrange,p1_x,[0.5 0.5],'b-'); +[cN, hN] = contour(xrange,yrange,yg,[0.5 0.5],'r-'); +set(hB, 'LineWidth', 2); +set(hN, 'LineWidth', 2); +Chandles = [hB(1) hN(1)]; +legend(Chandles, 'Bayes', ... + 'Network', 3); + +axis([x0 x1 y0 y1]) +set(gca,'Box','on','XTick',[],'YTick',[]) + +title('Training Data','FontSize',titleSize); +hold off + +subplot(1, 2, 2) +imagesc(X(:),Y(:),yg); +colormap hot +colorbar +axis(axis) +set(gca,'YDir','normal','XTick',[],'YTick',[]) +title('Network Output','FontSize',titleSize) + +clc +disp('This figure shows the training data with the decision boundary') +disp('produced by the trained network and the network''s prediction of') +disp('the posterior probability of the red class.') +disp(' ') +disp('Press any key to continue.') +pause + +% +% Now generate and classify a test data set +% +[testdata testlabel] = gmmsamp(mix, n); +testlab=[testlabel==1 testlabel>1]; + +% This is the Bayesian classification +tpx_j = gmmpost(mix, testdata); +Bpost = [tpx_j(:,1), tpx_j(:,2)+tpx_j(:,3)]; +[Bcon Brate]=confmat(Bpost, [testlabel==1 testlabel>1]); + +% Compute network classification +yt = mlpfwd(net, testdata); +% Convert single output to posteriors for both classes +testpost = [yt 1-yt]; +[C trate]=confmat(testpost,[testlabel==1 testlabel>1]); + +fh4 = figure; +set(fh4, 'Name', 'Decision Boundaries'); +whitebg(fh4, 'k'); +hold on +plot(testdata((testlabel==1),1),testdata((testlabel==1),2),... + ClassSymbol1, 'MarkerSize', PointSize) +plot(testdata((testlabel>1),1),testdata((testlabel>1),2),... + ClassSymbol2, 'MarkerSize', PointSize) +% Bayesian decision boundary +[cB, hB] = contour(xrange,yrange,p1_x,[0.5 0.5],'b-'); +set(hB, 'LineWidth', 2); +% Network decision boundary +[cN, hN] = contour(xrange,yrange,yg,[0.5 0.5],'r-'); +set(hN, 'LineWidth', 2); +Chandles = [hB(1) hN(1)]; +legend(Chandles, 'Bayes decision boundary', ... + 'Network decision boundary', -1); +axis([x0 x1 y0 y1]) +title('Test Data') +set(gca,'Box','On','Xtick',[],'YTick',[]) + +clc +disp('This figure shows the test data with the decision boundary') +disp('produced by the trained network and the optimal Bayes rule.') +disp(' ') +disp('Press any key to continue.') +pause + +fh5 = figure; +set(fh5, 'Name', 'Test Set Performance'); +whitebg(fh5, 'w'); +% Bayes rule performance +subplot(1,2,1) +plotmat(Bcon,'b','k',12) +set(gca,'XTick',[0.5 1.5]) +set(gca,'YTick',[0.5 1.5]) +grid('off') +set(gca,'XTickLabel',['Red ' ; 'Yellow']) +set(gca,'YTickLabel',['Yellow' ; 'Red ']) +ylabel('True') +xlabel('Predicted') +title(['Bayes Confusion Matrix (' num2str(Brate(1)) '%)']) + +% Network performance +subplot(1,2, 2) +plotmat(C,'b','k',12) +set(gca,'XTick',[0.5 1.5]) +set(gca,'YTick',[0.5 1.5]) +grid('off') +set(gca,'XTickLabel',['Red ' ; 'Yellow']) +set(gca,'YTickLabel',['Yellow' ; 'Red ']) +ylabel('True') +xlabel('Predicted') +title(['Network Confusion Matrix (' num2str(trate(1)) '%)']) + +disp('The final figure shows the confusion matrices for the') +disp('two rules on the test set.') +disp(' ') +disp('Press any key to exit.') +pause +whitebg(fh1, 'w'); +whitebg(fh2, 'w'); +whitebg(fh3, 'w'); +whitebg(fh4, 'w'); +whitebg(fh5, 'w'); +close(fh1); close(fh2); close(fh3); +close(fh4); close(fh5); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demnlab.m b/sourcecodes/bnt-master/netlab3.3/demnlab.m new file mode 100644 index 00000000..11fa3e87 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demnlab.m @@ -0,0 +1,380 @@ +function demnlab(action); +%DEMNLAB A front-end Graphical User Interface to the demos +% +% Description +% This function will start a user interface allowing the user to select +% different demonstration functions to view. The demos are divided into +% 4 groups, with the demo being executed by selecting the desired +% option from a pop-up menu. +% +% See also +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% If run without parameters, initialise gui. +if nargin<1, + action='initialise'; +end; + +if strcmp(action,'initialise'), + + % Create figure + fig = figure( ... + 'Name', 'Netlab Demos', ... + 'NumberTitle', 'off', ... + 'Color', [0.7529 0.7529 0.7529], ... + 'Visible', 'on'); + + % Create GROUPS + % Bottom of demo buttons + group1_bot = 0.20; + group1_top = 0.75; + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'Position', [0.03 group1_bot 0.94 group1_top - group1_bot], ... + 'BackgroundColor', [0.5 0.5 0.5]); + + % Bottom of help and close buttons + group2_bot = 0.04; + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'Position', [0.03 group2_bot 0.94 0.12], ... + 'BackgroundColor', [0.5 0.5 0.5]); + + % Draw title + hLogoAxis = axes( ... + 'Units', 'normalized', ... + 'Position', [0.05 0.82 0.90 0.14], ... + 'Box', 'off', ... + 'XColor', [0 0 0], ... + 'YColor', [0 0 0], ... + 'Visible', 'on'); + + load netlogo; % load image and colour map + colormap(netcmap(1:3,:)); % change colour map: don't need many entries + image(nlogo); % draw logo + axis('image'); % ensures pixels on axis are square + axis off; % turn axes off + + % Create static text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.05 group1_top-0.1 0.90 0.08], ... + 'String', 'Select demo to run:'); + + % First row text offset + tRow1Offset = 0.14; + % Offset between text and button + TBoffset = 0.07; + % First row button offset + bRow1Offset = tRow1Offset+TBoffset; + % ONE text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.08 group1_top-tRow1Offset 0.36 0.05], ... + 'String', 'Regression'); + + popup1str(1) = {'Select Option'}; + popup1str(2) = {'Multi-Layer Perceptron'}; + popup1str(3) = {'Radial Basis Function'}; + popup1str(4) = {'Mixture Density Network'}; + % ONE popup + hPop1 = uicontrol(fig, ... + 'Style','popup', ... + 'Units','normalized', ... + 'String', popup1str, ... + 'Position', [0.08 group1_top-bRow1Offset 0.36 0.08], ... + 'BackgroundColor', [0.7 0.7 0.7], ... + 'Callback', 'demnlab popup1'); + + % TWO text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.56 group1_top-tRow1Offset 0.36 0.05], ... + 'String', 'Classification'); + + popup2str(1) = popup1str(1); + popup2str(2) = {'Generalised Linear Model (2 class)'}; + popup2str(3) = {'Generalised Linear Model (3 class)'}; + popup2str(4) = {'Multi-Layer Perceptron'}; + popup2str(5) = {'K nearest neighbour'}; + % TWO popup + hPop2 = uicontrol(fig, ... + 'Style','popup', ... + 'Units','normalized', ... + 'String', popup2str, ... + 'Position', [0.56 group1_top-bRow1Offset 0.36 0.08], ... + 'BackgroundColor', [0.7 0.7 0.7], ... + 'Callback', 'demnlab popup2'); + + tRow2Offset = 0.30; + bRow2Offset = tRow2Offset+TBoffset; + % THREE text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.08 group1_top - tRow2Offset 0.36 0.05], ... + 'String', 'Density Modelling and Clustering'); + + popup3str(1) = popup1str(1); + popup3str(2) = {'Gaussian Mixture (EM training)'}; + popup3str(3) = {'Gaussian Mixture (spherical)'}; + popup3str(4) = {'Gaussian Mixture (diagonal)'}; + popup3str(5) = {'Gaussian Mixture (full)'}; + popup3str(6) = {'Neuroscale'}; + popup3str(7) = {'GTM (EM training)'}; + popup3str(8) = {'GTM (visualisation)'}; + popup3str(9) = {'K-means clustering'}; + popup3str(10) = {'Self-Organising Map'}; + % TWO popup + % THREE popup + hPop3 = uicontrol(fig, ... + 'Style','popup', ... + 'Units','normalized', ... + 'String', popup3str, ... + 'Position', [0.08 group1_top - bRow2Offset 0.36 0.08], ... + 'BackgroundColor', [0.7 0.7 0.7], ... + 'Callback', 'demnlab popup3'); + + % FOUR text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.56 group1_top - tRow2Offset 0.36 0.05], ... + 'String', 'Bayesian Methods'); + + popup4str(1) = popup1str(1); + popup4str(2) = {'Sampling the MLP Prior'}; + popup4str(3) = {'Evidence Approximation for MLP'}; + popup4str(4) = {'Evidence Approximation for RBF'}; + popup4str(5) = {'Evidence Approximation in Classification'}; + popup4str(6) = {'ARD for MLP'}; + popup4str(7) = {'Sampling the GP Prior'}; + popup4str(8) = {'GPs for Regression'}; + popup4str(9) = {'ARD for GP'}; + % FOUR popup + hPop4 = uicontrol(fig, ... + 'Style','popup', ... + 'Units','normalized', ... + 'String', popup4str, ... + 'Position', [0.56 group1_top - bRow2Offset 0.36 0.08], ... + 'BackgroundColor', [0.7 0.7 0.7], ... + 'Callback', 'demnlab popup4'); + + + tRow3Offset = 0.45; + bRow3Offset = tRow3Offset+TBoffset; + % FIVE text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.08 group1_top - tRow3Offset 0.36 0.05], ... + 'String', 'Optimisation and Visualisation'); + + popup5str(1) = popup1str(1); + popup5str(2) = {'Algorithm Comparison'}; + popup5str(3) = {'On-line Gradient Descent'}; + popup5str(4) = {'Hinton Diagrams'}; + % FIVE popup + hPop5 = uicontrol(fig, ... + 'Style','popup', ... + 'Units','normalized', ... + 'String',popup5str, ... + 'Position', [0.08 group1_top - bRow3Offset 0.36 0.08], ... + 'BackgroundColor', [0.7 0.7 0.7], ... + 'Callback', 'demnlab popup5'); + + % SIX text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.56 group1_top - tRow3Offset 0.36 0.05], ... + 'String', 'Sampling'); + + popup6str(1) = popup1str(1); + popup6str(2) = {'Sampling a Gaussian'}; + popup6str(3) = {'MCMC sampling (Metropolis)'}; + popup6str(4) = {'Hybrid MC (Gaussian mixture)'}; + popup6str(5) = {'Hybrid MC for MLP I'}; + popup6str(6) = {'Hybrid MC for MLP II'}; + % SIX popup + hPop6 = uicontrol(fig, ... + 'Style','popup', ... + 'Units','normalized', ... + 'String', popup6str, ... + 'Position', [0.56 group1_top - bRow3Offset 0.36 0.08], ... + 'BackgroundColor', [0.7 0.7 0.7], ... + 'Callback', 'demnlab popup6'); + + + % Create HELP button + uicontrol(fig, ... + 'Units', 'normalized', ... + 'Position' , [0.05 group2_bot+0.02 0.40 0.08], ... + 'String', 'Help', ... + 'Callback', 'demnlab help'); + + % Create CLOSE button + uicontrol(fig, ... + 'Units', 'normalized', ... + 'Position' , [0.55 group2_bot+0.02 0.40 0.08], ... + 'String', 'Close', ... + 'Callback', 'close(gcf)'); + + hndlList=[fig hPop1 hPop2 hPop3 hPop4 hPop5 hPop6]; + set(fig, 'UserData', hndlList); + set(fig, 'HandleVisibility', 'callback'); + +elseif strcmp(action, 'popup1'), + + hndlList=get(gcf,'UserData'); + hPop = hndlList(2); + + selected = get(hPop, 'Val'); + set(hPop, 'Val', [1]); + + switch selected + case 2 + demmlp1; + case 3 + demrbf1; + case 4 + demmdn1; + end; + +elseif strcmp(action,'popup2'), + + hndlList=get(gcf,'UserData'); + hPop = hndlList(3); + + selected = get(hPop, 'Val'); + set(hPop, 'Val', [1]); + + switch selected + case 2 + demglm1; + case 3 + demglm2; + case 4 + demmlp2; + case 5 + demknn1; + end + +elseif strcmp(action,'popup3'), + + hndlList=get(gcf,'UserData'); + hPop = hndlList(4); + + selected = get(hPop, 'Val'); + set(hPop, 'Val', [1]); + + switch selected + case 2 + demgmm1; + case 3 + demgmm2; + case 4 + demgmm3; + case 5 + demgmm4; + case 6 + demns1; + case 7 + demgtm1; + case 8 + demgtm2; + case 9 + demkmn1; + case 10 + demsom1; + end + +elseif strcmp(action,'popup4'), + + hndlList=get(gcf,'UserData'); + hPop = hndlList(5); + + selected = get(hPop, 'Val'); + set(hPop, 'Val', [1]); + + switch selected + case 2 + demprior; + case 3 + demev1; + case 4 + demev3; + case 5 + demev2; + case 6 + demard; + case 7 + demprgp; + case 8 + demgp; + case 9 + demgpard; + end + +elseif strcmp(action,'popup5'), + + hndlList=get(gcf,'UserData'); + hPop = hndlList(6); + + selected = get(hPop, 'Val'); + set(hPop, 'Val', [1]); + + switch selected + case 2 + demopt1; + case 3 + demolgd1; + case 4 + demhint; + end + + +elseif strcmp(action,'popup6'), + + hndlList=get(gcf,'UserData'); + hPop = hndlList(7); + + selected = get(hPop, 'Val'); + set(hPop, 'Val', [1]); + + switch selected + case 2 + demgauss; + case 3 + demmet1; + case 4 + demhmc1; + case 5 + demhmc2; + case 6 + demhmc3; + end + +elseif strcmp(action, 'help'), + + helpStr = {'To run a demo, press the appropriate button.'; ... + 'Instructions and information will appear in the Matlab';... + 'command window.'}; + + hHelpDlg = helpdlg(helpStr, 'Netlab Demo Help'); + +end; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demns1.m b/sourcecodes/bnt-master/netlab3.3/demns1.m new file mode 100644 index 00000000..029e1b48 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demns1.m @@ -0,0 +1,114 @@ +%DEMNS1 Demonstrate Neuroscale for visualisation. +% +% Description +% This script demonstrates the use of the Neuroscale algorithm for +% topographic projection and visualisation. A data sample is generated +% from a mixture of two Gaussians in 4d space, and an RBF is trained +% with the stress error function to project the data into 2d. The +% training data and a test sample are both plotted in this projection. +% +% See also +% RBF, RBFTRAIN, RBFPRIOR +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Generate the data +% Fix seeds for reproducible results +rand('state', 420); +randn('state', 420); + +input_dim = 4; +output_dim = 2; +mix = gmm(input_dim, 2, 'spherical'); +mix.centres = [1 1 1 1; 0 0 0 0]; +mix.priors = [0.5 0.5]; +mix.covars = [0.1 0.1]; + +ndata = 60; +[data, labels] = gmmsamp(mix, ndata); + +clc +disp('This demonstration illustrates the use of the Neuroscale model') +disp('to perform a topographic projection of data. We begin by generating') +disp('60 data points from a mixture of two Gaussians in 4 dimensional space.') +disp(' ') +disp('Press any key to continue') +pause + +ncentres = 10; +net = rbf(input_dim, ncentres, output_dim, 'tps', 'neuroscale'); +dstring = ['the Sammon mapping. The model has ', num2str(ncentres), ... + ' centres, two outputs, and uses']; +clc +disp('The Neuroscale model is an RBF with a Stress error measure as used in') +disp(dstring) +disp('thin plate spline basis functions.') +disp(' ') +disp('It is trained using the shadow targets algorithm for at most 60 iterations.') +disp(' ') +disp('Press any key to continue') +pause + +% First row controls shadow targets, second row controls rbfsetbf +options(1, :) = foptions; +options(2, :) = foptions; +options(1, 1) = 1; +options(1, 2) = 1e-2; +options(1, 3) = 1e-2; +options(1, 6) = 1; % Switch on PCA initialisation +options(1, 14) = 60; +options(2, 1) = -1; % Switch off all warnings +options(2, 5) = 1; +options(2, 14) = 10; +net2 = rbftrain(net, options, data); + +disp(' ') +disp('After training the model, we project the training data by a normal') +disp('forward propagation through the RBF network. Because there are two') +disp('outputs, the results can be plotted and visualised.') +disp(' ') +disp('Press any key to continue') +pause + +% Plot the result +y = rbffwd(net2, data); +ClassSymbol1 = 'r.'; +ClassSymbol2 = 'b.'; +PointSize = 12; +fh1 = figure; +hold on; +plot(y((labels==1),1),y(labels==1,2),ClassSymbol1, 'MarkerSize', PointSize) +plot(y((labels>1),1),y(labels>1,2),ClassSymbol2, 'MarkerSize', PointSize) + +disp(' ') +disp('In this plot, the red dots denote the first class and the blue') +disp('dots the second class.') +disp(' ') +disp('Press any key to continue.') +disp(' ') +pause + +disp('We now generate a further 100 data points from the original distribution') +disp('and plot their projection using star symbols. Note that a Sammon') +disp('mapping cannot be used to generalise to new data in this fashion.') + +[test_data, test_labels] = gmmsamp(mix, 100); +ytest = rbffwd(net2, test_data); +ClassSymbol1 = 'ro'; +ClassSymbol2 = 'bo'; +% Circles are rather large symbols +PointSize = 6; +hold on +plot(ytest((test_labels==1),1),ytest(test_labels==1,2), ... + ClassSymbol1, 'MarkerSize', PointSize) +plot(ytest((test_labels>1),1),ytest(test_labels>1,2),... + ClassSymbol2, 'MarkerSize', PointSize) +hold on +legend('Class 1', 'Class 2', 'Test Class 1', 'Test Class 2') +disp('Press any key to exit.') +pause + +close(fh1); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demolgd1.m b/sourcecodes/bnt-master/netlab3.3/demolgd1.m new file mode 100644 index 00000000..fb18f760 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demolgd1.m @@ -0,0 +1,115 @@ +%DEMOLGD1 Demonstrate simple MLP optimisation with on-line gradient descent +% +% Description +% The problem consists of one input variable X and one target variable +% T with data generated by sampling X at equal intervals and then +% generating target data by computing SIN(2*PI*X) and adding Gaussian +% noise. A 2-layer network with linear outputs is trained by minimizing +% a sum-of-squares error function using on-line gradient descent. +% +% See also +% DEMMLP1, OLGD +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the matrix of inputs x and targets t. + +ndata = 20; % Number of data points. +noise = 0.2; % Standard deviation of noise distribution. +x = [0:1/(ndata - 1):1]'; +randn('state', 42); +rand('state', 42); +t = sin(2*pi*x) + noise*randn(ndata, 1); + +clc +disp('This demonstration illustrates the use of the on-line gradient') +disp('descent algorithm to train a Multi-Layer Perceptron network for') +disp('regression problems. It is intended to illustrate the drawbacks') +disp('of this algorithm compared to more powerful non-linear optimisation') +disp('algorithms, such as conjugate gradients.') +disp(' ') +disp('First we generate the data from a noisy sine function and construct') +disp('the network.') +disp(' ') +disp('Press any key to continue.') +pause +% Set up network parameters. +nin = 1; % Number of inputs. +nhidden = 3; % Number of hidden units. +nout = 1; % Number of outputs. +alpha = 0.01; % Coefficient of weight-decay prior. + +% Create and initialize network weight vector. +net = mlp(nin, nhidden, nout, 'linear'); +% Initialise weights reasonably close to 0 +net = mlpinit(net, 10); + +% Set up vector of options for the optimiser. +options = foptions; +options(1) = 1; % This provides display of error values. +options(14) = 20; % Number of training cycles. +options(18) = 0.1; % Learning rate +%options(17) = 0.4; % Momentum +options(17) = 0.4; % Momentum +options(5) = 1; % Do randomise pattern order +clc +disp('Then we set the options for the training algorithm.') +disp(['In the first phase of training, which lasts for ',... + num2str(options(14)), ' cycles,']) +disp(['the learning rate is ', num2str(options(18)), ... + ' and the momentum is ', num2str(options(17)), '.']) +disp('The error values are displayed at the end of each pass through the') +disp('entire pattern set.') +disp(' ') +disp('Press any key to continue.') +pause + +% Train using online gradient descent +[net, options] = olgd(net, options, x, t); + +% Now allow learning rate to decay and remove momentum +options(2) = 0; +options(3) = 0; +options(17) = 0.4; % Turn off momentum +options(5) = 1; % Randomise pattern order +options(6) = 1; % Set learning rate decay on +options(14) = 200; +options(18) = 0.1; % Initial learning rate + +disp(['In the second phase of training, which lasts for up to ',... + num2str(options(14)), ' cycles,']) +disp(['the learning rate starts at ', num2str(options(18)), ... + ', decaying at 1/t and the momentum is ', num2str(options(17)), '.']) +disp(' ') +disp('Press any key to continue.') +pause +[net, options] = olgd(net, options, x, t); + +clc +disp('Now we plot the data, underlying function, and network outputs') +disp('on a single graph to compare the results.') +disp(' ') +disp('Press any key to continue.') +pause + +% Plot the data, the original function, and the trained network function. +plotvals = [0:0.01:1]'; +y = mlpfwd(net, plotvals); +fh1 = figure; +plot(x, t, 'ob') +hold on +axis([0 1 -1.5 1.5]) +fplot('sin(2*pi*x)', [0 1], '--g') +plot(plotvals, y, '-r') +legend('data', 'function', 'network'); +hold off + +disp('Note the very poor fit to the data: this should be compared with') +disp('the results obtained in demmlp1.') +disp(' ') +disp('Press any key to exit.') +pause +close(fh1); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demopt1.m b/sourcecodes/bnt-master/netlab3.3/demopt1.m new file mode 100644 index 00000000..31b4a421 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demopt1.m @@ -0,0 +1,170 @@ +function demopt1(xinit) +%DEMOPT1 Demonstrate different optimisers on Rosenbrock's function. +% +% Description +% The four general optimisers (quasi-Newton, conjugate gradients, +% scaled conjugate gradients, and gradient descent) are applied to the +% minimisation of Rosenbrock's well known `banana' function. Each +% optimiser is run for at most 100 cycles, and a stopping criterion of +% 1.0e-4 is used for both position and function value. At the end, the +% trajectory of each algorithm is shown on a contour plot of the +% function. +% +% DEMOPT1(XINIT) allows the user to specify a row vector with two +% columns as the starting point. The default is the point [-1 1]. Note +% that the contour plot has an x range of [-1.5, 1.5] and a y range of +% [-0.5, 2.1], so it is best to choose a starting point in the same +% region. +% +% See also +% CONJGRAD, GRADDESC, QUASINEW, SCG, ROSEN, ROSEGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Initialise start point for search +if nargin < 1 | size(xinit) ~= [1 2] + xinit = [-1 1]; % Traditional start point +end + +% Find out if flops is available (i.e. pre-version 6 Matlab) +v = version; +if (str2num(strtok(v, '.')) >= 6) + flops_works = logical(0); +else + flops_works = logical(1); +end + +% Set up options +options = foptions; % Standard options +options(1) = -1; % Turn off printing completely +options(3) = 1e-8; % Tolerance in value of function +options(14) = 100; % Max. 100 iterations of algorithm + +clc +disp('This demonstration compares the performance of four generic') +disp('optimisation routines when finding the minimum of Rosenbrock''s') +disp('function y = 100*(x2-x1^2)^2 + (1-x1)^2.') +disp(' ') +disp('The global minimum of this function is at [1 1].') +disp(['Each algorithm starts at the point [' num2str(xinit(1))... + ' ' num2str(xinit(2)) '].']) +disp(' ') +disp('Press any key to continue.') +pause + +% Generate a contour plot of the function +a = -1.5:.02:1.5; +b = -0.5:.02:2.1; +[A, B] = meshgrid(a, b); +Z = rosen([A(:), B(:)]); +Z = reshape(Z, length(b), length(a)); +l = -1:6; +v = 2.^l; +fh1 = figure; +contour(a, b, Z, v) +title('Contour plot of Rosenbrock''s function') +hold on + +clc +disp('We now use quasi-Newton, conjugate gradient, scaled conjugate') +disp('gradient, and gradient descent with line search algorithms') +disp('to find a local minimum of this function. Each algorithm is stopped') +disp('when 100 cycles have elapsed, or if the change in function value') +disp('is less than 1.0e-8 or the change in the input vector is less than') +disp('1.0e-4 in magnitude.') +disp(' ') +disp('Press any key to continue.') +pause + +clc +x = xinit; +flops(0) +[x, options, errlog, pointlog] = quasinew('rosen', x, options, 'rosegrad'); +fprintf(1, 'For quasi-Newton method:\n') +fprintf(1, 'Final point is (%f, %f), value is %f\n', x(1), x(2), options(8)) +fprintf(1, 'Number of function evaluations is %d\n', options(10)) +fprintf(1, 'Number of gradient evaluations is %d\n', options(11)) +if flops_works + opt_flops = flops; + fprintf(1, 'Number of floating point operations is %d\n', opt_flops) +end +fprintf(1, 'Number of cycles is %d\n', size(pointlog, 1) - 1); +disp(' ') + +x = xinit; +flops(0) +[x, options, errlog2, pointlog2] = conjgrad('rosen', x, options, 'rosegrad'); +fprintf(1, 'For conjugate gradient method:\n') +fprintf(1, 'Final point is (%f, %f), value is %f\n', x(1), x(2), options(8)) +fprintf(1, 'Number of function evaluations is %d\n', options(10)) +fprintf(1, 'Number of gradient evaluations is %d\n', options(11)) +if flops_works + opt_flops = flops; + fprintf(1, 'Number of floating point operations is %d\n', ... + opt_flops) +end +fprintf(1, 'Number of cycles is %d\n', size(pointlog2, 1) - 1); +disp(' ') + +x = xinit; +flops(0) +[x, options, errlog3, pointlog3] = scg('rosen', x, options, 'rosegrad'); +fprintf(1, 'For scaled conjugate gradient method:\n') +fprintf(1, 'Final point is (%f, %f), value is %f\n', x(1), x(2), options(8)) +fprintf(1, 'Number of function evaluations is %d\n', options(10)) +fprintf(1, 'Number of gradient evaluations is %d\n', options(11)) +if flops_works + opt_flops = flops; + fprintf(1, 'Number of floating point operations is %d\n', opt_flops) +end +fprintf(1, 'Number of cycles is %d\n', size(pointlog3, 1) - 1); +disp(' ') + +x = xinit; +options(7) = 1; % Line minimisation used +flops(0) +[x, options, errlog4, pointlog4] = graddesc('rosen', x, options, 'rosegrad'); +fprintf(1, 'For gradient descent method:\n') +fprintf(1, 'Final point is (%f, %f), value is %f\n', x(1), x(2), options(8)) +fprintf(1, 'Number of function evaluations is %d\n', options(10)) +fprintf(1, 'Number of gradient evaluations is %d\n', options(11)) +if flops_works + opt_flops = flops; + fprintf(1, 'Number of floating point operations is %d\n', opt_flops) +end +fprintf(1, 'Number of cycles is %d\n', size(pointlog4, 1) - 1); +disp(' ') +disp('Note that gradient descent does not reach a local minimum in') +disp('100 cycles.') +disp(' ') +disp('On this problem, where the function is cheap to evaluate, the') +disp('computational effort is dominated by the algorithm overhead.') +disp('However on more complex optimisation problems (such as those') +disp('involving neural networks), computational effort is dominated by') +disp('the number of function and gradient evaluations. Counting these,') +disp('we can rank the algorithms: quasi-Newton (the best), conjugate') +disp('gradient, scaled conjugate gradient, gradient descent (the worst)') +disp(' ') +disp('Press any key to continue.') +pause +clc +disp('We now plot the trajectory of search points for each algorithm') +disp('superimposed on the contour plot.') +disp(' ') +disp('Press any key to continue.') +pause +plot(pointlog4(:,1), pointlog4(:,2), 'bd', 'MarkerSize', 6) +plot(pointlog3(:,1), pointlog3(:,2), 'mx', 'MarkerSize', 6, 'LineWidth', 2) +plot(pointlog(:,1), pointlog(:,2), 'k.', 'MarkerSize', 18) +plot(pointlog2(:,1), pointlog2(:,2), 'g+', 'MarkerSize', 6, 'LineWidth', 2) +lh = legend( 'Gradient Descent', 'Scaled Conjugate Gradients', ... + 'Quasi Newton', 'Conjugate Gradients'); + +hold off + +clc +disp('Press any key to end.') +pause +close(fh1); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/dempot.m b/sourcecodes/bnt-master/netlab3.3/dempot.m new file mode 100644 index 00000000..87301bce --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/dempot.m @@ -0,0 +1,17 @@ +function e = dempot(x, mix) +%DEMPOT Computes the negative log likelihood for a mixture model. +% +% Description +% This function computes the negative log of the unconditional data +% density P(X) for a Gaussian mixture model. The data structure MIX +% defines the mixture model, while the matrix X contains the data +% vectors. +% +% See also +% DEMGPOT, DEMHMC1, DEMMET1 +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Computes the potential (negative log likelihood) +e = -log(gmmprob(mix, x)); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demprgp.m b/sourcecodes/bnt-master/netlab3.3/demprgp.m new file mode 100644 index 00000000..3cc7b8f5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demprgp.m @@ -0,0 +1,542 @@ +function demprgp(action); +%DEMPRGP Demonstrate sampling from a Gaussian Process prior. +% +% Description +% This function plots the functions represented by a Gaussian Process +% model. The hyperparameter values can be adjusted on a linear scale +% using the sliders (though the exponential of the parameters is used +% in the covariance function), or by typing values into the text boxes +% and pressing the return key. Both types of covariance function are +% supported. An extra function specific parameter is needed for the +% rational quadratic function. +% +% See also +% GP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargin<1, + action='initialize'; +end; + +if strcmp(action,'initialize') + + % Bounds on hyperparameter values + biasminval = -3.0; biasmaxval = 3.0; + noiseminval = -20; noisemaxval = -2; + fparminval = 0.0; fparmaxval = 2.0; + inwminval = 0; inwmaxval = 8; + % Initial hyperparameter values + bias = (biasminval+biasmaxval)/2; + noise = (noiseminval+noisemaxval)/2; + inweights = (inwminval+inwmaxval)/2; + fpar = (fparminval+fparmaxval)/2; + fpar2 = (fparminval+fparmaxval)/2; + + gptype = 'sqexp'; + + % Create FIGURE + fig=figure( ... + 'Name','Sampling from a Gaussian Process prior', ... + 'Position', [50 50 480 380], ... + 'NumberTitle','off', ... + 'Color', [0.8 0.8 0.8], ... + 'Visible','on'); + + % List box for covariance function type + nettype_box = uicontrol(fig, ... + 'Style', 'listbox', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'center', ... + 'Position', [0.52 0.77 0.40 0.12], ... + 'String', 'Squared Exponential|Rational Quadratic', ... + 'Max', 1, 'Min', 0, ... % Only allow one selection + 'Value', 1, ... % Initial value is squared exponential + 'BackgroundColor',[0.60 0.60 0.60],... + 'CallBack', 'demprgp GPtype'); + + % Title for list box + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'Position', [0.52 0.89 0.40 0.05], ... + 'String', 'Covariance Function Type', ... + 'BackgroundColor', get(fig, 'Color'), ... + 'HorizontalAlignment', 'center'); + + % Frames to enclose sliders + bottom_row = 0.04; + slider_frame_height = 0.15; + biasframe = uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'String', 'bias', ... + 'HorizontalAlignment', 'left', ... + 'Position', [0.05 bottom_row 0.35 slider_frame_height]); + + bpos = get(biasframe, 'Position'); + noise_frame_bottom = bpos(2) + bpos(4) + 0.02; + noiseframe = uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 noise_frame_bottom 0.35 slider_frame_height]); + + npos = get(noiseframe, 'Position'); + inw_frame_bottom = npos(2) + npos(4) + 0.02; + inwframe = uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 inw_frame_bottom 0.35 slider_frame_height]); + + inwpos = get(inwframe, 'Position'); + fpar_frame_bottom = inwpos(2) + inwpos(4) + 0.02; + % This frame sometimes has multiple parameters + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 fpar_frame_bottom 0.35 2*slider_frame_height]); + + % Frame text + slider_text_height = 0.05; + slider_text_voffset = 0.08; + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 bottom_row+slider_text_voffset ... + 0.06 slider_text_height], ... + 'String', 'bias'); + + % Frame text + noiseframe = uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 noise_frame_bottom+slider_text_voffset ... + 0.08 slider_text_height], ... + 'String', 'noise'); + + % Frame text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 inw_frame_bottom+slider_text_voffset ... + 0.14 slider_text_height], ... + 'String', 'inweights'); + + % Frame text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 fpar_frame_bottom+slider_frame_height+ ... + slider_text_voffset 0.06 slider_text_height], ... + 'String', 'fpar'); + + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 fpar_frame_bottom+slider_text_voffset ... + 0.06 slider_text_height], ... + 'String', 'fpar2', ... + 'Tag', 'fpar2text', ... + 'Enable', 'off'); + + % Slider + slider_left = 0.07; + slider_width = 0.31; + slider_frame_voffset = 0.02; + biasslide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', bias, ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [slider_left bottom_row+slider_frame_voffset ... + slider_width 0.05], ... + 'Min', biasminval, 'Max', biasmaxval, ... + 'Callback', 'demprgp update'); + + % Slider + noiseslide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', noise, ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [slider_left noise_frame_bottom+slider_frame_voffset ... + slider_width 0.05], ... + 'Min', noiseminval, 'Max', noisemaxval, ... + 'Callback', 'demprgp update'); + + % Slider + inweightsslide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', inweights, ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [slider_left inw_frame_bottom+slider_frame_voffset ... + slider_width 0.05], ... + 'Min', inwminval, 'Max', inwmaxval, ... + 'Callback', 'demprgp update'); + + % Slider + fparslide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', fpar, ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [slider_left fpar_frame_bottom+slider_frame_height+ ... + slider_frame_voffset slider_width 0.05], ... + 'Min', fparminval, 'Max', fparmaxval, ... + 'Callback', 'demprgp update'); + + fpar2slide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', fpar2, ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [slider_left fpar_frame_bottom+slider_frame_voffset ... + slider_width 0.05], ... + 'Min', fparminval, 'Max', fparmaxval, ... + 'Callback', 'demprgp update', ... + 'Tag', 'fpar2slider', ... + 'Enable', 'off'); + + % Text display of hyper-parameter values + + format = '%8f'; + + hp_left = 0.20; + hp_width = 0.17; + biasval = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [hp_left bottom_row+slider_text_voffset ... + hp_width slider_text_height], ... + 'String', sprintf(format, bias), ... + 'Callback', 'demprgp newval'); + + noiseval = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [hp_left noise_frame_bottom+slider_text_voffset ... + hp_width slider_text_height], ... + 'String', sprintf(format, noise), ... + 'Callback', 'demprgp newval'); + + inweightsval = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [hp_left inw_frame_bottom+slider_text_voffset ... + hp_width slider_text_height], ... + 'String', sprintf(format, inweights), ... + 'Callback', 'demprgp newval'); + + fparval = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [hp_left fpar_frame_bottom+slider_frame_height+ ... + slider_text_voffset hp_width slider_text_height], ... + 'String', sprintf(format, fpar), ... + 'Callback', 'demprgp newval'); + + fpar2val = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [hp_left fpar_frame_bottom+slider_text_voffset ... + hp_width slider_text_height], ... + 'String', sprintf(format, fpar), ... + 'Callback', 'demprgp newval', ... + 'Enable', 'off', ... + 'Tag', 'fpar2val'); + + + % The graph box + haxes = axes('Position', [0.5 0.28 0.45 0.45], ... + 'Units', 'normalized', ... + 'Visible', 'on'); + + % The SAMPLE button + uicontrol(fig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.5 bottom_row 0.13 0.1], ... + 'String','Sample', ... + 'Callback','demprgp replot'); + + % The CLOSE button + uicontrol(fig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.82 bottom_row 0.13 0.1], ... + 'String','Close', ... + 'Callback','close(gcf)'); + + % The HELP button + uicontrol(fig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.66 bottom_row 0.13 0.1], ... + 'String','Help', ... + 'Callback','demprgp help'); + + % Save handles to objects + + hndlList=[fig biasslide noiseslide inweightsslide fparslide ... + biasval noiseval inweightsval ... + fparval haxes nettype_box]; + set(fig, 'UserData', hndlList); + + demprgp('replot') + + +elseif strcmp(action, 'update'), + + % Update when a slider is moved. + + hndlList = get(gcf, 'UserData'); + biasslide = hndlList(2); + noiseslide = hndlList(3); + inweightsslide = hndlList(4); + fparslide = hndlList(5); + biasval = hndlList(6); + noiseval = hndlList(7); + inweightsval = hndlList(8); + fparval = hndlList(9); + haxes = hndlList(10); + nettype_box = hndlList(11); + + + bias = get(biasslide, 'Value'); + noise = get(noiseslide, 'Value'); + inweights = get(inweightsslide, 'Value'); + fpar = get(fparslide, 'Value'); + fpar2 = get(findobj('Tag', 'fpar2slider'), 'Value'); + + format = '%8f'; + set(biasval, 'String', sprintf(format, bias)); + set(noiseval, 'String', sprintf(format, noise)); + set(inweightsval, 'String', sprintf(format, inweights)); + set(fparval, 'String', sprintf(format, fpar)); + set(findobj('Tag', 'fpar2val'), 'String', ... + sprintf(format, fpar2)); + + demprgp('replot'); + +elseif strcmp(action, 'newval'), + + % Update when text is changed. + + hndlList = get(gcf, 'UserData'); + biasslide = hndlList(2); + noiseslide = hndlList(3); + inweightsslide = hndlList(4); + fparslide = hndlList(5); + biasval = hndlList(6); + noiseval = hndlList(7); + inweightsval = hndlList(8); + fparval = hndlList(9); + haxes = hndlList(10); + + bias = sscanf(get(biasval, 'String'), '%f'); + noise = sscanf(get(noiseval, 'String'), '%f'); + inweights = sscanf(get(inweightsval, 'String'), '%f'); + fpar = sscanf(get(fparval, 'String'), '%f'); + fpar2 = sscanf(get(findobj('Tag', 'fpar2val'), 'String'), '%f'); + + set(biasslide, 'Value', bias); + set(noiseslide, 'Value', noise); + set(inweightsslide, 'Value', inweights); + set(fparslide, 'Value', fpar); + set(findobj('Tag', 'fpar2slider'), 'Value', fpar2); + + demprgp('replot'); + +elseif strcmp(action, 'GPtype') + hndlList = get(gcf, 'UserData'); + nettype_box = hndlList(11); + gptval = get(nettype_box, 'Value'); + if gptval == 1 + % Squared exponential, so turn off fpar2 + set(findobj('Tag', 'fpar2text'), 'Enable', 'off'); + set(findobj('Tag', 'fpar2slider'), 'Enable', 'off'); + set(findobj('Tag', 'fpar2val'), 'Enable', 'off'); + else + % Rational quadratic, so turn on fpar2 + set(findobj('Tag', 'fpar2text'), 'Enable', 'on'); + set(findobj('Tag', 'fpar2slider'), 'Enable', 'on'); + set(findobj('Tag', 'fpar2val'), 'Enable', 'on'); + end + demprgp('replot'); + +elseif strcmp(action, 'replot'), + + % Re-sample from the prior and plot graphs. + + oldFigNumber=watchon; + + hndlList = get(gcf, 'UserData'); + biasslide = hndlList(2); + noiseslide = hndlList(3); + inweightsslide = hndlList(4); + fparslide = hndlList(5); + haxes = hndlList(10); + nettype_box = hndlList(11); + gptval = get(nettype_box, 'Value'); + if gptval == 1 + gptype = 'sqexp'; + else + gptype = 'ratquad'; + end + + bias = get(biasslide, 'Value'); + noise = get(noiseslide, 'Value'); + inweights = get(inweightsslide, 'Value'); + fpar = get(fparslide, 'Value'); + + + axes(haxes); + cla + set(gca, ... + 'Box', 'on', ... + 'Color', [0 0 0], ... + 'XColor', [0 0 0], ... + 'YColor', [0 0 0], ... + 'FontSize', 14); + ymin = -10; + ymax = 10; + axis([-1 1 ymin ymax]); + set(gca,'DefaultLineLineWidth', 2); + + xvals = (-1:0.01:1)'; + nsample = 10; % Number of samples from prior. + hold on + plot([-1 0; 1 0], [0 ymin; 0 ymax], 'b--'); + net = gp(1, gptype); + net.bias = bias; + net.noise = noise; + net.inweights = inweights; + if strcmp(gptype, 'sqexp') + net.fpar = fpar; + else + fpar2 = get(findobj('Tag', 'fpar2slider'), 'Value'); + net.fpar = [fpar fpar2]; + end + cn = gpcovar(net, xvals); + cninv = inv(cn); + cnchol = chol(cn); + set(gca, 'DefaultLineLineWidth', 1); + for n = 1:nsample + y = (cnchol') * randn(size(xvals)); + plot(xvals, y, 'y'); + end + + watchoff(oldFigNumber); + +elseif strcmp(action, 'help'), + + % Provide help to user. + + oldFigNumber=watchon; + + helpfig = figure('Position', [100 100 480 400], ... + 'Name', 'Help', ... + 'NumberTitle', 'off', ... + 'Color', [0.8 0.8 0.8], ... + 'Visible','on'); + + % The HELP TITLE BAR frame + uicontrol(helpfig, ... + 'Style','frame', ... + 'Units','normalized', ... + 'HorizontalAlignment', 'center', ... + 'Position', [0.05 0.82 0.9 0.1], ... + 'BackgroundColor',[0.60 0.60 0.60]); + + % The HELP TITLE BAR text + uicontrol(helpfig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.26 0.85 0.6 0.05], ... + 'HorizontalAlignment', 'left', ... + 'String', 'Help: Sampling from a Gaussian Process Prior'); + + helpstr1 = strcat(... + 'This demonstration shows the effects of sampling from a Gaussian', ... + ' process prior. The parameters bias, noise, inweights and fpar', ... + ' control the corresponding terms in the covariance function of the',... + ' Gaussian process. Their values can be adjusted on a linear scale',... + ' using the sliders, or by typing values into the text boxes and',... + ' pressing the return key. After setting these values, press the',... + ' ''Sample'' button to see a new sample from the prior.'); + + helpstr2 = strcat(... + 'Observe how inweights controls horizontal length-scale of the',... + ' variation in the functions, noise controls the roughness of the',... + ' functions, and the bias controls the size of the', ... + ' vertical offset of the signal.'); + helpstr3 = strcat(... + 'There are two types of covariance function supported by', ... + ' Netlab which can be selected using the ''Covariance Function', ... + ' Type'' menu.'); + helpstr4 = strcat(... + 'The squared exponential has a single fpar which', ... + ' controls the vertical scale of the process.'); + helpstr5 = strcat(... + 'The rational quadratic has two fpar values. The first is', ... + ' is a scale parameter inside the rational function like the',... + ' first fpar for the squared exponential covariance, while the', ... + ' second gives the exponent of the rational function (i.e. the',... + ' rate of decay of the covariance function.'); + % Set up cell array with help strings + hstr(1) = {helpstr1}; + hstr(2) = {''}; + hstr(3) = {helpstr2}; + hstr(4) = {''}; + hstr(5) = {helpstr3}; + hstr(6) = {''}; + hstr(7) = {helpstr4}; + hstr(8) = {''}; + hstr(9) = {helpstr5}; + + % The HELP text + helpui = uicontrol(helpfig, ... + 'Style', 'Text', ... + 'Units', 'normalized', ... + 'ForegroundColor', [0 0 0], ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [1 1 1], ... + 'Min', 0, ... + 'Max', 2, ... + 'Position', [0.05 0.2 0.9 0.57]); + [hstrw, newpos] = textwrap(helpui, hstr); + set(helpui, 'String', hstrw, 'Position', [0.05, 0.2, 0.9 newpos(4)]); + + % The CLOSE button + uicontrol(helpfig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.4 0.05 0.2 0.1], ... + 'String','Close', ... + 'Callback','close(gcf)'); + + watchoff(oldFigNumber); + +end; + diff --git a/sourcecodes/bnt-master/netlab3.3/demprior.m b/sourcecodes/bnt-master/netlab3.3/demprior.m new file mode 100644 index 00000000..4ad975a9 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demprior.m @@ -0,0 +1,406 @@ +function demprior(action); +%DEMPRIOR Demonstrate sampling from a multi-parameter Gaussian prior. +% +% Description +% This function plots the functions represented by a multi-layer +% perceptron network when the weights are set to values drawn from a +% Gaussian prior distribution. The parameters AW1, AB1 AW2 and AB2 +% control the inverse variances of the first-layer weights, the hidden +% unit biases, the second-layer weights and the output unit biases +% respectively. Their values can be adjusted on a logarithmic scale +% using the sliders, or by typing values into the text boxes and +% pressing the return key. +% +% See also +% MLP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargin<1, + action='initialize'; +end; + +if strcmp(action,'initialize') + + aw1 = 0.01; + ab1 = 0.1; + aw2 = 1.0; + ab2 = 1.0; + + % Create FIGURE + fig=figure( ... + 'Name','Sampling from a Gaussian prior', ... + 'Position', [50 50 480 380], ... + 'NumberTitle','off', ... + 'Color', [0.8 0.8 0.8], ... + 'Visible','on'); + + % The TITLE BAR frame + uicontrol(fig, ... + 'Style','frame', ... + 'Units','normalized', ... + 'HorizontalAlignment', 'center', ... + 'Position', [0.5 0.82 0.45 0.1], ... + 'BackgroundColor',[0.60 0.60 0.60]); + + % The TITLE BAR text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.54 0.85 0.40 0.05], ... + 'HorizontalAlignment', 'left', ... + 'String', 'Sampling from a Gaussian prior'); + + % Frames to enclose sliders + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 0.08 0.35 0.18]); + + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 0.3 0.35 0.18]); + + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 0.52 0.35 0.18]); + + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 0.74 0.35 0.18]); + + % Frame text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 0.17 0.06 0.07], ... + 'String', 'aw1'); + + % Frame text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 0.39 0.06 0.07], ... + 'String', 'ab1'); + + % Frame text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 0.61 0.06 0.07], ... + 'String', 'aw2'); + + % Frame text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 0.83 0.06 0.07], ... + 'String', 'ab2'); + + % Slider + minval = -5; maxval = 5; + aw1slide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', log10(aw1), ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [0.07 0.1 0.31 0.05], ... + 'Min', minval, 'Max', maxval, ... + 'Callback', 'demprior update'); + + % Slider + ab1slide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', log10(ab1), ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [0.07 0.32 0.31 0.05], ... + 'Min', minval, 'Max', maxval, ... + 'Callback', 'demprior update'); + + % Slider + aw2slide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', log10(aw2), ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [0.07 0.54 0.31 0.05], ... + 'Min', minval, 'Max', maxval, ... + 'Callback', 'demprior update'); + + % Slider + ab2slide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', log10(ab2), ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [0.07 0.76 0.31 0.05], ... + 'Min', minval, 'Max', maxval, ... + 'Callback', 'demprior update'); + + % The graph box + haxes = axes('Position', [0.5 0.28 0.45 0.45], ... + 'Units', 'normalized', ... + 'Visible', 'on'); + + % Text display of hyper-parameter values + + format = '%8f'; + + aw1val = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [0.15 0.17 0.23 0.07], ... + 'String', sprintf(format, aw1), ... + 'Callback', 'demprior newval'); + + ab1val = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [0.15 0.39 0.23 0.07], ... + 'String', sprintf(format, ab1), ... + 'Callback', 'demprior newval'); + + aw2val = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [0.15 0.61 0.23 0.07], ... + 'String', sprintf(format, aw2), ... + 'Callback', 'demprior newval'); + + ab2val = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [0.15 0.83 0.23 0.07], ... + 'String', sprintf(format, ab2), ... + 'Callback', 'demprior newval'); + + % The SAMPLE button + uicontrol(fig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.5 0.08 0.13 0.1], ... + 'String','Sample', ... + 'Callback','demprior replot'); + + % The CLOSE button + uicontrol(fig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.82 0.08 0.13 0.1], ... + 'String','Close', ... + 'Callback','close(gcf)'); + + % The HELP button + uicontrol(fig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.66 0.08 0.13 0.1], ... + 'String','Help', ... + 'Callback','demprior help'); + + % Save handles to objects + + hndlList=[fig aw1slide ab1slide aw2slide ab2slide aw1val ab1val aw2val ... + ab2val haxes]; + set(fig, 'UserData', hndlList); + + demprior('replot') + + +elseif strcmp(action, 'update'), + + % Update when a slider is moved. + + hndlList = get(gcf, 'UserData'); + aw1slide = hndlList(2); + ab1slide = hndlList(3); + aw2slide = hndlList(4); + ab2slide = hndlList(5); + aw1val = hndlList(6); + ab1val = hndlList(7); + aw2val = hndlList(8); + ab2val = hndlList(9); + haxes = hndlList(10); + + aw1 = 10^get(aw1slide, 'Value'); + ab1 = 10^get(ab1slide, 'Value'); + aw2 = 10^get(aw2slide, 'Value'); + ab2 = 10^get(ab2slide, 'Value'); + + format = '%8f'; + set(aw1val, 'String', sprintf(format, aw1)); + set(ab1val, 'String', sprintf(format, ab1)); + set(aw2val, 'String', sprintf(format, aw2)); + set(ab2val, 'String', sprintf(format, ab2)); + + demprior('replot'); + +elseif strcmp(action, 'newval'), + + % Update when text is changed. + + hndlList = get(gcf, 'UserData'); + aw1slide = hndlList(2); + ab1slide = hndlList(3); + aw2slide = hndlList(4); + ab2slide = hndlList(5); + aw1val = hndlList(6); + ab1val = hndlList(7); + aw2val = hndlList(8); + ab2val = hndlList(9); + haxes = hndlList(10); + + aw1 = sscanf(get(aw1val, 'String'), '%f'); + ab1 = sscanf(get(ab1val, 'String'), '%f'); + aw2 = sscanf(get(aw2val, 'String'), '%f'); + ab2 = sscanf(get(ab2val, 'String'), '%f'); + + set(aw1slide, 'Value', log10(aw1)); + set(ab1slide, 'Value', log10(ab1)); + set(aw2slide, 'Value', log10(aw2)); + set(ab2slide, 'Value', log10(ab2)); + + demprior('replot'); + +elseif strcmp(action, 'replot'), + + % Re-sample from the prior and plot graphs. + + oldFigNumber=watchon; + + hndlList = get(gcf, 'UserData'); + aw1slide = hndlList(2); + ab1slide = hndlList(3); + aw2slide = hndlList(4); + ab2slide = hndlList(5); + haxes = hndlList(10); + + aw1 = 10^get(aw1slide, 'Value'); + ab1 = 10^get(ab1slide, 'Value'); + aw2 = 10^get(aw2slide, 'Value'); + ab2 = 10^get(ab2slide, 'Value'); + + axes(haxes); + cla + set(gca, ... + 'Box', 'on', ... + 'Color', [0 0 0], ... + 'XColor', [0 0 0], ... + 'YColor', [0 0 0], ... + 'FontSize', 14); + axis([-1 1 -10 10]); + set(gca,'DefaultLineLineWidth', 2); + + nhidden = 12; + prior = mlpprior(1, nhidden, 1, aw1, ab1, aw2, ab2); + xvals = -1:0.005:1; + nsample = 10; % Number of samples from prior. + hold on + plot([-1 0; 1 0], [0 -10; 0 10], 'b--'); + net = mlp(1, nhidden, 1, 'linear', prior); + for i = 1:nsample + net = mlpinit(net, prior); + yvals = mlpfwd(net, xvals'); + plot(xvals', yvals, 'y'); + end + + watchoff(oldFigNumber); + +elseif strcmp(action, 'help'), + + % Provide help to user. + + oldFigNumber=watchon; + + helpfig = figure('Position', [100 100 480 400], ... + 'Name', 'Help', ... + 'NumberTitle', 'off', ... + 'Color', [0.8 0.8 0.8], ... + 'Visible','on'); + + % The HELP TITLE BAR frame + uicontrol(helpfig, ... + 'Style','frame', ... + 'Units','normalized', ... + 'HorizontalAlignment', 'center', ... + 'Position', [0.05 0.82 0.9 0.1], ... + 'BackgroundColor',[0.60 0.60 0.60]); + + % The HELP TITLE BAR text + uicontrol(helpfig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.26 0.85 0.6 0.05], ... + 'HorizontalAlignment', 'left', ... + 'String', 'Help: Sampling from a Gaussian Prior'); + + helpstr1 = strcat( ... + 'This demonstration shows the effects of sampling from a Gaussian', ... + ' prior over weights for a two-layer feed-forward network. The', ... + ' parameters aw1, ab1, aw2 and ab2 control the inverse variances of', ... + ' the first-layer weights, the hidden unit biases, the second-layer', ... + ' weights and the output unit biases respectively. Their values can', ... + ' be adjusted on a logarithmic scale using the sliders, or by', ... + ' typing values into the text boxes and pressing the return key.', ... + ' After setting these values, press the ''Sample'' button to see a', ... + ' new sample from the prior. '); + helpstr2 = strcat( ... + 'Observe how aw1 controls the horizontal length-scale of the', ... + ' variation in the functions, ab1 controls the input range over', ... + ' such variations occur, aw2 sets the vertical scale of the output', ... + ' and ab2 sets the vertical off-set of the output. The network has', ... + ' 12 hidden units. '); + hstr(1) = {helpstr1}; + hstr(2) = {''}; + hstr(3) = {helpstr2}; + + % The HELP text + helpui = uicontrol(helpfig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'ForegroundColor', [0 0 0], ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [1 1 1], ... + 'Min', 0, ... + 'Max', 2, ... + 'Position', [0.05 0.2 0.9 0.8]); + + [hstrw , newpos] = textwrap(helpui, hstr, 70); + set(helpui, 'String', hstrw, 'Position', [0.05, 0.2, 0.9, newpos(4)]); + + + % The CLOSE button + uicontrol(helpfig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.4 0.05 0.2 0.1], ... + 'String','Close', ... + 'Callback','close(gcf)'); + + watchoff(oldFigNumber); + +end; + diff --git a/sourcecodes/bnt-master/netlab3.3/demrbf1.m b/sourcecodes/bnt-master/netlab3.3/demrbf1.m new file mode 100644 index 00000000..ffd6fd59 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demrbf1.m @@ -0,0 +1,138 @@ +%DEMRBF1 Demonstrate simple regression using a radial basis function network. +% +% Description +% The problem consists of one input variable X and one target variable +% T with data generated by sampling X at equal intervals and then +% generating target data by computing SIN(2*PI*X) and adding Gaussian +% noise. This data is the same as that used in demmlp1. +% +% Three different RBF networks (with different activation functions) +% are trained in two stages. First, a Gaussian mixture model is trained +% using the EM algorithm, and the centres of this model are used to set +% the centres of the RBF. Second, the output weights (and biases) are +% determined using the pseudo-inverse of the design matrix. +% +% See also +% DEMMLP1, RBF, RBFFWD, GMM, GMMEM +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the matrix of inputs x and targets t. +randn('state', 42); +rand('state', 42); +ndata = 20; % Number of data points. +noise = 0.2; % Standard deviation of noise distribution. +x = (linspace(0, 1, ndata))'; +t = sin(2*pi*x) + noise*randn(ndata, 1); +mu = mean(x); +sigma = std(x); +tr_in = (x - mu)./(sigma); + +clc +disp('This demonstration illustrates the use of a Radial Basis Function') +disp('network for regression problems. The data is generated from a noisy') +disp('sine function.') +disp(' ') +disp('Press any key to continue.') +pause +% Set up network parameters. +nin = 1; % Number of inputs. +nhidden = 7; % Number of hidden units. +nout = 1; % Number of outputs. + +clc +disp('We assess the effect of three different activation functions.') +disp('First we create a network with Gaussian activations.') +disp(' ') +disp('Press any key to continue.') +pause +% Create and initialize network weight and parameter vectors. +net = rbf(nin, nhidden, nout, 'gaussian'); + +disp('A two-stage training algorithm is used: it uses a small number of') +disp('iterations of EM to position the centres, and then the pseudo-inverse') +disp('of the design matrix to find the second layer weights.') +disp(' ') +disp('Press any key to continue.') +pause +disp('Error values from EM training.') +% Use fast training method +options = foptions; +options(1) = 1; % Display EM training +options(14) = 10; % number of iterations of EM +net = rbftrain(net, options, tr_in, t); + +disp(' ') +disp('Press any key to continue.') +pause +clc +disp('The second RBF network has thin plate spline activations.') +disp('The same centres are used again, so we just need to calculate') +disp('the second layer weights.') +disp(' ') +disp('Press any key to continue.') +pause +% Create a second RBF with thin plate spline functions +net2 = rbf(nin, nhidden, nout, 'tps'); + +% Re-use previous centres rather than calling rbftrain again +net2.c = net.c; +[y, act2] = rbffwd(net2, tr_in); + +% Solve for new output weights and biases from RBF activations +temp = pinv([act2 ones(ndata, 1)]) * t; +net2.w2 = temp(1:nhidden, :); +net2.b2 = temp(nhidden+1, :); + +disp('The third RBF network has r^4 log r activations.') +disp(' ') +disp('Press any key to continue.') +pause +% Create a third RBF with r^4 log r functions +net3 = rbf(nin, nhidden, nout, 'r4logr'); + +% Overwrite weight vector with parameters from first RBF +net3.c = net.c; +[y, act3] = rbffwd(net3, tr_in); +temp = pinv([act3 ones(ndata, 1)]) * t; +net3.w2 = temp(1:nhidden, :); +net3.b2 = temp(nhidden+1, :); + +disp('Now we plot the data, underlying function, and network outputs') +disp('on a single graph to compare the results.') +disp(' ') +disp('Press any key to continue.') +pause +% Plot the data, the original function, and the trained network functions. +plotvals = [x(1):0.01:x(end)]'; +inputvals = (plotvals-mu)./sigma; +y = rbffwd(net, inputvals); +y2 = rbffwd(net2, inputvals); +y3 = rbffwd(net3, inputvals); +fh1 = figure; + +plot(x, t, 'ob') +hold on +xlabel('Input') +ylabel('Target') +axis([x(1) x(end) -1.5 1.5]) +[fx, fy] = fplot('sin(2*pi*x)', [x(1) x(end)]); +plot(fx, fy, '-r', 'LineWidth', 2) +plot(plotvals, y, '--g', 'LineWidth', 2) +plot(plotvals, y2, 'k--', 'LineWidth', 2) +plot(plotvals, y3, '-.c', 'LineWidth', 2) +legend('data', 'function', 'Gaussian RBF', 'Thin plate spline RBF', ... + 'r^4 log r RBF'); +hold off + +disp('RBF training errors are'); +disp(['Gaussian ', num2str(rbferr(net, tr_in, t)), ' TPS ', ... +num2str(rbferr(net2, tr_in, t)), ' R4logr ', num2str(rbferr(net3, tr_in, t))]); + +disp(' ') +disp('Press any key to end.') +pause +close(fh1); +clear all; diff --git a/sourcecodes/bnt-master/netlab3.3/demsom1.m b/sourcecodes/bnt-master/netlab3.3/demsom1.m new file mode 100644 index 00000000..2f1fe93c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demsom1.m @@ -0,0 +1,113 @@ +%DEMSOM1 Demonstrate SOM for visualisation. +% +% Description +% This script demonstrates the use of a SOM with a two-dimensional +% grid to map onto data in two-dimensional space. Both on-line and +% batch training algorithms are shown. +% +% See also +% SOM, SOMPAK, SOMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +randn('state', 42); +rand('state', 42); +nin = 2; +ndata = 300; +% Give data an offset so that network has something to learn. +x = rand(ndata, nin) + ones(ndata, 1)*[1.5 1.5]; + +clc; +disp('This demonstration of the SOM, or Kohonen network, shows how the') +disp('network units after training lie in regions of high data density.') +disp('First we show the data, which is generated uniformly from a square.') +disp('Red crosses denote the data and black dots are the initial locations') +disp('of the SOM units.') +disp(' ') +disp('Press any key to continue.') +pause +net = som(nin, [8, 7]); +c1 = sompak(net); +h1 = figure; +plot(x(:, 1), x(:, 2), 'r+'); +hold on +plot(c1(:,1), c1(:, 2), 'k.'); +drawnow; % Force figure to be drawn before training starts +options = foptions; + +% Ordering phase +options(1) = 1; +options(14) = 50; +%options(14) = 5; % Just for testing +options(18) = 0.9; % Initial learning rate +options(16) = 0.05; % Final learning rate +options(17) = 8; % Initial neighbourhood size +options(15) = 1; % Final neighbourhood size + +disp('The SOM network is trained in two phases using an on-line algorithm.') +disp('Initially the neighbourhood is set to 8 and is then reduced') +disp('linearly to 1 over the first 50 iterations.') +disp('Each iteration consists of a pass through the complete') +disp('dataset, while the weights are adjusted after each pattern.') +disp('The learning rate is reduced linearly from 0.9 to 0.05.') +disp('This ordering phase puts the units in a rough grid shape.') +disp('Blue circles denote the units at the end of this phase.') +disp(' ') +disp('Press any key to continue.') +pause +net2 = somtrain(net, options, x); +c2 = sompak(net2); +plot(c2(:, 1), c2(:, 2), 'bo'); +drawnow; + +% Convergence phase +options(1) = 1; +options(14) = 400; +options(18) = 0.05; +options(16) = 0.01; +options(17) = 0; +options(15) = 0; + +disp('The second, convergence, phase of learning just updates the winning node.') +disp('The learning rate is reduced from 0.05 to 0.01 over 400 iterations.') +disp('Note how the error value does not decrease monotonically; it is') +disp('difficult to decide when training is complete in a principled way.') +disp('The units are plotted as green stars.') +disp(' ') +disp('Press any key to continue.') +pause +net3 = somtrain(net2, options, x); +c3 = sompak(net3); +plot(c3(:, 1), c3(:, 2), 'g*'); +drawnow; + +% Now try batch training +options(1) = 1; +options(6) = 1; +options(14) = 50; +options(17) = 3; +options(15) = 0; +disp('An alternative approach to the on-line algorithm is a batch update') +disp('rule. Each unit is updated to be the average weights') +disp('in a neighbourhood (which reduces from 3 to 0) over 50 iterations.'); +disp('Note how the error is even more unstable at first, though eventually') +disp('it does converge.') +disp('The final units are shown as black triangles.') +disp(' ') +disp('Press any key to continue.') +pause +net4 = somtrain(net, options, x); +c4 = sompak(net4); +plot(c4(:, 1), c4(:, 2), 'k^') +legend('Data', 'Initial weights', 'Weights after ordering', ... + 'Weights after convergence', 'Batch weights', 2); +drawnow; + +disp(' ') +disp('Press any key to end.') +disp(' ') +pause + +close(h1); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demtrain.m b/sourcecodes/bnt-master/netlab3.3/demtrain.m new file mode 100644 index 00000000..61ebc16f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demtrain.m @@ -0,0 +1,362 @@ +function demtrain(action); +%DEMTRAIN Demonstrate training of MLP network. +% +% Description +% DEMTRAIN brings up a simple GUI to show the training of an MLP +% network on classification and regression problems. The user should +% load in a dataset (which should be in Netlab format: see DATREAD), +% select the output activation function, the number of cycles and +% hidden units and then train the network. The scaled conjugate +% gradient algorithm is used. A graph shows the evolution of the error: +% the value is shown MAX(CEIL(ITERATIONS / 50), 5) cycles. +% +% Once the network is trained, it is saved to the file MLPTRAIN.NET. +% The results can then be viewed as a confusion matrix (for +% classification problems) or a plot of output versus target (for +% regression problems). +% +% See also +% CONFMAT, DATREAD, MLP, NETOPT, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% If run without parameters, initialise gui. +if nargin<1, + action='initialise'; +end; + +% Global variable to reference GUI figure +global DEMTRAIN_FIG +% Global array to reference sub-figures for results plots +global DEMTRAIN_RES_FIGS +global NUM_DEMTRAIN_RES_FIGS + +if strcmp(action,'initialise'), + + file = ''; + path = '.'; + + % Create FIGURE + fig = figure( ... + 'Name', 'Netlab Demo', ... + 'NumberTitle', 'off', ... + 'Menubar', 'none', ... + 'Color', [0.7529 0.7529 0.7529], ... + 'Visible', 'on'); + % Initialise the globals + DEMTRAIN_FIG = fig; + DEMTRAIN_RES_FIGS = 0; + NUM_DEMTRAIN_RES_FIGS = 0; + + % Create GROUP for buttons + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'Position', [0.03 0.08 0.94 0.22], ... + 'BackgroundColor', [0.5 0.5 0.5]); + + % Create MAIN axis + hMain = axes( ... + 'Units', 'normalized', ... + 'Position', [0.10 0.5 0.80 0.40], ... + 'XColor', [0 0 0], ... + 'YColor', [0 0 0], ... + 'Visible', 'on'); + + % Create static text for FILENAME and PATH + hFilename = uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.7529 0.7529 0.7529], ... + 'Position', [0.05 0.32 0.90 0.05], ... + 'HorizontalAlignment', 'center', ... + 'String', 'Please load data file.', ... + 'Visible', 'on'); + hPath = uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.7529 0.7529 0.7529], ... + 'Position', [0.05 0.37 0.90 0.05], ... + 'HorizontalAlignment', 'center', ... + 'String', '', ... + 'Visible', 'on'); + + % Create NO OF HIDDEN UNITS slider and text + hSliderText = uicontrol(fig, ... + 'Style', 'text', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Units', 'normalized', ... + 'Position', [0.27 0.12 0.17 0.04], ... + 'HorizontalAlignment', 'right', ... + 'String', 'Hidden Units: 5'); + hSlider = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Position', [0.45 0.12 0.26 0.04], ... + 'String', 'Slider', ... + 'Min', 1, 'Max', 25, ... + 'Value', 5, ... + 'Callback', 'demtrain slider_moved'); + + % Create ITERATIONS slider and text + hIterationsText = uicontrol(fig, ... + 'Style', 'text', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Units', 'normalized', ... + 'Position', [0.27 0.21 0.17 0.04], ... + 'HorizontalAlignment', 'right', ... + 'String', 'Iterations: 50'); + hIterations = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Position', [0.45 0.21 0.26 0.04], ... + 'String', 'Slider', ... + 'Min', 10, 'Max', 500, ... + 'Value', 50, ... + 'Callback', 'demtrain iterations_moved'); + + % Create ACTIVATION FUNCTION popup and text + uicontrol(fig, ... + 'Style', 'text', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Units', 'normalized', ... + 'Position', [0.05 0.20 0.20 0.04], ... + 'HorizontalAlignment', 'center', ... + 'String', 'Activation Function:'); + hPopup = uicontrol(fig, ... + 'Style', 'popup', ... + 'Units', 'normalized', ... + 'Position' , [0.05 0.10 0.20 0.08], ... + 'String', 'Linear|Logistic|Softmax', ... + 'Callback', ''); + + % Create MENU + hMenu1 = uimenu('Label', 'Load Data file...', 'Callback', ''); + uimenu(hMenu1, 'Label', 'Select training data file', ... + 'Callback', 'demtrain get_ip_file'); + hMenu2 = uimenu('Label', 'Show Results...', 'Callback', ''); + uimenu(hMenu2, 'Label', 'Show classification results', ... + 'Callback', 'demtrain classify'); + uimenu(hMenu2, 'Label', 'Show regression results', ... + 'Callback', 'demtrain predict'); + + % Create START button + hStart = uicontrol(fig, ... + 'Units', 'normalized', ... + 'Position' , [0.75 0.2 0.20 0.08], ... + 'String', 'Start Training', ... + 'Enable', 'off',... + 'Callback', 'demtrain start'); + + % Create CLOSE button + uicontrol(fig, ... + 'Units', 'normalized', ... + 'Position' , [0.75 0.1 0.20 0.08], ... + 'String', 'Close', ... + 'Callback', 'demtrain close'); + + % Save handles of important UI objects + hndlList = [hSlider hSliderText hFilename hPath hPopup ... + hIterations hIterationsText hStart]; + set(fig, 'UserData', hndlList); + % Hide window from command line + set(fig, 'HandleVisibility', 'callback'); + + +elseif strcmp(action, 'slider_moved'), + + % Slider has been moved. + + hndlList = get(gcf, 'UserData'); + hSlider = hndlList(1); + hSliderText = hndlList(2); + + val = get(hSlider, 'Value'); + if rem(val, 1) < 0.5, % Force up and down arrows to work! + val = ceil(val); + else + val = floor(val); + end; + set(hSlider, 'Value', val); + set(hSliderText, 'String', ['Hidden Units: ' int2str(val)]); + + +elseif strcmp(action, 'iterations_moved'), + + % Slider has been moved. + + hndlList = get(gcf, 'UserData'); + hSlider = hndlList(6); + hSliderText = hndlList(7); + + val = get(hSlider, 'Value'); + set(hSliderText, 'String', ['Iterations: ' int2str(val)]); + +elseif strcmp(action, 'get_ip_file'), + + % Get data file button pressed. + + hndlList = get(gcf, 'UserData'); + + [file, path] = uigetfile('*.dat', 'Get Data File', 50, 50); + + if strcmp(file, '') | file == 0, + set(hndlList(3), 'String', 'No data file loaded.'); + set(hndlList(4), 'String', ''); + else + set(hndlList(3), 'String', file); + set(hndlList(4), 'String', path); + end; + + % Enable training button + set(hndlList(8), 'Enable', 'on'); + + set(gcf, 'UserData', hndlList); + +elseif strcmp(action, 'start'), + + % Start training + + % Get handles of and values from UI objects + hndlList = get(gcf, 'UserData'); + hSlider = hndlList(1); % No of hidden units + hIterations = hndlList(6); + iterations = get(hIterations, 'Value'); + + hFilename = hndlList(3); % Data file name + filename = get(hFilename, 'String'); + + hPath = hndlList(4); % Data file path + path = get(hPath, 'String'); + + hPopup = hndlList(5); % Activation function + if get(hPopup, 'Value') == 1, + act_fn = 'linear'; + elseif get(hPopup, 'Value') == 2, + act_fn = 'logistic'; + else + act_fn = 'softmax'; + end; + nhidden = get(hSlider, 'Value'); + + % Check data file exists + if fopen([path '/' filename]) == -1, + errordlg('Training data file has not been selected.', 'Error'); + else + % Load data file + [x,t,nin,nout,ndata] = datread([path filename]); + + % Call MLPTRAIN function repeatedly, while drawing training graph. + figure(DEMTRAIN_FIG); + hold on; + + title('Training - please wait.'); + + % Create net and find initial error + net = mlp(size(x, 2), nhidden, size(t, 2), act_fn); + % Initialise network with inverse variance of 10 + net = mlpinit(net, 10); + error = mlperr(net, x, t); + % Work out reporting step: should be sufficiently big to let training + % algorithm have a chance + step = max(ceil(iterations / 50), 5); + + % Refresh and rescale axis. + cla; + max = error; + min = max/10; + set(gca, 'YScale', 'log'); + ylabel('log Error'); + xlabel('No. iterations'); + axis([0 iterations min max+1]); + iold = 0; + errold = error; + % Plot circle to show error of last iteration + % Setting erase mode to none prevents screen flashing during + % training + plot(0, error, 'ro', 'EraseMode', 'none'); + hold on + drawnow; % Force redraw + for i = step-1:step:iterations, + [net, error] = mlptrain(net, x, t, step); + % Plot line from last point to new point. + line([iold i], [errold error], 'Color', 'r', 'EraseMode', 'none'); + iold = i; + errold = error; + + % If new point off scale, redraw axes. + if error > max, + max = error; + axis([0 iterations min max+1]); + end; + if error < min + min = error/10; + axis([0 iterations min max+1]); + end + % Plot circle to show error of last iteration + plot(i, error, 'ro', 'EraseMode', 'none'); + drawnow; % Force redraw + end; + save mlptrain.net net + zoom on; + + title(['Training complete. Final error=', num2str(error)]); + + end; + +elseif strcmp(action, 'close'), + + % Close all the figures we have created + close(DEMTRAIN_FIG); + for n = 1:NUM_DEMTRAIN_RES_FIGS + if ishandle(DEMTRAIN_RES_FIGS(n)) + close(DEMTRAIN_RES_FIGS(n)); + end + end + +elseif strcmp(action, 'classify'), + + if fopen('mlptrain.net') == -1, + errordlg('You have not yet trained the network.', 'Error'); + else + + hndlList = get(gcf, 'UserData'); + filename = get(hndlList(3), 'String'); + path = get(hndlList(4), 'String'); + [x,t,nin,nout,ndata] = datread([path filename]); + load mlptrain.net net -mat + y = mlpfwd(net, x); + + % Save results figure so that it can be closed later + NUM_DEMTRAIN_RES_FIGS = NUM_DEMTRAIN_RES_FIGS + 1; + DEMTRAIN_RES_FIGS(NUM_DEMTRAIN_RES_FIGS)=conffig(y,t); + + end; + +elseif strcmp(action, 'predict'), + + if fopen('mlptrain.net') == -1, + errordlg('You have not yet trained the network.', 'Error'); + else + + hndlList = get(gcf, 'UserData'); + filename = get(hndlList(3), 'String'); + path = get(hndlList(4), 'String'); + [x,t,nin,nout,ndata] = datread([path filename]); + load mlptrain.net net -mat + y = mlpfwd(net, x); + + for i = 1:size(y,2), + % Save results figure so that it can be closed later + NUM_DEMTRAIN_RES_FIGS = NUM_DEMTRAIN_RES_FIGS + 1; + DEMTRAIN_RES_FIGS(NUM_DEMTRAIN_RES_FIGS) = figure; + hold on; + title(['Output no ' num2str(i)]); + plot([0 1], [0 1], 'r:'); + plot(y(:,i),t(:,i), 'o'); + hold off; + end; + end; + +end; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/dist2.m b/sourcecodes/bnt-master/netlab3.3/dist2.m new file mode 100644 index 00000000..b5f1c646 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/dist2.m @@ -0,0 +1,31 @@ +function n2 = dist2(x, c) +%DIST2 Calculates squared distance between two sets of points. +% +% Description +% D = DIST2(X, C) takes two matrices of vectors and calculates the +% squared Euclidean distance between them. Both matrices must be of +% the same column dimension. If X has M rows and N columns, and C has +% L rows and N columns, then the result has M rows and L columns. The +% I, Jth entry is the squared distance from the Ith row of X to the +% Jth row of C. +% +% See also +% GMMACTIV, KMEANS, RBFFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[ndata, dimx] = size(x); +[ncentres, dimc] = size(c); +if dimx ~= dimc + error('Data dimension does not match dimension of centres') +end + +n2 = (ones(ncentres, 1) * sum((x.^2)', 1))' + ... + ones(ndata, 1) * sum((c.^2)',1) - ... + 2.*(x*(c')); + +% Rounding errors occasionally cause negative entries in n2 +if any(any(n2<0)) + n2(n2<0) = 0; +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/eigdec.m b/sourcecodes/bnt-master/netlab3.3/eigdec.m new file mode 100644 index 00000000..f5901d42 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/eigdec.m @@ -0,0 +1,55 @@ +function [evals, evec] = eigdec(x, N) +%EIGDEC Sorted eigendecomposition +% +% Description +% EVALS = EIGDEC(X, N computes the largest N eigenvalues of the +% matrix X in descending order. [EVALS, EVEC] = EIGDEC(X, N) also +% computes the corresponding eigenvectors. +% +% See also +% PCA, PPCA +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargout == 1 + evals_only = logical(1); +else + evals_only = logical(0); +end + +if N ~= round(N) | N < 1 | N > size(x, 2) + error('Number of PCs must be integer, >0, < dim'); +end + +% Find the eigenvalues of the data covariance matrix +if evals_only + % Use eig function as always more efficient than eigs here + temp_evals = eig(x); +else + % Use eig function unless fraction of eigenvalues required is tiny + if (N/size(x, 2)) > 0.04 + [temp_evec, temp_evals] = eig(x); + else + options.disp = 0; + [temp_evec, temp_evals] = eigs(x, N, 'LM', options); + end + temp_evals = diag(temp_evals); +end + +% Eigenvalues nearly always returned in descending order, but just +% to make sure..... +[evals perm] = sort(-temp_evals); +evals = -evals(1:N); +if ~evals_only + if evals == temp_evals(1:N) + % Originals were in order + evec = temp_evec(:, 1:N); + return + else + % Need to reorder the eigenvectors + for i=1:N + evec(:,i) = temp_evec(:,perm(i)); + end + end +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/errbayes.m b/sourcecodes/bnt-master/netlab3.3/errbayes.m new file mode 100644 index 00000000..7c2a330b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/errbayes.m @@ -0,0 +1,49 @@ +function [e, edata, eprior] = errbayes(net, edata) +%ERRBAYES Evaluate Bayesian error function for network. +% +% Description +% E = ERRBAYES(NET, EDATA) takes a network data structure NET together +% the data contribution to the error for a set of inputs and targets. +% It returns the regularised error using any zero mean Gaussian priors +% on the weights defined in NET. +% +% [E, EDATA, EPRIOR] = ERRBAYES(NET, X, T) additionally returns the +% data and prior components of the error. +% +% See also +% GLMERR, MLPERR, RBFERR +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Evaluate the data contribution to the error. +if isfield(net, 'beta') + e1 = net.beta*edata; +else + e1 = edata; +end + +% Evaluate the prior contribution to the error. +if isfield(net, 'alpha') + w = netpak(net); + if size(net.alpha) == [1 1] + eprior = 0.5*(w*w'); + e2 = eprior*net.alpha; + else + if (isfield(net, 'mask')) + nindx_cols = size(net.index, 2); + nmask_rows = size(find(net.mask), 1); + index = reshape(net.index(logical(repmat(net.mask, ... + 1, nindx_cols))), nmask_rows, nindx_cols); + else + index = net.index; + end + eprior = 0.5*(w.^2)*index; + e2 = eprior*net.alpha; + end +else + eprior = 0; + e2 = 0; +end + +e = e1 + e2; diff --git a/sourcecodes/bnt-master/netlab3.3/evidence.m b/sourcecodes/bnt-master/netlab3.3/evidence.m new file mode 100644 index 00000000..ab023cb8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/evidence.m @@ -0,0 +1,114 @@ +function [net, gamma, logev] = evidence(net, x, t, num) +%EVIDENCE Re-estimate hyperparameters using evidence approximation. +% +% Description +% [NET] = EVIDENCE(NET, X, T) re-estimates the hyperparameters ALPHA +% and BETA by applying Bayesian re-estimation formulae for NUM +% iterations. The hyperparameter ALPHA can be a simple scalar +% associated with an isotropic prior on the weights, or can be a vector +% in which each component is associated with a group of weights as +% defined by the INDEX matrix in the NET data structure. These more +% complex priors can be set up for an MLP using MLPPRIOR. Initial +% values for the iterative re-estimation are taken from the network +% data structure NET passed as an input argument, while the return +% argument NET contains the re-estimated values. +% +% [NET, GAMMA, LOGEV] = EVIDENCE(NET, X, T, NUM) allows the re- +% estimation formula to be applied for NUM cycles in which the re- +% estimated values for the hyperparameters from each cycle are used to +% re-evaluate the Hessian matrix for the next cycle. The return value +% GAMMA is the number of well-determined parameters and LOGEV is the +% log of the evidence. +% +% See also +% MLPPRIOR, NETGRAD, NETHESS, DEMEV1, DEMARD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, '', x, t); +if ~isempty(errstring) + error(errstring); +end + +ndata = size(x, 1); +if nargin == 3 + num = 1; +end + +% Extract weights from network +w = netpak(net); + +% Evaluate data-dependent contribution to the Hessian matrix. +[h, dh] = nethess(w, net, x, t); +clear h; % To save memory when Hessian is large +if (~isfield(net, 'beta')) + local_beta = 1; +end + +[evec, evl] = eig(dh); +% Now set the negative eigenvalues to zero. +evl = evl.*(evl > 0); +% safe_evl is used to avoid taking log of zero +safe_evl = evl + eps.*(evl <= 0); + +[e, edata, eprior] = neterr(w, net, x, t); + +if size(net.alpha) == [1 1] + % Form vector of eigenvalues + evl = diag(evl); + safe_evl = diag(safe_evl); +else + ngroups = size(net.alpha, 1); + gams = zeros(1, ngroups); + logas = zeros(1, ngroups); + % Reconstruct data hessian with negative eigenvalues set to zero. + dh = evec*evl*evec'; +end + +% Do the re-estimation. +for k = 1 : num + % Re-estimate alpha. + if size(net.alpha) == [1 1] + % Evaluate number of well-determined parameters. + L = evl; + if isfield(net, 'beta') + L = net.beta*L; + end + gamma = sum(L./(L + net.alpha)); + net.alpha = 0.5*gamma/eprior; + % Partially evaluate log evidence: only include unmasked weights + logev = 0.5*length(w)*log(net.alpha); + else + hinv = inv(hbayes(net, dh)); + for m = 1 : ngroups + group_nweights = sum(net.index(:, m)); + gams(m) = group_nweights - ... + net.alpha(m)*sum(diag(hinv).*net.index(:,m)); + net.alpha(m) = real(gams(m)/(2*eprior(m))); + % Weight alphas by number of weights in group + logas(m) = 0.5*group_nweights*log(net.alpha(m)); + end + gamma = sum(gams, 2); + logev = sum(logas); + end + % Re-estimate beta. + if isfield(net, 'beta') + net.beta = 0.5*(net.nout*ndata - gamma)/edata; + logev = logev + 0.5*ndata*log(net.beta) - 0.5*ndata*log(2*pi); + local_beta = net.beta; + end + + % Evaluate new log evidence + e = errbayes(net, edata); + if size(net.alpha) == [1 1] + logev = logev - e - 0.5*sum(log(local_beta*safe_evl+net.alpha)); + else + for m = 1:ngroups + logev = logev - e - ... + 0.5*sum(log(local_beta*(safe_evl*net.index(:, m))+... + net.alpha(m))); + end + end +end + diff --git a/sourcecodes/bnt-master/netlab3.3/fevbayes.m b/sourcecodes/bnt-master/netlab3.3/fevbayes.m new file mode 100644 index 00000000..d5681596 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/fevbayes.m @@ -0,0 +1,59 @@ +function [extra, invhess] = fevbayes(net, y, a, x, t, x_test, invhess) +%FEVBAYES Evaluate Bayesian regularisation for network forward propagation. +% +% Description +% EXTRA = FEVBAYES(NET, Y, A, X, T, X_TEST) takes a network data +% structure NET together with a set of hidden unit activations A from +% test inputs X_TEST, training data inputs X and T and outputs a matrix +% of extra information EXTRA that consists of error bars (variance) for +% a regression problem or moderated outputs for a classification +% problem. The optional argument (and return value) INVHESS is the +% inverse of the network Hessian computed on the training data inputs +% and targets. Passing it in avoids recomputing it, which can be a +% significant saving for large training sets. +% +% This is called by network-specific functions such as MLPEVFWD which +% are needed since the return values (predictions and hidden unit +% activations) for different network types are in different orders (for +% good reasons). +% +% See also +% MLPEVFWD, RBFEVFWD, GLMEVFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +w = netpak(net); +g = netderiv(w, net, x_test); +if nargin < 7 + % Need to compute inverse hessian + hess = nethess(w, net, x, t); + invhess = inv(hess); +end + +ntest = size(x_test, 1); +var = zeros(ntest, 1); +for idx = 1:1:net.nout, + for n = 1:1:ntest, + grad = squeeze(g(n,:,idx)); + var(n,idx) = grad*invhess*grad'; + end +end + +switch net.outfn + case 'linear' + % extra is variance + extra = ones(size(var))./net.beta + var; + case 'logistic' + % extra is moderated output + kappa = 1./(sqrt(ones(size(var)) + (pi.*var)./8)); + extra = 1./(1 + exp(-kappa.*a)); + case 'softmax' + % Use extended Mackay formula; beware that this may not + % be very accurate + kappa = 1./(sqrt(ones(size(var)) + (pi.*var)./8)); + temp = exp(kappa.*a); + extra = temp./(sum(temp, 2)*ones(1, net.nout)); + otherwise + error(['Unknown activation function ', net.outfn]); +end diff --git a/sourcecodes/bnt-master/netlab3.3/gauss.m b/sourcecodes/bnt-master/netlab3.3/gauss.m new file mode 100644 index 00000000..33e23136 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gauss.m @@ -0,0 +1,33 @@ +function y = gauss(mu, covar, x) +%GAUSS Evaluate a Gaussian distribution. +% +% Description +% +% Y = GAUSS(MU, COVAR, X) evaluates a multi-variate Gaussian density +% in D-dimensions at a set of points given by the rows of the matrix X. +% The Gaussian density has mean vector MU and covariance matrix COVAR. +% +% See also +% GSAMP, DEMGAUSS +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[n, d] = size(x); + +[j, k] = size(covar); + +% Check that the covariance matrix is the correct dimension +if ((j ~= d) | (k ~=d)) + error('Dimension of the covariance matrix and data should match'); +end + +invcov = inv(covar); +mu = reshape(mu, 1, d); % Ensure that mu is a row vector + +x = x - ones(n, 1)*mu; +fact = sum(((x*invcov).*x), 2); + +y = exp(-0.5*fact); + +y = y./sqrt((2*pi)^d*det(covar)); diff --git a/sourcecodes/bnt-master/netlab3.3/gbayes.m b/sourcecodes/bnt-master/netlab3.3/gbayes.m new file mode 100644 index 00000000..7b814e97 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gbayes.m @@ -0,0 +1,56 @@ +function [g, gdata, gprior] = gbayes(net, gdata) +%GBAYES Evaluate gradient of Bayesian error function for network. +% +% Description +% G = GBAYES(NET, GDATA) takes a network data structure NET together +% the data contribution to the error gradient for a set of inputs and +% targets. It returns the regularised error gradient using any zero +% mean Gaussian priors on the weights defined in NET. In addition, if +% a MASK is defined in NET, then the entries in G that correspond to +% weights with a 0 in the mask are removed. +% +% [G, GDATA, GPRIOR] = GBAYES(NET, GDATA) additionally returns the data +% and prior components of the error. +% +% See also +% ERRBAYES, GLMGRAD, MLPGRAD, RBFGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Evaluate the data contribution to the gradient. +if (isfield(net, 'mask')) + gdata = gdata(logical(net.mask)); +end +if isfield(net, 'beta') + g1 = gdata*net.beta; +else + g1 = gdata; +end + +% Evaluate the prior contribution to the gradient. +if isfield(net, 'alpha') + w = netpak(net); + if size(net.alpha) == [1 1] + gprior = w; + g2 = net.alpha*gprior; + else + if (isfield(net, 'mask')) + nindx_cols = size(net.index, 2); + nmask_rows = size(find(net.mask), 1); + index = reshape(net.index(logical(repmat(net.mask, ... + 1, nindx_cols))), nmask_rows, nindx_cols); + else + index = net.index; + end + + ngroups = size(net.alpha, 1); + gprior = index'.*(ones(ngroups, 1)*w); + g2 = net.alpha'*gprior; + end +else + gprior = 0; + g2 = 0; +end + +g = g1 + g2; diff --git a/sourcecodes/bnt-master/netlab3.3/glm.m b/sourcecodes/bnt-master/netlab3.3/glm.m new file mode 100644 index 00000000..7741171a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glm.m @@ -0,0 +1,82 @@ +function net = glm(nin, nout, outfunc, prior, beta) +%GLM Create a generalized linear model. +% +% Description +% +% NET = GLM(NIN, NOUT, FUNC) takes the number of inputs and outputs for +% a generalized linear model, together with a string FUNC which +% specifies the output unit activation function, and returns a data +% structure NET. The weights are drawn from a zero mean, isotropic +% Gaussian, with variance scaled by the fan-in of the output units. +% This makes use of the Matlab function RANDN and so the seed for the +% random weight initialization can be set using RANDN('STATE', S) +% where S is the seed value. The optional argument ALPHA sets the +% inverse variance for the weight initialization. +% +% The fields in NET are +% type = 'glm' +% nin = number of inputs +% nout = number of outputs +% nwts = total number of weights and biases +% actfn = string describing the output unit activation function: +% 'linear' +% 'logistic' +% 'softmax' +% w1 = first-layer weight matrix +% b1 = first-layer bias vector +% +% NET = GLM(NIN, NOUT, FUNC, PRIOR), in which PRIOR is a scalar, allows +% the field NET.ALPHA in the data structure NET to be set, +% corresponding to a zero-mean isotropic Gaussian prior with inverse +% variance with value PRIOR. Alternatively, PRIOR can consist of a data +% structure with fields ALPHA and INDEX, allowing individual Gaussian +% priors to be set over groups of weights in the network. Here ALPHA is +% a column vector in which each element corresponds to a separate +% group of weights, which need not be mutually exclusive. The +% membership of the groups is defined by the matrix INDEX in which the +% columns correspond to the elements of ALPHA. Each column has one +% element for each weight in the matrix, in the order defined by the +% function GLMPAK, and each element is 1 or 0 according to whether the +% weight is a member of the corresponding group or not. +% +% NET = GLM(NIN, NOUT, FUNC, PRIOR, BETA) also sets the additional +% field NET.BETA in the data structure NET, where beta corresponds to +% the inverse noise variance. +% +% See also +% GLMPAK, GLMUNPAK, GLMFWD, GLMERR, GLMGRAD, GLMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.type = 'glm'; +net.nin = nin; +net.nout = nout; +net.nwts = (nin + 1)*nout; + +outtfns = {'linear', 'logistic', 'softmax'}; + +if sum(strcmp(outfunc, outtfns)) == 0 + error('Undefined activation function. Exiting.'); +else + net.outfn = outfunc; +end + +if nargin > 3 + if isstruct(prior) + net.alpha = prior.alpha; + net.index = prior.index; + elseif size(prior) == [1 1] + net.alpha = prior; + else + error('prior must be a scalar or structure'); + end +end + +net.w1 = randn(nin, nout)/sqrt(nin + 1); +net.b1 = randn(1, nout)/sqrt(nin + 1); + +if nargin == 5 + net.beta = beta; +end + diff --git a/sourcecodes/bnt-master/netlab3.3/glmderiv.m b/sourcecodes/bnt-master/netlab3.3/glmderiv.m new file mode 100644 index 00000000..bc8de671 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmderiv.m @@ -0,0 +1,40 @@ +function g = glmderiv(net, x) +%GLMDERIV Evaluate derivatives of GLM outputs with respect to weights. +% +% Description +% G = GLMDERIV(NET, X) takes a network data structure NET and a matrix +% of input vectors X and returns a three-index matrix mat{g} whose I, +% J, K element contains the derivative of network output K with respect +% to weight or bias parameter J for input pattern I. The ordering of +% the weight and bias parameters is defined by GLMUNPAK. +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm', x); +if ~isempty(errstring) + error(errstring); +end + +ndata = size(x, 1); +if isfield(net, 'mask') + nwts = size(find(net.mask), 1); + mask_array = logical(net.mask)*ones(1, net.nout); +else + nwts = net.nwts; +end +g = zeros(ndata, nwts, net.nout); + +temp = zeros(net.nwts, net.nout); +for n = 1:ndata + % Weight matrix w1 + temp(1:(net.nin*net.nout), :) = kron(eye(net.nout), (x(n, :))'); + % Bias term b1 + temp(net.nin*net.nout+1:end, :) = eye(net.nout); + if isfield(net, 'mask') + g(n, :, :) = reshape(temp(find(mask_array)), nwts, net.nout); + else + g(n, :, :) = temp; + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/glmerr.m b/sourcecodes/bnt-master/netlab3.3/glmerr.m new file mode 100644 index 00000000..bab849c3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmerr.m @@ -0,0 +1,49 @@ +function [e, edata, eprior, y, a] = glmerr(net, x, t) +%GLMERR Evaluate error function for generalized linear model. +% +% Description +% E = GLMERR(NET, X, T) takes a generalized linear model data +% structure NET together with a matrix X of input vectors and a matrix +% T of target vectors, and evaluates the error function E. The choice +% of error function corresponds to the output unit activation function. +% Each row of X corresponds to one input vector and each row of T +% corresponds to one target vector. +% +% [E, EDATA, EPRIOR, Y, A] = GLMERR(NET, X, T) also returns the data +% and prior components of the total error. +% +% [E, EDATA, EPRIOR, Y, A] = GLMERR(NET, X) also returns a matrix Y +% giving the outputs of the models and a matrix A giving the summed +% inputs to each output unit, where each row corresponds to one +% pattern. +% +% See also +% GLM, GLMPAK, GLMUNPAK, GLMFWD, GLMGRAD, GLMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm', x, t); +if ~isempty(errstring); + error(errstring); +end + +[y, a] = glmfwd(net, x); + +switch net.outfn + + case 'linear' % Linear outputs + edata = 0.5*sum(sum((y - t).^2)); + + case 'logistic' % Logistic outputs + edata = - sum(sum(t.*log(y) + (1 - t).*log(1 - y))); + + case 'softmax' % Softmax outputs + edata = - sum(sum(t.*log(y))); + + otherwise + error(['Unknown activation function ', net.outfn]); +end + +[e, edata, eprior] = errbayes(net, edata); diff --git a/sourcecodes/bnt-master/netlab3.3/glmevfwd.m b/sourcecodes/bnt-master/netlab3.3/glmevfwd.m new file mode 100644 index 00000000..d445a367 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmevfwd.m @@ -0,0 +1,28 @@ +function [y, extra, invhess] = glmevfwd(net, x, t, x_test, invhess) +%GLMEVFWD Forward propagation with evidence for GLM +% +% Description +% Y = GLMEVFWD(NET, X, T, X_TEST) takes a network data structure NET +% together with the input X and target T training data and input test +% data X_TEST. It returns the normal forward propagation through the +% network Y together with a matrix EXTRA which consists of error bars +% (variance) for a regression problem or moderated outputs for a +% classification problem. +% +% The optional argument (and return value) INVHESS is the inverse of +% the network Hessian computed on the training data inputs and targets. +% Passing it in avoids recomputing it, which can be a significant +% saving for large training sets. +% +% See also +% FEVBAYES +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[y, a] = glmfwd(net, x_test); +if nargin == 4 + [extra, invhess] = fevbayes(net, y, a, x, t, x_test); +else + [extra, invhess] = fevbayes(net, y, a, x, t, x_test, invhess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/glmfwd.m b/sourcecodes/bnt-master/netlab3.3/glmfwd.m new file mode 100644 index 00000000..5a9519cd --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmfwd.m @@ -0,0 +1,62 @@ +function [y, a] = glmfwd(net, x) +%GLMFWD Forward propagation through generalized linear model. +% +% Description +% Y = GLMFWD(NET, X) takes a generalized linear model data structure +% NET together with a matrix X of input vectors, and forward propagates +% the inputs through the network to generate a matrix Y of output +% vectors. Each row of X corresponds to one input vector and each row +% of Y corresponds to one output vector. +% +% [Y, A] = GLMFWD(NET, X) also returns a matrix A giving the summed +% inputs to each output unit, where each row corresponds to one +% pattern. +% +% See also +% GLM, GLMPAK, GLMUNPAK, GLMERR, GLMGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm', x); +if ~isempty(errstring); + error(errstring); +end + +ndata = size(x, 1); + +a = x*net.w1 + ones(ndata, 1)*net.b1; + +switch net.outfn + + case 'linear' % Linear outputs + y = a; + + case 'logistic' % Logistic outputs + % Prevent overflow and underflow: use same bounds as glmerr + % Ensure that log(1-y) is computable: need exp(a) > eps + maxcut = -log(eps); + % Ensure that log(y) is computable + mincut = -log(1/realmin - 1); + a = min(a, maxcut); + a = max(a, mincut); + y = 1./(1 + exp(-a)); + + case 'softmax' % Softmax outputs + nout = size(a,2); + % Prevent overflow and underflow: use same bounds as glmerr + % Ensure that sum(exp(a), 2) does not overflow + maxcut = log(realmax) - log(nout); + % Ensure that exp(a) > 0 + mincut = log(realmin); + a = min(a, maxcut); + a = max(a, mincut); + temp = exp(a); + y = temp./(sum(temp, 2)*ones(1,nout)); + % Ensure that log(y) is computable + y(y<realmin) = realmin; + + otherwise + error(['Unknown activation function ', net.outfn]); +end diff --git a/sourcecodes/bnt-master/netlab3.3/glmgrad.m b/sourcecodes/bnt-master/netlab3.3/glmgrad.m new file mode 100644 index 00000000..d967804e --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmgrad.m @@ -0,0 +1,36 @@ +function [g, gdata, gprior] = glmgrad(net, x, t) +%GLMGRAD Evaluate gradient of error function for generalized linear model. +% +% Description +% G = GLMGRAD(NET, X, T) takes a generalized linear model data +% structure NET together with a matrix X of input vectors and a matrix +% T of target vectors, and evaluates the gradient G of the error +% function with respect to the network weights. The error function +% corresponds to the choice of output unit activation function. Each +% row of X corresponds to one input vector and each row of T +% corresponds to one target vector. +% +% [G, GDATA, GPRIOR] = GLMGRAD(NET, X, T) also returns separately the +% data and prior contributions to the gradient. +% +% See also +% GLM, GLMPAK, GLMUNPAK, GLMFWD, GLMERR, GLMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm', x, t); +if ~isempty(errstring); + error(errstring); +end + +y = glmfwd(net, x); +delout = y - t; + +gw1 = x'*delout; +gb1 = sum(delout, 1); + +gdata = [gw1(:)', gb1]; + +[g, gdata, gprior] = gbayes(net, gdata); diff --git a/sourcecodes/bnt-master/netlab3.3/glmhess.m b/sourcecodes/bnt-master/netlab3.3/glmhess.m new file mode 100644 index 00000000..21fc8ad2 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmhess.m @@ -0,0 +1,106 @@ +function [h, hdata] = glmhess(net, x, t, hdata) +%GLMHESS Evaluate the Hessian matrix for a generalised linear model. +% +% Description +% H = GLMHESS(NET, X, T) takes a GLM network data structure NET, a +% matrix X of input values, and a matrix T of target values and returns +% the full Hessian matrix H corresponding to the second derivatives of +% the negative log posterior distribution, evaluated for the current +% weight and bias values as defined by NET. Note that the target data +% is not required in the calculation, but is included to make the +% interface uniform with NETHESS. For linear and logistic outputs, the +% computation is very simple and is done (in effect) in one line in +% GLMTRAIN. +% +% [H, HDATA] = GLMHESS(NET, X, T) returns both the Hessian matrix H and +% the contribution HDATA arising from the data dependent term in the +% Hessian. +% +% H = GLMHESS(NET, X, T, HDATA) takes a network data structure NET, a +% matrix X of input values, and a matrix T of target values, together +% with the contribution HDATA arising from the data dependent term in +% the Hessian, and returns the full Hessian matrix H corresponding to +% the second derivatives of the negative log posterior distribution. +% This version saves computation time if HDATA has already been +% evaluated for the current weight and bias values. +% +% See also +% GLM, GLMTRAIN, HESSCHEK, NETHESS +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm', x, t); +if ~isempty(errstring); + error(errstring); +end + +ndata = size(x, 1); +nparams = net.nwts; +nout = net.nout; +p = glmfwd(net, x); +inputs = [x ones(ndata, 1)]; + +if nargin == 3 + hdata = zeros(nparams); % Full Hessian matrix + % Calculate data component of Hessian + switch net.outfn + + case 'linear' + % No weighting function here + out_hess = [x ones(ndata, 1)]'*[x ones(ndata, 1)]; + for j = 1:nout + hdata = rearrange_hess(net, j, out_hess, hdata); + end + case 'logistic' + % Each output is independent + e = ones(1, net.nin+1); + link_deriv = p.*(1-p); + out_hess = zeros(net.nin+1); + for j = 1:nout + inputs = [x ones(ndata, 1)].*(sqrt(link_deriv(:,j))*e); + out_hess = inputs'*inputs; % Hessian for this output + hdata = rearrange_hess(net, j, out_hess, hdata); + end + + case 'softmax' + bb_start = nparams - nout + 1; % Start of bias weights block + ex_hess = zeros(nparams); % Contribution to Hessian from single example + for m = 1:ndata + X = x(m,:)'*x(m,:); + a = diag(p(m,:))-((p(m,:)')*p(m,:)); + ex_hess(1:nparams-nout,1:nparams-nout) = kron(a, X); + ex_hess(bb_start:nparams, bb_start:nparams) = a.*ones(net.nout, net.nout); + temp = kron(a, x(m,:)); + ex_hess(bb_start:nparams, 1:nparams-nout) = temp; + ex_hess(1:nparams-nout, bb_start:nparams) = temp'; + hdata = hdata + ex_hess; + end + otherwise + error(['Unknown activation function ', net.outfn]); + end +end + +[h, hdata] = hbayes(net, hdata); + +function hdata = rearrange_hess(net, j, out_hess, hdata) + +% Because all the biases come after all the input weights, +% we have to rearrange the blocks that make up the network Hessian. +% This function assumes that we are on the jth output and that all outputs +% are independent. + +bb_start = net.nwts - net.nout + 1; % Start of bias weights block +ob_start = 1+(j-1)*net.nin; % Start of weight block for jth output +ob_end = j*net.nin; % End of weight block for jth output +b_index = bb_start+(j-1); % Index of bias weight +% Put input weight block in right place +hdata(ob_start:ob_end, ob_start:ob_end) = out_hess(1:net.nin, 1:net.nin); +% Put second derivative of bias weight in right place +hdata(b_index, b_index) = out_hess(net.nin+1, net.nin+1); +% Put cross terms (input weight v bias weight) in right place +hdata(b_index, ob_start:ob_end) = out_hess(net.nin+1,1:net.nin); +hdata(ob_start:ob_end, b_index) = out_hess(1:net.nin, net.nin+1); + +return \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/glminit.m b/sourcecodes/bnt-master/netlab3.3/glminit.m new file mode 100644 index 00000000..3dc47bc7 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glminit.m @@ -0,0 +1,36 @@ +function net = glminit(net, prior) +%GLMINIT Initialise the weights in a generalized linear model. +% +% Description +% +% NET = GLMINIT(NET, PRIOR) takes a generalized linear model NET and +% sets the weights and biases by sampling from a Gaussian distribution. +% If PRIOR is a scalar, then all of the parameters (weights and biases) +% are sampled from a single isotropic Gaussian with inverse variance +% equal to PRIOR. If PRIOR is a data structure similar to that in +% MLPPRIOR but for a single layer of weights, then the parameters are +% sampled from multiple Gaussians according to their groupings (defined +% by the INDEX field) with corresponding variances (defined by the +% ALPHA field). +% +% See also +% GLM, GLMPAK, GLMUNPAK, MLPINIT, MLPPRIOR +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'glm'); +if ~isempty(errstring); + error(errstring); +end +if isstruct(prior) + sig = 1./sqrt(prior.index*prior.alpha); + w = sig'.*randn(1, net.nwts); +elseif size(prior) == [1 1] + w = randn(1, net.nwts).*sqrt(1/prior); +else + error('prior must be a scalar or a structure'); +end + +net = glmunpak(net, w); + diff --git a/sourcecodes/bnt-master/netlab3.3/glmpak.m b/sourcecodes/bnt-master/netlab3.3/glmpak.m new file mode 100644 index 00000000..300960fc --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmpak.m @@ -0,0 +1,20 @@ +function w = glmpak(net) +%GLMPAK Combines weights and biases into one weights vector. +% +% Description +% W = GLMPAK(NET) takes a network data structure NET and combines them +% into a single row vector W. +% +% See also +% GLM, GLMUNPAK, GLMFWD, GLMERR, GLMGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'glm'); +if ~errstring + error(errstring); +end + +w = [net.w1(:)', net.b1]; + diff --git a/sourcecodes/bnt-master/netlab3.3/glmtrain.m b/sourcecodes/bnt-master/netlab3.3/glmtrain.m new file mode 100644 index 00000000..8dd0e3db --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmtrain.m @@ -0,0 +1,206 @@ +function [net, options] = glmtrain(net, options, x, t) +%GLMTRAIN Specialised training of generalized linear model +% +% Description +% NET = GLMTRAIN(NET, OPTIONS, X, T) uses the iterative reweighted +% least squares (IRLS) algorithm to set the weights in the generalized +% linear model structure NET. This is a more efficient alternative to +% using GLMERR and GLMGRAD and a non-linear optimisation routine +% through NETOPT. Note that for linear outputs, a single pass through +% the algorithm is all that is required, since the error function is +% quadratic in the weights. The algorithm also handles scalar ALPHA +% and BETA terms. If you want to use more complicated priors, you +% should use general-purpose non-linear optimisation algorithms. +% +% For logistic and softmax outputs, general priors can be handled, +% although this requires the pseudo-inverse of the Hessian, giving up +% the better conditioning and some of the speed advantage of the normal +% form equations. +% +% The error function value at the final set of weights is returned in +% OPTIONS(8). Each row of X corresponds to one input vector and each +% row of T corresponds to one target vector. +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values during training. If +% OPTIONS(1) is set to 0, then only warning messages are displayed. If +% OPTIONS(1) is -1, then nothing is displayed. +% +% OPTIONS(2) is a measure of the precision required for the value of +% the weights W at the solution. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. Both this and the previous condition must +% be satisfied for termination. +% +% OPTIONS(5) is set to 1 if an approximation to the Hessian (which +% assumes that all outputs are independent) is used for softmax +% outputs. With the default value of 0 the exact Hessian (which is more +% expensive to compute) is used. +% +% OPTIONS(14) is the maximum number of iterations for the IRLS +% algorithm; default 100. +% +% See also +% GLM, GLMERR, GLMGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm', x, t); +if ~errstring + error(errstring); +end + +if(~options(14)) + options(14) = 100; +end + +display = options(1); +% Do we need to test for termination? +test = (options(2) | options(3)); + +ndata = size(x, 1); +% Add a column of ones for the bias +inputs = [x ones(ndata, 1)]; + +% Linear outputs are a special case as they can be found in one step +if strcmp(net.outfn, 'linear') + if ~isfield(net, 'alpha') + % Solve for the weights and biases using left matrix divide + temp = inputs\t; + elseif size(net.alpha == [1 1]) + if isfield(net, 'beta') + beta = net.beta; + else + beta = 1.0; + end + % Use normal form equation + hessian = beta*(inputs'*inputs) + net.alpha*eye(net.nin+1); + temp = pinv(hessian)*(beta*(inputs'*t)); + else + error('Only scalar alpha allowed'); + end + net.w1 = temp(1:net.nin, :); + net.b1 = temp(net.nin+1, :); + % Store error value in options vector + options(8) = glmerr(net, x, t); + return; +end + +% Otherwise need to use iterative reweighted least squares +e = ones(1, net.nin+1); +for n = 1:options(14) + + switch net.outfn + case 'logistic' + if n == 1 + % Initialise model + p = (t+0.5)/2; + act = log(p./(1-p)); + wold = glmpak(net); + end + link_deriv = p.*(1-p); + weights = sqrt(link_deriv); % sqrt of weights + if (min(min(weights)) < eps) + warning('ill-conditioned weights in glmtrain') + return + end + z = act + (t-p)./link_deriv; + if ~isfield(net, 'alpha') + % Treat each output independently with relevant set of weights + for j = 1:net.nout + indep = inputs.*(weights(:,j)*e); + dep = z(:,j).*weights(:,j); + temp = indep\dep; + net.w1(:,j) = temp(1:net.nin); + net.b1(j) = temp(net.nin+1); + end + else + gradient = glmgrad(net, x, t); + Hessian = glmhess(net, x, t); + deltaw = -gradient*pinv(Hessian); + w = wold + deltaw; + net = glmunpak(net, w); + end + [err, edata, eprior, p, act] = glmerr(net, x, t); + if n == 1 + errold = err; + wold = netpak(net); + else + w = netpak(net); + end + case 'softmax' + if n == 1 + % Initialise model: ensure that row sum of p is one no matter + % how many classes there are + p = (t + (1/size(t, 2)))/2; + act = log(p./(1-p)); + end + if options(5) == 1 | n == 1 + link_deriv = p.*(1-p); + weights = sqrt(link_deriv); % sqrt of weights + if (min(min(weights)) < eps) + warning('ill-conditioned weights in glmtrain') + return + end + z = act + (t-p)./link_deriv; + % Treat each output independently with relevant set of weights + for j = 1:net.nout + indep = inputs.*(weights(:,j)*e); + dep = z(:,j).*weights(:,j); + temp = indep\dep; + net.w1(:,j) = temp(1:net.nin); + net.b1(j) = temp(net.nin+1); + end + [err, edata, eprior, p, act] = glmerr(net, x, t); + if n == 1 + errold = err; + wold = netpak(net); + else + w = netpak(net); + end + else + % Exact method of calculation after w first initialised + % Start by working out Hessian + Hessian = glmhess(net, x, t); + gradient = glmgrad(net, x, t); + % Now compute modification to weights + deltaw = -gradient*pinv(Hessian); + w = wold + deltaw; + net = glmunpak(net, w); + [err, edata, eprior, p] = glmerr(net, x, t); + end + + otherwise + error(['Unknown activation function ', net.outfn]); + end + if options(1) + fprintf(1, 'Cycle %4d Error %11.6f\n', n, err) + end + % Test for termination + % Terminate if error increases + if err > errold + errold = err; + w = wold; + options(8) = err; + fprintf(1, 'Error has increased: terminating\n') + return; + end + if test && n > 1 + if (max(abs(w - wold)) < options(2) && abs(err-errold) < options(3)) + options(8) = err; + return; + else + errold = err; + wold = w; + end + end +end + +options(8) = err; +if (options(1) >= 0) + disp(maxitmess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/glmunpak.m b/sourcecodes/bnt-master/netlab3.3/glmunpak.m new file mode 100644 index 00000000..478ef657 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmunpak.m @@ -0,0 +1,30 @@ +function net = glmunpak(net, w) +%GLMUNPAK Separates weights vector into weight and bias matrices. +% +% Description +% NET = GLMUNPAK(NET, W) takes a glm network data structure NET and a +% weight vector W, and returns a network data structure identical to +% the input network, except that the first-layer weight matrix W1 and +% the first-layer bias vector B1 have been set to the corresponding +% elements of W. +% +% See also +% GLM, GLMPAK, GLMFWD, GLMERR, GLMGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm'); +if ~errstring + error(errstring); +end + +if net.nwts ~= length(w) + error('Invalid weight vector length') +end + +nin = net.nin; +nout = net.nout; +net.w1 = reshape(w(1:nin*nout), nin, nout); +net.b1 = reshape(w(nin*nout + 1: (nin + 1)*nout), 1, nout); diff --git a/sourcecodes/bnt-master/netlab3.3/gmm.m b/sourcecodes/bnt-master/netlab3.3/gmm.m new file mode 100644 index 00000000..248de8f5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmm.m @@ -0,0 +1,113 @@ +function mix = gmm(dim, ncentres, covar_type, ppca_dim) +%GMM Creates a Gaussian mixture model with specified architecture. +% +% Description +% MIX = GMM(DIM, NCENTRES, COVARTYPE) takes the dimension of the space +% DIM, the number of centres in the mixture model and the type of the +% mixture model, and returns a data structure MIX. The mixture model +% type defines the covariance structure of each component Gaussian: +% 'spherical' = single variance parameter for each component: stored as a vector +% 'diag' = diagonal matrix for each component: stored as rows of a matrix +% 'full' = full matrix for each component: stored as 3d array +% 'ppca' = probabilistic PCA: stored as principal components (in a 3d array +% and associated variances and off-subspace noise +% MIX = GMM(DIM, NCENTRES, COVARTYPE, PPCA_DIM) also sets the +% dimension of the PPCA sub-spaces: the default value is one. +% +% The priors are initialised to equal values summing to one, and the +% covariances are all the identity matrix (or equivalent). The centres +% are initialised randomly from a zero mean unit variance Gaussian. +% This makes use of the MATLAB function RANDN and so the seed for the +% random weight initialisation can be set using RANDN('STATE', S) where +% S is the state value. +% +% The fields in MIX are +% +% type = 'gmm' +% nin = the dimension of the space +% ncentres = number of mixture components +% covartype = string for type of variance model +% priors = mixing coefficients +% centres = means of Gaussians: stored as rows of a matrix +% covars = covariances of Gaussians +% The additional fields for mixtures of PPCA are +% U = principal component subspaces +% lambda = in-space covariances: stored as rows of a matrix +% The off-subspace noise is stored in COVARS. +% +% See also +% GMMPAK, GMMUNPAK, GMMSAMP, GMMINIT, GMMEM, GMMACTIV, GMMPOST, +% GMMPROB +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if ncentres < 1 + error('Number of centres must be greater than zero') +end + +mix.type = 'gmm'; +mix.nin = dim; +mix.ncentres = ncentres; + +vartypes = {'spherical', 'diag', 'full', 'ppca'}; + +if sum(strcmp(covar_type, vartypes)) == 0 + error('Undefined covariance type') +else + mix.covar_type = covar_type; +end + +% Make default dimension of PPCA subspaces one. +if strcmp(covar_type, 'ppca') + if nargin < 4 + ppca_dim = 1; + end + if ppca_dim > dim + error('Dimension of PPCA subspaces must be less than data.') + end + mix.ppca_dim = ppca_dim; +end + +% Initialise priors to be equal and summing to one +mix.priors = ones(1,mix.ncentres) ./ mix.ncentres; + +% Initialise centres +mix.centres = randn(mix.ncentres, mix.nin); + +% Initialise all the variances to unity +switch mix.covar_type + +case 'spherical' + mix.covars = ones(1, mix.ncentres); + mix.nwts = mix.ncentres + mix.ncentres*mix.nin + mix.ncentres; +case 'diag' + % Store diagonals of covariance matrices as rows in a matrix + mix.covars = ones(mix.ncentres, mix.nin); + mix.nwts = mix.ncentres + mix.ncentres*mix.nin + ... + mix.ncentres*mix.nin; +case 'full' + % Store covariance matrices in a row vector of matrices + mix.covars = repmat(eye(mix.nin), [1 1 mix.ncentres]); + mix.nwts = mix.ncentres + mix.ncentres*mix.nin + ... + mix.ncentres*mix.nin*mix.nin; +case 'ppca' + % This is the off-subspace noise: make it smaller than + % lambdas + mix.covars = 0.1*ones(1, mix.ncentres); + % Also set aside storage for principal components and + % associated variances + init_space = eye(mix.nin); + init_space = init_space(:, 1:mix.ppca_dim); + init_space(mix.ppca_dim+1:mix.nin, :) = ... + ones(mix.nin - mix.ppca_dim, mix.ppca_dim); + mix.U = repmat(init_space , [1 1 mix.ncentres]); + mix.lambda = ones(mix.ncentres, mix.ppca_dim); + % Take account of additional parameters + mix.nwts = mix.ncentres + mix.ncentres*mix.nin + ... + mix.ncentres + mix.ncentres*mix.ppca_dim + ... + mix.ncentres*mix.nin*mix.ppca_dim; +otherwise + error(['Unknown covariance type ', mix.covar_type]); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/gmmactiv.m b/sourcecodes/bnt-master/netlab3.3/gmmactiv.m new file mode 100644 index 00000000..117e1cdc --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmactiv.m @@ -0,0 +1,77 @@ +function a = gmmactiv(mix, x) +%GMMACTIV Computes the activations of a Gaussian mixture model. +% +% Description +% This function computes the activations A (i.e. the probability +% P(X|J) of the data conditioned on each component density) for a +% Gaussian mixture model. For the PPCA model, each activation is the +% conditional probability of X given that it is generated by the +% component subspace. The data structure MIX defines the mixture model, +% while the matrix X contains the data vectors. Each row of X +% represents a single vector. +% +% See also +% GMM, GMMPOST, GMMPROB +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check that inputs are consistent +errstring = consist(mix, 'gmm', x); +if ~isempty(errstring) + error(errstring); +end + +ndata = size(x, 1); +a = zeros(ndata, mix.ncentres); % Preallocate matrix + +switch mix.covar_type + +case 'spherical' + % Calculate squared norm matrix, of dimension (ndata, ncentres) + n2 = dist2(x, mix.centres); + + % Calculate width factors + wi2 = ones(ndata, 1) * (2 .* mix.covars); + normal = (pi .* wi2) .^ (mix.nin/2); + + % Now compute the activations + a = exp(-(n2./wi2))./ normal; + +case 'diag' + normal = (2*pi)^(mix.nin/2); + s = prod(sqrt(mix.covars), 2); + for j = 1:mix.ncentres + diffs = x - (ones(ndata, 1) * mix.centres(j, :)); + a(:, j) = exp(-0.5*sum((diffs.*diffs)./(ones(ndata, 1) * ... + mix.covars(j, :)), 2)) ./ (normal*s(j)); + end + +case 'full' + normal = (2*pi)^(mix.nin/2); + for j = 1:mix.ncentres + diffs = x - (ones(ndata, 1) * mix.centres(j, :)); + % Use Cholesky decomposition of covariance matrix to speed computation + c = chol(mix.covars(:, :, j)); + temp = diffs/c; + a(:, j) = exp(-0.5*sum(temp.*temp, 2))./(normal*prod(diag(c))); + end +case 'ppca' + log_normal = mix.nin*log(2*pi); + d2 = zeros(ndata, mix.ncentres); + logZ = zeros(1, mix.ncentres); + for i = 1:mix.ncentres + k = 1 - mix.covars(i)./mix.lambda(i, :); + logZ(i) = log_normal + mix.nin*log(mix.covars(i)) - ... + sum(log(1 - k)); + diffs = x - ones(ndata, 1)*mix.centres(i, :); + proj = diffs*mix.U(:, :, i); + d2(:,i) = (sum(diffs.*diffs, 2) - ... + sum((proj.*(ones(ndata, 1)*k)).*proj, 2)) / ... + mix.covars(i); + end + a = exp(-0.5*(d2 + ones(ndata, 1)*logZ)); +otherwise + error(['Unknown covariance type ', mix.covar_type]); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/gmmem.m b/sourcecodes/bnt-master/netlab3.3/gmmem.m new file mode 100644 index 00000000..42066538 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmem.m @@ -0,0 +1,181 @@ +function [mix, options, errlog] = gmmem(mix, x, options) +%GMMEM EM algorithm for Gaussian mixture model. +% +% Description +% [MIX, OPTIONS, ERRLOG] = GMMEM(MIX, X, OPTIONS) uses the Expectation +% Maximization algorithm of Dempster et al. to estimate the parameters +% of a Gaussian mixture model defined by a data structure MIX. The +% matrix X represents the data whose expectation is maximized, with +% each row corresponding to a vector. The optional parameters have +% the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG. If OPTIONS(1) is set to 0, then +% only warning messages are displayed. If OPTIONS(1) is -1, then +% nothing is displayed. +% +% OPTIONS(3) is a measure of the absolute precision required of the +% error function at the solution. If the change in log likelihood +% between two steps of the EM algorithm is less than this value, then +% the function terminates. +% +% OPTIONS(5) is set to 1 if a covariance matrix is reset to its +% original value when any of its singular values are too small (less +% than MIN_COVAR which has the value eps). With the default value of +% 0 no action is taken. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% The optional return value OPTIONS contains the final error value +% (i.e. data log likelihood) in OPTIONS(8). +% +% See also +% GMM, GMMINIT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check that inputs are consistent +errstring = consist(mix, 'gmm', x); +if ~isempty(errstring) + error(errstring); +end + +[ndata, xdim] = size(x); + +% Sort out the options +if (options(14)) + niters = options(14); +else + niters = 100; +end + +display = options(1); +store = 0; +if (nargout > 2) + store = 1; % Store the error values to return them + errlog = zeros(1, niters); +end +test = 0; +if options(3) > 0.0 + test = 1; % Test log likelihood for termination +end + +check_covars = 0; +if options(5) >= 1 + if display >= 0 + disp('check_covars is on'); + end + check_covars = 1; % Ensure that covariances don't collapse + MIN_COVAR = eps; % Minimum singular value of covariance matrix + init_covars = mix.covars; +end + +% Main loop of algorithm +for n = 1:niters + + % Calculate posteriors based on old parameters + [post, act] = gmmpost(mix, x); + + % Calculate error value if needed + if (display || store || test) + prob = act*(mix.priors)'; + % Error value is negative log likelihood of data + e = - sum(log(prob)); + if store + errlog(n) = e; + end + if display > 0 + fprintf(1, 'Cycle %4d Error %11.6f\n', n, e); + end + if test + if (n > 1 && abs(e - eold) < options(3)) + options(8) = e; + return; + else + eold = e; + end + end + end + + % Adjust the new estimates for the parameters + new_pr = sum(post, 1); + new_c = post' * x; + + % Now move new estimates to old parameter vectors + mix.priors = new_pr ./ ndata; + + mix.centres = new_c ./ (new_pr' * ones(1, mix.nin)); + + switch mix.covar_type + case 'spherical' + n2 = dist2(x, mix.centres); + for j = 1:mix.ncentres + v(j) = (post(:,j)'*n2(:,j)); + end + mix.covars = ((v./new_pr))./mix.nin; + if check_covars + % Ensure that no covariance is too small + for j = 1:mix.ncentres + if mix.covars(j) < MIN_COVAR + mix.covars(j) = init_covars(j); + end + end + end + case 'diag' + for j = 1:mix.ncentres + diffs = x - (ones(ndata, 1) * mix.centres(j,:)); + mix.covars(j,:) = sum((diffs.*diffs).*(post(:,j)*ones(1, ... + mix.nin)), 1)./new_pr(j); + end + if check_covars + % Ensure that no covariance is too small + for j = 1:mix.ncentres + if min(mix.covars(j,:)) < MIN_COVAR + mix.covars(j,:) = init_covars(j,:); + end + end + end + case 'full' + for j = 1:mix.ncentres + diffs = x - (ones(ndata, 1) * mix.centres(j,:)); + diffs = diffs.*(sqrt(post(:,j))*ones(1, mix.nin)); + mix.covars(:,:,j) = (diffs'*diffs)/new_pr(j); + end + if check_covars + % Ensure that no covariance is too small + for j = 1:mix.ncentres + if min(svd(mix.covars(:,:,j))) < MIN_COVAR + mix.covars(:,:,j) = init_covars(:,:,j); + end + end + end + case 'ppca' + for j = 1:mix.ncentres + diffs = x - (ones(ndata, 1) * mix.centres(j,:)); + diffs = diffs.*(sqrt(post(:,j))*ones(1, mix.nin)); + [tempcovars, tempU, templambda] = ... + ppca((diffs'*diffs)/new_pr(j), mix.ppca_dim); + if length(templambda) ~= mix.ppca_dim + error('Unable to extract enough components'); + else + mix.covars(j) = tempcovars; + mix.U(:, :, j) = tempU; + mix.lambda(j, :) = templambda; + end + end + if check_covars + if mix.covars(j) < MIN_COVAR + mix.covars(j) = init_covars(j); + end + end + otherwise + error(['Unknown covariance type ', mix.covar_type]); + end +end + +options(8) = -sum(log(gmmprob(mix, x))); +if (display >= 0) + disp(maxitmess); +end + \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gmminit.m b/sourcecodes/bnt-master/netlab3.3/gmminit.m new file mode 100644 index 00000000..30dc974d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmminit.m @@ -0,0 +1,94 @@ +function mix = gmminit(mix, x, options) +%GMMINIT Initialises Gaussian mixture model from data +% +% Description +% MIX = GMMINIT(MIX, X, OPTIONS) uses a dataset X to initialise the +% parameters of a Gaussian mixture model defined by the data structure +% MIX. The k-means algorithm is used to determine the centres. The +% priors are computed from the proportion of examples belonging to each +% cluster. The covariance matrices are calculated as the sample +% covariance of the points associated with (i.e. closest to) the +% corresponding centres. For a mixture of PPCA model, the PPCA +% decomposition is calculated for the points closest to a given centre. +% This initialisation can be used as the starting point for training +% the model using the EM algorithm. +% +% See also +% GMM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[ndata, xdim] = size(x); + +% Check that inputs are consistent +errstring = consist(mix, 'gmm', x); +if ~isempty(errstring) + error(errstring); +end + +% Arbitrary width used if variance collapses to zero: make it 'large' so +% that centre is responsible for a reasonable number of points. +GMM_WIDTH = 1.0; + +% Use kmeans algorithm to set centres +options(5) = 1; +[mix.centres, options, post] = kmeansNetlab(mix.centres, x, options); + +% Set priors depending on number of points in each cluster +cluster_sizes = max(sum(post, 1), 1); % Make sure that no prior is zero +mix.priors = cluster_sizes/sum(cluster_sizes); % Normalise priors + +switch mix.covar_type +case 'spherical' + if mix.ncentres > 1 + % Determine widths as distance to nearest centre + % (or a constant if this is zero) + cdist = dist2(mix.centres, mix.centres); + cdist = cdist + diag(ones(mix.ncentres, 1)*realmax); + mix.covars = min(cdist); + mix.covars = mix.covars + GMM_WIDTH*(mix.covars < eps); + else + % Just use variance of all data points averaged over all + % dimensions + mix.covars = mean(diag(cov(x))); + end + case 'diag' + for j = 1:mix.ncentres + % Pick out data points belonging to this centre + c = x(find(post(:, j)),:); + diffs = c - (ones(size(c, 1), 1) * mix.centres(j, :)); + mix.covars(j, :) = sum((diffs.*diffs), 1)/size(c, 1); + % Replace small entries by GMM_WIDTH value + mix.covars(j, :) = mix.covars(j, :) + GMM_WIDTH.*(mix.covars(j, :)<eps); + end + case 'full' + for j = 1:mix.ncentres + % Pick out data points belonging to this centre + c = x(find(post(:, j)),:); + diffs = c - (ones(size(c, 1), 1) * mix.centres(j, :)); + mix.covars(:,:,j) = (diffs'*diffs)/(size(c, 1)); + % Add GMM_WIDTH*Identity to rank-deficient covariance matrices + if rank(mix.covars(:,:,j)) < mix.nin + mix.covars(:,:,j) = mix.covars(:,:,j) + GMM_WIDTH.*eye(mix.nin); + end + end + case 'ppca' + for j = 1:mix.ncentres + % Pick out data points belonging to this centre + c = x(find(post(:,j)),:); + diffs = c - (ones(size(c, 1), 1) * mix.centres(j, :)); + [tempcovars, tempU, templambda] = ... + ppca((diffs'*diffs)/size(c, 1), mix.ppca_dim); + if length(templambda) ~= mix.ppca_dim + error('Unable to extract enough components'); + else + mix.covars(j) = tempcovars; + mix.U(:, :, j) = tempU; + mix.lambda(j, :) = templambda; + end + end + otherwise + error(['Unknown covariance type ', mix.covar_type]); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/gmmpak.m b/sourcecodes/bnt-master/netlab3.3/gmmpak.m new file mode 100644 index 00000000..8e307fab --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmpak.m @@ -0,0 +1,22 @@ +function p = gmmpak(mix) +%GMMPAK Combines all the parameters in a Gaussian mixture model into one vector. +% +% Description +% P = GMMPAK(NET) takes a mixture data structure MIX and combines the +% component parameter matrices into a single row vector P. +% +% See also +% GMM, GMMUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(mix, 'gmm'); +if ~errstring + error(errstring); +end + +p = [mix.priors, mix.centres(:)', mix.covars(:)']; +if strcmp(mix.covar_type, 'ppca') + p = [p, mix.lambda(:)', mix.U(:)']; +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gmmpost.m b/sourcecodes/bnt-master/netlab3.3/gmmpost.m new file mode 100644 index 00000000..f2b7e224 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmpost.m @@ -0,0 +1,36 @@ +function [post, a] = gmmpost(mix, x) +%GMMPOST Computes the class posterior probabilities of a Gaussian mixture model. +% +% Description +% This function computes the posteriors POST (i.e. the probability of +% each component conditioned on the data P(J|X)) for a Gaussian mixture +% model. The data structure MIX defines the mixture model, while the +% matrix X contains the data vectors. Each row of X represents a +% single vector. +% +% See also +% GMM, GMMACTIV, GMMPROB +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check that inputs are consistent +errstring = consist(mix, 'gmm', x); +if ~isempty(errstring) + error(errstring); +end + +ndata = size(x, 1); + +a = gmmactiv(mix, x); + +post = (ones(ndata, 1)*mix.priors).*a; +s = sum(post, 2); +if any(s==0) + warning('Some zero posterior probabilities') + % Set any zeros to one before dividing + zero_rows = find(s==0); + s = s + (s==0); + post(zero_rows, :) = 1/mix.ncentres; +end +post = post./(s*ones(1, mix.ncentres)); diff --git a/sourcecodes/bnt-master/netlab3.3/gmmprob.m b/sourcecodes/bnt-master/netlab3.3/gmmprob.m new file mode 100644 index 00000000..8f438b66 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmprob.m @@ -0,0 +1,26 @@ +function prob = gmmprob(mix, x) +%GMMPROB Computes the data probability for a Gaussian mixture model. +% +% Description +% This function computes the unconditional data density P(X) for a +% Gaussian mixture model. The data structure MIX defines the mixture +% model, while the matrix X contains the data vectors. Each row of X +% represents a single vector. +% +% See also +% GMM, GMMPOST, GMMACTIV +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check that inputs are consistent +errstring = consist(mix, 'gmm', x); +if ~isempty(errstring) + error(errstring); +end + +% Compute activations +a = gmmactiv(mix, x); + +% Form dot product with priors +prob = a * (mix.priors)'; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gmmsamp.m b/sourcecodes/bnt-master/netlab3.3/gmmsamp.m new file mode 100644 index 00000000..8550ddf9 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmsamp.m @@ -0,0 +1,66 @@ +function [data, label] = gmmsamp(mix, n) +%GMMSAMP Sample from a Gaussian mixture distribution. +% +% Description +% +% DATA = GSAMP(MIX, N) generates a sample of size N from a Gaussian +% mixture distribution defined by the MIX data structure. The matrix X +% has N rows in which each row represents a MIX.NIN-dimensional sample +% vector. +% +% [DATA, LABEL] = GMMSAMP(MIX, N) also returns a column vector of +% classes (as an index 1..N) LABEL. +% +% See also +% GSAMP, GMM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check input arguments +errstring = consist(mix, 'gmm'); +if ~isempty(errstring) + error(errstring); +end +if n < 1 + error('Number of data points must be positive') +end + +% Determine number to sample from each component +priors = rand(1, n); + +% Pre-allocate data array +data = zeros(n, mix.nin); +if nargout > 1 + label = zeros(n, 1); +end +cum_prior = 0; % Cumulative sum of priors +total_samples = 0; % Cumulative sum of number of sampled points +for j = 1:mix.ncentres + num_samples = sum(priors >= cum_prior & ... + priors < cum_prior + mix.priors(j)); + % Form a full covariance matrix + switch mix.covar_type + case 'spherical' + covar = mix.covars(j) * eye(mix.nin); + case 'diag' + covar = diag(mix.covars(j, :)); + case 'full' + covar = mix.covars(:, :, j); + case 'ppca' + covar = mix.covars(j) * eye(mix.nin) + ... + mix.U(:, :, j)* ... + (diag(mix.lambda(j, :))-(mix.covars(j)*eye(mix.ppca_dim)))* ... + (mix.U(:, :, j)'); + otherwise + error(['Unknown covariance type ', mix.covar_type]); + end + data(total_samples+1:total_samples+num_samples, :) = ... + gsamp(mix.centres(j, :), covar, num_samples); + if nargout > 1 + label(total_samples+1:total_samples+num_samples) = j; + end + cum_prior = cum_prior + mix.priors(j); + total_samples = total_samples + num_samples; +end + diff --git a/sourcecodes/bnt-master/netlab3.3/gmmunpak.m b/sourcecodes/bnt-master/netlab3.3/gmmunpak.m new file mode 100644 index 00000000..9a503816 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmunpak.m @@ -0,0 +1,54 @@ +function mix = gmmunpak(mix, p) +%GMMUNPAK Separates a vector of Gaussian mixture model parameters into its components. +% +% Description +% MIX = GMMUNPAK(MIX, P) takes a GMM data structure MIX and a single +% row vector of parameters P and returns a mixture data structure +% identical to the input MIX, except that the mixing coefficients +% PRIORS, centres CENTRES and covariances COVARS (and, for PPCA, the +% lambdas and U (PCA sub-spaces)) are all set to the corresponding +% elements of P. +% +% See also +% GMM, GMMPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(mix, 'gmm'); +if ~errstring + error(errstring); +end +if mix.nwts ~= length(p) + error('Invalid weight vector length') +end + +mark1 = mix.ncentres; +mark2 = mark1 + mix.ncentres*mix.nin; + +mix.priors = reshape(p(1:mark1), 1, mix.ncentres); +mix.centres = reshape(p(mark1 + 1:mark2), mix.ncentres, mix.nin); +switch mix.covar_type + case 'spherical' + mark3 = mix.ncentres*(2 + mix.nin); + mix.covars = reshape(p(mark2 + 1:mark3), 1, mix.ncentres); + case 'diag' + mark3 = mix.ncentres*(1 + mix.nin + mix.nin); + mix.covars = reshape(p(mark2 + 1:mark3), mix.ncentres, mix.nin); + case 'full' + mark3 = mix.ncentres*(1 + mix.nin + mix.nin*mix.nin); + mix.covars = reshape(p(mark2 + 1:mark3), mix.nin, mix.nin, ... + mix.ncentres); + case 'ppca' + mark3 = mix.ncentres*(2 + mix.nin); + mix.covars = reshape(p(mark2 + 1:mark3), 1, mix.ncentres); + % Now also extract k and eigenspaces + mark4 = mark3 + mix.ncentres*mix.ppca_dim; + mix.lambda = reshape(p(mark3 + 1:mark4), mix.ncentres, ... + mix.ppca_dim); + mix.U = reshape(p(mark4 + 1:end), mix.nin, mix.ppca_dim, ... + mix.ncentres); + otherwise + error(['Unknown covariance type ', mix.covar_type]); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/gp.m b/sourcecodes/bnt-master/netlab3.3/gp.m new file mode 100644 index 00000000..54643faa --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gp.m @@ -0,0 +1,93 @@ +function net = gp(nin, covar_fn, prior) +%GP Create a Gaussian Process. +% +% Description +% +% NET = GP(NIN, COVARFN) takes the number of inputs NIN for a Gaussian +% Process model with a single output, together with a string COVARFN +% which specifies the type of the covariance function, and returns a +% data structure NET. The parameters are set to zero. +% +% The fields in NET are +% type = 'gp' +% nin = number of inputs +% nout = number of outputs: always 1 +% nwts = total number of weights and covariance function parameters +% bias = logarithm of constant offset in covariance function +% noise = logarithm of output noise variance +% inweights = logarithm of inverse length scale for each input +% covarfn = string describing the covariance function: +% 'sqexp' +% 'ratquad' +% fpar = covariance function specific parameters (1 for squared exponential, +% 2 for rational quadratic) +% trin = training input data (initially empty) +% trtargets = training target data (initially empty) +% +% NET = GP(NIN, COVARFN, PRIOR) sets a Gaussian prior on the parameters +% of the model. PRIOR must contain the fields PR_MEAN and PR_VARIANCE. +% If PR_MEAN is a scalar, then the Gaussian is assumed to be isotropic +% and the additional fields NET.PR_MEAN and PR_VARIANCE are set. +% Otherwise, the Gaussian prior has a mean defined by a column vector +% of parameters PRIOR.PR_MEAN and covariance defined by a column vector +% of parameters PRIOR.PR_VARIANCE. Each element of PRMEAN corresponds +% to a separate group of parameters, which need not be mutually +% exclusive. The membership of the groups is defined by the matrix +% PRIOR.INDEX in which the columns correspond to the elements of +% PRMEAN. Each column has one element for each weight in the matrix, in +% the order defined by the function GPPAK, and each element is 1 or 0 +% according to whether the parameter is a member of the corresponding +% group or not. The additional field NET.INDEX is set in this case. +% +% See also +% GPPAK, GPUNPAK, GPFWD, GPERR, GPCOVAR, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.type = 'gp'; +net.nin = nin; +net.nout = 1; % Only do single output GP + +% Store log parameters +net.bias = 0; +net.min_noise = sqrt(eps); % Prevent output noise collapsing completely +net.noise = 0; +net.inweights = zeros(1,nin); % Weights on inputs in covariance function + +covarfns = {'sqexp', 'ratquad'}; + +if sum(strcmp(covar_fn, covarfns)) == 0 + error('Undefined activation function. Exiting.'); +else + net.covar_fn = covar_fn; +end + +switch covar_fn + + case 'sqexp' % Squared exponential + net.fpar = zeros(1,1); % One function specific parameter + + case 'ratquad' % Rational quadratic + net.fpar = zeros(1, 2); % Two function specific parameters + + otherwise + error(['Unknown covariance function ', covar_fn]); +end + +net.nwts = 2 + nin + length(net.fpar); + +if nargin >= 3 + if size(prior.pr_mean) == [1 1] + net.pr_mean = prior.pr_mean; + net.pr_var = prior.pr_var; + else + net.pr_mean = prior.pr_mean; + net.pr_var = prior.pr_var; + net.index = prior.index; + end +end + +% Store training data as needed for gpfwd +net.tr_in = []; +net.tr_targets = []; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gpcovar.m b/sourcecodes/bnt-master/netlab3.3/gpcovar.m new file mode 100644 index 00000000..f71fa4db --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpcovar.m @@ -0,0 +1,38 @@ +function [cov, covf] = gpcovar(net, x) +%GPCOVAR Calculate the covariance for a Gaussian Process. +% +% Description +% +% COV = GPCOVAR(NET, X) takes a Gaussian Process data structure NET +% together with a matrix X of input vectors, and computes the +% covariance matrix COV. The inverse of this matrix is used when +% calculating the mean and variance of the predictions made by NET. +% +% [COV, COVF] = GPCOVAR(NET, X) also generates the covariance matrix +% due to the covariance function specified by NET.COVARFN as calculated +% by GPCOVARF. +% +% See also +% GP, GPPAK, GPUNPAK, GPCOVARP, GPCOVARF, GPFWD, GPERR, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'gp', x); +if ~isempty(errstring); + error(errstring); +end + +ndata = size(x, 1); + +% Compute prior covariance +if nargout >= 2 + [covp, covf] = gpcovarp(net, x, x); +else + covp = gpcovarp(net, x, x); +end + +% Add output noise variance +cov = covp + (net.min_noise + exp(net.noise))*eye(ndata); + diff --git a/sourcecodes/bnt-master/netlab3.3/gpcovarf.m b/sourcecodes/bnt-master/netlab3.3/gpcovarf.m new file mode 100644 index 00000000..ed66e60e --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpcovarf.m @@ -0,0 +1,44 @@ +function covf = gpcovarf(net, x1, x2) +%GPCOVARF Calculate the covariance function for a Gaussian Process. +% +% Description +% +% COVF = GPCOVARF(NET, X1, X2) takes a Gaussian Process data structure +% NET together with two matrices X1 and X2 of input vectors, and +% computes the matrix of the covariance function values COVF. +% +% See also +% GP, GPCOVAR, GPCOVARP, GPERR, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gp', x1); +if ~isempty(errstring); + error(errstring); +end + +if size(x1, 2) ~= size(x2, 2) + error('Number of variables in x1 and x2 must be the same'); +end + +n1 = size(x1, 1); +n2 = size(x2, 1); +beta = diag(exp(net.inweights)); + +% Compute the weighted squared distances between x1 and x2 +z = (x1.*x1)*beta*ones(net.nin, n2) - 2*x1*beta*x2' ... + + ones(n1, net.nin)*beta*(x2.*x2)'; + +switch net.covar_fn + + case 'sqexp' % Squared exponential + covf = exp(net.fpar(1) - 0.5*z); + + case 'ratquad' % Rational quadratic + nu = exp(net.fpar(2)); + covf = exp(net.fpar(1))*((ones(size(z)) + z).^(-nu)); + + otherwise + error(['Unknown covariance function ', net.covar_fn]); +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gpcovarp.m b/sourcecodes/bnt-master/netlab3.3/gpcovarp.m new file mode 100644 index 00000000..203c405c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpcovarp.m @@ -0,0 +1,30 @@ +function [covp, covf] = gpcovarp(net, x1, x2) +%GPCOVARP Calculate the prior covariance for a Gaussian Process. +% +% Description +% +% COVP = GPCOVARP(NET, X1, X2) takes a Gaussian Process data structure +% NET together with two matrices X1 and X2 of input vectors, and +% computes the matrix of the prior covariance. This is the function +% component of the covariance plus the exponential of the bias term. +% +% [COVP, COVF] = GPCOVARP(NET, X1, X2) also returns the function +% component of the covariance. +% +% See also +% GP, GPCOVAR, GPCOVARF, GPERR, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gp', x1); +if ~isempty(errstring); + error(errstring); +end + +if size(x1, 2) ~= size(x2, 2) + error('Number of variables in x1 and x2 must be the same'); +end + +covf = gpcovarf(net, x1, x2); +covp = covf + exp(net.bias); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gperr.m b/sourcecodes/bnt-master/netlab3.3/gperr.m new file mode 100644 index 00000000..402d63bd --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gperr.m @@ -0,0 +1,51 @@ +function [e, edata, eprior] = gperr(net, x, t) +%GPERR Evaluate error function for Gaussian Process. +% +% Description +% E = GPERR(NET, X, T) takes a Gaussian Process data structure NET +% together with a matrix X of input vectors and a matrix T of target +% vectors, and evaluates the error function E. Each row of X +% corresponds to one input vector and each row of T corresponds to one +% target vector. +% +% [E, EDATA, EPRIOR] = GPERR(NET, X, T) additionally returns the data +% and hyperprior components of the error, assuming a Gaussian prior on +% the weights with mean and variance parameters PRMEAN and PRVARIANCE +% taken from the network data structure NET. +% +% See also +% GP, GPCOVAR, GPFWD, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gp', x, t); +if ~isempty(errstring); + error(errstring); +end + +cn = gpcovar(net, x); + +edata = 0.5*(sum(log(eig(cn, 'nobalance'))) + t'*inv(cn)*t); + +% Evaluate the hyperprior contribution to the error. +% The hyperprior is Gaussian with mean pr_mean and variance +% pr_variance +if isfield(net, 'pr_mean') + w = gppak(net); + m = repmat(net.pr_mean, size(w)); + if size(net.pr_mean) == [1 1] + eprior = 0.5*((w-m)*(w-m)'); + e2 = eprior/net.pr_var; + else + wpr = repmat(w, size(net.pr_mean, 1), 1)'; + eprior = 0.5*(((wpr - m').^2).*net.index); + e2 = (sum(eprior, 1))*(1./net.pr_var); + end +else + e2 = 0; + eprior = 0; +end + +e = edata + e2; + diff --git a/sourcecodes/bnt-master/netlab3.3/gpfwd.m b/sourcecodes/bnt-master/netlab3.3/gpfwd.m new file mode 100644 index 00000000..b2d390e5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpfwd.m @@ -0,0 +1,51 @@ +function [y, sigsq] = gpfwd(net, x, cninv) +%GPFWD Forward propagation through Gaussian Process. +% +% Description +% Y = GPFWD(NET, X) takes a Gaussian Process data structure NET +% together with a matrix X of input vectors, and forward propagates +% the inputs through the model to generate a matrix Y of output +% vectors. Each row of X corresponds to one input vector and each row +% of Y corresponds to one output vector. This assumes that the +% training data (both inputs and targets) has been stored in NET by a +% call to GPINIT; these are needed to compute the training data +% covariance matrix. +% +% [Y, SIGSQ] = GPFWD(NET, X) also generates a column vector SIGSQ of +% conditional variances (or squared error bars) where each value +% corresponds to a pattern. +% +% [Y, SIGSQ] = GPFWD(NET, X, CNINV) uses the pre-computed inverse +% covariance matrix CNINV in the forward propagation. This increases +% efficiency if several calls to GPFWD are made. +% +% See also +% GP, DEMGP, GPINIT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gp', x); +if ~isempty(errstring); + error(errstring); +end + +if ~(isfield(net, 'tr_in') & isfield(net, 'tr_targets')) + error('Require training inputs and targets'); +end + +if nargin == 2 + % Inverse covariance matrix not supplied. + cninv = inv(gpcovar(net, net.tr_in)); +end +ktest = gpcovarp(net, x, net.tr_in); + +% Predict mean +y = ktest*cninv*net.tr_targets; + +if nargout >= 2 + % Predict error bar + ndata = size(x, 1); + sigsq = (ones(ndata, 1) * gpcovarp(net, x(1,:), x(1,:))) ... + - sum((ktest*cninv).*ktest, 2); +end diff --git a/sourcecodes/bnt-master/netlab3.3/gpgrad.m b/sourcecodes/bnt-master/netlab3.3/gpgrad.m new file mode 100644 index 00000000..7ea531bc --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpgrad.m @@ -0,0 +1,100 @@ +function g = gpgrad(net, x, t) +%GPGRAD Evaluate error gradient for Gaussian Process. +% +% Description +% G = GPGRAD(NET, X, T) takes a Gaussian Process data structure NET +% together with a matrix X of input vectors and a matrix T of target +% vectors, and evaluates the error gradient G. Each row of X +% corresponds to one input vector and each row of T corresponds to one +% target vector. +% +% See also +% GP, GPCOVAR, GPFWD, GPERR +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gp', x, t); +if ~isempty(errstring); + error(errstring); +end + +% Evaluate derivatives with respect to each hyperparameter in turn. +ndata = size(x, 1); +[cov, covf] = gpcovar(net, x); +cninv = inv(cov); +trcninv = trace(cninv); +cninvt = cninv*t; + +% Function parameters +switch net.covar_fn + + case 'sqexp' % Squared exponential + gfpar = trace(cninv*covf) - cninvt'*covf*cninvt; + + case 'ratquad' % Rational quadratic + beta = diag(exp(net.inweights)); + gfpar(1) = trace(cninv*covf) - cninvt'*covf*cninvt; + D2 = (x.*x)*beta*ones(net.nin, ndata) - 2*x*beta*x' ... + + ones(ndata, net.nin)*beta*(x.*x)'; + E = ones(size(D2)); + L = - exp(net.fpar(2)) * covf .* log(E + D2); % d(cn)/d(nu) + gfpar(2) = trace(cninv*L) - cninvt'*L*cninvt; + + otherwise + error(['Unknown covariance function ', net.covar_fn]); +end + +% Bias derivative +ndata = size(x, 1); +fac = exp(net.bias)*ones(ndata); +gbias = trace(cninv*fac) - cninvt'*fac*cninvt; + +% Noise derivative +gnoise = exp(net.noise)*(trcninv - cninvt'*cninvt); + +% Input weight derivatives +if strcmp(net.covar_fn, 'ratquad') + F = (exp(net.fpar(2))*E)./(E + D2); +end + +nparams = length(net.inweights); +for l = 1 : nparams + vect = x(:, l); + matx = (vect.*vect)*ones(1, ndata) ... + - 2.0*vect*vect' ... + + ones(ndata, 1)*(vect.*vect)'; + switch net.covar_fn + case 'sqexp' % Squared exponential + dmat = -0.5*exp(net.inweights(l))*covf.*matx; + + case 'ratquad' % Rational quadratic + dmat = - exp(net.inweights(l))*covf.*matx.*F; + otherwise + error(['Unknown covariance function ', net.covar_fn]); + end + + gw1(l) = trace(cninv*dmat) - cninvt'*dmat*cninvt; +end + +g1 = [gbias, gnoise, gw1, gfpar]; +g1 = 0.5*g1; + +% Evaluate the prior contribution to the gradient. +if isfield(net, 'pr_mean') + w = gppak(net); + m = repmat(net.pr_mean, size(w)); + if size(net.pr_mean) == [1 1] + gprior = w - m; + g2 = gprior/net.pr_var; + else + ngroups = size(net.pr_mean, 1); + gprior = net.index'.*(ones(ngroups, 1)*w - m); + g2 = (1./net.pr_var)'*gprior; + end +else + gprior = 0; + g2 = 0; +end + +g = g1 + g2; diff --git a/sourcecodes/bnt-master/netlab3.3/gpinit.m b/sourcecodes/bnt-master/netlab3.3/gpinit.m new file mode 100644 index 00000000..c355c0d5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpinit.m @@ -0,0 +1,43 @@ +function net = gpinit(net, tr_in, tr_targets, prior) +%GPINIT Initialise Gaussian Process model. +% +% Description +% NET = GPINIT(NET, TRIN, TRTARGETS) takes a Gaussian Process data +% structure NET together with a matrix TRIN of training input vectors +% and a matrix TRTARGETS of training target vectors, and stores them +% in NET. These datasets are required if the corresponding inverse +% covariance matrix is not supplied to GPFWD. This is important if the +% data structure is saved and then reloaded before calling GPFWD. Each +% row of TRIN corresponds to one input vector and each row of TRTARGETS +% corresponds to one target vector. +% +% NET = GPINIT(NET, TRIN, TRTARGETS, PRIOR) additionally initialises +% the parameters in NET from the PRIOR data structure which contains +% the mean and variance of the Gaussian distribution which is sampled +% from. +% +% See also +% GP, GPFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gp', tr_in, tr_targets); +if ~isempty(errstring); + error(errstring); +end + +if nargin >= 4 + % Initialise weights at random + if size(prior.pr_mean) == [1 1] + w = randn(1, net.nwts).*sqrt(prior.pr_var) + ... + repmat(prior.pr_mean, 1, net.nwts); + else + sig = sqrt(prior.index*prior.pr_var); + w = sig'.*randn(1, net.nwts) + (prior.index*prior.pr_mean)'; + end + net = gpunpak(net, w); +end + +net.tr_in = tr_in; +net.tr_targets = tr_targets; diff --git a/sourcecodes/bnt-master/netlab3.3/gppak.m b/sourcecodes/bnt-master/netlab3.3/gppak.m new file mode 100644 index 00000000..bb1c52a8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gppak.m @@ -0,0 +1,19 @@ +function hp = gppak(net) +%GPPAK Combines GP hyperparameters into one vector. +% +% Description +% HP = GPPAK(NET) takes a Gaussian Process data structure NET and +% combines the hyperparameters into a single row vector HP. +% +% See also +% GP, GPUNPAK, GPFWD, GPERR, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'gp'); +if ~isempty(errstring); + error(errstring); +end +hp = [net.bias, net.noise, net.inweights, net.fpar]; diff --git a/sourcecodes/bnt-master/netlab3.3/gpunpak.m b/sourcecodes/bnt-master/netlab3.3/gpunpak.m new file mode 100644 index 00000000..8e1058e4 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpunpak.m @@ -0,0 +1,36 @@ +function net = gpunpak(net, hp) +%GPUNPAK Separates hyperparameter vector into components. +% +% Description +% NET = GPUNPAK(NET, HP) takes an Gaussian Process data structure NET +% and a hyperparameter vector HP, and returns a Gaussian Process data +% structure identical to the input model, except that the covariance +% bias BIAS, output noise NOISE, the input weight vector INWEIGHTS and +% the vector of covariance function specific parameters FPAR have all +% been set to the corresponding elements of HP. +% +% See also +% GP, GPPAK, GPFWD, GPERR, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'gp'); +if ~isempty(errstring); + error(errstring); +end +if net.nwts ~= length(hp) + error('Invalid weight vector length'); +end + +net.bias = hp(1); +net.noise = hp(2); + +% Unpack input weights +mark1 = 2 + net.nin; +net.inweights = hp(3:mark1); + +% Unpack function specific parameters +net.fpar = hp(mark1 + 1:size(hp, 2)); + diff --git a/sourcecodes/bnt-master/netlab3.3/gradchek.m b/sourcecodes/bnt-master/netlab3.3/gradchek.m new file mode 100644 index 00000000..3f9298fb --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gradchek.m @@ -0,0 +1,49 @@ +function [gradient, delta] = gradchek(w, func, grad, varargin) +%GRADCHEK Checks a user-defined gradient function using finite differences. +% +% Description +% This function is intended as a utility for other netlab functions +% (particularly optimisation functions) to use. It enables the user to +% check whether a gradient calculation has been correctly implmented +% for a given function. GRADCHEK(W, FUNC, GRAD) checks how accurate the +% gradient GRAD of a function FUNC is at a parameter vector X. A +% central difference formula with step size 1.0e-6 is used, and the +% results for both gradient function and finite difference +% approximation are printed. The optional return value GRADIENT is the +% gradient calculated using the function GRAD and the return value +% DELTA is the difference between the functional and finite difference +% methods of calculating the graident. +% +% GRADCHEK(X, FUNC, GRAD, P1, P2, ...) allows additional arguments to +% be passed to FUNC and GRAD. +% +% See also +% CONJGRAD, GRADDESC, HMC, OLGD, QUASINEW, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Reasonable value for step size +epsilon = 1.0e-6; + +func = fcnchk(func, length(varargin)); +grad = fcnchk(grad, length(varargin)); + +% Treat +nparams = length(w); +deltaf = zeros(1, nparams); +step = zeros(1, nparams); +for i = 1:nparams + % Move a small way in the ith coordinate of w + step(i) = 1.0; + fplus = feval('linef', epsilon, func, w, step, varargin{:}); + fminus = feval('linef', -epsilon, func, w, step, varargin{:}); + % Use central difference formula for approximation + deltaf(i) = 0.5*(fplus - fminus)/epsilon; + step(i) = 0.0; +end +gradient = feval(grad, w, varargin{:}); +fprintf(1, 'Checking gradient ...\n\n'); +delta = gradient - deltaf; +fprintf(1, ' analytic diffs delta\n\n'); +disp([gradient', deltaf', delta']) diff --git a/sourcecodes/bnt-master/netlab3.3/graddesc.m b/sourcecodes/bnt-master/netlab3.3/graddesc.m new file mode 100644 index 00000000..3cc64141 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/graddesc.m @@ -0,0 +1,180 @@ +function [x, options, flog, pointlog] = graddesc(f, x, options, gradf, ... + varargin) +%GRADDESC Gradient descent optimization. +% +% Description +% [X, OPTIONS, FLOG, POINTLOG] = GRADDESC(F, X, OPTIONS, GRADF) uses +% batch gradient descent to find a local minimum of the function F(X) +% whose gradient is given by GRADF(X). A log of the function values +% after each cycle is (optionally) returned in ERRLOG, and a log of the +% points visited is (optionally) returned in POINTLOG. +% +% Note that X is a row vector and F returns a scalar value. The point +% at which F has a local minimum is returned as X. The function value +% at that point is returned in OPTIONS(8). +% +% GRADDESC(F, X, OPTIONS, GRADF, P1, P2, ...) allows additional +% arguments to be passed to F() and GRADF(). +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG, and the points visited in the +% return argument POINTSLOG. If OPTIONS(1) is set to 0, then only +% warning messages are displayed. If OPTIONS(1) is -1, then nothing is +% displayed. +% +% OPTIONS(2) is the absolute precision required for the value of X at +% the solution. If the absolute difference between the values of X +% between two successive steps is less than OPTIONS(2), then this +% condition is satisfied. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. If the absolute difference between the +% objective function values between two successive steps is less than +% OPTIONS(3), then this condition is satisfied. Both this and the +% previous condition must be satisfied for termination. +% +% OPTIONS(7) determines the line minimisation method used. If it is +% set to 1 then a line minimiser is used (in the direction of the +% negative gradient). If it is 0 (the default), then each parameter +% update is a fixed multiple (the learning rate) of the negative +% gradient added to a fixed multiple (the momentum) of the previous +% parameter update. +% +% OPTIONS(9) should be set to 1 to check the user defined gradient +% function GRADF with GRADCHEK. This is carried out at the initial +% parameter vector X. +% +% OPTIONS(10) returns the total number of function evaluations +% (including those in any line searches). +% +% OPTIONS(11) returns the total number of gradient evaluations. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% OPTIONS(15) is the precision in parameter space of the line search; +% default FOPTIONS(2). +% +% OPTIONS(17) is the momentum; default 0.5. It should be scaled by the +% inverse of the number of data points. +% +% OPTIONS(18) is the learning rate; default 0.01. It should be scaled +% by the inverse of the number of data points. +% +% See also +% CONJGRAD, LINEMIN, OLGD, MINBRACK, QUASINEW, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set up the options. +if length(options) < 18 + error('Options vector too short') +end + +if (options(14)) + niters = options(14); +else + niters = 100; +end + +line_min_flag = 0; % Flag for line minimisation option +if (round(options(7)) == 1) + % Use line minimisation + line_min_flag = 1; + % Set options for line minimiser + line_options = foptions; + if options(15) > 0 + line_options(2) = options(15); + end +else + % Learning rate: must be positive + if (options(18) > 0) + eta = options(18); + else + eta = 0.01; + end + % Momentum term: allow zero momentum + if (options(17) >= 0) + mu = options(17); + else + mu = 0.5; + end +end + +% Check function string +f = fcnchk(f, length(varargin)); +gradf = fcnchk(gradf, length(varargin)); + +% Display information if options(1) > 0 +display = options(1) > 0; + +% Work out if we need to compute f at each iteration. +% Needed if using line search or if display results or if termination +% criterion requires it. +fcneval = (options(7) | display | options(3)); + +% Check gradients +if (options(9) > 0) + feval('gradchek', x, f, gradf, varargin{:}); +end + +dxold = zeros(1, size(x, 2)); +xold = x; +fold = 0; % Must be initialised so that termination test can be performed +if fcneval + fnew = feval(f, x, varargin{:}); + options(10) = options(10) + 1; + fold = fnew; +end + +% Main optimization loop. +for j = 1:niters + xold = x; + grad = feval(gradf, x, varargin{:}); + options(11) = options(11) + 1; % Increment gradient evaluation counter + if (line_min_flag ~= 1) + dx = mu*dxold - eta*grad; + x = x + dx; + dxold = dx; + if fcneval + fold = fnew; + fnew = feval(f, x, varargin{:}); + options(10) = options(10) + 1; + end + else + sd = - grad./norm(grad); % New search direction. + fold = fnew; + % Do a line search: normalise search direction to have length 1 + [lmin, line_options] = feval('linemin', f, x, sd, fold, ... + line_options, varargin{:}); + options(10) = options(10) + line_options(10); + x = xold + lmin*sd; + fnew = line_options(8); + end + if nargout >= 3 + flog(j) = fnew; + if nargout >= 4 + pointlog(j, :) = x; + end + end + if display + fprintf(1, 'Cycle %5d Function %11.8f\n', j, fnew); + end + if (max(abs(x - xold)) < options(2) & abs(fnew - fold) < options(3)) + % Termination criteria are met + options(8) = fnew; + return; + end +end + +if fcneval + options(8) = fnew; +else + options(8) = feval(f, x, varargin{:}); + options(10) = options(10) + 1; +end +if (options(1) >= 0) + disp(maxitmess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/gsamp.m b/sourcecodes/bnt-master/netlab3.3/gsamp.m new file mode 100644 index 00000000..58b1ba1d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gsamp.m @@ -0,0 +1,32 @@ +function x = gsamp(mu, covar, nsamp) +%GSAMP Sample from a Gaussian distribution. +% +% Description +% +% X = GSAMP(MU, COVAR, NSAMP) generates a sample of size NSAMP from a +% D-dimensional Gaussian distribution. The Gaussian density has mean +% vector MU and covariance matrix COVAR, and the matrix X has NSAMP +% rows in which each row represents a D-dimensional sample vector. +% +% See also +% GAUSS, DEMGAUSS +% + +% Copyright (c) Ian T Nabney (1996-2001) + +d = size(covar, 1); + +mu = reshape(mu, 1, d); % Ensure that mu is a row vector + +[evec, eval] = eig(covar); + +deig=diag(eval); + +if (~isreal(deig)) | any(deig<0), + warning('Covariance Matrix is not OK, redefined to be positive definite'); + eval=abs(eval); +end + +coeffs = randn(nsamp, d)*sqrt(eval); + +x = ones(nsamp, 1)*mu + coeffs*evec'; diff --git a/sourcecodes/bnt-master/netlab3.3/gtm.m b/sourcecodes/bnt-master/netlab3.3/gtm.m new file mode 100644 index 00000000..9d1b3ab3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtm.m @@ -0,0 +1,61 @@ +function net = gtm(dim_latent, nlatent, dim_data, ncentres, rbfunc, ... + prior) +%GTM Create a Generative Topographic Map. +% +% Description +% +% NET = GTM(DIMLATENT, NLATENT, DIMDATA, NCENTRES, RBFUNC), takes the +% dimension of the latent space DIMLATENT, the number of data points +% sampled in the latent space NLATENT, the dimension of the data space +% DIMDATA, the number of centres in the RBF model NCENTRES, the +% activation function for the RBF RBFUNC and returns a data structure +% NET. The parameters in the RBF and GMM sub-models are set by calls to +% the corresponding creation routines RBF and GMM. +% +% The fields in NET are +% type = 'gtm' +% nin = dimension of data space +% dimlatent = dimension of latent space +% rbfnet = RBF network data structure +% gmmnet = GMM data structure +% X = sample of latent points +% +% NET = GTM(DIMLATENT, NLATENT, DIMDATA, NCENTRES, RBFUNC, PRIOR), +% sets a Gaussian zero mean prior on the parameters of the RBF model. +% PRIOR must be a scalar and represents the inverse variance of the +% prior distribution. This gives rise to a weight decay term in the +% error function. +% +% See also +% GTMFWD, GTMPOST, RBF, GMM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.type = 'gtm'; +% Input to functions is data +net.nin = dim_data; +net.dim_latent = dim_latent; + +% Default is no regularisation +if nargin == 5 + prior = 0.0; +end + +% Only allow scalar prior +if isstruct(prior) | size(prior) ~= [1 1] + error('Prior must be a scalar'); +end + +% Create RBF network +net.rbfnet = rbf(dim_latent, ncentres, dim_data, rbfunc, ... + 'linear', prior); + +% Mask all but output weights +net.rbfnet.mask = rbfprior(rbfunc, dim_latent, ncentres, dim_data); + +% Create field for GMM output model +net.gmmnet = gmm(dim_data, nlatent, 'spherical'); + +% Create empty latent data sample +net.X = []; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gtmem.m b/sourcecodes/bnt-master/netlab3.3/gtmem.m new file mode 100644 index 00000000..47dd84a5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmem.m @@ -0,0 +1,135 @@ +function [net, options, errlog] = gtmem(net, t, options) +%GTMEM EM algorithm for Generative Topographic Mapping. +% +% Description +% [NET, OPTIONS, ERRLOG] = GTMEM(NET, T, OPTIONS) uses the Expectation +% Maximization algorithm to estimate the parameters of a GTM defined by +% a data structure NET. The matrix T represents the data whose +% expectation is maximized, with each row corresponding to a vector. +% It is assumed that the latent data NET.X has been set following a +% call to GTMINIT, for example. The optional parameters have the +% following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG. If OPTIONS(1) is set to 0, then +% only warning messages are displayed. If OPTIONS(1) is -1, then +% nothing is displayed. +% +% OPTIONS(3) is a measure of the absolute precision required of the +% error function at the solution. If the change in log likelihood +% between two steps of the EM algorithm is less than this value, then +% the function terminates. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% The optional return value OPTIONS contains the final error value +% (i.e. data log likelihood) in OPTIONS(8). +% +% See also +% GTM, GTMINIT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check that inputs are consistent +errstring = consist(net, 'gtm', t); +if ~isempty(errstring) + error(errstring); +end + +% Sort out the options +if (options(14)) + niters = options(14); +else + niters = 100; +end + +display = options(1); +store = 0; +if (nargout > 2) + store = 1; % Store the error values to return them + errlog = zeros(1, niters); +end +test = 0; +if options(3) > 0.0 + test = 1; % Test log likelihood for termination +end + +% Calculate various quantities that remain constant during training +[ndata, tdim] = size(t); +ND = ndata*tdim; +[net.gmmnet.centres, Phi] = rbffwd(net.rbfnet, net.X); +Phi = [Phi ones(size(net.X, 1), 1)]; +PhiT = Phi'; +[K, Mplus1] = size(Phi); + +A = zeros(Mplus1, Mplus1); +cholDcmp = zeros(Mplus1, Mplus1); +% Use a sparse representation for the weight regularizing matrix. +if (net.rbfnet.alpha > 0) + Alpha = net.rbfnet.alpha*speye(Mplus1); + Alpha(Mplus1, Mplus1) = 0; +end + +for n = 1:niters + % Calculate responsibilities + [R, act] = gtmpost(net, t); + % Calculate error value if needed + if (display | store | test) + prob = act*(net.gmmnet.priors)'; + % Error value is negative log likelihood of data + e = - sum(log(max(prob,eps))); + if store + errlog(n) = e; + end + if display > 0 + fprintf(1, 'Cycle %4d Error %11.6f\n', n, e); + end + if test + if (n > 1 & abs(e - eold) < options(3)) + options(8) = e; + return; + else + eold = e; + end + end + end + + % Calculate matrix be inverted (Phi'*G*Phi + alpha*I in the papers). + % Sparse representation of G normally executes faster and saves + % memory + if (net.rbfnet.alpha > 0) + A = full(PhiT*spdiags(sum(R)', 0, K, K)*Phi + ... + (Alpha.*net.gmmnet.covars(1))); + else + A = full(PhiT*spdiags(sum(R)', 0, K, K)*Phi); + end + % A is a symmetric matrix likely to be positive definite, so try + % fast Cholesky decomposition to calculate W, otherwise use SVD. + % (PhiT*(R*t)) is computed right-to-left, as R + % and t are normally (much) larger than PhiT. + [cholDcmp singular] = chol(A); + if (singular) + if (display) + fprintf(1, ... + 'gtmem: Warning -- M-Step matrix singular, using pinv.\n'); + end + W = pinv(A)*(PhiT*(R'*t)); + else + W = cholDcmp \ (cholDcmp' \ (PhiT*(R'*t))); + end + % Put new weights into network to calculate responsibilities + % net.rbfnet = netunpak(net.rbfnet, W); + net.rbfnet.w2 = W(1:net.rbfnet.nhidden, :); + net.rbfnet.b2 = W(net.rbfnet.nhidden+1, :); + % Calculate new distances + d = dist2(t, Phi*W); + + % Calculate new value for beta + net.gmmnet.covars = ones(1, net.gmmnet.ncentres)*(sum(sum(d.*R))/ND); +end + +options(8) = -sum(log(gtmprob(net, t))); +if (display >= 0) + disp(maxitmess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/gtmfwd.m b/sourcecodes/bnt-master/netlab3.3/gtmfwd.m new file mode 100644 index 00000000..bb84d344 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmfwd.m @@ -0,0 +1,17 @@ +function mix = gtmfwd(net) +%GTMFWD Forward propagation through GTM. +% +% Description +% MIX = GTMFWD(NET) takes a GTM structure NET, and forward propagates +% the latent data sample NET.X through the GTM to generate the +% structure MIX which represents the Gaussian mixture model in data +% space. +% +% See also +% GTM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.gmmnet.centres = rbffwd(net.rbfnet, net.X); +mix = net.gmmnet; diff --git a/sourcecodes/bnt-master/netlab3.3/gtminit.m b/sourcecodes/bnt-master/netlab3.3/gtminit.m new file mode 100644 index 00000000..ae646a09 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtminit.m @@ -0,0 +1,153 @@ +function net = gtminit(net, options, data, samp_type, varargin) +%GTMINIT Initialise the weights and latent sample in a GTM. +% +% Description +% NET = GTMINIT(NET, OPTIONS, DATA, SAMPTYPE) takes a GTM NET and +% generates a sample of latent data points and sets the centres (and +% widths if appropriate) of NET.RBFNET. +% +% If the SAMPTYPE is 'REGULAR', then regular grids of latent data +% points and RBF centres are created. The dimension of the latent data +% space must be 1 or 2. For one-dimensional latent space, the +% LSAMPSIZE parameter gives the number of latent points and the +% RBFSAMPSIZE parameter gives the number of RBF centres. For a two- +% dimensional latent space, these parameters must be vectors of length +% 2 with the number of points in each of the x and y directions to +% create a rectangular grid. The widths of the RBF basis functions are +% set by a call to RBFSETFW passing OPTIONS(7) as the scaling +% parameter. +% +% If the SAMPTYPE is 'UNIFORM' or 'GAUSSIAN' then the latent data is +% found by sampling from a uniform or Gaussian distribution +% correspondingly. The RBF basis function parameters are set by a call +% to RBFSETBF with the DATA parameter as dataset and the OPTIONS +% vector. +% +% Finally, the output layer weights of the RBF are initialised by +% mapping the mean of the latent variable to the mean of the target +% variable, and the L-dimensional latent variale variance to the +% variance of the targets along the first L principal components. +% +% See also +% GTM, GTMEM, PCA, RBFSETBF, RBFSETFW +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check for consistency +errstring = consist(net, 'gtm', data); +if ~isempty(errstring) + error(errstring); +end + +% Check type of sample +stypes = {'regular', 'uniform', 'gaussian'}; +if (strcmp(samp_type, stypes)) == 0 + error('Undefined sample type.') +end + +if net.dim_latent > size(data, 2) + error('Latent space dimension must not be greater than data dimension') +end +nlatent = net.gmmnet.ncentres; +nhidden = net.rbfnet.nhidden; + +% Create latent data sample and set RBF centres + +switch samp_type +case 'regular' + if nargin ~= 6 + error('Regular type must specify latent and RBF shapes'); + end + l_samp_size = varargin{1}; + rbf_samp_size = varargin{2}; + if round(l_samp_size) ~= l_samp_size + error('Latent sample specification must contain integers') + end + % Check existence and size of rbf specification + if any(size(rbf_samp_size) ~= [1 net.dim_latent]) | ... + prod(rbf_samp_size) ~= nhidden + error('Incorrect specification of RBF centres') + end + % Check dimension and type of latent data specification + if any(size(l_samp_size) ~= [1 net.dim_latent]) | ... + prod(l_samp_size) ~= nlatent + error('Incorrect dimension of latent sample spec.') + end + if net.dim_latent == 1 + net.X = [-1:2/(l_samp_size-1):1]'; + net.rbfnet.c = [-1:2/(rbf_samp_size-1):1]'; + net.rbfnet = rbfsetfw(net.rbfnet, options(7)); + elseif net.dim_latent == 2 + net.X = gtm_rctg(l_samp_size); + net.rbfnet.c = gtm_rctg(rbf_samp_size); + net.rbfnet = rbfsetfw(net.rbfnet, options(7)); + else + error('For regular sample, input dimension must be 1 or 2.') + end + + +case {'uniform', 'gaussian'} + if strcmp(samp_type, 'uniform') + net.X = 2 * (rand(nlatent, net.dim_latent) - 0.5); + else + % Sample from N(0, 0.25) distribution to ensure most latent + % data is inside square + net.X = randn(nlatent, net.dim_latent)/2; + end + net.rbfnet = rbfsetbf(net.rbfnet, options, net.X); +otherwise + % Shouldn't get here + error('Invalid sample type'); + +end + +% Latent data sample and basis function parameters chosen. +% Now set output weights +[PCcoeff, PCvec] = pca(data); + +% Scale PCs by eigenvalues +A = PCvec(:, 1:net.dim_latent)*diag(sqrt(PCcoeff(1:net.dim_latent))); + +[temp, Phi] = rbffwd(net.rbfnet, net.X); +% Normalise X to ensure 1:1 mapping of variances and calculate weights +% as solution of Phi*W = normX*A' +normX = (net.X - ones(size(net.X))*diag(mean(net.X)))*diag(1./std(net.X)); +net.rbfnet.w2 = Phi \ (normX*A'); +% Bias is mean of target data +net.rbfnet.b2 = mean(data); + +% Must also set initial value of variance +% Find average distance between nearest centres +% Ensure that distance of centre to itself is excluded by setting diagonal +% entries to realmax +net.gmmnet.centres = rbffwd(net.rbfnet, net.X); +d = dist2(net.gmmnet.centres, net.gmmnet.centres) + ... + diag(ones(net.gmmnet.ncentres, 1)*realmax); +sigma = mean(min(d))/2; + +% Now set covariance to minimum of this and next largest eigenvalue +if net.dim_latent < size(data, 2) + sigma = min(sigma, PCcoeff(net.dim_latent+1)); +end +net.gmmnet.covars = sigma*ones(1, net.gmmnet.ncentres); + +% Sub-function to create the sample data in 2d +function sample = gtm_rctg(samp_size) + +xDim = samp_size(1); +yDim = samp_size(2); +% Produce a grid with the right number of rows and columns +[X, Y] = meshgrid([0:1:(xDim-1)], [(yDim-1):-1:0]); + +% Change grid representation +sample = [X(:), Y(:)]; + +% Shift grid to correct position and scale it +maxXY= max(sample); +sample(:,1) = 2*(sample(:,1) - maxXY(1)/2)./maxXY(1); +sample(:,2) = 2*(sample(:,2) - maxXY(2)/2)./maxXY(2); +return; + + + diff --git a/sourcecodes/bnt-master/netlab3.3/gtmlmean.m b/sourcecodes/bnt-master/netlab3.3/gtmlmean.m new file mode 100644 index 00000000..8aad8103 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmlmean.m @@ -0,0 +1,22 @@ +function means = gtmlmean(net, data) +%GTMLMEAN Mean responsibility for data in a GTM. +% +% Description +% MEANS = GTMLMEAN(NET, DATA) takes a GTM structure NET, and computes +% the means of the responsibility distributions for each data point in +% DATA. +% +% See also +% GTM, GTMPOST, GTMLMODE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check for consistency +errstring = consist(net, 'gtm', data); +if ~isempty(errstring) + error(errstring); +end + +R = gtmpost(net, data); +means = R*net.X; diff --git a/sourcecodes/bnt-master/netlab3.3/gtmlmode.m b/sourcecodes/bnt-master/netlab3.3/gtmlmode.m new file mode 100644 index 00000000..6fee04a8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmlmode.m @@ -0,0 +1,25 @@ +function modes = gtmlmode(net, data) +%GTMLMODE Mode responsibility for data in a GTM. +% +% Description +% MODES = GTMLMODE(NET, DATA) takes a GTM structure NET, and computes +% the modes of the responsibility distributions for each data point in +% DATA. These will always lie at one of the latent space sample points +% NET.X. +% +% See also +% GTM, GTMPOST, GTMLMEAN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check for consistency +errstring = consist(net, 'gtm', data); +if ~isempty(errstring) + error(errstring); +end + +R = gtmpost(net, data); +% Mode is maximum responsibility +[max_resp, max_index] = max(R, [], 2); +modes = net.X(max_index, :); diff --git a/sourcecodes/bnt-master/netlab3.3/gtmmag.m b/sourcecodes/bnt-master/netlab3.3/gtmmag.m new file mode 100644 index 00000000..f0014971 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmmag.m @@ -0,0 +1,27 @@ +function mags = gtmmag(net, latent_data) +%GTMMAG Magnification factors for a GTM +% +% Description +% MAGS = GTMMAG(NET, LATENTDATA) takes a GTM structure NET, and +% computes the magnification factors for each point the latent space +% contained in LATENTDATA. +% +% See also +% GTM, GTMPOST, GTMLMEAN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gtm'); +if ~isempty(errstring) + error(errstring); +end + +Jacs = rbfjacob(net.rbfnet, latent_data); +nlatent = size(latent_data, 1); +mags = zeros(nlatent, 1); +temp = zeros(net.rbfnet.nin, net.rbfnet.nout); +for m = 1:nlatent + temp = squeeze(Jacs(m, :, :)); % Turn into a 2d matrix + mags(m) = sqrt(det(temp*temp')); +end diff --git a/sourcecodes/bnt-master/netlab3.3/gtmpost.m b/sourcecodes/bnt-master/netlab3.3/gtmpost.m new file mode 100644 index 00000000..d9182af0 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmpost.m @@ -0,0 +1,26 @@ +function [post, a] = gtmpost(net, data) +%GTMPOST Latent space responsibility for data in a GTM. +% +% Description +% POST = GTMPOST(NET, DATA) takes a GTM structure NET, and computes +% the responsibility at each latent space sample point NET.X for each +% data point in DATA. +% +% [POST, A] = GTMPOST(NET, DATA) also returns the activations A of the +% GMM NET.GMMNET as computed by GMMPOST. +% +% See also +% GTM, GTMEM, GTMLMEAN, GMLMODE, GMMPROB +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check for consistency +errstring = consist(net, 'gtm', data); +if ~isempty(errstring) + error(errstring); +end + +net.gmmnet.centres = rbffwd(net.rbfnet, net.X); + +[post, a] = gmmpost(net.gmmnet, data); diff --git a/sourcecodes/bnt-master/netlab3.3/gtmprob.m b/sourcecodes/bnt-master/netlab3.3/gtmprob.m new file mode 100644 index 00000000..203216a2 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmprob.m @@ -0,0 +1,22 @@ +function prob = gtmprob(net, data) +%GTMPROB Probability for data under a GTM. +% +% Description +% PROB = GTMPROB(NET, DATA) takes a GTM structure NET, and computes +% the probability of each point in the dataset DATA. +% +% See also +% GTM, GTMEM, GTMLMEAN, GTMLMODE, GTMPOST +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check for consistency +errstring = consist(net, 'gtm', data); +if ~isempty(errstring) + error(errstring); +end + +net.gmmnet.centres = rbffwd(net.rbfnet, net.X); + +prob = gmmprob(net.gmmnet, data); diff --git a/sourcecodes/bnt-master/netlab3.3/hbayes.m b/sourcecodes/bnt-master/netlab3.3/hbayes.m new file mode 100644 index 00000000..65864783 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/hbayes.m @@ -0,0 +1,49 @@ +function [h, hdata] = hbayes(net, hdata) +%HBAYES Evaluate Hessian of Bayesian error function for network. +% +% Description +% H = HBAYES(NET, HDATA) takes a network data structure NET together +% the data contribution to the Hessian for a set of inputs and targets. +% It returns the regularised Hessian using any zero mean Gaussian +% priors on the weights defined in NET. In addition, if a MASK is +% defined in NET, then the entries in H that correspond to weights with +% a 0 in the mask are removed. +% +% [H, HDATA] = HBAYES(NET, HDATA) additionally returns the data +% component of the Hessian. +% +% See also +% GBAYES, GLMHESS, MLPHESS, RBFHESS +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if (isfield(net, 'mask')) + % Extract relevant entries in Hessian + nmask_rows = size(find(net.mask), 1); + hdata = reshape(hdata(logical(net.mask*(net.mask'))), ... + nmask_rows, nmask_rows); + nwts = nmask_rows; +else + nwts = net.nwts; +end +if isfield(net, 'beta') + h = net.beta*hdata; +else + h = hdata; +end + +if isfield(net, 'alpha') + if size(net.alpha) == [1 1] + h = h + net.alpha*eye(nwts); + else + if isfield(net, 'mask') + nindx_cols = size(net.index, 2); + index = reshape(net.index(logical(repmat(net.mask, ... + 1, nindx_cols))), nmask_rows, nindx_cols); + else + index = net.index; + end + h = h + diag(index*net.alpha); + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/hesschek.m b/sourcecodes/bnt-master/netlab3.3/hesschek.m new file mode 100644 index 00000000..ffd7253d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/hesschek.m @@ -0,0 +1,60 @@ +function h = hesschek(net, x, t) +%HESSCHEK Use central differences to confirm correct evaluation of Hessian matrix. +% +% Description +% +% HESSCHEK(NET, X, T) takes a network data structure NET, together with +% input and target data matrices X and T, and compares the evaluation +% of the Hessian matrix using the function NETHESS and using central +% differences with the function NETERR. +% +% The optional return value H is the Hessian computed using NETHESS. +% +% See also +% NETHESS, NETERR +% + +% Copyright (c) Ian T Nabney (1996-2001) + +w0 = netpak(net); +nwts = length(w0); +h = nethess(w0, net, x, t); + +w = w0; +hcent = zeros(nwts, nwts); +h1 = 0.0; h2 = 0.0; h3 = 0.0; h4 = 0.0; +epsilon = 1.0e-4; +fprintf(1, 'Checking Hessian ...\n\n'); +for k = 1:nwts; + for l = 1:nwts; + if(l == k) + w(k) = w0(k) + 2.0*epsilon; + h1 = neterr(w, net, x, t); + w(k) = w0(k) - 2.0*epsilon; + h2 = neterr(w, net, x, t); + w(k) = w0(k); + h3 = neterr(w, net, x, t); + hcent(k, k) = (h1 + h2 - 2.0*h3)/(4.0*epsilon^2); + else + w(k) = w0(k) + epsilon; + w(l) = w0(l) + epsilon; + h1 = neterr(w, net, x, t); + w(k) = w0(k) - epsilon; + w(l) = w0(l) - epsilon; + h2 = neterr(w, net, x, t); + w(k) = w0(k) + epsilon; + w(l) = w0(l) - epsilon; + h3 = neterr(w, net, x, t); + w(k) = w0(k) - epsilon; + w(l) = w0(l) + epsilon; + h4 = neterr(w, net, x, t); + hcent(k, l) = (h1 + h2 - h3 - h4)/(4.0*epsilon^2); + w(k) = w0(k); + w(l) = w0(l); + end + end +end + +fprintf(1, ' analytical numerical delta\n\n'); +temp = [h(:), hcent(:), (h(:) - hcent(:))]; +fprintf(1, '%12.6f %12.6f %12.6f\n', temp'); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/hintmat.m b/sourcecodes/bnt-master/netlab3.3/hintmat.m new file mode 100644 index 00000000..24617921 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/hintmat.m @@ -0,0 +1,41 @@ +function [xvals, yvals, color] = hintmat(w); +%HINTMAT Evaluates the coordinates of the patches for a Hinton diagram. +% +% Description +% [xvals, yvals, color] = hintmat(w) +% takes a matrix W and returns coordinates XVALS, YVALS for the +% patches comrising the Hinton diagram, together with a vector COLOR +% labelling the color (black or white) of the corresponding elements +% according to their sign. +% +% See also +% HINTON +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set scale to be up to 0.9 of maximum absolute weight value, where scale +% defined so that area of box proportional to weight value. + +w = flipud(w); +[nrows, ncols] = size(w); + +scale = 0.45*sqrt(abs(w)/max(max(abs(w)))); +scale = scale(:); +color = 0.5*(sign(w(:)) + 3); + +delx = 1; +dely = 1; +[X, Y] = meshgrid(0.5*delx:delx:(ncols-0.5*delx), 0.5*dely:dely:(nrows-0.5*dely)); + +% Now convert from matrix format to column vector format, and then duplicate +% columns with appropriate offsets determined by normalized weight magnitudes. + +xtemp = X(:); +ytemp = Y(:); + +xvals = [xtemp-delx*scale, xtemp+delx*scale, ... + xtemp+delx*scale, xtemp-delx*scale]; +yvals = [ytemp-dely*scale, ytemp-dely*scale, ... + ytemp+dely*scale, ytemp+dely*scale]; + diff --git a/sourcecodes/bnt-master/netlab3.3/hinton.m b/sourcecodes/bnt-master/netlab3.3/hinton.m new file mode 100644 index 00000000..c2c4b2be --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/hinton.m @@ -0,0 +1,57 @@ +function h = hinton(w); +%HINTON Plot Hinton diagram for a weight matrix. +% +% Description +% +% HINTON(W) takes a matrix W and plots the Hinton diagram. +% +% H = HINTON(NET) also returns the figure handle H which can be used, +% for instance, to delete the figure when it is no longer needed. +% +% To print the figure correctly in black and white, you should call +% SET(H, 'INVERTHARDCOPY', 'OFF') before printing. +% +% See also +% DEMHINT, HINTMAT, MLPHINT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set scale to be up to 0.9 of maximum absolute weight value, where scale +% defined so that area of box proportional to weight value. + +% Use no more than 640x480 pixels +xmax = 640; ymax = 480; + +% Offset bottom left hand corner +x01 = 40; y01 = 40; +x02 = 80; y02 = 80; + +% Need to allow 5 pixels border for window frame: but 30 at top +border = 5; +top_border = 30; + +ymax = ymax - top_border; +xmax = xmax - border; + +% First layer + +[xvals, yvals, color] = hintmat(w); +% Try to preserve aspect ratio approximately +if (8*size(w, 1) < 6*size(w, 2)) + delx = xmax; dely = xmax*size(w, 1)/(size(w, 2)); +else + delx = ymax*size(w, 2)/size(w, 1); dely = ymax; +end + +h = figure('Color', [0.5 0.5 0.5], ... + 'Name', 'Hinton diagram', ... + 'NumberTitle', 'off', ... + 'Colormap', [0 0 0; 1 1 1], ... + 'Units', 'pixels', ... + 'Position', [x01 y01 delx dely]); +set(gca, 'Visible', 'off', 'Position', [0 0 1 1]); +hold on +patch(xvals', yvals', color', 'Edgecolor', 'none'); +axis equal; + diff --git a/sourcecodes/bnt-master/netlab3.3/histp.m b/sourcecodes/bnt-master/netlab3.3/histp.m new file mode 100644 index 00000000..7b7fb31b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/histp.m @@ -0,0 +1,29 @@ +function h = histp(x, xmin, xmax, nbins) +%HISTP Histogram estimate of 1-dimensional probability distribution. +% +% Description +% +% HISTP(X, XMIN, XMAX, NBINS) takes a column vector X of data values +% and generates a normalized histogram plot of the distribution. The +% histogram has NBINS bins lying in the range XMIN to XMAX. +% +% H = HISTP(...) returns a vector of patch handles. +% +% See also +% DEMGAUSS +% + +% Copyright (c) Ian T Nabney (1996-2001) + +ndata = length(x); + +bins = linspace(xmin, xmax, nbins); + +binwidth = (xmax - xmin)/nbins; + +num = hist(x, bins); + +num = num/(ndata*binwidth); + +h = bar(bins, num, 0.6); + diff --git a/sourcecodes/bnt-master/netlab3.3/hmc.m b/sourcecodes/bnt-master/netlab3.3/hmc.m new file mode 100644 index 00000000..dab42be8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/hmc.m @@ -0,0 +1,281 @@ +function [samples, energies, diagn] = hmc(f, x, options, gradf, varargin) +%HMC Hybrid Monte Carlo sampling. +% +% Description +% SAMPLES = HMC(F, X, OPTIONS, GRADF) uses a hybrid Monte Carlo +% algorithm to sample from the distribution P ~ EXP(-F), where F is the +% first argument to HMC. The Markov chain starts at the point X, and +% the function GRADF is the gradient of the `energy' function F. +% +% HMC(F, X, OPTIONS, GRADF, P1, P2, ...) allows additional arguments to +% be passed to F() and GRADF(). +% +% [SAMPLES, ENERGIES, DIAGN] = HMC(F, X, OPTIONS, GRADF) also returns a +% log of the energy values (i.e. negative log probabilities) for the +% samples in ENERGIES and DIAGN, a structure containing diagnostic +% information (position, momentum and acceptance threshold) for each +% step of the chain in DIAGN.POS, DIAGN.MOM and DIAGN.ACC respectively. +% All candidate states (including rejected ones) are stored in +% DIAGN.POS. +% +% [SAMPLES, ENERGIES, DIAGN] = HMC(F, X, OPTIONS, GRADF) also returns +% the ENERGIES (i.e. negative log probabilities) corresponding to the +% samples. The DIAGN structure contains three fields: +% +% POS the position vectors of the dynamic process. +% +% MOM the momentum vectors of the dynamic process. +% +% ACC the acceptance thresholds. +% +% S = HMC('STATE') returns a state structure that contains the state of +% the two random number generators RAND and RANDN and the momentum of +% the dynamic process. These are contained in fields randstate, +% randnstate and mom respectively. The momentum state is only used for +% a persistent momentum update. +% +% HMC('STATE', S) resets the state to S. If S is an integer, then it +% is passed to RAND and RANDN and the momentum variable is randomised. +% If S is a structure returned by HMC('STATE') then it resets the +% generator to exactly the same state. +% +% The optional parameters in the OPTIONS vector have the following +% interpretations. +% +% OPTIONS(1) is set to 1 to display the energy values and rejection +% threshold at each step of the Markov chain. If the value is 2, then +% the position vectors at each step are also displayed. +% +% OPTIONS(5) is set to 1 if momentum persistence is used; default 0, +% for complete replacement of momentum variables. +% +% OPTIONS(7) defines the trajectory length (i.e. the number of leap- +% frog steps at each iteration). Minimum value 1. +% +% OPTIONS(9) is set to 1 to check the user defined gradient function. +% +% OPTIONS(14) is the number of samples retained from the Markov chain; +% default 100. +% +% OPTIONS(15) is the number of samples omitted from the start of the +% chain; default 0. +% +% OPTIONS(17) defines the momentum used when a persistent update of +% (leap-frog) momentum is used. This is bounded to the interval [0, +% 1). +% +% OPTIONS(18) is the step size used in leap-frogs; default 1/trajectory +% length. +% +% See also +% METROP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Global variable to store state of momentum variables: set by set_state +% Used to initialise variable if set +global HMC_MOM +if nargin <= 2 + if ~strcmp(f, 'state') + error('Unknown argument to hmc'); + end + switch nargin + case 1 + samples = get_state(f); + return; + case 2 + set_state(f, x); + return; + end +end + +display = options(1); +if (round(options(5) == 1)) + persistence = 1; + % Set alpha to lie in [0, 1) + alpha = max(0, options(17)); + alpha = min(1, alpha); + salpha = sqrt(1-alpha*alpha); +else + persistence = 0; +end +L = max(1, options(7)); % At least one step in leap-frogging +if options(14) > 0 + nsamples = options(14); +else + nsamples = 100; % Default +end +if options(15) >= 0 + nomit = options(15); +else + nomit = 0; +end +if options(18) > 0 + step_size = options(18); % Step size. +else + step_size = 1/L; % Default +end +x = x(:)'; % Force x to be a row vector +nparams = length(x); + +% Set up strings for evaluating potential function and its gradient. +f = fcnchk(f, length(varargin)); +gradf = fcnchk(gradf, length(varargin)); + +% Check the gradient evaluation. +if (options(9)) + % Check gradients + feval('gradchek', x, f, gradf, varargin{:}); +end + +samples = zeros(nsamples, nparams); % Matrix of returned samples. +if nargout >= 2 + en_save = 1; + energies = zeros(nsamples, 1); +else + en_save = 0; +end +if nargout >= 3 + diagnostics = 1; + diagn_pos = zeros(nsamples, nparams); + diagn_mom = zeros(nsamples, nparams); + diagn_acc = zeros(nsamples, 1); +else + diagnostics = 0; +end + +n = - nomit + 1; +Eold = feval(f, x, varargin{:}); % Evaluate starting energy. +nreject = 0; +if (~persistence | isempty(HMC_MOM)) + p = randn(1, nparams); % Initialise momenta at random +else + p = HMC_MOM; % Initialise momenta from stored state +end +lambda = 1; + +% Main loop. +while n <= nsamples + + xold = x; % Store starting position. + pold = p; % Store starting momenta + Hold = Eold + 0.5*(p*p'); % Recalculate Hamiltonian as momenta have changed + + if ~persistence + % Choose a direction at random + if (rand < 0.5) + lambda = -1; + else + lambda = 1; + end + end + % Perturb step length. + epsilon = lambda*step_size*(1.0 + 0.1*randn(1)); + + % First half-step of leapfrog. + p = p - 0.5*epsilon*feval(gradf, x, varargin{:}); + x = x + epsilon*p; + + % Full leapfrog steps. + for m = 1 : L - 1 + p = p - epsilon*feval(gradf, x, varargin{:}); + x = x + epsilon*p; + end + + % Final half-step of leapfrog. + p = p - 0.5*epsilon*feval(gradf, x, varargin{:}); + + % Now apply Metropolis algorithm. + Enew = feval(f, x, varargin{:}); % Evaluate new energy. + p = -p; % Negate momentum + Hnew = Enew + 0.5*p*p'; % Evaluate new Hamiltonian. + a = exp(Hold - Hnew); % Acceptance threshold. + if (diagnostics & n > 0) + diagn_pos(n,:) = x; + diagn_mom(n,:) = p; + diagn_acc(n,:) = a; + end + if (display > 1) + fprintf(1, 'New position is\n'); + disp(x); + end + + if a > rand(1) % Accept the new state. + Eold = Enew; % Update energy + if (display > 0) + fprintf(1, 'Finished step %4d Threshold: %g\n', n, a); + end + else % Reject the new state. + if n > 0 + nreject = nreject + 1; + end + x = xold; % Reset position + p = pold; % Reset momenta + if (display > 0) + fprintf(1, ' Sample rejected %4d. Threshold: %g\n', n, a); + end + end + if n > 0 + samples(n,:) = x; % Store sample. + if en_save + energies(n) = Eold; % Store energy. + end + end + + % Set momenta for next iteration + if persistence + p = -p; + % Adjust momenta by a small random amount. + p = alpha.*p + salpha.*randn(1, nparams); + else + p = randn(1, nparams); % Replace all momenta. + end + + n = n + 1; +end + +if (display > 0) + fprintf(1, '\nFraction of samples rejected: %g\n', ... + nreject/(nsamples)); +end +if diagnostics + diagn.pos = diagn_pos; + diagn.mom = diagn_mom; + diagn.acc = diagn_acc; +end +% Store final momentum value in global so that it can be retrieved later +HMC_MOM = p; +return + +% Return complete state of sampler (including momentum) +function state = get_state(f) + +global HMC_MOM +state.randstate = rand('state'); +state.randnstate = randn('state'); +state.mom = HMC_MOM; +return + +% Set complete state of sampler (including momentum) or just set randn +% and rand with integer argument. +function set_state(f, x) + +global HMC_MOM +if isnumeric(x) + rand('state', x); + randn('state', x); + HMC_MOM = []; +else + if ~isstruct(x) + error('Second argument to hmc must be number or state structure'); + end + if (~isfield(x, 'randstate') | ~isfield(x, 'randnstate') ... + | ~isfield(x, 'mom')) + error('Second argument to hmc must contain correct fields') + end + rand('state', x.randstate); + randn('state', x.randnstate); + HMC_MOM = x.mom; +end +return diff --git a/sourcecodes/bnt-master/netlab3.3/kmeansNetlab.m b/sourcecodes/bnt-master/netlab3.3/kmeansNetlab.m new file mode 100644 index 00000000..09935f7b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/kmeansNetlab.m @@ -0,0 +1,124 @@ +function [centres, options, post, errlog] = kmeans(centres, data, options) +%KMEANS Trains a k means cluster model. +% +% Description +% CENTRES = KMEANS(CENTRES, DATA, OPTIONS) uses the batch K-means +% algorithm to set the centres of a cluster model. The matrix DATA +% represents the data which is being clustered, with each row +% corresponding to a vector. The sum of squares error function is used. +% The point at which a local minimum is achieved is returned as +% CENTRES. The error value at that point is returned in OPTIONS(8). +% +% [CENTRES, OPTIONS, POST, ERRLOG] = KMEANS(CENTRES, DATA, OPTIONS) +% also returns the cluster number (in a one-of-N encoding) for each +% data point in POST and a log of the error values after each cycle in +% ERRLOG. The optional parameters have the following +% interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG. If OPTIONS(1) is set to 0, then +% only warning messages are displayed. If OPTIONS(1) is -1, then +% nothing is displayed. +% +% OPTIONS(2) is a measure of the absolute precision required for the +% value of CENTRES at the solution. If the absolute difference between +% the values of CENTRES between two successive steps is less than +% OPTIONS(2), then this condition is satisfied. +% +% OPTIONS(3) is a measure of the precision required of the error +% function at the solution. If the absolute difference between the +% error functions between two successive steps is less than OPTIONS(3), +% then this condition is satisfied. Both this and the previous +% condition must be satisfied for termination. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% See also +% GMMINIT, GMMEM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[ndata, data_dim] = size(data); +[ncentres, dim] = size(centres); + +if dim ~= data_dim + error('Data dimension does not match dimension of centres') +end + +if (ncentres > ndata) + error('More centres than data') +end + +% Sort out the options +if (options(14)) + niters = options(14); +else + niters = 100; +end + +store = 0; +if (nargout > 3) + store = 1; + errlog = zeros(1, niters); +end + +% Check if centres and posteriors need to be initialised from data +if (options(5) == 1) + % Do the initialisation + perm = randperm(ndata); + perm = perm(1:ncentres); + + % Assign first ncentres (permuted) data points as centres + centres = data(perm, :); +end +% Matrix to make unit vectors easy to construct +id = eye(ncentres); + +% Main loop of algorithm +for n = 1:niters + + % Save old centres to check for termination + old_centres = centres; + + % Calculate posteriors based on existing centres + d2 = dist2(data, centres); + % Assign each point to nearest centre + [minvals, index] = min(d2', [], 1); + post = id(index,:); + + num_points = sum(post, 1); + % Adjust the centres based on new posteriors + for j = 1:ncentres + if (num_points(j) > 0) + centres(j,:) = sum(data(find(post(:,j)),:), 1)/num_points(j); + end + end + + % Error value is total squared distance from cluster centres + e = sum(minvals); + if store + errlog(n) = e; + end + if options(1) > 0 + fprintf(1, 'Cycle %4d Error %11.6f\n', n, e); + end + + if n > 1 + % Test for termination + if max(max(abs(centres - old_centres))) < options(2) & ... + abs(old_e - e) < options(3) + options(8) = e; + return; + end + end + old_e = e; +end + +% If we get here, then we haven't terminated in the given number of +% iterations. +options(8) = e; +if (options(1) >= 0) + disp(maxitmess); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/knn.m b/sourcecodes/bnt-master/netlab3.3/knn.m new file mode 100644 index 00000000..2a171804 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/knn.m @@ -0,0 +1,34 @@ +function net = knn(nin, nout, k, tr_in, tr_targets) +%KNN Creates a K-nearest-neighbour classifier. +% +% Description +% NET = KNN(NIN, NOUT, K, TR_IN, TR_TARGETS) creates a KNN model NET +% with input dimension NIN, output dimension NOUT and K neighbours. +% The training data is also stored in the data structure and the +% targets are assumed to be using a 1-of-N coding. +% +% The fields in NET are +% type = 'knn' +% nin = number of inputs +% nout = number of outputs +% tr_in = training input data +% tr_targets = training target data +% +% See also +% KMEANS, KNNFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +net.type = 'knn'; +net.nin = nin; +net.nout = nout; +net.k = k; +errstring = consist(net, 'knn', tr_in, tr_targets); +if ~isempty(errstring) + error(errstring); +end +net.tr_in = tr_in; +net.tr_targets = tr_targets; + diff --git a/sourcecodes/bnt-master/netlab3.3/knnfwd.m b/sourcecodes/bnt-master/netlab3.3/knnfwd.m new file mode 100644 index 00000000..db450bae --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/knnfwd.m @@ -0,0 +1,53 @@ +function [y, l] = knnfwd(net, x) +%KNNFWD Forward propagation through a K-nearest-neighbour classifier. +% +% Description +% [Y, L] = KNNFWD(NET, X) takes a matrix X of input vectors (one vector +% per row) and uses the K-nearest-neighbour rule on the training data +% contained in NET to produce a matrix Y of outputs and a matrix L of +% classification labels. The nearest neighbours are determined using +% Euclidean distance. The IJth entry of Y counts the number of +% occurrences that an example from class J is among the K closest +% training examples to example I from X. The matrix L contains the +% predicted class labels as an index 1..N, not as 1-of-N coding. +% +% See also +% KMEANS, KNN +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +errstring = consist(net, 'knn', x); +if ~isempty(errstring) + error(errstring); +end + +ntest = size(x, 1); % Number of input vectors. +nclass = size(net.tr_targets, 2); % Number of classes. + +% Compute matrix of squared distances between input vectors from the training +% and test sets. The matrix distsq has dimensions (ntrain, ntest). + +distsq = dist2(net.tr_in, x); + +% Now sort the distances. This generates a matrix kind of the same +% dimensions as distsq, in which each column gives the indices of the +% elements in the corresponding column of distsq in ascending order. + +[vals, kind] = sort(distsq); +y = zeros(ntest, nclass); + +for k=1:net.k + % We now look at the predictions made by the Kth nearest neighbours alone, + % and represent this as a 1-of-N coded matrix, and then accumulate the + % predictions so far. + + y = y + net.tr_targets(kind(k,:),:); + +end + +if nargout == 2 + % Convert this set of outputs to labels, randomly breaking ties + [temp, l] = max((y + 0.1*rand(size(y))), [], 2); +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/linef.m b/sourcecodes/bnt-master/netlab3.3/linef.m new file mode 100644 index 00000000..4f0ddd32 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/linef.m @@ -0,0 +1,21 @@ +function y = linef(lambda, fn, x, d, varargin) +%LINEF Calculate function value along a line. +% +% Description +% LINEF(LAMBDA, FN, X, D) calculates the value of the function FN at +% the point X+LAMBDA*D. Here X is a row vector and LAMBDA is a scalar. +% +% LINEF(LAMBDA, FN, X, D, P1, P2, ...) allows additional arguments to +% be passed to FN(). This function is used for convenience in some of +% the optimisation routines. +% +% See also +% GRADCHEK, LINEMIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check function string +fn = fcnchk(fn, length(varargin)); + +y = feval(fn, x+lambda.*d, varargin{:}); diff --git a/sourcecodes/bnt-master/netlab3.3/linemin.m b/sourcecodes/bnt-master/netlab3.3/linemin.m new file mode 100644 index 00000000..8e390a43 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/linemin.m @@ -0,0 +1,152 @@ +function [x, options] = linemin(f, pt, dir, fpt, options, ... + varargin) +%LINEMIN One dimensional minimization. +% +% Description +% [X, OPTIONS] = LINEMIN(F, PT, DIR, FPT, OPTIONS) uses Brent's +% algorithm to find the minimum of the function F(X) along the line DIR +% through the point PT. The function value at the starting point is +% FPT. The point at which F has a local minimum is returned as X. The +% function value at that point is returned in OPTIONS(8). +% +% LINEMIN(F, PT, DIR, FPT, OPTIONS, P1, P2, ...) allows additional +% arguments to be passed to F(). +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values. +% +% OPTIONS(2) is a measure of the absolute precision required for the +% value of X at the solution. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. Both this and the previous condition must +% be satisfied for termination. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% See also +% CONJGRAD, MINBRACK, QUASINEW +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set up the options. +if(options(14)) + niters = options(14); +else + niters = 100; +end +options(10) = 0; % Initialise count of function evaluations + +display = options(1); + +% Check function string +f = fcnchk(f, length(varargin)); + +% Value of golden section (1 + sqrt(5))/2.0 +phi = 1.6180339887499; +cphi = 1 - 1/phi; +TOL = sqrt(eps); % Maximal fractional precision +TINY = 1.0e-10; % Can't use fractional precision when minimum is at 0 + +% Bracket the minimum +[br_min, br_mid, br_max, num_evals] = feval('minbrack', 'linef', ... + 0.0, 1.0, fpt, f, pt, dir, varargin{:}); +options(10) = options(10) + num_evals; % Increment number of fn. evals + % No gradient evals in minbrack + +% Use Brent's algorithm to find minimum +% Initialise the points and function values +w = br_mid; % Where second from minimum is +v = br_mid; % Previous value of w +x = v; % Where current minimum is +e = 0.0; % Distance moved on step before last +fx = feval('linef', x, f, pt, dir, varargin{:}); +options(10) = options(10) + 1; +fv = fx; fw = fx; + +for n = 1:niters + xm = 0.5.*(br_min+br_max); % Middle of bracket + % Make sure that tolerance is big enough + tol1 = TOL * (max(abs(x))) + TINY; + % Decide termination on absolute precision required by options(2) + if (max(abs(x - xm)) <= options(2) & br_max-br_min < 4*options(2)) + options(8) = fx; + return; + end + % Check if step before last was big enough to try a parabolic step. + % Note that this will fail on first iteration, which must be a golden + % section step. + if (max(abs(e)) > tol1) + % Construct a trial parabolic fit through x, v and w + r = (fx - fv) .* (x - w); + q = (fx - fw) .* (x - v); + p = (x - v).*q - (x - w).*r; + q = 2.0 .* (q - r); + if (q > 0.0) p = -p; end + q = abs(q); + % Test if the parabolic fit is OK + if (abs(p) >= abs(0.5*q*e) | p <= q*(br_min-x) | p >= q*(br_max-x)) + % No it isn't, so take a golden section step + if (x >= xm) + e = br_min-x; + else + e = br_max-x; + end + d = cphi*e; + else + % Yes it is, so take the parabolic step + e = d; + d = p/q; + u = x+d; + if (u-br_min < 2*tol1 | br_max-u < 2*tol1) + d = sign(xm-x)*tol1; + end + end + else + % Step before last not big enough, so take a golden section step + if (x >= xm) + e = br_min - x; + else + e = br_max - x; + end + d = cphi*e; + end + % Make sure that step is big enough + if (abs(d) >= tol1) + u = x+d; + else + u = x + sign(d)*tol1; + end + % Evaluate function at u + fu = feval('linef', u, f, pt, dir, varargin{:}); + options(10) = options(10) + 1; + % Reorganise bracket + if (fu <= fx) + if (u >= x) + br_min = x; + else + br_max = x; + end + v = w; w = x; x = u; + fv = fw; fw = fx; fx = fu; + else + if (u < x) + br_min = u; + else + br_max = u; + end + if (fu <= fw | w == x) + v = w; w = u; + fv = fw; fw = fu; + elseif (fu <= fv | v == x | v == w) + v = u; + fv = fu; + end + end + if (display == 1) + fprintf(1, 'Cycle %4d Error %11.6f\n', n, fx); + end +end +options(8) = fx; diff --git a/sourcecodes/bnt-master/netlab3.3/maxitmess.m b/sourcecodes/bnt-master/netlab3.3/maxitmess.m new file mode 100644 index 00000000..338be6d7 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/maxitmess.m @@ -0,0 +1,16 @@ +function s = maxitmess() +%MAXITMESS Create a standard error message when training reaches max. iterations. +% +% Description +% S = MAXITMESS returns a standard string that it used by training +% algorithms when the maximum number of iterations (as specified in +% OPTIONS(14) is reached. +% +% See also +% CONJGRAD, GLMTRAIN, GMMEM, GRADDESC, GTMEM, KMEANS, OLGD, QUASINEW, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +s = 'Maximum number of iterations has been exceeded'; + diff --git a/sourcecodes/bnt-master/netlab3.3/mdn.m b/sourcecodes/bnt-master/netlab3.3/mdn.m new file mode 100644 index 00000000..964baa1d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdn.m @@ -0,0 +1,77 @@ +function net = mdn(nin, nhidden, ncentres, dim_target, mix_type, ... + prior, beta) +%MDN Creates a Mixture Density Network with specified architecture. +% +% Description +% NET = MDN(NIN, NHIDDEN, NCENTRES, DIMTARGET) takes the number of +% inputs, hidden units for a 2-layer feed-forward network and the +% number of centres and target dimension for the mixture model whose +% parameters are set from the outputs of the neural network. The fifth +% argument MIXTYPE is used to define the type of mixture model. +% (Currently there is only one type supported: a mixture of Gaussians +% with a single covariance parameter for each component.) For this +% model, the mixture coefficients are computed from a group of softmax +% outputs, the centres are equal to a group of linear outputs, and the +% variances are obtained by applying the exponential function to a +% third group of outputs. +% +% The network is initialised by a call to MLP, and the arguments PRIOR, +% and BETA have the same role as for that function. Weight +% initialisation uses the Matlab function RANDN and so the seed for +% the random weight initialization can be set using RANDN('STATE', S) +% where S is the seed value. A specialised data structure (rather than +% GMM) is used for the mixture model outputs to improve the efficiency +% of error and gradient calculations in network training. The fields +% are described in MDNFWD where they are set up. +% +% The fields in NET are +% +% type = 'mdn' +% nin = number of input variables +% nout = dimension of target space (not number of network outputs) +% nwts = total number of weights and biases +% mdnmixes = data structure for mixture model output +% mlp = data structure for MLP network +% +% See also +% MDNFWD, MDNERR, MDN2GMM, MDNGRAD, MDNPAK, MDNUNPAK, MLP +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Currently ignore type argument: reserved for future use +net.type = 'mdn'; + +% Set up the mixture model part of the structure +% For efficiency we use a specialised data structure in place of GMM +mdnmixes.type = 'mdnmixes'; +mdnmixes.ncentres = ncentres; +mdnmixes.dim_target = dim_target; + +% This calculation depends on spherical variances +mdnmixes.nparams = ncentres + ncentres*dim_target + ncentres; + +% Make the weights in the mdnmixes structure null +mdnmixes.mixcoeffs = []; +mdnmixes.centres = []; +mdnmixes.covars = []; + +% Number of output nodes = number of parameters in mixture model +nout = mdnmixes.nparams; + +% Set up the MLP part of the network +if (nargin == 5) + mlpnet = mlp(nin, nhidden, nout, 'linear'); +elseif (nargin == 6) + mlpnet = mlp(nin, nhidden, nout, 'linear', prior); +elseif (nargin == 7) + mlpnet = mlp(nin, nhidden, nout, 'linear', prior, beta); +end + +% Create descriptor +net.mdnmixes = mdnmixes; +net.mlp = mlpnet; +net.nin = nin; +net.nout = dim_target; +net.nwts = mlpnet.nwts; diff --git a/sourcecodes/bnt-master/netlab3.3/mdn2gmm.m b/sourcecodes/bnt-master/netlab3.3/mdn2gmm.m new file mode 100644 index 00000000..bad5ed9f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdn2gmm.m @@ -0,0 +1,41 @@ +function gmmmixes = mdn2gmm(mdnmixes) +%MDN2GMM Converts an MDN mixture data structure to array of GMMs. +% +% Description +% GMMMIXES = MDN2GMM(MDNMIXES) takes an MDN mixture data structure +% MDNMIXES containing three matrices (for priors, centres and +% variances) where each row represents the corresponding parameter +% values for a different mixture model and creates an array of GMMs. +% These can then be used with the standard Netlab Gaussian mixture +% model functions. +% +% See also +% GMM, MDN, MDNFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Check argument for consistency +errstring = consist(mdnmixes, 'mdnmixes'); +if ~isempty(errstring) + error(errstring); +end + +nmixes = size(mdnmixes.centres, 1); +% Construct ndata structures containing the mixture model information. +% First allocate the memory. +tempmix = gmm(mdnmixes.dim_target, mdnmixes.ncentres, 'spherical'); +f = fieldnames(tempmix); +gmmmixes = cell(size(f, 1), 1, nmixes); +gmmmixes = cell2struct(gmmmixes, f,1); + +% Then fill each structure in turn using gmmunpak. Assume that spherical +% covariance structure is used. +for i = 1:nmixes + centres = reshape(mdnmixes.centres(i, :), mdnmixes.dim_target, ... + mdnmixes.ncentres)'; + gmmmixes(i) = gmmunpak(tempmix, [mdnmixes.mixcoeffs(i,:), ... + centres(:)', mdnmixes.covars(i,:)]); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/mdndist2.m b/sourcecodes/bnt-master/netlab3.3/mdndist2.m new file mode 100644 index 00000000..d7ed93d3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdndist2.m @@ -0,0 +1,51 @@ +function n2 = mdndist2(mixparams, t) +%MDNDIST2 Calculates squared distance between centres of Gaussian kernels and data +% +% Description +% N2 = MDNDIST2(MIXPARAMS, T) takes takes the centres of the Gaussian +% contained in MIXPARAMS and the target data matrix, T, and computes +% the squared Euclidean distance between them. If T has M rows and N +% columns, then the CENTRES field in the MIXPARAMS structure should +% have M rows and N*MIXPARAMS.NCENTRES columns: the centres in each row +% relate to the corresponding row in T. The result has M rows and +% MIXPARAMS.NCENTRES columns. The I, Jth entry is the squared distance +% from the Ith row of X to the Jth centre in the Ith row of +% MIXPARAMS.CENTRES. +% +% See also +% MDNFWD, MDNPROB +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Check arguments for consistency +errstring = consist(mixparams, 'mdnmixes'); +if ~isempty(errstring) + error(errstring); +end + +ncentres = mixparams.ncentres; +dim_target = mixparams.dim_target; +ntarget = size(t, 1); +if ntarget ~= size(mixparams.centres, 1) + error('Number of targets does not match number of mixtures') +end +if size(t, 2) ~= mixparams.dim_target + error('Target dimension does not match mixture dimension') +end + +% Build t that suits parameters, that is repeat t for each centre +t = kron(ones(1, ncentres), t); + +% Do subtraction and square +diff2 = (t - mixparams.centres).^2; + +% Reshape and sum each component +diff2 = reshape(diff2', dim_target, (ntarget*ncentres))'; +n2 = sum(diff2, 2); + +% Calculate the sum of distance, and reshape +% so that we have a distance for each centre per target +n2 = reshape(n2, ncentres, ntarget)'; + diff --git a/sourcecodes/bnt-master/netlab3.3/mdnerr.m b/sourcecodes/bnt-master/netlab3.3/mdnerr.m new file mode 100644 index 00000000..a0b686e0 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnerr.m @@ -0,0 +1,33 @@ +function e = mdnerr(net, x, t) +%MDNERR Evaluate error function for Mixture Density Network. +% +% Description +% E = MDNERR(NET, X, T) takes a mixture density network data structure +% NET, a matrix X of input vectors and a matrix T of target vectors, +% and evaluates the error function E. The error function is the +% negative log likelihood of the target data under the conditional +% density given by the mixture model parameterised by the MLP. Each +% row of X corresponds to one input vector and each row of T +% corresponds to one target vector. +% +% See also +% MDN, MDNFWD, MDNGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Check arguments for consistency +errstring = consist(net, 'mdn', x, t); +if ~isempty(errstring) + error(errstring); +end + +% Get the output mixture models +mixparams = mdnfwd(net, x); + +% Compute the probabilities of mixtures +probs = mdnprob(mixparams, t); +% Compute the error +e = sum( -log(max(eps, sum(probs, 2)))); + diff --git a/sourcecodes/bnt-master/netlab3.3/mdnfwd.m b/sourcecodes/bnt-master/netlab3.3/mdnfwd.m new file mode 100644 index 00000000..962fbe08 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnfwd.m @@ -0,0 +1,93 @@ +function [mixparams, y, z, a] = mdnfwd(net, x) +%MDNFWD Forward propagation through Mixture Density Network. +% +% Description +% MIXPARAMS = MDNFWD(NET, X) takes a mixture density network data +% structure NET and a matrix X of input vectors, and forward propagates +% the inputs through the network to generate a structure MIXPARAMS +% which contains the parameters of several mixture models. Each row +% of X represents one input vector and the corresponding row of the +% matrices in MIXPARAMS represents the parameters of a mixture model +% for the conditional probability of target vectors given the input +% vector. This is not represented as an array of GMM structures to +% improve the efficiency of MDN training. +% +% The fields in MIXPARAMS are +% type = 'mdnmixes' +% ncentres = number of mixture components +% dimtarget = dimension of target space +% mixcoeffs = mixing coefficients +% centres = means of Gaussians: stored as one row per pattern +% covars = covariances of Gaussians +% nparams = number of parameters +% +% [MIXPARAMS, Y, Z] = MDNFWD(NET, X) also generates a matrix Y of the +% outputs of the MLP and a matrix Z of the hidden unit activations +% where each row corresponds to one pattern. +% +% [MIXPARAMS, Y, Z, A] = MLPFWD(NET, X) also returns a matrix A giving +% the summed inputs to each output unit, where each row corresponds to +% one pattern. +% +% See also +% MDN, MDN2GMM, MDNERR, MDNGRAD, MLPFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Check arguments for consistency +errstring = consist(net, 'mdn', x); +if ~isempty(errstring) + error(errstring); +end + +% Extract mlp and mixture model descriptors +mlpnet = net.mlp; +mixes = net.mdnmixes; + +ncentres = mixes.ncentres; % Number of components in mixture model +dim_target = mixes.dim_target; % Dimension of targets +nparams = mixes.nparams; % Number of parameters in mixture model + +% Propagate forwards through MLP +[y, z, a] = mlpfwd(mlpnet, x); + +% Compute the postion for each parameter in the whole +% matrix. Used to define the mixparams structure +mixcoeff = [1:1:ncentres]; +centres = [ncentres+1:1:(ncentres*(1+dim_target))]; +variances = [(ncentres*(1+dim_target)+1):1:nparams]; + +% Convert output values into mixture model parameters + +% Use softmax to calculate priors +% Prevent overflow and underflow: use same bounds as glmfwd +% Ensure that sum(exp(y), 2) does not overflow +maxcut = log(realmax) - log(ncentres); +% Ensure that exp(y) > 0 +mincut = log(realmin); +temp = min(y(:,1:ncentres), maxcut); +temp = max(temp, mincut); +temp = exp(temp); +mixpriors = temp./(sum(temp, 2)*ones(1,ncentres)); + +% Centres are just copies of network outputs +mixcentres = y(:,(ncentres+1):ncentres*(1+dim_target)); + +% Variances are exp of network outputs +mixwidths = exp(y(:,(ncentres*(1+dim_target)+1):nparams)); + +% Now build up all the mixture model weight vectors +ndata = size(x, 1); + +% Return parameters +mixparams.type = mixes.type; +mixparams.ncentres = mixes.ncentres; +mixparams.dim_target = mixes.dim_target; +mixparams.nparams = mixes.nparams; + +mixparams.mixcoeffs = mixpriors; +mixparams.centres = mixcentres; +mixparams.covars = mixwidths; + diff --git a/sourcecodes/bnt-master/netlab3.3/mdngrad.m b/sourcecodes/bnt-master/netlab3.3/mdngrad.m new file mode 100644 index 00000000..6a0b6ea8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdngrad.m @@ -0,0 +1,66 @@ +function g = mdngrad(net, x, t) +%MDNGRAD Evaluate gradient of error function for Mixture Density Network. +% +% Description +% G = MDNGRAD(NET, X, T) takes a mixture density network data +% structure NET, a matrix X of input vectors and a matrix T of target +% vectors, and evaluates the gradient G of the error function with +% respect to the network weights. The error function is negative log +% likelihood of the target data. Each row of X corresponds to one +% input vector and each row of T corresponds to one target vector. +% +% See also +% MDN, MDNFWD, MDNERR, MDNPROB, MLPBKP +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Check arguments for consistency +errstring = consist(net, 'mdn', x, t); +if ~isempty(errstring) + error(errstring); +end + +[mixparams, y, z] = mdnfwd(net, x); + +% Compute gradients at MLP outputs: put the answer in deltas +ncentres = net.mdnmixes.ncentres; +dim_target = net.mdnmixes.dim_target; +nmixparams = net.mdnmixes.nparams; +ntarget = size(t, 1); +deltas = zeros(ntarget, net.mlp.nout); +e = ones(ncentres, 1); +f = ones(1, dim_target); + +post = mdnpost(mixparams, t); + +% Calculate prior derivatives +deltas(:,1:ncentres) = mixparams.mixcoeffs - post; + +% Calculate centre derivatives +long_t = kron(ones(1, ncentres), t); +centre_err = mixparams.centres - long_t; + +% Get the post to match each u_jk: +% this array will be (ntarget, (ncentres*dim_target)) +long_post = kron(ones(dim_target, 1), post); +long_post = reshape(long_post, ntarget, (ncentres*dim_target)); + +% Get the variance to match each u_jk: +var = mixparams.covars; +var = kron(ones(dim_target, 1), var); +var = reshape(var, ntarget, (ncentres*dim_target)); + +% Compute centre deltas +deltas(:, (ncentres+1):(ncentres*(1+dim_target))) = ... + (centre_err.*long_post)./var; + +% Compute variance deltas +dist2 = mdndist2(mixparams, t); +c = dim_target*ones(ntarget, ncentres); +deltas(:, (ncentres*(1+dim_target)+1):nmixparams) = ... + post.*((dist2./mixparams.covars)-c)./(-2); + +% Now back-propagate deltas through MLP +g = mlpbkp(net.mlp, x, z, deltas); diff --git a/sourcecodes/bnt-master/netlab3.3/mdninit.m b/sourcecodes/bnt-master/netlab3.3/mdninit.m new file mode 100644 index 00000000..20f23d0e --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdninit.m @@ -0,0 +1,53 @@ +function net = mdninit(net, prior, t, options) +%MDNINIT Initialise the weights in a Mixture Density Network. +% +% Description +% +% NET = MDNINIT(NET, PRIOR) takes a Mixture Density Network NET and +% sets the weights and biases by sampling from a Gaussian distribution. +% It calls MLPINIT for the MLP component of NET. +% +% NET = MDNINIT(NET, PRIOR, T, OPTIONS) uses the target data T to +% initialise the biases for the output units after initialising the +% other weights as above. It calls GMMINIT, with T and OPTIONS as +% arguments, to obtain a model of the unconditional density of T. The +% biases are then set so that NET will output the values in the +% Gaussian mixture model. +% +% See also +% MDN, MLP, MLPINIT, GMMINIT +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Initialise network weights from prior: this gives noise around values +% determined later +net.mlp = mlpinit(net.mlp, prior); + +if nargin > 2 + % Initialise priors, centres and variances from target data + temp_mix = gmm(net.mdnmixes.dim_target, net.mdnmixes.ncentres, 'spherical'); + temp_mix = gmminit(temp_mix, t, options); + + ncentres = net.mdnmixes.ncentres; + dim_target = net.mdnmixes.dim_target; + + % Now set parameters in MLP to yield the right values. + % This involves setting the biases correctly. + + % Priors + net.mlp.b2(1:ncentres) = temp_mix.priors; + + % Centres are arranged in mlp such that we have + % u11, u12, u13, ..., u1c, ... , uj1, uj2, uj3, ..., ujc, ..., um1, uM2, + % ..., uMc + % This is achieved by transposing temp_mix.centres before reshaping + end_centres = ncentres*(dim_target+1); + net.mlp.b2(ncentres+1:end_centres) = ... + reshape(temp_mix.centres', 1, ncentres*dim_target); + + % Variances + net.mlp.b2((end_centres+1):net.mlp.nout) = ... + log(temp_mix.covars); +end diff --git a/sourcecodes/bnt-master/netlab3.3/mdnnet.mat b/sourcecodes/bnt-master/netlab3.3/mdnnet.mat new file mode 100644 index 00000000..724f8d43 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnnet.mat Binary files differdiff --git a/sourcecodes/bnt-master/netlab3.3/mdnpak.m b/sourcecodes/bnt-master/netlab3.3/mdnpak.m new file mode 100644 index 00000000..47da11a6 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnpak.m @@ -0,0 +1,19 @@ +function w = mdnpak(net) +%MDNPAK Combines weights and biases into one weights vector. +% +% Description +% W = MDNPAK(NET) takes a mixture density network data structure NET +% and combines the network weights into a single row vector W. +% +% See also +% MDN, MDNUNPAK, MDNFWD, MDNERR, MDNGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +errstring = consist(net, 'mdn'); +if ~errstring + error(errstring); +end +w = mlppak(net.mlp); diff --git a/sourcecodes/bnt-master/netlab3.3/mdnpost.m b/sourcecodes/bnt-master/netlab3.3/mdnpost.m new file mode 100644 index 00000000..2a27849e --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnpost.m @@ -0,0 +1,26 @@ +function [post, a] = mdnpost(mixparams, t) +%MDNPOST Computes the posterior probability for each MDN mixture component. +% +% Description +% POST = MDNPOST(MIXPARAMS, T) computes the posterior probability +% P(J|T) of each data vector in T under the Gaussian mixture model +% represented by the corresponding entries in MIXPARAMS. Each row of T +% represents a single vector. +% +% [POST, A] = MDNPOST(MIXPARAMS, T) also computes the activations A +% (i.e. the probability P(T|J) of the data conditioned on each +% component density) for a Gaussian mixture model. +% +% See also +% MDNGRAD, MDNPROB +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +[prob a] = mdnprob(mixparams, t); + +s = sum(prob, 2); +% Set any zeros to one before dividing +s = s + (s==0); +post = prob./(s*ones(1, mixparams.ncentres)); diff --git a/sourcecodes/bnt-master/netlab3.3/mdnprob.m b/sourcecodes/bnt-master/netlab3.3/mdnprob.m new file mode 100644 index 00000000..3c828eac --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnprob.m @@ -0,0 +1,52 @@ +function [prob,a] = mdnprob(mixparams, t) +%MDNPROB Computes the data probability likelihood for an MDN mixture structure. +% +% Description +% PROB = MDNPROB(MIXPARAMS, T) computes the probability P(T) of each +% data vector in T under the Gaussian mixture model represented by the +% corresponding entries in MIXPARAMS. Each row of T represents a single +% vector. +% +% [PROB, A] = MDNPROB(MIXPARAMS, T) also computes the activations A +% (i.e. the probability P(T|J) of the data conditioned on each +% component density) for a Gaussian mixture model. +% +% See also +% MDNERR, MDNPOST +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Check arguments for consistency +errstring = consist(mixparams, 'mdnmixes'); +if ~isempty(errstring) + error(errstring); +end + +ntarget = size(t, 1); +if ntarget ~= size(mixparams.centres, 1) + error('Number of targets does not match number of mixtures') +end +if size(t, 2) ~= mixparams.dim_target + error('Target dimension does not match mixture dimension') +end + +dim_target = mixparams.dim_target; +ntarget = size(t, 1); + +% Calculate squared norm matrix, of dimension (ndata, ncentres) +% vector (ntarget * ncentres) +dist2 = mdndist2(mixparams, t); + +% Calculate variance factors +variance = 2.*mixparams.covars; + +% Compute the normalisation term +normal = ((2.*pi).*mixparams.covars).^(dim_target./2); + +% Now compute the activations +a = exp(-(dist2./variance))./normal; + +% Accumulate negative log likelihood of targets +prob = mixparams.mixcoeffs.*a; diff --git a/sourcecodes/bnt-master/netlab3.3/mdnunpak.m b/sourcecodes/bnt-master/netlab3.3/mdnunpak.m new file mode 100644 index 00000000..0b88cb7a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnunpak.m @@ -0,0 +1,25 @@ +function net = mdnunpak(net, w) +%MDNUNPAK Separates weights vector into weight and bias matrices. +% +% Description +% NET = MDNUNPAK(NET, W) takes an mdn network data structure NET and a +% weight vector W, and returns a network data structure identical to +% the input network, except that the weights in the MLP sub-structure +% are set to the corresponding elements of W. +% +% See also +% MDN, MDNPAK, MDNFWD, MDNERR, MDNGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +errstring = consist(net, 'mdn'); +if ~errstring + error(errstring); +end +if net.nwts ~= length(w) + error('Invalid weight vector length') +end + +net.mlp = mlpunpak(net.mlp, w); diff --git a/sourcecodes/bnt-master/netlab3.3/metrop.m b/sourcecodes/bnt-master/netlab3.3/metrop.m new file mode 100644 index 00000000..730afc6b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/metrop.m @@ -0,0 +1,193 @@ +function [samples, energies, diagn] = metrop(f, x, options, gradf, varargin) +%METROP Markov Chain Monte Carlo sampling with Metropolis algorithm. +% +% Description +% SAMPLES = METROP(F, X, OPTIONS) uses the Metropolis algorithm to +% sample from the distribution P ~ EXP(-F), where F is the first +% argument to METROP. The Markov chain starts at the point X and each +% candidate state is picked from a Gaussian proposal distribution and +% accepted or rejected according to the Metropolis criterion. +% +% SAMPLES = METROP(F, X, OPTIONS, [], P1, P2, ...) allows additional +% arguments to be passed to F(). The fourth argument is ignored, but +% is included for compatibility with HMC and the optimisers. +% +% [SAMPLES, ENERGIES, DIAGN] = METROP(F, X, OPTIONS) also returns a log +% of the energy values (i.e. negative log probabilities) for the +% samples in ENERGIES and DIAGN, a structure containing diagnostic +% information (position and acceptance threshold) for each step of the +% chain in DIAGN.POS and DIAGN.ACC respectively. All candidate states +% (including rejected ones) are stored in DIAGN.POS. +% +% S = METROP('STATE') returns a state structure that contains the state +% of the two random number generators RAND and RANDN. These are +% contained in fields randstate, randnstate. +% +% METROP('STATE', S) resets the state to S. If S is an integer, then +% it is passed to RAND and RANDN. If S is a structure returned by +% METROP('STATE') then it resets the generator to exactly the same +% state. +% +% The optional parameters in the OPTIONS vector have the following +% interpretations. +% +% OPTIONS(1) is set to 1 to display the energy values and rejection +% threshold at each step of the Markov chain. If the value is 2, then +% the position vectors at each step are also displayed. +% +% OPTIONS(14) is the number of samples retained from the Markov chain; +% default 100. +% +% OPTIONS(15) is the number of samples omitted from the start of the +% chain; default 0. +% +% OPTIONS(18) is the variance of the proposal distribution; default 1. +% +% See also +% HMC +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargin <= 2 + if ~strcmp(f, 'state') + error('Unknown argument to metrop'); + end + switch nargin + case 1 + % Return state of sampler + samples = get_state(f); % Function defined in this module + return; + case 2 + % Set the state of the sampler + set_state(f, x); % Function defined in this module + return; + end +end + +if 0 +seed = 42; +randn('state', seed); +rand('state', seed) +end + +display = options(1); +if options(14) > 0 + nsamples = options(14); +else + nsamples = 100; +end +if options(15) >= 0 + nomit = options(15); +else + nomit = 0; +end +if options(18) > 0.0 + std_dev = sqrt(options(18)); +else + std_dev = 1.0; % default +end +nparams = length(x); + +% Set up string for evaluating potential function. +f = fcnchk(f, length(varargin)); + +samples = zeros(nsamples, nparams); % Matrix of returned samples. +if nargout >= 2 + en_save = 1; + energies = zeros(nsamples, 1); +else + en_save = 0; +end +if nargout >= 3 + diagnostics = 1; + diagn_pos = zeros(nsamples, nparams); + diagn_acc = zeros(nsamples, 1); +else + diagnostics = 0; +end + +% Main loop. +n = - nomit + 1; +Eold = feval(f, x, varargin{:}); % Evaluate starting energy. +nreject = 0; % Initialise count of rejected states. +while n <= nsamples + + xold = x; + % Sample a new point from the proposal distribution + x = xold + randn(1, nparams)*std_dev; + %fprintf('netlab propose: xold = %5.3f,%5.3f, xnew = %5.3f,%5.3f\n',... + % xold(1), xold(2), x(1), x(2)); + + % Now apply Metropolis algorithm. + Enew = feval(f, x, varargin{:}); % Evaluate new energy. + a = exp(Eold - Enew); % Acceptance threshold. + if (diagnostics & n > 0) + diagn_pos(n,:) = x; + diagn_acc(n,:) = a; + end + if (display > 1) + fprintf(1, 'New position is\n'); + disp(x); + end + + r = rand(1); + %fprintf('netlab: n=%d, a=%f/%f=%5.3f (%5.3f), r=%5.3f\n',... + % n, exp(-Enew), exp(-Eold), a, exp(-Enew)/exp(-Eold), r); + if a > r % Accept the new state. + Eold = Enew; + if (display > 0) + fprintf(1, 'Finished step %4d Threshold: %g\n', n, a); + end + else % Reject the new state + if n > 0 + nreject = nreject + 1; + end + x = xold; % Reset position + if (display > 0) + fprintf(1, ' Sample rejected %4d. Threshold: %g\n', n, a); + end + end + if n > 0 + samples(n,:) = x; % Store sample. + if en_save + energies(n) = Eold; % Store energy. + end + end + n = n + 1; +end + +if (display > 0) + fprintf(1, '\nFraction of samples rejected: %g\n', ... + nreject/(nsamples)); +end + +if diagnostics + diagn.pos = diagn_pos; + diagn.acc = diagn_acc; +end + +% Return complete state of the sampler. +function state = get_state(f) + +state.randstate = rand('state'); +state.randnstate = randn('state'); +return + +% Set state of sampler, either from full state, or with an integer +function set_state(f, x) + +if isnumeric(x) + rand('state', x); + randn('state', x); +else + if ~isstruct(x) + error('Second argument to metrop must be number or state structure'); + end + if (~isfield(x, 'randstate') | ~isfield(x, 'randnstate')) + error('Second argument to metrop must contain correct fields') + end + rand('state', x.randstate); + randn('state', x.randnstate); +end +return diff --git a/sourcecodes/bnt-master/netlab3.3/minbrack.m b/sourcecodes/bnt-master/netlab3.3/minbrack.m new file mode 100644 index 00000000..31b93c42 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/minbrack.m @@ -0,0 +1,127 @@ +function [br_min, br_mid, br_max, num_evals] = minbrack(f, a, b, fa, ... + varargin) +%MINBRACK Bracket a minimum of a function of one variable. +% +% Description +% BRMIN, BRMID, BRMAX, NUMEVALS] = MINBRACK(F, A, B, FA) finds a +% bracket of three points around a local minimum of F. The function F +% must have a one dimensional domain. A < B is an initial guess at the +% minimum and maximum points of a bracket, but MINBRACK will search +% outside this interval if necessary. The bracket consists of three +% points (in increasing order) such that F(BRMID) < F(BRMIN) and +% F(BRMID) < F(BRMAX). FA is the value of the function at A: it is +% included to avoid unnecessary function evaluations in the +% optimization routines. The return value NUMEVALS is the number of +% function evaluations in MINBRACK. +% +% MINBRACK(F, A, B, FA, P1, P2, ...) allows additional arguments to be +% passed to F +% +% See also +% LINEMIN, LINEF +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check function string +f = fcnchk(f, length(varargin)); + +% Value of golden section (1 + sqrt(5))/2.0 +phi = 1.6180339887499; + +% Initialise count of number of function evaluations +num_evals = 0; + +% A small non-zero number to avoid dividing by zero in quadratic interpolation +TINY = 1.e-10; + +% Maximal proportional step to take: don't want to make this too big +% as then spend a lot of time finding the minimum inside the bracket +max_step = 10.0; + +fb = feval(f, b, varargin{:}); +num_evals = num_evals + 1; + +% Assume that we know going from a to b is downhill initially +% (usually because gradf(a) < 0). +if (fb > fa) + % Minimum must lie between a and b: do golden section until we find point + % low enough to be middle of bracket + c = b; + b = a + (c-a)/phi; + fb = feval(f, b, varargin{:}); + num_evals = num_evals + 1; + while (fb > fa) + c = b; + b = a + (c-a)/phi; + fb = feval(f, b, varargin{:}); + num_evals = num_evals + 1; + end +else + % There is a valid bracket upper bound greater than b + c = b + phi*(b-a); + fc = feval(f, c, varargin{:}); + num_evals = num_evals + 1; + bracket_found = 0; + + while (fb > fc) + % Do a quadratic interpolation (i.e. to minimum of quadratic) + r = (b-a).*(fb-fc); + q = (b-c).*(fb-fa); + u = b - ((b-c)*q - (b-a)*r)/(2.0*(sign(q-r)*max([abs(q-r), TINY]))); + ulimit = b + max_step*(c-b); + + if ((b-u)'*(u-c) > 0.0) + % Interpolant lies between b and c + fu = feval(f, u, varargin{:}); + num_evals = num_evals + 1; + if (fu < fc) + % Have a minimum between b and c + br_min = b; + br_mid = u; + br_max = c; + return; + elseif (fu > fb) + % Have a minimum between a and u + br_min = a; + br_mid = c; + br_max = u; + return; + end + % Quadratic interpolation didn't give a bracket, so take a golden step + u = c + phi*(c-b); + elseif ((c-u)'*(u-ulimit) > 0.0) + % Interpolant lies between c and limit + fu = feval(f, u, varargin{:}); + num_evals = num_evals + 1; + if (fu < fc) + % Move bracket along, and then take a golden section step + b = c; + c = u; + u = c + phi*(c-b); + else + bracket_found = 1; + end + elseif ((u-ulimit)'*(ulimit-c) >= 0.0) + % Limit parabolic u to maximum value + u = ulimit; + else + % Reject parabolic u and use golden section step + u = c + phi*(c-b); + end + if ~bracket_found + fu = feval(f, u, varargin{:}); + num_evals = num_evals + 1; + end + a = b; b = c; c = u; + fa = fb; fb = fc; fc = fu; + end % while loop +end % bracket found +br_mid = b; +if (a < c) + br_min = a; + br_max = c; +else + br_min = c; + br_max = a; +end diff --git a/sourcecodes/bnt-master/netlab3.3/mlp.m b/sourcecodes/bnt-master/netlab3.3/mlp.m new file mode 100644 index 00000000..d1dc1cf1 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlp.m @@ -0,0 +1,92 @@ +function net = mlp(nin, nhidden, nout, outfunc, prior, beta) +%MLP Create a 2-layer feedforward network. +% +% Description +% NET = MLP(NIN, NHIDDEN, NOUT, FUNC) takes the number of inputs, +% hidden units and output units for a 2-layer feed-forward network, +% together with a string FUNC which specifies the output unit +% activation function, and returns a data structure NET. The weights +% are drawn from a zero mean, unit variance isotropic Gaussian, with +% varianced scaled by the fan-in of the hidden or output units as +% appropriate. This makes use of the Matlab function RANDN and so the +% seed for the random weight initialization can be set using +% RANDN('STATE', S) where S is the seed value. The hidden units use +% the TANH activation function. +% +% The fields in NET are +% type = 'mlp' +% nin = number of inputs +% nhidden = number of hidden units +% nout = number of outputs +% nwts = total number of weights and biases +% actfn = string describing the output unit activation function: +% 'linear' +% 'logistic +% 'softmax' +% w1 = first-layer weight matrix +% b1 = first-layer bias vector +% w2 = second-layer weight matrix +% b2 = second-layer bias vector +% Here W1 has dimensions NIN times NHIDDEN, B1 has dimensions 1 times +% NHIDDEN, W2 has dimensions NHIDDEN times NOUT, and B2 has dimensions +% 1 times NOUT. +% +% NET = MLP(NIN, NHIDDEN, NOUT, FUNC, PRIOR), in which PRIOR is a +% scalar, allows the field NET.ALPHA in the data structure NET to be +% set, corresponding to a zero-mean isotropic Gaussian prior with +% inverse variance with value PRIOR. Alternatively, PRIOR can consist +% of a data structure with fields ALPHA and INDEX, allowing individual +% Gaussian priors to be set over groups of weights in the network. Here +% ALPHA is a column vector in which each element corresponds to a +% separate group of weights, which need not be mutually exclusive. The +% membership of the groups is defined by the matrix INDX in which the +% columns correspond to the elements of ALPHA. Each column has one +% element for each weight in the matrix, in the order defined by the +% function MLPPAK, and each element is 1 or 0 according to whether the +% weight is a member of the corresponding group or not. A utility +% function MLPPRIOR is provided to help in setting up the PRIOR data +% structure. +% +% NET = MLP(NIN, NHIDDEN, NOUT, FUNC, PRIOR, BETA) also sets the +% additional field NET.BETA in the data structure NET, where beta +% corresponds to the inverse noise variance. +% +% See also +% MLPPRIOR, MLPPAK, MLPUNPAK, MLPFWD, MLPERR, MLPBKP, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.type = 'mlp'; +net.nin = nin; +net.nhidden = nhidden; +net.nout = nout; +net.nwts = (nin + 1)*nhidden + (nhidden + 1)*nout; + +outfns = {'linear', 'logistic', 'softmax'}; + +if sum(strcmp(outfunc, outfns)) == 0 + error('Undefined output function. Exiting.'); +else + net.outfn = outfunc; +end + +if nargin > 4 + if isstruct(prior) + net.alpha = prior.alpha; + net.index = prior.index; + elseif size(prior) == [1 1] + net.alpha = prior; + else + error('prior must be a scalar or a structure'); + end +end + +net.w1 = randn(nin, nhidden)/sqrt(nin + 1); +net.b1 = randn(1, nhidden)/sqrt(nin + 1); +net.w2 = randn(nhidden, nout)/sqrt(nhidden + 1); +net.b2 = randn(1, nout)/sqrt(nhidden + 1); + +if nargin == 6 + net.beta = beta; +end diff --git a/sourcecodes/bnt-master/netlab3.3/mlpbkp.m b/sourcecodes/bnt-master/netlab3.3/mlpbkp.m new file mode 100644 index 00000000..5e45c8b8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpbkp.m @@ -0,0 +1,37 @@ +function g = mlpbkp(net, x, z, deltas) +%MLPBKP Backpropagate gradient of error function for 2-layer network. +% +% Description +% G = MLPBKP(NET, X, Z, DELTAS) takes a network data structure NET +% together with a matrix X of input vectors, a matrix Z of hidden unit +% activations, and a matrix DELTAS of the gradient of the error +% function with respect to the values of the output units (i.e. the +% summed inputs to the output units, before the activation function is +% applied). The return value is the gradient G of the error function +% with respect to the network weights. Each row of X corresponds to one +% input vector. +% +% This function is provided so that the common backpropagation +% algorithm can be used by multi-layer perceptron network models to +% compute gradients for mixture density networks as well as standard +% error functions. +% +% See also +% MLP, MLPGRAD, MLPDERIV, MDNGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Evaluate second-layer gradients. +gw2 = z'*deltas; +gb2 = sum(deltas, 1); + +% Now do the backpropagation. +delhid = deltas*net.w2'; +delhid = delhid.*(1.0 - z.*z); + +% Finally, evaluate the first-layer gradients. +gw1 = x'*delhid; +gb1 = sum(delhid, 1); + +g = [gw1(:)', gb1, gw2(:)', gb2]; diff --git a/sourcecodes/bnt-master/netlab3.3/mlpderiv.m b/sourcecodes/bnt-master/netlab3.3/mlpderiv.m new file mode 100644 index 00000000..9dd96c03 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpderiv.m @@ -0,0 +1,47 @@ +function g = mlpderiv(net, x) +%MLPDERIV Evaluate derivatives of network outputs with respect to weights. +% +% Description +% G = MLPDERIV(NET, X) takes a network data structure NET and a matrix +% of input vectors X and returns a three-index matrix G whose I, J, K +% element contains the derivative of network output K with respect to +% weight or bias parameter J for input pattern I. The ordering of the +% weight and bias parameters is defined by MLPUNPAK. +% +% See also +% MLP, MLPPAK, MLPGRAD, MLPBKP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp', x); +if ~isempty(errstring); + error(errstring); +end + +[y, z] = mlpfwd(net, x); + +ndata = size(x, 1); + +if isfield(net, 'mask') + nwts = size(find(net.mask), 1); + temp = zeros(1, net.nwts); +else + nwts = net.nwts; +end + +g = zeros(ndata, nwts, net.nout); +for k = 1 : net.nout + delta = zeros(1, net.nout); + delta(1, k) = 1; + for n = 1 : ndata + if isfield(net, 'mask') + temp = mlpbkp(net, x(n, :), z(n, :), delta); + g(n, :, k) = temp(logical(net.mask)); + else + g(n, :, k) = mlpbkp(net, x(n, :), z(n, :),... + delta); + end + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/mlperr.m b/sourcecodes/bnt-master/netlab3.3/mlperr.m new file mode 100644 index 00000000..7e3efe47 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlperr.m @@ -0,0 +1,62 @@ +function [e, edata, eprior] = mlperr(net, x, t) +%MLPERR Evaluate error function for 2-layer network. +% +% Description +% E = MLPERR(NET, X, T) takes a network data structure NET together +% with a matrix X of input vectors and a matrix T of target vectors, +% and evaluates the error function E. The choice of error function +% corresponds to the output unit activation function. Each row of X +% corresponds to one input vector and each row of T corresponds to one +% target vector. +% +% [E, EDATA, EPRIOR] = MLPERR(NET, X, T) additionally returns the data +% and prior components of the error, assuming a zero mean Gaussian +% prior on the weights with inverse variance parameters ALPHA and BETA +% taken from the network data structure NET. +% +% See also +% MLP, MLPPAK, MLPUNPAK, MLPFWD, MLPBKP, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp', x, t); +if ~isempty(errstring); + error(errstring); +end +[y, z, a] = mlpfwd(net, x); + +switch net.outfn + + case 'linear' % Linear outputs + edata = 0.5*sum(sum((y - t).^2)); + + case 'logistic' % Logistic outputs + % Ensure that log(1-y) is computable: need exp(a) > eps + maxcut = -log(eps); + % Ensure that log(y) is computable + mincut = -log(1/realmin - 1); + a = min(a, maxcut); + a = max(a, mincut); + y = 1./(1 + exp(-a)); + edata = - sum(sum(t.*log(y) + (1 - t).*log(1 - y))); + + case 'softmax' % Softmax outputs + nout = size(a,2); + % Ensure that sum(exp(a), 2) does not overflow + maxcut = log(realmax) - log(nout); + % Ensure that exp(a) > 0 + mincut = log(realmin); + a = min(a, maxcut); + a = max(a, mincut); + temp = exp(a); + y = temp./(sum(temp, 2)*ones(1,nout)); + % Ensure that log(y) is computable + y(y<realmin) = realmin; + edata = - sum(sum(t.*log(y))); + + otherwise + error(['Unknown activation function ', net.outfn]); +end +[e, edata, eprior] = errbayes(net, edata); diff --git a/sourcecodes/bnt-master/netlab3.3/mlpevfwd.m b/sourcecodes/bnt-master/netlab3.3/mlpevfwd.m new file mode 100644 index 00000000..e6d6898e --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpevfwd.m @@ -0,0 +1,26 @@ +function [y, extra, invhess] = mlpevfwd(net, x, t, x_test, invhess) +%MLPEVFWD Forward propagation with evidence for MLP +% +% Description +% Y = MLPEVFWD(NET, X, T, X_TEST) takes a network data structure NET +% together with the input X and target T training data and input test +% data X_TEST. It returns the normal forward propagation through the +% network Y together with a matrix EXTRA which consists of error bars +% (variance) for a regression problem or moderated outputs for a +% classification problem. The optional argument (and return value) +% INVHESS is the inverse of the network Hessian computed on the +% training data inputs and targets. Passing it in avoids recomputing +% it, which can be a significant saving for large training sets. +% +% See also +% FEVBAYES +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[y, z, a] = mlpfwd(net, x_test); +if nargin == 4 + [extra, invhess] = fevbayes(net, y, a, x, t, x_test); +else + [extra, invhess] = fevbayes(net, y, a, x, t, x_test, invhess); +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/mlpfwd.m b/sourcecodes/bnt-master/netlab3.3/mlpfwd.m new file mode 100644 index 00000000..d470ac96 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpfwd.m @@ -0,0 +1,65 @@ +function [y, z, a] = mlpfwd(net, x) +%MLPFWD Forward propagation through 2-layer network. +% +% Description +% Y = MLPFWD(NET, X) takes a network data structure NET together with a +% matrix X of input vectors, and forward propagates the inputs through +% the network to generate a matrix Y of output vectors. Each row of X +% corresponds to one input vector and each row of Y corresponds to one +% output vector. +% +% [Y, Z] = MLPFWD(NET, X) also generates a matrix Z of the hidden unit +% activations where each row corresponds to one pattern. +% +% [Y, Z, A] = MLPFWD(NET, X) also returns a matrix A giving the summed +% inputs to each output unit, where each row corresponds to one +% pattern. +% +% See also +% MLP, MLPPAK, MLPUNPAK, MLPERR, MLPBKP, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp', x); +if ~isempty(errstring); + error(errstring); +end + +ndata = size(x, 1); + +z = tanh(x*net.w1 + ones(ndata, 1)*net.b1); +a = z*net.w2 + ones(ndata, 1)*net.b2; + +switch net.outfn + + case 'linear' % Linear outputs + + y = a; + + case 'logistic' % Logistic outputs + % Prevent overflow and underflow: use same bounds as mlperr + % Ensure that log(1-y) is computable: need exp(a) > eps + maxcut = -log(eps); + % Ensure that log(y) is computable + mincut = -log(1/realmin - 1); + a = min(a, maxcut); + a = max(a, mincut); + y = 1./(1 + exp(-a)); + + case 'softmax' % Softmax outputs + + % Prevent overflow and underflow: use same bounds as glmerr + % Ensure that sum(exp(a), 2) does not overflow + maxcut = log(realmax) - log(net.nout); + % Ensure that exp(a) > 0 + mincut = log(realmin); + a = min(a, maxcut); + a = max(a, mincut); + temp = exp(a); + y = temp./(sum(temp, 2)*ones(1, net.nout)); + + otherwise + error(['Unknown activation function ', net.outfn]); +end diff --git a/sourcecodes/bnt-master/netlab3.3/mlpgrad.m b/sourcecodes/bnt-master/netlab3.3/mlpgrad.m new file mode 100644 index 00000000..c5a2a349 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpgrad.m @@ -0,0 +1,33 @@ +function [g, gdata, gprior] = mlpgrad(net, x, t) +%MLPGRAD Evaluate gradient of error function for 2-layer network. +% +% Description +% G = MLPGRAD(NET, X, T) takes a network data structure NET together +% with a matrix X of input vectors and a matrix T of target vectors, +% and evaluates the gradient G of the error function with respect to +% the network weights. The error funcion corresponds to the choice of +% output unit activation function. Each row of X corresponds to one +% input vector and each row of T corresponds to one target vector. +% +% [G, GDATA, GPRIOR] = MLPGRAD(NET, X, T) also returns separately the +% data and prior contributions to the gradient. In the case of multiple +% groups in the prior, GPRIOR is a matrix with a row for each group and +% a column for each weight parameter. +% +% See also +% MLP, MLPPAK, MLPUNPAK, MLPFWD, MLPERR, MLPBKP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp', x, t); +if ~isempty(errstring); + error(errstring); +end +[y, z] = mlpfwd(net, x); +delout = y - t; + +gdata = mlpbkp(net, x, z, delout); + +[g, gdata, gprior] = gbayes(net, gdata); diff --git a/sourcecodes/bnt-master/netlab3.3/mlphdotv.m b/sourcecodes/bnt-master/netlab3.3/mlphdotv.m new file mode 100644 index 00000000..63b54622 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlphdotv.m @@ -0,0 +1,79 @@ +function hdv = mlphdotv(net, x, t, v) +%MLPHDOTV Evaluate the product of the data Hessian with a vector. +% +% Description +% +% HDV = MLPHDOTV(NET, X, T, V) takes an MLP network data structure NET, +% together with the matrix X of input vectors, the matrix T of target +% vectors and an arbitrary row vector V whose length equals the number +% of parameters in the network, and returns the product of the data- +% dependent contribution to the Hessian matrix with V. The +% implementation is based on the R-propagation algorithm of +% Pearlmutter. +% +% See also +% MLP, MLPHESS, HESSCHEK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp', x, t); +if ~isempty(errstring); + error(errstring); +end + +ndata = size(x, 1); + +[y, z] = mlpfwd(net, x); % Standard forward propagation. +zprime = (1 - z.*z); % Hidden unit first derivatives. +zpprime = -2.0*z.*zprime; % Hidden unit second derivatives. + +vnet = mlpunpak(net, v); % Unpack the v vector. + +% Do the R-forward propagation. + +ra1 = x*vnet.w1 + ones(ndata, 1)*vnet.b1; +rz = zprime.*ra1; +ra2 = rz*net.w2 + z*vnet.w2 + ones(ndata, 1)*vnet.b2; + +switch net.outfn + + case 'linear' % Linear outputs + + ry = ra2; + + case 'logistic' % Logistic outputs + + ry = y.*(1 - y).*ra2; + + case 'softmax' % Softmax outputs + + nout = size(t, 2); + ry = y.*ra2 - y.*(sum(y.*ra2, 2)*ones(1, nout)); + + otherwise + error(['Unknown activation function ', net.outfn]); +end + +% Evaluate delta for the output units. + +delout = y - t; + +% Do the standard backpropagation. + +delhid = zprime.*(delout*net.w2'); + +% Now do the R-backpropagation. + +rdelhid = zpprime.*ra1.*(delout*net.w2') + zprime.*(delout*vnet.w2') + ... + zprime.*(ry*net.w2'); + +% Finally, evaluate the components of hdv and then merge into long vector. + +hw1 = x'*rdelhid; +hb1 = sum(rdelhid, 1); +hw2 = z'*ry + rz'*delout; +hb2 = sum(ry, 1); + +hdv = [hw1(:)', hb1, hw2(:)', hb2]; diff --git a/sourcecodes/bnt-master/netlab3.3/mlphess.m b/sourcecodes/bnt-master/netlab3.3/mlphess.m new file mode 100644 index 00000000..bd7c44e9 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlphess.m @@ -0,0 +1,51 @@ +function [h, hdata] = mlphess(net, x, t, hdata) +%MLPHESS Evaluate the Hessian matrix for a multi-layer perceptron network. +% +% Description +% H = MLPHESS(NET, X, T) takes an MLP network data structure NET, a +% matrix X of input values, and a matrix T of target values and returns +% the full Hessian matrix H corresponding to the second derivatives of +% the negative log posterior distribution, evaluated for the current +% weight and bias values as defined by NET. +% +% [H, HDATA] = MLPHESS(NET, X, T) returns both the Hessian matrix H and +% the contribution HDATA arising from the data dependent term in the +% Hessian. +% +% H = MLPHESS(NET, X, T, HDATA) takes a network data structure NET, a +% matrix X of input values, and a matrix T of target values, together +% with the contribution HDATA arising from the data dependent term in +% the Hessian, and returns the full Hessian matrix H corresponding to +% the second derivatives of the negative log posterior distribution. +% This version saves computation time if HDATA has already been +% evaluated for the current weight and bias values. +% +% See also +% MLP, HESSCHEK, MLPHDOTV, EVIDENCE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp', x, t); +if ~isempty(errstring); + error(errstring); +end + +if nargin == 3 + % Data term in Hessian needs to be computed + hdata = datahess(net, x, t); +end + +[h, hdata] = hbayes(net, hdata); + +% Sub-function to compute data part of Hessian +function hdata = datahess(net, x, t) + +hdata = zeros(net.nwts, net.nwts); + +for v = eye(net.nwts); + hdata(find(v),:) = mlphdotv(net, x, t, v); +end + +return diff --git a/sourcecodes/bnt-master/netlab3.3/mlphint.m b/sourcecodes/bnt-master/netlab3.3/mlphint.m new file mode 100644 index 00000000..3546dfd3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlphint.m @@ -0,0 +1,92 @@ +function [h1, h2] = mlphint(net); +%MLPHINT Plot Hinton diagram for 2-layer feed-forward network. +% +% Description +% +% MLPHINT(NET) takes a network structure NET and plots the Hinton +% diagram comprised of two figure windows, one displaying the first- +% layer weights and biases, and one displaying the second-layer weights +% and biases. +% +% [H1, H2] = MLPHINT(NET) also returns handles H1 and H2 to the +% figures which can be used, for instance, to delete the figures when +% they are no longer needed. +% +% To print the figure correctly, you should call SET(H, +% 'INVERTHARDCOPY', 'ON') before printing. +% +% See also +% DEMHINT, HINTMAT, MLP, MLPPAK, MLPUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set scale to be up to 0.9 of maximum absolute weight value, where scale +% defined so that area of box proportional to weight value. + +% Use no more than 640x480 pixels +xmax = 640; ymax = 480; + +% Offset bottom left hand corner +x01 = 40; y01 = 40; +x02 = 80; y02 = 80; + +% Need to allow 5 pixels border for window frame: but 30 at top +border = 5; +top_border = 30; + +ymax = ymax - top_border; +xmax = xmax - border; + +% First layer + +wb1 = [net.w1; net.b1]; +[xvals, yvals, color] = hintmat(wb1'); +% Try to preserve aspect ratio approximately +if (8*net.nhidden < 6*(net.nin + 1)) + delx = xmax; dely = xmax*net.nhidden/(net.nin + 1); +else + delx = ymax*(net.nin + 1)/net.nhidden; dely = ymax; +end + +h1 = figure('Color', [0.5 0.5 0.5], ... + 'Name', 'Hinton diagram: first-layer weights and biases', ... + 'NumberTitle', 'off', ... + 'Colormap', [0 0 0; 1 1 1], ... + 'Units', 'pixels', ... + 'Position', [x01 y01 delx dely]); +set(gca, 'Visible', 'off', 'Position', [0 0 1 1]); +hold on + +cmap = [0 0 0; 1 1 1]; +colors(1, :, :) = cmap(color, :); +patch(xvals', yvals', colors, 'Edgecolor', 'none'); +axis equal; +xpos = net.nin; +line([xpos xpos], [0 net.nhidden], 'color', 'red', 'linewidth', 3); + +% Second layer + +wb2 = [net.w2; net.b2]; +[xvals, yvals, color] = hintmat(wb2'); +if (8*net.nout < 6*(net.nhidden + 1)) + delx = xmax; dely = xmax*net.nout/(net.nhidden + 1); +else + delx = ymax*(net.nhidden + 1)/net.nout; dely = ymax; +end + +h2 = figure('Color', [0.5 0.5 0.5], ... + 'Name', 'Hinton diagram: second-layer weights and biases', ... + 'NumberTitle', 'off', ... + 'Colormap', [0 0 0; 1 1 1], ... + 'Units', 'pixels', ... + 'Position', [x02 y02 delx dely]); +set(gca, 'Visible', 'off', 'Position', [0 0 1 1]); + +hold on +colors2(1, :, :) = cmap(color, :); +patch(xvals', yvals', colors2, 'Edgecolor', 'none'); +axis equal; +xpos = net.nhidden; +line([xpos xpos], [0 net.nout], 'color', 'red', 'linewidth', 3); + diff --git a/sourcecodes/bnt-master/netlab3.3/mlpinit.m b/sourcecodes/bnt-master/netlab3.3/mlpinit.m new file mode 100644 index 00000000..ae4e80b5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpinit.m @@ -0,0 +1,31 @@ +function net = mlpinit(net, prior) +%MLPINIT Initialise the weights in a 2-layer feedforward network. +% +% Description +% +% NET = MLPINIT(NET, PRIOR) takes a 2-layer feedforward network NET and +% sets the weights and biases by sampling from a Gaussian distribution. +% If PRIOR is a scalar, then all of the parameters (weights and biases) +% are sampled from a single isotropic Gaussian with inverse variance +% equal to PRIOR. If PRIOR is a data structure of the kind generated by +% MLPPRIOR, then the parameters are sampled from multiple Gaussians +% according to their groupings (defined by the INDEX field) with +% corresponding variances (defined by the ALPHA field). +% +% See also +% MLP, MLPPRIOR, MLPPAK, MLPUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if isstruct(prior) + sig = 1./sqrt(prior.index*prior.alpha); + w = sig'.*randn(1, net.nwts); +elseif size(prior) == [1 1] + w = randn(1, net.nwts).*sqrt(1/prior); +else + error('prior must be a scalar or a structure'); +end + +net = mlpunpak(net, w); + diff --git a/sourcecodes/bnt-master/netlab3.3/mlppak.m b/sourcecodes/bnt-master/netlab3.3/mlppak.m new file mode 100644 index 00000000..30ec375d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlppak.m @@ -0,0 +1,31 @@ +function w = mlppak(net) +%MLPPAK Combines weights and biases into one weights vector. +% +% Description +% W = MLPPAK(NET) takes a network data structure NET and combines the +% component weight matrices bias vectors into a single row vector W. +% The facility to switch between these two representations for the +% network parameters is useful, for example, in training a network by +% error function minimization, since a single vector of parameters can +% be handled by general-purpose optimization routines. +% +% The ordering of the paramters in W is defined by +% w = [net.w1(:)', net.b1, net.w2(:)', net.b2]; +% where W1 is the first-layer weight matrix, B1 is the first-layer +% bias vector, W2 is the second-layer weight matrix, and B2 is the +% second-layer bias vector. +% +% See also +% MLP, MLPUNPAK, MLPFWD, MLPERR, MLPBKP, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp'); +if ~isempty(errstring); + error(errstring); +end + +w = [net.w1(:)', net.b1, net.w2(:)', net.b2]; + diff --git a/sourcecodes/bnt-master/netlab3.3/mlpprior.m b/sourcecodes/bnt-master/netlab3.3/mlpprior.m new file mode 100644 index 00000000..8f7415c5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpprior.m @@ -0,0 +1,62 @@ +function prior = mlpprior(nin, nhidden, nout, aw1, ab1, aw2, ab2) +%MLPPRIOR Create Gaussian prior for mlp. +% +% Description +% PRIOR = MLPPRIOR(NIN, NHIDDEN, NOUT, AW1, AB1, AW2, AB2) generates a +% data structure PRIOR, with fields PRIOR.ALPHA and PRIOR.INDEX, which +% specifies a Gaussian prior distribution for the network weights in a +% two-layer feedforward network. Two different cases are possible. In +% the first case, AW1, AB1, AW2 and AB2 are all scalars and represent +% the regularization coefficients for four groups of parameters in the +% network corresponding to first-layer weights, first-layer biases, +% second-layer weights, and second-layer biases respectively. Then +% PRIOR.ALPHA represents a column vector of length 4 containing the +% parameters, and PRIOR.INDEX is a matrix specifying which weights +% belong in each group. Each column has one element for each weight in +% the matrix, using the standard ordering as defined in MLPPAK, and +% each element is 1 or 0 according to whether the weight is a member of +% the corresponding group or not. In the second case the parameter AW1 +% is a vector of length equal to the number of inputs in the network, +% and the corresponding matrix PRIOR.INDEX now partitions the first- +% layer weights into groups corresponding to the weights fanning out of +% each input unit. This prior is appropriate for the technique of +% automatic relevance determination. +% +% See also +% MLP, MLPERR, MLPGRAD, EVIDENCE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +nextra = nhidden + (nhidden + 1)*nout; +nwts = nin*nhidden + nextra; + +if size(aw1) == [1,1] + + indx = [ones(1, nin*nhidden), zeros(1, nextra)]'; + +elseif size(aw1) == [1, nin] + + indx = kron(ones(nhidden, 1), eye(nin)); + indx = [indx; zeros(nextra, nin)]; + +else + + error('Parameter aw1 of invalid dimensions'); + +end + +extra = zeros(nwts, 3); + +mark1 = nin*nhidden; +mark2 = mark1 + nhidden; +extra(mark1 + 1:mark2, 1) = ones(nhidden,1); +mark3 = mark2 + nhidden*nout; +extra(mark2 + 1:mark3, 2) = ones(nhidden*nout,1); +mark4 = mark3 + nout; +extra(mark3 + 1:mark4, 3) = ones(nout,1); + +indx = [indx, extra]; + +prior.index = indx; +prior.alpha = [aw1, ab1, aw2, ab2]'; diff --git a/sourcecodes/bnt-master/netlab3.3/mlptrain.m b/sourcecodes/bnt-master/netlab3.3/mlptrain.m new file mode 100644 index 00000000..395689e9 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlptrain.m @@ -0,0 +1,24 @@ +function [net, error] = mlptrain(net, x, t, its); +%MLPTRAIN Utility to train an MLP network for demtrain +% +% Description +% +% [NET, ERROR] = MLPTRAIN(NET, X, T, ITS) trains a network data +% structure NET using the scaled conjugate gradient algorithm for ITS +% cycles with input data X, target data T. +% +% See also +% DEMTRAIN, SCG, NETOPT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +options = zeros(1,18); +options(1) = -1; % To prevent any messages at all +options(9) = 0; +options(14) = its; + +[net, options] = netopt(net, options, x, t, 'scg'); + +error = options(8); + diff --git a/sourcecodes/bnt-master/netlab3.3/mlpunpak.m b/sourcecodes/bnt-master/netlab3.3/mlpunpak.m new file mode 100644 index 00000000..c2b16b5f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpunpak.m @@ -0,0 +1,39 @@ +function net = mlpunpak(net, w) +%MLPUNPAK Separates weights vector into weight and bias matrices. +% +% Description +% NET = MLPUNPAK(NET, W) takes an mlp network data structure NET and a +% weight vector W, and returns a network data structure identical to +% the input network, except that the first-layer weight matrix W1, the +% first-layer bias vector B1, the second-layer weight matrix W2 and the +% second-layer bias vector B2 have all been set to the corresponding +% elements of W. +% +% See also +% MLP, MLPPAK, MLPFWD, MLPERR, MLPBKP, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp'); +if ~isempty(errstring); + error(errstring); +end + +if net.nwts ~= length(w) + error('Invalid weight vector length') +end + +nin = net.nin; +nhidden = net.nhidden; +nout = net.nout; + +mark1 = nin*nhidden; +net.w1 = reshape(w(1:mark1), nin, nhidden); +mark2 = mark1 + nhidden; +net.b1 = reshape(w(mark1 + 1: mark2), 1, nhidden); +mark3 = mark2 + nhidden*nout; +net.w2 = reshape(w(mark2 + 1: mark3), nhidden, nout); +mark4 = mark3 + nout; +net.b2 = reshape(w(mark3 + 1: mark4), 1, nout); diff --git a/sourcecodes/bnt-master/netlab3.3/netderiv.m b/sourcecodes/bnt-master/netlab3.3/netderiv.m new file mode 100644 index 00000000..2c6b379c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netderiv.m @@ -0,0 +1,19 @@ +function g = netderiv(w, net, x) +%NETDERIV Evaluate derivatives of network outputs by weights generically. +% +% Description +% +% G = NETDERIV(W, NET, X) takes a weight vector W and a network data +% structure NET, together with the matrix X of input vectors, and +% returns the gradient of the outputs with respect to the weights +% evaluated at W. +% +% See also +% NETEVFWD, NETOPT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +fstr = [net.type, 'deriv']; +net = netunpak(net, w); +g = feval(fstr, net, x); diff --git a/sourcecodes/bnt-master/netlab3.3/neterr.m b/sourcecodes/bnt-master/netlab3.3/neterr.m new file mode 100644 index 00000000..6679630c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/neterr.m @@ -0,0 +1,29 @@ +function [e, varargout] = neterr(w, net, x, t) +%NETERR Evaluate network error function for generic optimizers +% +% Description +% +% E = NETERR(W, NET, X, T) takes a weight vector W and a network data +% structure NET, together with the matrix X of input vectors and the +% matrix T of target vectors, and returns the value of the error +% function evaluated at W. +% +% [E, VARARGOUT] = NETERR(W, NET, X, T) also returns any additional +% return values from the error function. +% +% See also +% NETGRAD, NETHESS, NETOPT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstr = [net.type, 'err']; +net = netunpak(net, w); + +[s{1:nargout}] = feval(errstr, net, x, t); +e = s{1}; +if nargout > 1 + for i = 2:nargout + varargout{i-1} = s{i}; + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/netevfwd.m b/sourcecodes/bnt-master/netlab3.3/netevfwd.m new file mode 100644 index 00000000..26bb25aa --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netevfwd.m @@ -0,0 +1,29 @@ +function [y, extra, invhess] = netevfwd(w, net, x, t, x_test, invhess) +%NETEVFWD Generic forward propagation with evidence for network +% +% Description +% [Y, EXTRA] = NETEVFWD(W, NET, X, T, X_TEST) takes a network data +% structure NET together with the input X and target T training data +% and input test data X_TEST. It returns the normal forward propagation +% through the network Y together with a matrix EXTRA which consists of +% error bars (variance) for a regression problem or moderated outputs +% for a classification problem. +% +% The optional argument (and return value) INVHESS is the inverse of +% the network Hessian computed on the training data inputs and targets. +% Passing it in avoids recomputing it, which can be a significant +% saving for large training sets. +% +% See also +% MLPEVFWD, RBFEVFWD, GLMEVFWD, FEVBAYES +% + +% Copyright (c) Ian T Nabney (1996-2001) + +func = [net.type, 'evfwd']; +net = netunpak(net, w); +if nargin == 5 + [y, extra, invhess] = feval(func, net, x, t, x_test); +else + [y, extra, invhess] = feval(func, net, x, t, x_test, invhess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/netgrad.m b/sourcecodes/bnt-master/netlab3.3/netgrad.m new file mode 100644 index 00000000..afab88e0 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netgrad.m @@ -0,0 +1,21 @@ +function g = netgrad(w, net, x, t) +%NETGRAD Evaluate network error gradient for generic optimizers +% +% Description +% +% G = NETGRAD(W, NET, X, T) takes a weight vector W and a network data +% structure NET, together with the matrix X of input vectors and the +% matrix T of target vectors, and returns the gradient of the error +% function evaluated at W. +% +% See also +% MLP, NETERR, NETOPT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +gradstr = [net.type, 'grad']; + +net = netunpak(net, w); + +g = feval(gradstr, net, x, t); diff --git a/sourcecodes/bnt-master/netlab3.3/nethess.m b/sourcecodes/bnt-master/netlab3.3/nethess.m new file mode 100644 index 00000000..119fe502 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/nethess.m @@ -0,0 +1,29 @@ +function [h, varargout] = nethess(w, net, x, t, varargin) +%NETHESS Evaluate network Hessian +% +% Description +% +% H = NETHESS(W, NET, X, T) takes a weight vector W and a network data +% structure NET, together with the matrix X of input vectors and the +% matrix T of target vectors, and returns the value of the Hessian +% evaluated at W. +% +% [E, VARARGOUT] = NETHESS(W, NET, X, T, VARARGIN) also returns any +% additional return values from the network Hessian function, and +% passes additional arguments to that function. +% +% See also +% NETERR, NETGRAD, NETOPT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +hess_str = [net.type, 'hess']; + +net = netunpak(net, w); + +[s{1:nargout}] = feval(hess_str, net, x, t, varargin{:}); +h = s{1}; +for i = 2:nargout + varargout{i-1} = s{i}; +end diff --git a/sourcecodes/bnt-master/netlab3.3/netinit.m b/sourcecodes/bnt-master/netlab3.3/netinit.m new file mode 100644 index 00000000..f94e30b3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netinit.m @@ -0,0 +1,45 @@ +function net = netinit(net, prior) +%NETINIT Initialise the weights in a network. +% +% Description +% +% NET = NETINIT(NET, PRIOR) takes a network data structure NET and sets +% the weights and biases by sampling from a Gaussian distribution. If +% PRIOR is a scalar, then all of the parameters (weights and biases) +% are sampled from a single isotropic Gaussian with inverse variance +% equal to PRIOR. If PRIOR is a data structure of the kind generated by +% MLPPRIOR, then the parameters are sampled from multiple Gaussians +% according to their groupings (defined by the INDEX field) with +% corresponding variances (defined by the ALPHA field). +% +% See also +% MLPPRIOR, NETUNPAK, RBFPRIOR +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if isstruct(prior) + if (isfield(net, 'mask')) + if find(sum(prior.index, 2)) ~= find(net.mask) + error('Index does not match mask'); + end + sig = sqrt(prior.index*prior.alpha); + % Weights corresponding to zeros in mask will not be used anyway + % Set their priors to one to avoid division by zero + sig = sig + (sig == 0); + sig = 1./sqrt(sig); + else + sig = 1./sqrt(prior.index*prior.alpha); + end + w = sig'.*randn(1, net.nwts); +elseif size(prior) == [1 1] + w = randn(1, net.nwts).*sqrt(1/prior); +else + error('prior must be a scalar or a structure'); +end + +if (isfield(net, 'mask')) + w = w(logical(net.mask)); +end +net = netunpak(net, w); + diff --git a/sourcecodes/bnt-master/netlab3.3/netlab3.3.zip b/sourcecodes/bnt-master/netlab3.3/netlab3.3.zip new file mode 100644 index 00000000..96bcfaa3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netlab3.3.zip Binary files differdiff --git a/sourcecodes/bnt-master/netlab3.3/netlogo.mat b/sourcecodes/bnt-master/netlab3.3/netlogo.mat new file mode 100644 index 00000000..743716f8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netlogo.mat Binary files differdiff --git a/sourcecodes/bnt-master/netlab3.3/netopt.m b/sourcecodes/bnt-master/netlab3.3/netopt.m new file mode 100644 index 00000000..76807903 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netopt.m @@ -0,0 +1,52 @@ +function [net, options, varargout] = netopt(net, options, x, t, alg); +%NETOPT Optimize the weights in a network model. +% +% Description +% +% NETOPT is a helper function which facilitates the training of +% networks using the general purpose optimizers as well as sampling +% from the posterior distribution of parameters using general purpose +% Markov chain Monte Carlo sampling algorithms. It can be used with any +% function that searches in parameter space using error and gradient +% functions. +% +% [NET, OPTIONS] = NETOPT(NET, OPTIONS, X, T, ALG) takes a network +% data structure NET, together with a vector OPTIONS of parameters +% governing the behaviour of the optimization algorithm, a matrix X of +% input vectors and a matrix T of target vectors, and returns the +% trained network as well as an updated OPTIONS vector. The string ALG +% determines which optimization algorithm (CONJGRAD, QUASINEW, SCG, +% etc.) or Monte Carlo algorithm (such as HMC) will be used. +% +% [NET, OPTIONS, VARARGOUT] = NETOPT(NET, OPTIONS, X, T, ALG) also +% returns any additional return values from the optimisation algorithm. +% +% See also +% NETGRAD, BFGS, CONJGRAD, GRADDESC, HMC, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +optstring = [alg, '(''neterr'', w, options, ''netgrad'', net, x, t)']; + +% Extract weights from network as single vector +w = netpak(net); + +% Carry out optimisation +[s{1:nargout}] = eval(optstring); +w = s{1}; + +if nargout > 1 + options = s{2}; + + % If there are additional arguments, extract them + nextra = nargout - 2; + if nextra > 0 + for i = 1:nextra + varargout{i} = s{i+2}; + end + end +end + +% Pack the weights back into the network +net = netunpak(net, w); diff --git a/sourcecodes/bnt-master/netlab3.3/netpak.m b/sourcecodes/bnt-master/netlab3.3/netpak.m new file mode 100644 index 00000000..03d9bd58 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netpak.m @@ -0,0 +1,25 @@ +function w = netpak(net) +%NETPAK Combines weights and biases into one weights vector. +% +% Description +% W = NETPAK(NET) takes a network data structure NET and combines the +% component weight matrices into a single row vector W. The facility +% to switch between these two representations for the network +% parameters is useful, for example, in training a network by error +% function minimization, since a single vector of parameters can be +% handled by general-purpose optimization routines. This function also +% takes into account a MASK defined as a field in NET by removing any +% weights that correspond to entries of 0 in the mask. +% +% See also +% NET, NETUNPAK, NETFWD, NETERR, NETGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +pakstr = [net.type, 'pak']; +w = feval(pakstr, net); +% Return masked subset of weights +if (isfield(net, 'mask')) + w = w(logical(net.mask)); +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/netunpak.m b/sourcecodes/bnt-master/netlab3.3/netunpak.m new file mode 100644 index 00000000..7f253d2f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netunpak.m @@ -0,0 +1,34 @@ +function net = netunpak(net, w) +%NETUNPAK Separates weights vector into weight and bias matrices. +% +% Description +% NET = NETUNPAK(NET, W) takes an net network data structure NET and a +% weight vector W, and returns a network data structure identical to +% the input network, except that the componenet weight matrices have +% all been set to the corresponding elements of W. If there is a MASK +% field in the NET data structure, then the weights in W are placed in +% locations corresponding to non-zero entries in the mask (so W should +% have the same length as the number of non-zero entries in the MASK). +% +% See also +% NETPAK, NETFWD, NETERR, NETGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +unpakstr = [net.type, 'unpak']; + +% Check if we are being passed a masked set of weights +if (isfield(net, 'mask')) + if length(w) ~= size(find(net.mask), 1) + error('Weight vector length does not match mask length') + end + % Do a full pack of all current network weights + pakstr = [net.type, 'pak']; + fullw = feval(pakstr, net); + % Replace current weights with new ones + fullw(logical(net.mask)) = w; + w = fullw; +end + +net = feval(unpakstr, net, w); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/oilTrn.dat b/sourcecodes/bnt-master/netlab3.3/oilTrn.dat new file mode 100644 index 00000000..ba12723b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/oilTrn.dat @@ -0,0 +1,503 @@ + nin 12 + nout 2 + ndata 500 + 3.315000e-01 2.156000e-01 6.802000e-01 1.434000e-01 6.825000e-01 2.720000e-01 6.223000e-01 2.092000e-01 7.961000e-01 1.530000e-01 5.856000e-01 2.573000e-01 3.440000e-01 1.401000e-01 + 9.390000e-02 1.008900e+00 3.650000e-02 6.944000e-01 9.080000e-02 4.961000e-01 7.220000e-02 6.521000e-01 -1.300000e-02 6.085000e-01 6.310000e-02 6.597000e-01 5.140000e-02 4.459000e-01 + 5.184000e-01 2.283000e-01 5.300000e-01 6.884000e-01 7.456000e-01 6.171000e-01 6.136000e-01 5.928000e-01 7.678000e-01 6.130000e-01 6.705000e-01 5.202000e-01 3.710000e-01 3.214000e-01 + 4.208000e-01 6.740000e-01 1.651000e-01 7.592000e-01 1.810000e-01 5.448000e-01 1.707000e-01 7.554000e-01 1.635000e-01 5.492000e-01 2.598000e-01 6.455000e-01 1.667000e-01 4.177000e-01 + 3.130000e-01 6.465000e-01 5.908000e-01 6.924000e-01 7.664000e-01 6.262000e-01 1.717700e+00 1.500000e-02 8.510000e-02 1.904600e+00 -1.650000e-02 2.210000e-02 3.378000e-01 4.184000e-01 + 1.145800e+00 -4.670000e-02 4.056000e-01 5.662000e-01 3.123000e-01 4.580000e-01 3.636000e-01 6.134000e-01 3.305000e-01 4.132000e-01 4.167000e-01 5.514000e-01 3.249000e-01 2.790000e-01 +-1.900000e-03 1.732000e-01 5.700000e-03 4.882000e-01 2.076000e-01 3.910000e-01 8.600000e-03 1.719800e+00 2.150000e-02 -2.580000e-02 6.730000e-02 -8.290000e-02 5.100000e-02 2.123000e-01 + 7.800000e-03 4.615000e-01 1.181000e-01 6.590000e-01 2.587000e-01 6.352000e-01 -1.910000e-02 1.749100e+00 1.098000e-01 -1.315000e-01 4.070000e-02 -4.850000e-02 6.210000e-02 3.856000e-01 + 7.305000e-01 1.189000e-01 1.062600e+00 1.013000e-01 1.138500e+00 1.486000e-01 1.763100e+00 -4.160000e-02 2.073100e+00 -8.720000e-02 -4.390000e-02 5.300000e-02 6.070000e-01 8.130000e-02 + 6.199000e-01 4.779000e-01 9.528000e-01 7.463000e-01 9.741000e-01 8.633000e-01 1.677100e+00 6.200000e-02 2.027100e+00 -3.010000e-02 1.510000e-02 1.714700e+00 5.065000e-01 4.589000e-01 + 1.130800e+00 -2.860000e-02 1.000500e+00 7.387000e-01 8.390000e-01 1.874000e-01 1.084800e+00 6.374000e-01 8.534000e-01 1.373000e-01 1.042200e+00 6.836000e-01 6.215000e-01 1.323000e-01 + 5.093000e-01 6.038000e-01 2.874000e-01 8.162000e-01 2.267000e-01 6.181000e-01 2.646000e-01 8.490000e-01 3.365000e-01 4.648000e-01 2.815000e-01 8.254000e-01 2.280000e-01 4.243000e-01 + 7.600000e-02 5.010000e-01 1.870000e-01 7.011000e-01 1.728000e-01 8.475000e-01 2.300000e-01 6.536000e-01 1.616000e-01 8.732000e-01 1.603000e-01 7.331000e-01 9.550000e-02 4.203000e-01 + 4.662000e-01 1.209000e-01 7.657000e-01 1.390000e-01 8.886000e-01 1.304000e-01 1.694000e+00 4.530000e-02 -1.170000e-02 2.011100e+00 -1.940000e-02 2.170000e-02 4.422000e-01 7.900000e-02 + 1.125400e+00 -2.520000e-02 6.776000e-01 1.793000e-01 4.287000e-01 2.804000e-01 6.003000e-01 2.737000e-01 4.541000e-01 2.370000e-01 6.775000e-01 1.850000e-01 4.369000e-01 1.284000e-01 + 3.334000e-01 5.468000e-01 6.072000e-01 7.549000e-01 6.294000e-01 9.322000e-01 5.850000e-01 7.859000e-01 6.324000e-01 9.568000e-01 6.026000e-01 7.616000e-01 3.277000e-01 4.629000e-01 + 3.760000e-02 7.113000e-01 4.992000e-01 5.377000e-01 6.049000e-01 5.530000e-01 1.663500e+00 8.210000e-02 -1.890000e-02 2.024100e+00 1.400000e-03 -3.000000e-03 2.534000e-01 3.568000e-01 + 1.703000e-01 2.234000e-01 2.677000e-01 3.477000e-01 2.734000e-01 4.324000e-01 1.222000e-01 5.213000e-01 2.333000e-01 4.926000e-01 2.044000e-01 4.215000e-01 1.407000e-01 2.175000e-01 + 1.328000e-01 2.144000e-01 2.466000e-01 2.912000e-01 3.244000e-01 2.815000e-01 1.586000e-01 3.982000e-01 3.165000e-01 3.005000e-01 3.038000e-01 2.239000e-01 1.300000e-01 1.831000e-01 + 1.106900e+00 -1.500000e-03 1.787100e+00 -7.230000e-02 1.038900e+00 7.086000e-01 1.672800e+00 7.010000e-02 1.008300e+00 5.901000e-01 1.740500e+00 -1.880000e-02 7.498000e-01 2.096000e-01 + 1.083700e+00 2.940000e-02 6.675000e-01 5.421000e-01 5.040000e-01 3.997000e-01 7.271000e-01 4.673000e-01 6.061000e-01 2.526000e-01 7.957000e-01 3.861000e-01 4.794000e-01 1.997000e-01 + 6.379000e-01 3.709000e-01 9.139000e-01 6.785000e-01 1.117100e+00 6.825000e-01 9.074000e-01 6.917000e-01 1.152700e+00 6.739000e-01 9.866000e-01 5.957000e-01 5.590000e-01 3.562000e-01 + 6.520000e-02 4.575000e-01 5.170000e-01 2.900000e-01 6.257000e-01 3.041000e-01 1.700700e+00 3.410000e-02 -1.700000e-03 2.400000e-03 -4.430000e-02 5.270000e-02 2.800000e-01 1.838000e-01 + 5.460000e-02 4.317000e-01 2.260000e-02 8.026000e-01 2.228000e-01 7.077000e-01 1.370000e-02 1.717800e+00 -5.640000e-02 6.540000e-02 -9.100000e-03 1.070000e-02 7.470000e-02 3.874000e-01 + 6.250000e-02 1.036100e+00 -3.160000e-02 9.835000e-01 1.020000e-02 7.331000e-01 -5.260000e-02 1.005500e+00 -5.360000e-02 7.915000e-01 -2.270000e-02 9.754000e-01 3.200000e-03 5.922000e-01 + 8.813000e-01 2.321000e-01 4.637000e-01 1.117000e-01 2.749000e-01 2.270000e-01 4.328000e-01 1.497000e-01 3.223000e-01 1.589000e-01 4.639000e-01 1.134000e-01 2.964000e-01 1.282000e-01 + 2.830000e-02 1.087400e+00 1.069000e-01 6.261000e-01 1.089000e-01 4.879000e-01 -3.670000e-02 7.978000e-01 -6.910000e-02 6.871000e-01 4.170000e-02 7.048000e-01 3.750000e-02 4.698000e-01 + 1.116000e+00 -1.490000e-02 1.075900e+00 6.281000e-01 6.518000e-01 9.179000e-01 9.002000e-01 8.356000e-01 6.325000e-01 8.442000e-01 8.748000e-01 8.595000e-01 5.796000e-01 3.492000e-01 + 3.371000e-01 7.714000e-01 7.103000e-01 8.678000e-01 8.329000e-01 8.652000e-01 1.701600e+00 3.270000e-02 -1.300000e-01 2.157200e+00 -8.280000e-02 1.834700e+00 3.773000e-01 5.367000e-01 + 7.327000e-01 3.195000e-01 9.908000e-01 6.799000e-01 1.154500e+00 7.437000e-01 1.039000e+00 6.200000e-01 1.251800e+00 6.617000e-01 9.425000e-01 7.395000e-01 5.773000e-01 3.877000e-01 + 3.568000e-01 4.784000e-01 7.597000e-01 3.709000e-01 7.185000e-01 5.614000e-01 1.689700e+00 4.820000e-02 -5.500000e-02 2.069800e+00 1.110000e-02 -1.070000e-02 3.974000e-01 2.757000e-01 + 8.451000e-01 2.537000e-01 1.175900e+00 3.158000e-01 1.157700e+00 4.833000e-01 1.690800e+00 5.060000e-02 1.869400e+00 1.534000e-01 -8.700000e-02 1.834000e+00 6.629000e-01 2.139000e-01 + 1.053400e+00 6.240000e-02 9.920000e-01 7.462000e-01 6.545000e-01 9.606000e-01 9.666000e-01 7.742000e-01 7.771000e-01 7.086000e-01 9.877000e-01 7.512000e-01 6.243000e-01 3.112000e-01 + 7.150000e-02 1.045000e+00 1.133000e-01 8.252000e-01 4.200000e-02 7.057000e-01 1.016000e-01 8.422000e-01 7.360000e-02 6.540000e-01 7.840000e-02 8.680000e-01 7.220000e-02 5.227000e-01 + 1.002600e+00 1.198000e-01 6.942000e-01 1.048700e+00 5.669000e-01 5.787000e-01 7.905000e-01 9.402000e-01 6.875000e-01 3.929000e-01 7.361000e-01 1.000800e+00 5.116000e-01 2.856000e-01 + 4.843000e-01 6.102000e-01 2.827000e-01 1.441700e+00 2.176000e-01 8.856000e-01 3.216000e-01 1.399100e+00 8.600000e-02 1.002300e+00 2.466000e-01 1.485100e+00 1.914000e-01 5.923000e-01 + 3.972000e-01 1.078000e-01 5.178000e-01 2.948000e-01 5.923000e-01 3.344000e-01 5.037000e-01 3.121000e-01 6.118000e-01 3.270000e-01 5.425000e-01 2.633000e-01 3.099000e-01 1.574000e-01 + 3.522000e-01 3.868000e-01 5.201000e-01 6.377000e-01 5.522000e-01 7.776000e-01 5.076000e-01 6.553000e-01 6.258000e-01 7.105000e-01 4.515000e-01 7.196000e-01 3.029000e-01 3.665000e-01 + 2.230000e-01 8.879000e-01 1.593000e-01 7.129000e-01 1.089000e-01 5.914000e-01 1.294000e-01 7.501000e-01 1.590000e-01 5.180000e-01 1.132000e-01 7.677000e-01 1.232000e-01 4.428000e-01 + 4.409000e-01 -1.820000e-02 6.190000e-01 5.930000e-02 5.689000e-01 2.286000e-01 4.717000e-01 2.311000e-01 5.974000e-01 2.056000e-01 4.849000e-01 2.192000e-01 3.067000e-01 9.330000e-02 + 7.310000e-02 4.657000e-01 1.022000e-01 7.503000e-01 5.510000e-02 9.277000e-01 7.300000e-02 7.852000e-01 1.180000e-01 8.651000e-01 1.219000e-01 7.260000e-01 5.260000e-02 4.397000e-01 + 6.670000e-02 5.461000e-01 4.370000e-01 4.789000e-01 5.523000e-01 4.830000e-01 1.668600e+00 7.700000e-02 1.280000e-02 1.978700e+00 3.340000e-02 -4.030000e-02 2.290000e-01 3.043000e-01 + 8.427000e-01 2.270000e-01 1.094300e+00 3.023000e-01 1.294900e+00 2.022000e-01 1.886300e+00 -1.803000e-01 2.044700e+00 -4.780000e-02 2.900000e-03 1.732800e+00 6.900000e-01 1.222000e-01 + 1.126600e+00 -2.560000e-02 5.215000e-01 1.210700e+00 4.669000e-01 6.685000e-01 5.571000e-01 1.170500e+00 4.676000e-01 6.308000e-01 4.948000e-01 1.244900e+00 3.906000e-01 4.083000e-01 + 5.600000e-01 3.640000e-01 8.255000e-01 6.307000e-01 9.831000e-01 6.679000e-01 8.842000e-01 5.626000e-01 9.357000e-01 7.488000e-01 8.266000e-01 6.312000e-01 4.738000e-01 3.681000e-01 + 4.810000e-02 1.007400e+00 4.296000e-01 9.304000e-01 4.478000e-01 1.047200e+00 1.756000e+00 -3.380000e-02 -7.480000e-02 2.089300e+00 -4.250000e-02 5.140000e-02 2.391000e-01 5.596000e-01 + 1.095200e+00 1.350000e-02 8.862000e-01 8.475000e-01 7.144000e-01 3.892000e-01 9.740000e-01 7.529000e-01 7.114000e-01 3.571000e-01 9.248000e-01 8.077000e-01 5.859000e-01 1.979000e-01 + 2.006000e-01 3.454000e-01 3.986000e-01 4.469000e-01 3.417000e-01 6.387000e-01 2.808000e-01 5.793000e-01 4.392000e-01 5.374000e-01 3.424000e-01 5.096000e-01 2.043000e-01 2.865000e-01 +-2.480000e-02 8.079000e-01 3.514000e-01 7.237000e-01 4.603000e-01 7.386000e-01 8.570000e-02 1.632300e+00 6.240000e-02 1.922700e+00 -2.600000e-02 3.120000e-02 1.639000e-01 4.770000e-01 + 4.297000e-01 1.825000e-01 5.939000e-01 3.794000e-01 7.233000e-01 3.816000e-01 6.731000e-01 2.893000e-01 7.694000e-01 3.435000e-01 7.308000e-01 2.192000e-01 3.835000e-01 1.722000e-01 + 6.357000e-01 4.547000e-01 2.637000e-01 7.217000e-01 2.175000e-01 5.563000e-01 2.751000e-01 7.127000e-01 1.261000e-01 6.451000e-01 2.460000e-01 7.426000e-01 2.248000e-01 3.850000e-01 + 1.124800e+00 -2.590000e-02 4.916000e-01 1.229300e+00 3.932000e-01 9.462000e-01 5.271000e-01 1.185500e+00 4.211000e-01 8.591000e-01 5.142000e-01 1.206400e+00 3.246000e-01 5.502000e-01 + 1.131100e+00 -2.980000e-02 1.013700e+00 7.144000e-01 8.036000e-01 2.317000e-01 1.076000e+00 6.498000e-01 7.927000e-01 2.099000e-01 9.650000e-01 7.713000e-01 6.183000e-01 1.358000e-01 + 3.026000e-01 4.889000e-01 6.114000e-01 4.941000e-01 6.889000e-01 5.464000e-01 1.615600e+00 1.361000e-01 5.000000e-04 2.004300e+00 3.290000e-02 -3.970000e-02 3.152000e-01 3.408000e-01 +-1.580000e-02 4.553000e-01 1.158000e-01 6.442000e-01 3.486000e-01 5.106000e-01 -1.019000e-01 1.854600e+00 2.450000e-02 -3.050000e-02 6.260000e-02 -7.460000e-02 1.096000e-01 3.147000e-01 +-8.610000e-02 1.207600e+00 3.407000e-01 1.090100e+00 5.373000e-01 9.981000e-01 -3.530000e-02 1.776400e+00 -1.200000e-03 2.001700e+00 2.530000e-02 1.701800e+00 1.707000e-01 6.720000e-01 + 3.425000e-01 1.699000e-01 6.039000e-01 1.853000e-01 5.608000e-01 3.606000e-01 5.748000e-01 2.185000e-01 6.062000e-01 3.205000e-01 4.801000e-01 3.313000e-01 3.026000e-01 1.613000e-01 + 8.624000e-01 2.389000e-01 1.129600e+00 5.381000e-01 1.292500e+00 4.896000e-01 1.810500e+00 -9.300000e-02 2.041200e+00 -4.980000e-02 4.800000e-03 1.730400e+00 6.792000e-01 2.692000e-01 +-2.360000e-02 6.987000e-01 1.493000e-01 8.415000e-01 2.933000e-01 8.121000e-01 -3.410000e-02 1.772800e+00 -7.390000e-02 2.085800e+00 -5.190000e-02 6.100000e-02 1.021000e-01 4.729000e-01 + 1.140600e+00 -3.760000e-02 5.592000e-01 1.172900e+00 3.989000e-01 8.188000e-01 6.176000e-01 1.108100e+00 4.037000e-01 7.717000e-01 5.580000e-01 1.172800e+00 4.104000e-01 4.151000e-01 + 4.442000e-01 2.989000e-01 6.370000e-01 5.436000e-01 8.215000e-01 5.003000e-01 7.841000e-01 3.647000e-01 7.794000e-01 5.730000e-01 6.997000e-01 4.653000e-01 3.936000e-01 2.806000e-01 + 7.410000e-02 3.967000e-01 4.133000e-01 3.645000e-01 4.413000e-01 4.767000e-01 -4.360000e-02 1.781800e+00 4.830000e-02 -5.760000e-02 1.240000e-02 -1.260000e-02 1.716000e-01 2.843000e-01 + 3.583000e-01 7.460000e-01 1.489000e-01 6.681000e-01 1.327000e-01 5.254000e-01 1.705000e-01 6.434000e-01 1.733000e-01 4.643000e-01 1.570000e-01 6.552000e-01 1.646000e-01 3.708000e-01 + 5.144000e-01 7.100000e-03 7.310000e-01 1.166000e-01 8.438000e-01 1.292000e-01 1.745400e+00 -2.040000e-02 5.240000e-02 -6.220000e-02 2.000000e-04 -1.200000e-03 4.207000e-01 6.780000e-02 + 6.726000e-01 1.323000e-01 1.144800e+00 9.590000e-02 1.362000e+00 3.870000e-02 1.169700e+00 6.840000e-02 1.369600e+00 5.110000e-02 1.182100e+00 5.370000e-02 6.784000e-01 3.530000e-02 + 1.674000e-01 3.897000e-01 4.111000e-01 4.741000e-01 6.153000e-01 3.688000e-01 1.697900e+00 3.140000e-02 -5.520000e-02 2.065200e+00 -4.320000e-02 5.020000e-02 2.435000e-01 2.632000e-01 + 3.785000e-01 7.282000e-01 1.824000e-01 7.266000e-01 1.605000e-01 5.580000e-01 1.435000e-01 7.714000e-01 1.922000e-01 5.056000e-01 3.317000e-01 5.522000e-01 1.777000e-01 3.997000e-01 + 3.145000e-01 7.818000e-01 1.241000e-01 5.976000e-01 1.230000e-01 4.677000e-01 1.220000e-01 6.060000e-01 1.649000e-01 4.058000e-01 7.690000e-02 6.546000e-01 1.322000e-01 3.602000e-01 + 2.918000e-01 2.719000e-01 6.459000e-01 2.241000e-01 7.378000e-01 2.574000e-01 1.683400e+00 6.250000e-02 2.860000e-02 1.964400e+00 -5.600000e-03 6.500000e-03 3.631000e-01 1.400000e-01 + 1.151300e+00 -5.120000e-02 6.183000e-01 3.049000e-01 6.627000e-01 3.790000e-02 7.937000e-01 9.420000e-02 5.698000e-01 1.317000e-01 8.385000e-01 4.390000e-02 4.815000e-01 9.850000e-02 + 1.269000e-01 9.731000e-01 4.120000e-01 1.036700e+00 5.733000e-01 9.856000e-01 1.699400e+00 3.680000e-02 -2.470000e-02 2.024300e+00 -1.100000e-02 1.743800e+00 2.299000e-01 6.160000e-01 +-9.200000e-03 6.616000e-01 5.080000e-02 9.117000e-01 1.767000e-01 9.075000e-01 2.000000e-02 1.709800e+00 -6.340000e-02 2.076700e+00 -2.020000e-02 2.450000e-02 4.400000e-02 5.188000e-01 + 3.096000e-01 3.670000e-01 4.894000e-01 5.677000e-01 5.287000e-01 6.841000e-01 4.190000e-01 6.544000e-01 5.844000e-01 6.375000e-01 5.973000e-01 4.401000e-01 2.783000e-01 3.331000e-01 + 5.330000e-02 8.748000e-01 1.818000e-01 1.072900e+00 2.553000e-01 1.123700e+00 -2.450000e-02 1.760300e+00 -3.000000e-03 2.009000e+00 4.320000e-02 -5.350000e-02 1.072000e-01 6.315000e-01 + 1.054700e+00 5.700000e-02 9.906000e-01 9.390000e-02 8.846000e-01 -8.220000e-02 1.024500e+00 4.920000e-02 7.418000e-01 6.930000e-02 1.035600e+00 3.830000e-02 6.174000e-01 2.600000e-02 + 4.399000e-01 3.581000e-01 8.365000e-01 3.885000e-01 9.079000e-01 4.923000e-01 7.725000e-01 4.573000e-01 7.929000e-01 6.507000e-01 7.882000e-01 4.434000e-01 4.279000e-01 2.874000e-01 + 4.167000e-01 5.007000e-01 6.382000e-01 7.986000e-01 8.046000e-01 8.237000e-01 7.404000e-01 6.831000e-01 7.403000e-01 9.233000e-01 6.321000e-01 8.118000e-01 3.938000e-01 4.348000e-01 + 1.641000e-01 5.116000e-01 5.404000e-01 4.425000e-01 5.521000e-01 5.661000e-01 1.699300e+00 3.610000e-02 2.810000e-02 1.964300e+00 6.130000e-02 -7.280000e-02 2.508000e-01 3.306000e-01 + 6.288000e-01 4.748000e-01 3.036000e-01 1.429400e+00 1.956000e-01 1.249500e+00 3.526000e-01 1.368400e+00 2.819000e-01 1.080300e+00 2.704000e-01 1.468300e+00 2.507000e-01 6.484000e-01 + 2.369000e-01 6.611000e-01 5.288000e-01 6.834000e-01 6.557000e-01 6.726000e-01 1.709300e+00 2.570000e-02 -3.230000e-02 2.038300e+00 -7.010000e-02 8.680000e-02 2.784000e-01 4.395000e-01 + 8.934000e-01 2.195000e-01 4.395000e-01 9.475000e-01 3.611000e-01 6.075000e-01 3.924000e-01 1.003200e+00 3.487000e-01 5.886000e-01 4.447000e-01 9.433000e-01 2.960000e-01 4.255000e-01 + 3.127000e-01 7.936000e-01 2.430000e-01 1.480400e+00 1.331000e-01 1.843300e+00 1.919000e-01 1.538600e+00 8.090000e-02 1.620200e+00 2.307000e-01 1.493000e+00 1.468000e-01 8.283000e-01 + 4.778000e-01 5.678000e-01 8.714000e-01 7.462000e-01 9.055000e-01 9.499000e-01 8.423000e-01 7.811000e-01 9.141000e-01 9.747000e-01 7.781000e-01 8.622000e-01 4.561000e-01 4.868000e-01 + 8.896000e-01 2.207000e-01 2.578000e-01 1.349300e+00 2.169000e-01 8.216000e-01 4.123000e-01 1.165400e+00 3.797000e-01 5.948000e-01 4.365000e-01 1.129600e+00 2.961000e-01 4.484000e-01 + 1.164000e-01 5.170000e-01 9.500000e-03 1.017200e+00 1.090000e-02 1.156900e+00 -2.770000e-02 1.060900e+00 -2.000000e-03 1.186900e+00 7.900000e-02 9.352000e-01 2.830000e-02 5.596000e-01 + 4.084000e-01 6.935000e-01 7.813000e-01 7.456000e-01 8.580000e-01 7.981000e-01 1.763400e+00 -3.380000e-02 3.120000e-02 1.962900e+00 4.640000e-02 1.680600e+00 4.056000e-01 4.892000e-01 + 4.115000e-01 3.150000e-01 6.251000e-01 5.168000e-01 6.864000e-01 6.216000e-01 6.157000e-01 5.283000e-01 5.501000e-01 8.035000e-01 5.848000e-01 5.660000e-01 3.420000e-01 3.209000e-01 + 5.793000e-01 5.256000e-01 2.275000e-01 6.283000e-01 2.350000e-01 4.414000e-01 3.038000e-01 5.316000e-01 2.404000e-01 4.183000e-01 2.991000e-01 5.356000e-01 2.314000e-01 3.192000e-01 + 1.988000e-01 4.499000e-01 2.277000e-01 8.006000e-01 2.805000e-01 8.852000e-01 2.238000e-01 8.069000e-01 2.915000e-01 8.895000e-01 2.605000e-01 7.604000e-01 1.524000e-01 4.373000e-01 + 5.380000e-02 4.115000e-01 1.528000e-01 5.627000e-01 5.870000e-02 7.814000e-01 1.204000e-01 6.025000e-01 2.900000e-03 8.575000e-01 8.210000e-02 6.511000e-01 4.320000e-02 3.794000e-01 + 5.143000e-01 4.670000e-01 6.954000e-01 6.313000e-01 8.395000e-01 5.981000e-01 1.833600e+00 -1.195000e-01 6.730000e-02 1.927100e+00 4.320000e-02 -5.020000e-02 4.050000e-01 3.727000e-01 + 1.093200e+00 1.700000e-02 9.920000e-01 2.131000e-01 8.428000e-01 4.310000e-02 1.037400e+00 1.593000e-01 8.353000e-01 2.400000e-02 1.127100e+00 5.930000e-02 6.564000e-01 1.860000e-02 + 1.529000e-01 7.166000e-01 4.760000e-01 7.066000e-01 6.900000e-01 5.941000e-01 1.790100e+00 -6.710000e-02 8.600000e-03 1.987800e+00 -4.000000e-03 4.600000e-03 3.005000e-01 3.884000e-01 + 6.414000e-01 1.886000e-01 9.415000e-01 2.036000e-01 9.966000e-01 2.832000e-01 1.774400e+00 -5.000000e-02 2.061900e+00 -7.050000e-02 2.750000e-02 -3.310000e-02 5.474000e-01 1.283000e-01 + 1.100400e+00 1.080000e-02 1.541000e+00 1.791000e-01 9.895000e-01 8.366000e-01 1.446100e+00 2.887000e-01 9.364000e-01 7.076000e-01 1.512700e+00 2.114000e-01 7.341000e-01 2.329000e-01 + 1.063300e+00 4.970000e-02 6.328000e-01 1.103200e+00 4.612000e-01 1.008900e+00 7.123000e-01 1.013200e+00 5.231000e-01 8.612000e-01 7.706000e-01 9.377000e-01 4.556000e-01 4.502000e-01 + 9.390000e-02 2.852000e-01 1.502000e-01 5.744000e-01 3.306000e-01 4.972000e-01 2.500000e-02 1.699300e+00 2.940000e-02 -3.570000e-02 -5.610000e-02 6.750000e-02 1.023000e-01 3.008000e-01 + 6.954000e-01 2.670000e-01 1.011800e+00 2.593000e-01 1.137400e+00 2.561000e-01 1.706800e+00 3.340000e-02 2.083900e+00 -9.270000e-02 1.300000e-02 -1.780000e-02 6.024000e-01 1.478000e-01 + 1.190800e+00 -9.960000e-02 3.961000e-01 1.285800e+00 3.905000e-01 6.359000e-01 4.426000e-01 1.233500e+00 3.446000e-01 6.614000e-01 4.912000e-01 1.180000e+00 3.361000e-01 4.130000e-01 + 5.374000e-01 3.805000e-01 8.830000e-01 5.444000e-01 1.049100e+00 5.771000e-01 9.036000e-01 5.234000e-01 1.049500e+00 5.964000e-01 9.324000e-01 4.881000e-01 5.132000e-01 3.125000e-01 + 1.039100e+00 7.860000e-02 1.266200e+00 4.742000e-01 9.161000e-01 1.626000e-01 1.197900e+00 5.470000e-01 9.137000e-01 1.269000e-01 1.213700e+00 5.333000e-01 6.993000e-01 7.170000e-02 + 6.458000e-01 4.649000e-01 2.589000e-01 9.217000e-01 3.329000e-01 5.323000e-01 2.311000e-01 9.548000e-01 1.899000e-01 6.802000e-01 3.269000e-01 8.420000e-01 2.573000e-01 4.138000e-01 + 1.052100e+00 6.650000e-02 7.528000e-01 9.807000e-01 6.024000e-01 8.703000e-01 7.803000e-01 9.460000e-01 7.222000e-01 6.507000e-01 7.261000e-01 1.012000e+00 5.141000e-01 3.938000e-01 + 5.840000e-02 5.279000e-01 8.240000e-02 8.389000e-01 8.580000e-02 9.652000e-01 7.990000e-02 8.364000e-01 1.640000e-02 1.055800e+00 1.320000e-01 7.804000e-01 6.160000e-02 4.665000e-01 + 4.050000e-02 9.263000e-01 3.432000e-01 9.408000e-01 4.826000e-01 9.161000e-01 -1.620000e-02 1.747900e+00 -1.530000e-02 2.022200e+00 6.260000e-02 -7.430000e-02 1.872000e-01 5.699000e-01 + 5.268000e-01 3.336000e-01 6.997000e-01 4.976000e-01 9.724000e-01 3.158000e-01 1.717100e+00 1.850000e-02 1.060000e-02 1.983800e+00 -2.130000e-02 2.460000e-02 4.486000e-01 2.498000e-01 + 1.870000e-02 6.936000e-01 1.526000e-01 8.764000e-01 2.685000e-01 8.772000e-01 -2.400000e-02 1.761100e+00 5.660000e-02 1.933300e+00 3.520000e-02 -4.000000e-02 7.850000e-02 5.252000e-01 + 1.103300e+00 3.900000e-03 8.285000e-01 3.832000e-01 6.649000e-01 2.246000e-01 6.963000e-01 5.402000e-01 5.935000e-01 2.886000e-01 7.667000e-01 4.494000e-01 5.473000e-01 1.339000e-01 + 8.410000e-02 1.032800e+00 4.392000e-01 1.048400e+00 6.082000e-01 9.935000e-01 1.738900e+00 -3.800000e-03 -2.100000e-02 2.026900e+00 -1.067000e-01 1.858800e+00 2.398000e-01 6.295000e-01 + 4.689000e-01 4.092000e-01 6.791000e-01 7.029000e-01 8.472000e-01 7.169000e-01 6.373000e-01 7.540000e-01 7.336000e-01 8.768000e-01 7.611000e-01 6.114000e-01 3.997000e-01 3.986000e-01 + 5.368000e-01 2.121000e-01 7.638000e-01 3.146000e-01 8.883000e-01 3.072000e-01 1.674800e+00 6.920000e-02 9.300000e-03 1.989600e+00 6.700000e-03 -9.100000e-03 4.426000e-01 1.885000e-01 + 4.240000e-01 4.900000e-01 7.232000e-01 5.014000e-01 8.591000e-01 4.838000e-01 1.772600e+00 -4.870000e-02 4.240000e-02 1.954800e+00 -1.780000e-02 2.100000e-02 4.348000e-01 2.854000e-01 + 1.050000e-01 1.003300e+00 2.230000e-02 5.880000e-01 3.430000e-02 4.697000e-01 1.249000e-01 4.665000e-01 6.710000e-02 4.225000e-01 1.030000e-01 4.903000e-01 6.130000e-02 3.688000e-01 + 3.111000e-01 3.751000e-01 4.651000e-01 6.121000e-01 6.362000e-01 5.681000e-01 5.020000e-01 5.630000e-01 6.580000e-01 5.583000e-01 5.059000e-01 5.646000e-01 3.052000e-01 3.081000e-01 + 4.478000e-01 -1.400000e-03 6.501000e-01 6.110000e-02 7.771000e-01 2.590000e-02 6.717000e-01 3.060000e-02 7.627000e-01 5.250000e-02 6.493000e-01 5.720000e-02 3.869000e-01 2.020000e-02 + 7.303000e-01 3.720000e-01 3.490000e-01 6.100000e-01 2.788000e-01 4.756000e-01 2.771000e-01 6.963000e-01 3.090000e-01 4.241000e-01 2.837000e-01 6.863000e-01 2.655000e-01 3.350000e-01 + 5.140000e-02 1.062500e+00 3.940000e-02 4.118000e-01 -9.000000e-03 4.001000e-01 1.314000e-01 3.037000e-01 7.130000e-02 2.978000e-01 7.100000e-02 3.778000e-01 4.670000e-02 2.925000e-01 + 4.980000e-01 5.100000e-01 8.111000e-01 5.115000e-01 8.700000e-01 5.826000e-01 1.682700e+00 5.570000e-02 7.700000e-03 1.987000e+00 3.500000e-02 -4.180000e-02 4.798000e-01 2.986000e-01 + 2.034000e-01 6.543000e-01 2.535000e-01 1.109500e+00 3.152000e-01 1.231200e+00 2.384000e-01 1.129500e+00 3.808000e-01 1.179800e+00 2.254000e-01 1.140700e+00 1.744000e-01 6.073000e-01 + 3.153000e-01 4.310000e-01 6.277000e-01 5.140000e-01 6.222000e-01 6.974000e-01 6.016000e-01 5.469000e-01 7.290000e-01 5.904000e-01 5.691000e-01 5.835000e-01 3.592000e-01 3.007000e-01 + 8.961000e-01 2.310000e-02 1.048800e+00 2.182000e-01 1.305600e+00 6.030000e-02 1.701700e+00 3.300000e-02 2.162900e+00 -1.954000e-01 -5.500000e-03 6.700000e-03 6.851000e-01 5.080000e-02 + 1.099300e+00 6.800000e-03 4.745000e-01 1.250200e+00 3.631000e-01 8.284000e-01 4.291000e-01 1.304800e+00 3.799000e-01 7.691000e-01 5.049000e-01 1.217200e+00 3.288000e-01 4.910000e-01 + 1.124000e-01 4.549000e-01 1.585000e-01 7.363000e-01 7.680000e-02 9.628000e-01 1.012000e-01 8.069000e-01 9.650000e-02 9.527000e-01 1.245000e-01 7.763000e-01 5.500000e-02 4.688000e-01 + 2.914000e-01 8.203000e-01 1.384000e-01 9.877000e-01 1.531000e-01 6.968000e-01 2.093000e-01 9.102000e-01 2.100000e-01 6.084000e-01 1.684000e-01 9.538000e-01 1.537000e-01 5.024000e-01 + 2.503000e-01 5.377000e-01 4.508000e-01 7.689000e-01 6.041000e-01 7.665000e-01 4.468000e-01 7.721000e-01 6.211000e-01 7.645000e-01 4.442000e-01 7.789000e-01 2.925000e-01 4.055000e-01 + 5.522000e-01 9.600000e-02 8.623000e-01 9.830000e-02 9.783000e-01 1.028000e-01 1.732200e+00 1.400000e-03 6.040000e-02 1.928700e+00 -1.110000e-02 1.420000e-02 4.638000e-01 1.047000e-01 + 6.990000e-01 1.695000e-01 1.112800e+00 2.379000e-01 1.287200e+00 2.463000e-01 1.054400e+00 3.047000e-01 1.330800e+00 2.267000e-01 1.103400e+00 2.486000e-01 6.255000e-01 1.588000e-01 + 1.206000e-01 5.177000e-01 2.957000e-01 6.815000e-01 2.742000e-01 8.485000e-01 2.064000e-01 7.865000e-01 3.213000e-01 8.104000e-01 2.374000e-01 7.500000e-01 1.375000e-01 4.325000e-01 + 7.560000e-02 4.799000e-01 1.069000e-01 7.654000e-01 2.251000e-01 7.443000e-01 2.162000e-01 6.340000e-01 1.732000e-01 8.273000e-01 1.715000e-01 6.874000e-01 9.100000e-02 4.062000e-01 + 2.584000e-01 8.386000e-01 7.620000e-02 7.278000e-01 1.070000e-01 5.378000e-01 7.110000e-02 7.384000e-01 7.870000e-02 5.573000e-01 4.850000e-02 7.629000e-01 1.041000e-01 4.274000e-01 + 7.270000e-01 3.828000e-01 1.070200e+00 4.521000e-01 1.323100e+00 3.022000e-01 1.663400e+00 8.400000e-02 2.080600e+00 -9.090000e-02 7.430000e-02 1.642100e+00 6.176000e-01 2.725000e-01 + 2.311000e-01 8.728000e-01 1.756000e-01 5.309000e-01 1.360000e-01 4.464000e-01 1.481000e-01 5.650000e-01 6.090000e-02 5.215000e-01 1.231000e-01 5.944000e-01 1.064000e-01 3.859000e-01 + 6.257000e-01 4.252000e-01 8.745000e-01 5.017000e-01 8.145000e-01 7.101000e-01 1.767800e+00 -4.060000e-02 3.790000e-02 1.954800e+00 -3.610000e-02 1.777500e+00 4.962000e-01 3.155000e-01 + 3.711000e-01 3.448000e-01 6.690000e-01 3.564000e-01 7.493000e-01 4.080000e-01 1.665500e+00 8.190000e-02 4.510000e-02 1.949600e+00 1.120000e-02 -1.470000e-02 3.586000e-01 2.472000e-01 + 1.091400e+00 1.500000e-02 7.516000e-01 9.932000e-01 6.739000e-01 4.224000e-01 9.437000e-01 7.683000e-01 6.294000e-01 4.399000e-01 8.063000e-01 9.298000e-01 5.482000e-01 2.326000e-01 + 4.256000e-01 2.780000e-02 6.816000e-01 2.220000e-02 7.522000e-01 5.630000e-02 6.717000e-01 3.390000e-02 8.997000e-01 -1.075000e-01 6.933000e-01 9.500000e-03 3.776000e-01 3.240000e-02 + 1.028000e+00 9.510000e-02 5.619000e-01 4.529000e-01 5.648000e-01 2.058000e-01 6.508000e-01 3.510000e-01 4.077000e-01 3.744000e-01 5.686000e-01 4.476000e-01 4.042000e-01 2.186000e-01 +-4.200000e-02 7.610000e-01 1.212000e-01 9.064000e-01 1.753000e-01 9.878000e-01 5.340000e-02 1.670300e+00 -1.660000e-02 2.013600e+00 6.680000e-02 -7.970000e-02 7.740000e-02 5.239000e-01 + 3.907000e-01 5.587000e-01 6.918000e-01 5.787000e-01 7.508000e-01 6.533000e-01 1.765900e+00 -3.900000e-02 -4.400000e-03 2.007300e+00 1.150000e-02 -1.440000e-02 3.759000e-01 3.750000e-01 + 5.883000e-01 4.444000e-01 9.442000e-01 6.742000e-01 1.223800e+00 5.907000e-01 1.045900e+00 5.541000e-01 1.129100e+00 7.246000e-01 1.068700e+00 5.271000e-01 5.945000e-01 3.288000e-01 + 1.261000e-01 4.338000e-01 2.383000e-01 6.331000e-01 1.393000e-01 8.796000e-01 7.800000e-02 8.233000e-01 1.462000e-01 8.875000e-01 1.771000e-01 7.083000e-01 8.940000e-02 4.235000e-01 + 2.951000e-01 7.971000e-01 2.091000e-01 7.775000e-01 8.550000e-02 6.993000e-01 1.987000e-01 7.923000e-01 5.940000e-02 7.144000e-01 1.031000e-01 9.043000e-01 1.108000e-01 5.057000e-01 + 1.100700e+00 5.800000e-03 5.735000e-01 5.508000e-01 4.843000e-01 3.643000e-01 5.151000e-01 6.179000e-01 4.244000e-01 4.098000e-01 6.194000e-01 4.928000e-01 4.094000e-01 2.485000e-01 + 8.864000e-01 1.600000e-03 1.375200e+00 2.020000e-02 1.571500e+00 1.050000e-02 1.415500e+00 -3.190000e-02 1.655200e+00 -6.190000e-02 1.409500e+00 -2.720000e-02 7.862000e-01 1.790000e-02 + 1.152100e+00 -5.580000e-02 7.192000e-01 1.030300e+00 7.694000e-01 4.801000e-01 8.318000e-01 8.979000e-01 6.653000e-01 5.582000e-01 9.178000e-01 7.947000e-01 5.415000e-01 3.093000e-01 + 5.044000e-01 1.540000e-02 8.126000e-01 -2.700000e-03 8.906000e-01 4.030000e-02 7.963000e-01 1.780000e-02 9.854000e-01 -5.490000e-02 7.248000e-01 1.000000e-01 4.532000e-01 1.820000e-02 +-2.180000e-02 8.299000e-01 1.599000e-01 9.649000e-01 3.780000e-01 8.483000e-01 2.180000e-02 1.707200e+00 -4.570000e-02 2.054800e+00 -1.020000e-02 1.240000e-02 1.121000e-01 5.458000e-01 + 9.160000e-02 1.036900e+00 1.755000e-01 6.179000e-01 8.330000e-02 5.723000e-01 1.340000e-01 6.709000e-01 5.480000e-02 5.914000e-01 6.290000e-02 7.533000e-01 1.001000e-01 4.355000e-01 + 3.507000e-01 4.682000e-01 6.535000e-01 4.860000e-01 7.487000e-01 5.119000e-01 1.678900e+00 5.970000e-02 -5.820000e-02 2.068200e+00 -2.410000e-02 2.870000e-02 3.420000e-01 3.332000e-01 + 1.034700e+00 8.190000e-02 5.016000e-01 1.232800e+00 4.562000e-01 1.218400e+00 4.686000e-01 1.269500e+00 4.205000e-01 1.132200e+00 5.104000e-01 1.223000e+00 3.742000e-01 5.762000e-01 + 6.282000e-01 1.157000e-01 8.571000e-01 3.314000e-01 1.049100e+00 2.918000e-01 8.451000e-01 3.437000e-01 1.130400e+00 2.179000e-01 9.457000e-01 2.238000e-01 5.257000e-01 1.544000e-01 + 1.087900e+00 1.780000e-02 5.849000e-01 9.080000e-02 4.400000e-01 1.295000e-01 5.795000e-01 1.014000e-01 4.978000e-01 4.730000e-02 6.493000e-01 2.220000e-02 4.243000e-01 4.870000e-02 + 1.043200e+00 7.570000e-02 3.362000e-01 8.387000e-01 3.922000e-01 4.720000e-01 4.663000e-01 6.843000e-01 3.137000e-01 5.435000e-01 4.333000e-01 7.240000e-01 3.211000e-01 3.463000e-01 + 1.763000e-01 5.283000e-01 4.547000e-01 5.701000e-01 6.420000e-01 4.928000e-01 1.767600e+00 -4.320000e-02 3.570000e-02 1.956900e+00 -3.270000e-02 3.990000e-02 2.514000e-01 3.485000e-01 + 4.510000e-01 6.430000e-02 6.271000e-01 2.025000e-01 7.699000e-01 1.555000e-01 7.216000e-01 8.540000e-02 8.444000e-01 8.560000e-02 8.332000e-01 -4.750000e-02 4.062000e-01 6.300000e-02 + 7.900000e-02 5.267000e-01 8.690000e-02 8.697000e-01 1.671000e-01 9.115000e-01 5.880000e-02 9.035000e-01 6.870000e-02 1.042700e+00 8.080000e-02 8.789000e-01 6.730000e-02 4.822000e-01 + 2.188000e-01 3.369000e-01 3.669000e-01 5.010000e-01 3.112000e-01 6.960000e-01 2.375000e-01 6.537000e-01 3.345000e-01 6.777000e-01 2.797000e-01 6.031000e-01 1.660000e-01 3.429000e-01 +-9.450000e-02 1.214000e+00 -7.660000e-02 1.790700e+00 1.769000e-01 1.628300e+00 -1.730000e-02 1.752100e+00 2.320000e-02 1.976100e+00 -8.350000e-02 1.829900e+00 2.030000e-02 9.413000e-01 + 1.486000e-01 6.692000e-01 1.849000e-01 1.112100e+00 2.417000e-01 1.229800e+00 3.131000e-01 9.606000e-01 3.608000e-01 1.105200e+00 1.707000e-01 1.123700e+00 1.341000e-01 6.086000e-01 + 1.151900e+00 -5.420000e-02 6.338000e-01 3.940000e-01 4.542000e-01 3.527000e-01 5.905000e-01 4.522000e-01 5.814000e-01 1.835000e-01 6.035000e-01 4.320000e-01 4.249000e-01 2.042000e-01 + 1.479000e-01 2.251000e-01 1.986000e-01 3.961000e-01 2.674000e-01 3.986000e-01 1.340000e-01 4.690000e-01 2.304000e-01 4.534000e-01 1.999000e-01 3.911000e-01 1.029000e-01 2.407000e-01 + 5.890000e-02 1.864000e-01 3.178000e-01 2.535000e-01 4.522000e-01 2.345000e-01 9.190000e-02 1.622500e+00 -7.700000e-03 9.000000e-03 -7.670000e-02 9.070000e-02 1.755000e-01 1.388000e-01 + 9.400000e-02 6.238000e-01 3.969000e-01 6.334000e-01 4.998000e-01 6.599000e-01 1.790100e+00 -6.620000e-02 -7.930000e-02 2.093800e+00 2.700000e-03 -4.400000e-03 2.079000e-01 3.997000e-01 + 2.319000e-01 5.927000e-01 3.022000e-01 1.001700e+00 3.991000e-01 1.072700e+00 4.055000e-01 8.795000e-01 4.504000e-01 1.036000e+00 3.112000e-01 9.839000e-01 1.925000e-01 5.553000e-01 + 3.815000e-01 3.332000e-01 6.462000e-01 3.913000e-01 7.058000e-01 4.557000e-01 1.709500e+00 2.520000e-02 -6.140000e-02 2.076500e+00 2.690000e-02 -3.110000e-02 3.664000e-01 2.403000e-01 + 1.078300e+00 3.410000e-02 8.201000e-01 9.103000e-01 6.938000e-01 1.278200e+00 7.104000e-01 1.039200e+00 5.932000e-01 1.278000e+00 7.361000e-01 1.003500e+00 5.368000e-01 4.586000e-01 + 9.140000e-02 5.724000e-01 3.504000e-01 6.380000e-01 4.148000e-01 7.028000e-01 1.752300e+00 -2.260000e-02 -2.290000e-02 2.024300e+00 -1.770000e-02 1.910000e-02 2.003000e-01 3.734000e-01 + 4.475000e-01 6.681000e-01 3.219000e-01 5.937000e-01 2.144000e-01 5.219000e-01 2.775000e-01 6.470000e-01 1.795000e-01 5.440000e-01 2.650000e-01 6.645000e-01 2.055000e-01 3.837000e-01 + 1.052400e+00 6.480000e-02 5.225000e-01 8.387000e-01 4.788000e-01 4.712000e-01 5.038000e-01 8.560000e-01 3.832000e-01 5.550000e-01 5.543000e-01 7.987000e-01 3.811000e-01 3.338000e-01 + 9.849000e-01 1.433000e-01 5.062000e-01 2.203000e-01 5.441000e-01 2.260000e-02 5.696000e-01 1.404000e-01 4.719000e-01 9.920000e-02 4.997000e-01 2.260000e-01 3.843000e-01 1.097000e-01 + 5.748000e-01 1.579000e-01 9.100000e-01 2.326000e-01 9.980000e-01 3.141000e-01 9.007000e-01 2.419000e-01 9.729000e-01 3.628000e-01 8.942000e-01 2.515000e-01 5.003000e-01 1.643000e-01 + 4.395000e-01 4.679000e-01 8.962000e-01 2.943000e-01 9.701000e-01 3.516000e-01 1.817900e+00 -9.650000e-02 2.730000e-02 1.960900e+00 2.100000e-03 -1.700000e-03 4.865000e-01 2.228000e-01 + 4.840000e-01 2.094000e-01 7.491000e-01 3.479000e-01 9.198000e-01 3.134000e-01 7.565000e-01 3.362000e-01 7.934000e-01 4.768000e-01 6.991000e-01 4.020000e-01 4.212000e-01 2.110000e-01 + 4.958000e-01 1.443000e-01 8.970000e-01 8.420000e-02 8.679000e-01 2.843000e-01 7.258000e-01 2.898000e-01 8.606000e-01 3.076000e-01 7.934000e-01 2.101000e-01 4.459000e-01 1.359000e-01 + 1.079100e+00 3.290000e-02 5.166000e-01 7.448000e-01 4.626000e-01 4.487000e-01 5.806000e-01 6.720000e-01 4.977000e-01 3.901000e-01 5.378000e-01 7.246000e-01 4.012000e-01 2.923000e-01 + 5.550000e-02 7.274000e-01 3.617000e-01 7.326000e-01 5.602000e-01 6.382000e-01 1.931200e+00 -2.333000e-01 7.000000e-03 1.992700e+00 -7.110000e-02 8.280000e-02 2.292000e-01 4.078000e-01 + 1.089100e+00 2.140000e-02 4.152000e-01 8.053000e-01 3.788000e-01 5.185000e-01 4.597000e-01 7.495000e-01 3.572000e-01 5.192000e-01 4.332000e-01 7.823000e-01 3.524000e-01 3.287000e-01 + 5.581000e-01 5.780000e-02 7.462000e-01 2.461000e-01 8.369000e-01 2.983000e-01 8.010000e-01 1.832000e-01 9.550000e-01 1.747000e-01 8.798000e-01 8.960000e-02 4.791000e-01 8.590000e-02 + 1.096500e+00 3.900000e-03 4.117000e-01 4.921000e-01 3.664000e-01 3.479000e-01 4.433000e-01 4.572000e-01 3.868000e-01 3.088000e-01 4.363000e-01 4.653000e-01 3.154000e-01 2.638000e-01 + 1.103600e+00 -9.000000e-04 5.596000e-01 1.163800e+00 4.281000e-01 9.927000e-01 5.554000e-01 1.170500e+00 4.449000e-01 9.107000e-01 5.766000e-01 1.139700e+00 3.750000e-01 5.223000e-01 + 4.741000e-01 2.965000e-01 7.502000e-01 4.499000e-01 7.687000e-01 6.150000e-01 6.862000e-01 5.282000e-01 8.357000e-01 5.573000e-01 6.292000e-01 5.912000e-01 4.165000e-01 2.794000e-01 + 3.807000e-01 4.936000e-01 5.540000e-01 8.272000e-01 8.500000e-01 6.754000e-01 6.042000e-01 7.640000e-01 7.195000e-01 8.542000e-01 6.354000e-01 7.275000e-01 3.726000e-01 4.133000e-01 + 8.364000e-01 3.270000e-02 1.186400e+00 1.968000e-01 1.436000e+00 1.226000e-01 1.236400e+00 1.442000e-01 1.383800e+00 2.110000e-01 1.171300e+00 2.177000e-01 7.280000e-01 6.490000e-02 +-1.560000e-02 4.196000e-01 4.650000e-02 5.764000e-01 6.000000e-02 6.460000e-01 3.050000e-02 5.935000e-01 4.570000e-02 6.741000e-01 -3.280000e-02 6.699000e-01 1.860000e-02 3.420000e-01 + 5.736000e-01 5.383000e-01 2.750000e-01 1.314900e+00 2.674000e-01 7.521000e-01 3.316000e-01 1.253500e+00 2.474000e-01 7.465000e-01 2.832000e-01 1.306000e+00 2.464000e-01 4.982000e-01 + 7.007000e-01 3.733000e-01 9.031000e-01 7.899000e-01 1.038900e+00 7.671000e-01 1.784000e+00 -5.930000e-02 2.500000e-02 1.970800e+00 2.790000e-02 1.699700e+00 4.946000e-01 4.653000e-01 + 1.101300e+00 3.700000e-03 4.778000e-01 5.740000e-02 3.817000e-01 6.810000e-02 3.378000e-01 2.228000e-01 3.602000e-01 8.800000e-02 3.097000e-01 2.582000e-01 3.379000e-01 5.670000e-02 +-5.860000e-02 7.000000e-02 2.469000e-01 5.640000e-02 2.655000e-01 1.748000e-01 -4.000000e-03 5.900000e-03 2.730000e-02 -3.150000e-02 5.900000e-03 -7.100000e-03 1.159000e-01 4.810000e-02 + 1.120600e+00 -1.640000e-02 5.178000e-01 1.212100e+00 3.926000e-01 1.017600e+00 5.052000e-01 1.224700e+00 3.007000e-01 1.063800e+00 4.734000e-01 1.256600e+00 3.547000e-01 5.385000e-01 + 2.983000e-01 8.036000e-01 2.176000e-01 9.156000e-01 1.722000e-01 6.839000e-01 1.845000e-01 9.522000e-01 1.204000e-01 7.224000e-01 2.503000e-01 8.755000e-01 1.339000e-01 5.293000e-01 + 1.063100e+00 5.430000e-02 8.058000e-01 9.270000e-01 5.826000e-01 9.213000e-01 7.334000e-01 1.003300e+00 6.819000e-01 7.321000e-01 7.287000e-01 1.006000e+00 5.168000e-01 4.008000e-01 + 4.254000e-01 3.643000e-01 7.034000e-01 5.253000e-01 6.943000e-01 7.253000e-01 8.134000e-01 4.001000e-01 7.932000e-01 6.344000e-01 7.578000e-01 4.625000e-01 4.057000e-01 3.053000e-01 + 4.290000e-02 7.170000e-01 2.490000e-01 8.308000e-01 4.208000e-01 7.661000e-01 -2.320000e-02 1.762900e+00 1.860000e-02 1.978800e+00 -6.870000e-02 8.130000e-02 1.143000e-01 5.234000e-01 +-3.980000e-02 6.514000e-01 1.588000e-01 7.603000e-01 2.672000e-01 7.724000e-01 -1.780000e-02 1.750400e+00 1.890000e-02 1.978200e+00 -5.620000e-02 6.840000e-02 9.140000e-02 4.407000e-01 + 2.062000e-01 4.098000e-01 2.551000e-01 7.148000e-01 4.466000e-01 6.359000e-01 2.715000e-01 6.969000e-01 2.490000e-01 8.829000e-01 2.201000e-01 7.591000e-01 1.590000e-01 4.010000e-01 + 3.410000e-01 4.307000e-01 7.206000e-01 3.524000e-01 7.949000e-01 4.069000e-01 1.602900e+00 1.539000e-01 -1.459000e-01 2.171600e+00 1.000000e-02 -1.250000e-02 3.856000e-01 2.496000e-01 +-2.410000e-02 6.119000e-01 2.913000e-01 5.930000e-01 2.959000e-01 7.341000e-01 6.990000e-02 1.644200e+00 -4.070000e-02 2.050400e+00 9.500000e-02 -1.145000e-01 1.324000e-01 3.856000e-01 + 8.864000e-01 1.457000e-01 1.449500e+00 1.612000e-01 1.558100e+00 2.872000e-01 1.467800e+00 1.383000e-01 1.645400e+00 2.126000e-01 1.511700e+00 8.060000e-02 8.334000e-01 9.520000e-02 + 4.516000e-01 3.031000e-01 6.539000e-01 4.383000e-01 8.696000e-01 3.244000e-01 1.766400e+00 -3.990000e-02 -4.020000e-02 2.044000e+00 5.800000e-02 -6.990000e-02 3.998000e-01 2.355000e-01 + 1.500000e-03 1.039600e+00 -6.270000e-02 1.707800e+00 -1.400000e-02 1.880800e+00 6.470000e-02 1.564000e+00 1.572000e-01 1.698000e+00 1.540000e-02 1.620800e+00 1.990000e-02 9.212000e-01 + 4.896000e-01 2.424000e-01 8.585000e-01 1.687000e-01 1.021900e+00 1.227000e-01 1.728400e+00 -1.600000e-03 3.230000e-02 1.961700e+00 -3.790000e-02 4.570000e-02 4.665000e-01 1.439000e-01 + 1.028500e+00 2.200000e-03 1.519100e+00 1.153000e-01 1.891400e+00 -6.680000e-02 1.574600e+00 4.710000e-02 1.882900e+00 -2.460000e-02 1.660200e+00 -5.800000e-02 9.215000e-01 1.180000e-02 + 5.700000e-01 1.949000e-01 6.662000e-01 5.775000e-01 7.901000e-01 6.218000e-01 7.056000e-01 5.306000e-01 7.961000e-01 6.351000e-01 7.592000e-01 4.700000e-01 4.177000e-01 2.949000e-01 + 2.964000e-01 2.839000e-01 3.532000e-01 5.773000e-01 4.329000e-01 6.187000e-01 3.322000e-01 6.025000e-01 4.204000e-01 6.566000e-01 3.175000e-01 6.181000e-01 2.053000e-01 3.326000e-01 + 1.110100e+00 -6.900000e-03 5.430000e-01 7.135000e-01 3.813000e-01 5.501000e-01 4.775000e-01 7.901000e-01 4.455000e-01 4.477000e-01 5.519000e-01 7.022000e-01 3.929000e-01 3.006000e-01 + 4.494000e-01 3.016000e-01 6.661000e-01 5.118000e-01 8.272000e-01 5.039000e-01 7.380000e-01 4.324000e-01 8.066000e-01 5.531000e-01 7.674000e-01 3.959000e-01 4.210000e-01 2.538000e-01 + 1.201700e+00 -1.156000e-01 3.379000e-01 6.891000e-01 3.519000e-01 4.317000e-01 4.171000e-01 5.947000e-01 2.714000e-01 5.115000e-01 3.865000e-01 6.303000e-01 3.125000e-01 3.083000e-01 + 4.175000e-01 6.920000e-01 7.238000e-01 8.324000e-01 8.213000e-01 8.528000e-01 1.684100e+00 5.680000e-02 9.100000e-03 1.987300e+00 -1.150000e-02 1.749900e+00 4.226000e-01 4.766000e-01 + 4.360000e-01 3.713000e-01 6.214000e-01 6.574000e-01 7.203000e-01 7.235000e-01 5.825000e-01 6.956000e-01 7.206000e-01 7.458000e-01 6.731000e-01 5.917000e-01 3.731000e-01 3.595000e-01 + 3.848000e-01 2.801000e-01 5.886000e-01 4.593000e-01 7.417000e-01 4.387000e-01 5.876000e-01 4.592000e-01 7.156000e-01 4.903000e-01 6.483000e-01 3.882000e-01 3.588000e-01 2.423000e-01 + 1.096400e+00 1.260000e-02 8.966000e-01 3.421000e-01 6.822000e-01 2.301000e-01 8.664000e-01 3.788000e-01 6.956000e-01 1.860000e-01 8.982000e-01 3.447000e-01 5.707000e-01 1.193000e-01 + 4.805000e-01 5.688000e-01 5.959000e-01 1.079200e+00 6.938000e-01 1.213100e+00 6.770000e-01 9.835000e-01 7.859000e-01 1.129700e+00 7.537000e-01 8.900000e-01 3.702000e-01 5.925000e-01 + 1.090700e+00 1.510000e-02 4.532000e-01 8.947000e-01 3.829000e-01 5.780000e-01 4.569000e-01 8.886000e-01 3.970000e-01 5.348000e-01 5.703000e-01 7.581000e-01 3.893000e-01 3.213000e-01 + 4.580000e-02 5.139000e-01 -1.840000e-02 3.196000e-01 -3.710000e-02 3.004000e-01 9.080000e-02 1.905000e-01 -1.390000e-02 2.675000e-01 1.530000e-02 2.785000e-01 9.600000e-03 2.236000e-01 + 1.131200e+00 -3.020000e-02 8.929000e-01 7.040000e-02 7.703000e-01 -2.560000e-02 8.951000e-01 6.580000e-02 6.570000e-01 8.810000e-02 8.834000e-01 8.170000e-02 5.877000e-01 1.170000e-02 +-5.820000e-02 8.150000e-01 3.495000e-01 7.002000e-01 4.372000e-01 7.411000e-01 -8.800000e-02 1.829700e+00 6.840000e-02 1.920900e+00 -3.980000e-02 4.890000e-02 1.860000e-01 4.308000e-01 + 3.479000e-01 6.417000e-01 6.093000e-01 7.009000e-01 7.565000e-01 6.621000e-01 1.628200e+00 1.224000e-01 1.492000e-01 1.828500e+00 -1.960000e-02 2.300000e-02 3.743000e-01 3.924000e-01 + 4.160000e-01 1.654000e-01 6.349000e-01 2.783000e-01 6.370000e-01 4.172000e-01 6.487000e-01 2.577000e-01 6.379000e-01 4.257000e-01 6.367000e-01 2.739000e-01 3.315000e-01 2.005000e-01 + 7.600000e-02 1.032000e+00 3.440000e-02 1.042800e+00 -1.570000e-02 8.486000e-01 4.830000e-02 1.026200e+00 9.170000e-02 7.007000e-01 -1.420000e-02 1.099300e+00 4.420000e-02 5.973000e-01 + 6.870000e-02 6.446000e-01 4.410000e-01 5.699000e-01 4.881000e-01 6.571000e-01 1.674200e+00 6.730000e-02 1.860000e-02 1.976100e+00 2.750000e-02 -3.180000e-02 2.199000e-01 3.772000e-01 + 1.838000e-01 9.222000e-01 1.668000e-01 8.231000e-01 1.093000e-01 6.645000e-01 1.597000e-01 8.303000e-01 1.604000e-01 5.899000e-01 1.905000e-01 7.935000e-01 9.970000e-02 5.155000e-01 + 5.985000e-01 2.114000e-01 8.599000e-01 2.747000e-01 8.155000e-01 4.715000e-01 1.754300e+00 -2.590000e-02 -1.890000e-02 2.017000e+00 -6.550000e-02 7.960000e-02 4.398000e-01 2.384000e-01 + 9.812000e-01 1.464000e-01 9.873000e-01 7.388000e-01 7.230000e-01 1.253800e+00 9.414000e-01 7.927000e-01 7.645000e-01 1.026800e+00 9.855000e-01 7.375000e-01 6.030000e-01 3.864000e-01 + 2.771000e-01 4.934000e-01 4.557000e-01 7.448000e-01 4.533000e-01 9.261000e-01 3.633000e-01 8.519000e-01 3.589000e-01 1.055700e+00 4.432000e-01 7.585000e-01 2.485000e-01 4.471000e-01 + 2.765000e-01 5.446000e-01 4.058000e-01 8.792000e-01 5.765000e-01 8.749000e-01 4.498000e-01 8.381000e-01 6.530000e-01 8.068000e-01 4.016000e-01 8.954000e-01 2.742000e-01 4.652000e-01 + 8.580000e-01 2.206000e-01 3.943000e-01 1.329800e+00 3.000000e-01 7.321000e-01 4.228000e-01 1.296500e+00 3.049000e-01 6.911000e-01 3.720000e-01 1.352200e+00 2.668000e-01 4.835000e-01 + 4.178000e-01 5.168000e-01 5.407000e-01 9.438000e-01 6.523000e-01 1.029500e+00 6.214000e-01 8.466000e-01 7.659000e-01 9.212000e-01 5.444000e-01 9.391000e-01 3.318000e-01 5.214000e-01 + 5.398000e-01 3.048000e-01 8.779000e-01 4.404000e-01 1.011700e+00 4.854000e-01 8.146000e-01 5.131000e-01 9.718000e-01 5.539000e-01 8.771000e-01 4.340000e-01 4.961000e-01 2.664000e-01 + 1.737000e-01 1.020000e-01 2.218000e-01 2.243000e-01 3.025000e-01 1.940000e-01 2.421000e-01 1.973000e-01 2.738000e-01 2.345000e-01 1.940000e-01 2.585000e-01 1.483000e-01 1.043000e-01 + 2.552000e-01 6.242000e-01 5.095000e-01 6.932000e-01 7.394000e-01 5.597000e-01 1.650500e+00 1.011000e-01 -2.090000e-02 2.025200e+00 4.510000e-02 -5.310000e-02 3.175000e-01 3.858000e-01 + 6.043000e-01 4.847000e-01 2.698000e-01 7.133000e-01 1.207000e-01 6.650000e-01 1.966000e-01 7.995000e-01 3.183000e-01 4.146000e-01 3.367000e-01 6.292000e-01 2.181000e-01 3.906000e-01 + 7.783000e-01 4.700000e-03 1.048700e+00 2.047000e-01 1.169700e+00 2.627000e-01 1.024000e+00 2.338000e-01 1.229200e+00 2.135000e-01 1.067300e+00 1.827000e-01 5.997000e-01 1.258000e-01 + 1.056900e+00 5.520000e-02 7.505000e-01 1.643000e-01 5.627000e-01 1.735000e-01 7.464000e-01 1.701000e-01 5.599000e-01 1.603000e-01 7.443000e-01 1.746000e-01 4.886000e-01 9.790000e-02 + 1.082000e+00 2.940000e-02 6.796000e-01 1.778000e-01 5.755000e-01 1.053000e-01 7.075000e-01 1.409000e-01 5.595000e-01 1.105000e-01 8.247000e-01 -2.300000e-03 5.124000e-01 3.890000e-02 + 5.710000e-02 1.050300e+00 -1.100000e-03 4.328000e-01 1.623000e-01 1.709000e-01 -3.400000e-02 4.719000e-01 -1.950000e-02 3.805000e-01 -9.330000e-02 5.434000e-01 3.350000e-02 2.892000e-01 + 1.309000e-01 3.577000e-01 1.534000e-01 6.240000e-01 1.369000e-01 7.509000e-01 2.430000e-01 5.122000e-01 2.289000e-01 6.538000e-01 2.045000e-01 5.573000e-01 1.022000e-01 3.412000e-01 + 3.363000e-01 4.877000e-01 6.284000e-01 5.175000e-01 6.639000e-01 6.105000e-01 1.634700e+00 1.160000e-01 -7.740000e-02 2.089800e+00 1.650000e-02 -2.060000e-02 3.415000e-01 3.319000e-01 + 8.430000e-02 3.520000e-01 5.280000e-02 6.516000e-01 6.420000e-02 7.334000e-01 5.900000e-02 6.430000e-01 6.910000e-02 7.387000e-01 1.051000e-01 5.891000e-01 3.390000e-02 3.710000e-01 + 2.419000e-01 2.554000e-01 3.898000e-01 3.872000e-01 4.288000e-01 4.575000e-01 3.504000e-01 4.362000e-01 4.903000e-01 3.988000e-01 3.969000e-01 3.784000e-01 2.171000e-01 2.331000e-01 + 6.307000e-01 4.740000e-01 9.798000e-01 5.770000e-01 9.152000e-01 7.924000e-01 1.912900e+00 -2.140000e-01 2.007500e+00 -1.050000e-02 -5.000000e-04 1.730100e+00 5.307000e-01 3.768000e-01 + 4.460000e-01 -1.960000e-02 7.009000e-01 -2.960000e-02 7.206000e-01 5.770000e-02 6.415000e-01 4.150000e-02 7.693000e-01 1.520000e-02 7.258000e-01 -5.950000e-02 3.754000e-01 1.790000e-02 + 1.226000e-01 8.072000e-01 5.387000e-01 6.823000e-01 6.626000e-01 6.829000e-01 1.689300e+00 4.540000e-02 -1.990000e-02 2.019300e+00 -3.810000e-02 4.740000e-02 2.932000e-01 4.304000e-01 + 7.825000e-01 9.250000e-02 1.127900e+00 2.562000e-01 1.325700e+00 2.437000e-01 1.183100e+00 1.954000e-01 1.290500e+00 3.130000e-01 1.240500e+00 1.253000e-01 6.916000e-01 1.009000e-01 + 4.689000e-01 1.481000e-01 7.357000e-01 2.329000e-01 7.442000e-01 3.792000e-01 6.927000e-01 2.833000e-01 8.208000e-01 3.054000e-01 7.793000e-01 1.818000e-01 4.031000e-01 1.613000e-01 + 6.073000e-01 1.505000e-01 8.918000e-01 3.078000e-01 1.123600e+00 2.238000e-01 8.878000e-01 3.141000e-01 1.023600e+00 3.635000e-01 8.919000e-01 3.122000e-01 4.959000e-01 2.008000e-01 + 1.084500e+00 2.560000e-02 1.255700e+00 4.653000e-01 9.001000e-01 1.065200e+00 1.240200e+00 4.783000e-01 9.773000e-01 7.541000e-01 1.280700e+00 4.358000e-01 6.714000e-01 3.143000e-01 + 1.122400e+00 -2.090000e-02 6.288000e-01 1.111500e+00 5.239000e-01 8.923000e-01 7.640000e-01 9.513000e-01 6.113000e-01 7.272000e-01 6.948000e-01 1.037500e+00 4.859000e-01 4.066000e-01 + 8.826000e-01 2.210000e-01 3.047000e-01 6.431000e-01 3.247000e-01 4.102000e-01 4.222000e-01 5.054000e-01 3.085000e-01 4.134000e-01 3.644000e-01 5.701000e-01 2.931000e-01 2.968000e-01 +-7.180000e-02 6.558000e-01 2.116000e-01 6.708000e-01 3.328000e-01 6.744000e-01 3.570000e-02 1.688300e+00 5.100000e-02 1.946700e+00 -7.030000e-02 8.540000e-02 1.075000e-01 4.049000e-01 + 3.613000e-01 7.365000e-01 2.189000e-01 5.576000e-01 1.782000e-01 4.514000e-01 2.020000e-01 5.775000e-01 1.855000e-01 4.275000e-01 2.536000e-01 5.144000e-01 1.511000e-01 3.725000e-01 + 1.074900e+00 4.140000e-02 9.121000e-01 1.876000e-01 7.667000e-01 5.760000e-02 9.258000e-01 1.722000e-01 7.291000e-01 8.450000e-02 8.942000e-01 2.083000e-01 5.681000e-01 8.440000e-02 + 2.760000e-02 1.086800e+00 -4.250000e-02 5.219000e-01 3.250000e-02 3.607000e-01 2.460000e-02 4.429000e-01 8.690000e-02 2.859000e-01 -1.100000e-03 4.726000e-01 3.560000e-02 3.120000e-01 +-4.950000e-02 5.970000e-02 5.990000e-02 1.809000e-01 1.120000e-02 3.660000e-01 1.340000e-02 -1.290000e-02 7.630000e-02 -9.200000e-02 5.320000e-02 -6.370000e-02 6.000000e-03 1.293000e-01 +-7.640000e-02 1.198800e+00 3.247000e-01 1.192000e+00 3.850000e-01 1.257300e+00 -8.280000e-02 1.832100e+00 4.500000e-03 1.991200e+00 -1.990000e-02 1.757700e+00 1.692000e-01 7.139000e-01 + 3.735000e-01 7.355000e-01 1.927000e-01 5.480000e-01 2.143000e-01 3.824000e-01 2.458000e-01 4.837000e-01 1.067000e-01 4.974000e-01 1.687000e-01 5.756000e-01 1.769000e-01 3.251000e-01 + 2.741000e-01 5.164000e-01 3.749000e-01 8.773000e-01 4.355000e-01 9.832000e-01 4.299000e-01 8.103000e-01 5.146000e-01 9.149000e-01 3.877000e-01 8.587000e-01 2.248000e-01 4.949000e-01 +-7.800000e-03 7.603000e-01 -4.110000e-02 1.110900e+00 -3.300000e-03 1.193900e+00 -7.110000e-02 1.819100e+00 6.250000e-02 1.926700e+00 4.510000e-02 -5.140000e-02 1.150000e-02 6.132000e-01 + 1.184000e+00 -9.540000e-02 4.044000e-01 6.970000e-01 2.919000e-01 5.550000e-01 5.266000e-01 5.560000e-01 3.209000e-01 5.018000e-01 3.657000e-01 7.450000e-01 3.112000e-01 3.396000e-01 + 4.732000e-01 3.046000e-01 6.683000e-01 4.455000e-01 8.095000e-01 4.224000e-01 1.736500e+00 1.300000e-03 3.190000e-02 1.964000e+00 -2.400000e-03 3.500000e-03 4.121000e-01 2.376000e-01 + 1.384000e-01 6.302000e-01 1.225000e-01 1.096600e+00 2.138000e-01 1.162500e+00 1.260000e-01 1.090700e+00 2.173000e-01 1.177400e+00 2.158000e-01 9.860000e-01 1.137000e-01 5.832000e-01 + 5.437000e-01 1.828000e-01 8.222000e-01 3.264000e-01 1.067400e+00 2.131000e-01 8.137000e-01 3.349000e-01 1.033500e+00 2.770000e-01 8.017000e-01 3.519000e-01 5.158000e-01 1.387000e-01 + 8.353000e-01 1.856000e-01 1.130800e+00 2.115000e-01 1.142400e+00 3.409000e-01 1.648800e+00 9.690000e-02 1.939400e+00 7.250000e-02 -3.960000e-02 4.650000e-02 6.589000e-01 1.338000e-01 + 5.665000e-01 4.000000e-01 8.040000e-01 4.914000e-01 9.725000e-01 4.351000e-01 1.770100e+00 -5.090000e-02 -6.530000e-02 2.077400e+00 8.140000e-02 -9.730000e-02 4.666000e-01 2.948000e-01 + 1.780000e-02 6.441000e-01 1.934000e-01 8.153000e-01 2.080000e-01 9.453000e-01 2.370000e-01 7.618000e-01 2.103000e-01 9.556000e-01 8.110000e-02 9.460000e-01 8.990000e-02 4.970000e-01 + 3.716000e-01 5.727000e-01 5.762000e-01 9.028000e-01 7.666000e-01 8.992000e-01 6.574000e-01 8.045000e-01 8.085000e-01 8.724000e-01 6.284000e-01 8.446000e-01 3.867000e-01 4.575000e-01 + 9.650000e-02 8.800000e-02 5.990000e-02 2.461000e-01 1.445000e-01 1.877000e-01 1.420000e-01 1.516000e-01 1.662000e-01 1.680000e-01 1.162000e-01 1.799000e-01 6.860000e-02 1.019000e-01 + 5.024000e-01 5.639000e-01 7.328000e-01 9.492000e-01 8.707000e-01 1.036400e+00 6.174000e-01 1.084500e+00 8.037000e-01 1.142800e+00 7.794000e-01 8.958000e-01 4.507000e-01 5.156000e-01 + 5.465000e-01 4.334000e-01 9.130000e-01 6.068000e-01 8.931000e-01 8.647000e-01 8.490000e-01 6.866000e-01 9.978000e-01 7.642000e-01 8.058000e-01 7.353000e-01 4.696000e-01 4.187000e-01 + 1.090000e-01 7.328000e-01 2.592000e-01 1.042400e+00 3.453000e-01 1.126100e+00 2.120000e-01 1.099000e+00 3.552000e-01 1.139400e+00 2.400000e-01 1.065500e+00 1.622000e-01 5.869000e-01 + 3.045000e-01 4.043000e-01 4.679000e-01 6.469000e-01 5.784000e-01 6.798000e-01 4.442000e-01 6.740000e-01 5.287000e-01 7.581000e-01 4.444000e-01 6.719000e-01 2.540000e-01 3.910000e-01 + 5.597000e-01 2.298000e-01 8.849000e-01 3.551000e-01 1.034300e+00 3.724000e-01 1.094000e+00 1.069000e-01 1.116900e+00 3.034000e-01 8.565000e-01 3.830000e-01 5.458000e-01 1.637000e-01 + 1.135400e+00 -3.660000e-02 9.930000e-01 7.328000e-01 7.396000e-01 4.829000e-01 1.096200e+00 6.153000e-01 7.750000e-01 4.016000e-01 1.057200e+00 6.568000e-01 6.032000e-01 2.289000e-01 + 2.453000e-01 4.808000e-01 4.855000e-01 6.343000e-01 6.267000e-01 6.316000e-01 5.035000e-01 6.100000e-01 6.372000e-01 6.430000e-01 4.382000e-01 6.838000e-01 2.897000e-01 3.544000e-01 + 1.590000e-01 9.624000e-01 5.616000e-01 1.167600e+00 7.067000e-01 1.198000e+00 1.750800e+00 -2.140000e-02 1.230000e-02 1.983100e+00 1.710000e-02 1.712500e+00 3.037000e-01 6.852000e-01 + 1.016200e+00 1.034000e-01 4.259000e-01 4.574000e-01 3.615000e-01 3.437000e-01 3.998000e-01 4.866000e-01 3.650000e-01 3.214000e-01 4.585000e-01 4.220000e-01 3.193000e-01 2.518000e-01 +-6.100000e-02 1.181100e+00 1.040000e-02 1.661100e+00 9.170000e-02 1.701800e+00 1.160000e-02 1.715900e+00 -5.730000e-02 2.068800e+00 -1.860000e-02 1.750300e+00 4.010000e-02 9.064000e-01 + 1.116900e+00 -1.100000e-02 1.668300e+00 7.910000e-02 1.079600e+00 2.620000e-02 1.753600e+00 -3.070000e-02 9.317000e-01 1.614000e-01 1.719400e+00 1.680000e-02 7.741000e-01 1.160000e-02 + 5.425000e-01 5.662000e-01 4.065000e-01 7.119000e-01 1.688000e-01 7.062000e-01 3.222000e-01 8.160000e-01 7.430000e-02 7.979000e-01 1.885000e-01 9.741000e-01 2.290000e-01 4.339000e-01 + 1.092100e+00 1.910000e-02 6.402000e-01 6.883000e-01 4.099000e-01 5.550000e-01 5.433000e-01 8.057000e-01 4.109000e-01 5.229000e-01 5.550000e-01 7.881000e-01 4.032000e-01 3.093000e-01 + 4.276000e-01 5.982000e-01 8.580000e-01 4.580000e-01 9.666000e-01 4.712000e-01 1.702400e+00 3.460000e-02 9.500000e-03 1.983900e+00 3.450000e-02 -4.050000e-02 4.654000e-01 3.163000e-01 + 1.115200e+00 -1.240000e-02 8.336000e-01 9.390000e-02 7.419000e-01 -2.350000e-02 7.926000e-01 1.436000e-01 6.870000e-01 2.140000e-02 8.857000e-01 3.260000e-02 5.579000e-01 2.920000e-02 + 6.695000e-01 2.592000e-01 1.030400e+00 1.922000e-01 1.123800e+00 2.317000e-01 1.701900e+00 3.900000e-02 1.941600e+00 6.780000e-02 -6.900000e-03 9.600000e-03 5.872000e-01 1.409000e-01 + 1.106000e+00 1.000000e-04 5.493000e-01 1.184000e+00 5.373000e-01 6.650000e-01 4.143000e-01 1.344100e+00 4.731000e-01 6.990000e-01 5.029000e-01 1.236600e+00 4.045000e-01 4.258000e-01 + 2.478000e-01 6.458000e-01 4.043000e-01 8.337000e-01 7.193000e-01 6.084000e-01 1.761800e+00 -2.940000e-02 3.350000e-02 1.959700e+00 -3.270000e-02 4.190000e-02 2.817000e-01 4.376000e-01 + 7.105000e-01 3.832000e-01 9.548000e-01 4.648000e-01 1.065400e+00 4.734000e-01 1.780600e+00 -5.460000e-02 1.931100e+00 7.980000e-02 -8.090000e-02 1.827600e+00 5.788000e-01 2.513000e-01 + 9.968000e-01 1.254000e-01 3.771000e-01 8.535000e-01 3.739000e-01 5.202000e-01 4.012000e-01 8.273000e-01 3.794000e-01 4.933000e-01 3.189000e-01 9.200000e-01 3.228000e-01 3.621000e-01 + 8.350000e-02 5.562000e-01 -2.360000e-02 9.990000e-01 1.625000e-01 8.986000e-01 -2.440000e-02 1.758700e+00 9.700000e-02 1.884700e+00 -6.000000e-03 8.400000e-03 4.000000e-03 5.609000e-01 + 7.489000e-01 2.026000e-01 1.055900e+00 4.588000e-01 1.076900e+00 6.637000e-01 9.010000e-01 6.377000e-01 1.106200e+00 6.628000e-01 9.712000e-01 5.537000e-01 5.625000e-01 3.189000e-01 + 6.852000e-01 4.216000e-01 2.922000e-01 1.446000e+00 2.703000e-01 1.491500e+00 3.944000e-01 1.328700e+00 2.530000e-01 1.362000e+00 3.464000e-01 1.382300e+00 2.611000e-01 7.005000e-01 + 1.096600e+00 1.160000e-02 5.389000e-01 6.236000e-01 4.291000e-01 4.441000e-01 4.939000e-01 6.776000e-01 5.015000e-01 3.356000e-01 5.481000e-01 6.099000e-01 3.832000e-01 2.869000e-01 + 4.674000e-01 3.366000e-01 6.300000e-01 6.509000e-01 8.707000e-01 5.648000e-01 8.028000e-01 4.547000e-01 7.896000e-01 6.800000e-01 6.882000e-01 5.842000e-01 3.931000e-01 3.419000e-01 + 3.740000e-02 3.977000e-01 1.968000e-01 5.588000e-01 3.630000e-01 5.027000e-01 5.700000e-03 1.723800e+00 5.880000e-02 -6.790000e-02 4.370000e-02 -5.240000e-02 9.980000e-02 3.309000e-01 +-8.640000e-02 7.606000e-01 1.386000e-01 8.285000e-01 1.761000e-01 9.310000e-01 1.890000e-02 1.707500e+00 8.900000e-02 1.899300e+00 -2.260000e-02 2.660000e-02 7.470000e-02 4.927000e-01 + 9.280000e-02 4.511000e-01 2.259000e-01 6.111000e-01 2.024000e-01 7.561000e-01 2.166000e-01 6.206000e-01 2.474000e-01 7.191000e-01 3.004000e-01 5.213000e-01 1.155000e-01 3.700000e-01 + 1.129200e+00 -2.850000e-02 1.839300e+00 -1.256000e-01 1.044800e+00 3.002000e-01 1.690200e+00 4.770000e-02 1.060200e+00 2.250000e-01 1.756400e+00 -2.550000e-02 7.967000e-01 7.160000e-02 + 4.576000e-01 6.481000e-01 8.842000e-01 6.654000e-01 8.400000e-01 8.585000e-01 1.776800e+00 -5.000000e-02 -2.600000e-02 2.027400e+00 -3.770000e-02 1.776600e+00 4.504000e-01 4.539000e-01 + 9.950000e-01 1.179000e-01 4.039000e-01 5.380000e-02 3.093000e-01 8.520000e-02 3.899000e-01 6.880000e-02 3.294000e-01 5.420000e-02 3.797000e-01 8.240000e-02 3.046000e-01 4.090000e-02 + 3.339000e-01 6.875000e-01 4.558000e-01 1.153400e+00 6.172000e-01 1.192000e+00 5.502000e-01 1.038800e+00 6.202000e-01 1.225200e+00 5.811000e-01 1.007600e+00 3.185000e-01 6.004000e-01 +-4.820000e-02 6.359000e-01 1.800000e-02 8.810000e-01 2.713000e-01 7.287000e-01 -7.900000e-03 1.742900e+00 -7.300000e-02 2.082400e+00 1.420000e-02 -1.830000e-02 5.610000e-02 4.595000e-01 + 2.457000e-01 8.729000e-01 5.142000e-01 1.241100e+00 6.712000e-01 1.218500e+00 1.593200e+00 1.650000e-01 -7.070000e-02 2.076500e+00 -4.660000e-02 1.782600e+00 3.449000e-01 6.335000e-01 + 1.229000e-01 4.821000e-01 1.893000e-01 7.599000e-01 1.586000e-01 9.281000e-01 1.126000e-01 8.487000e-01 1.617000e-01 9.455000e-01 1.438000e-01 8.120000e-01 8.710000e-02 4.641000e-01 + 1.070200e+00 4.060000e-02 9.576000e-01 7.759000e-01 7.389000e-01 6.740000e-01 9.694000e-01 7.624000e-01 6.454000e-01 7.210000e-01 8.924000e-01 8.504000e-01 5.947000e-01 3.001000e-01 + 1.169300e+00 -7.410000e-02 4.090000e-01 1.405000e-01 3.912000e-01 5.770000e-02 3.398000e-01 2.218000e-01 3.632000e-01 8.360000e-02 3.703000e-01 1.829000e-01 3.115000e-01 8.760000e-02 + 1.105400e+00 -1.500000e-03 8.807000e-01 8.586000e-01 7.397000e-01 7.915000e-01 8.299000e-01 9.165000e-01 6.437000e-01 8.200000e-01 8.560000e-01 8.872000e-01 5.826000e-01 3.420000e-01 + 7.364000e-01 3.606000e-01 3.379000e-01 1.395600e+00 3.809000e-01 6.602000e-01 3.742000e-01 1.349900e+00 2.766000e-01 7.495000e-01 3.453000e-01 1.385800e+00 2.625000e-01 5.014000e-01 + 2.111000e-01 4.445000e-01 3.171000e-01 7.100000e-01 2.735000e-01 9.087000e-01 2.507000e-01 7.881000e-01 3.714000e-01 8.123000e-01 3.026000e-01 7.286000e-01 1.915000e-01 3.995000e-01 + 1.103900e+00 1.500000e-03 1.563900e+00 1.408000e-01 1.042800e+00 2.625000e-01 1.342200e+00 4.042000e-01 9.119000e-01 3.649000e-01 1.385600e+00 3.508000e-01 7.253000e-01 1.404000e-01 + 4.195000e-01 8.830000e-02 7.209000e-01 9.850000e-02 9.006000e-01 3.060000e-02 1.774500e+00 -4.860000e-02 4.910000e-02 -5.700000e-02 -4.180000e-02 4.740000e-02 4.286000e-01 3.880000e-02 + 5.653000e-01 2.091000e-01 8.336000e-01 2.673000e-01 9.022000e-01 3.250000e-01 1.754900e+00 -2.750000e-02 -4.610000e-02 2.045700e+00 9.900000e-03 -1.070000e-02 4.766000e-01 1.708000e-01 + 1.073400e+00 3.650000e-02 1.302600e+00 4.334000e-01 9.583000e-01 1.296000e-01 1.387600e+00 3.254000e-01 9.933000e-01 4.790000e-02 1.318700e+00 4.104000e-01 7.018000e-01 7.740000e-02 + 1.012300e+00 9.170000e-02 1.274500e+00 4.656000e-01 1.394300e+00 5.197000e-01 1.596500e+00 1.580000e-01 1.963400e+00 4.470000e-02 -3.710000e-02 1.775800e+00 7.757000e-01 2.109000e-01 + 3.312000e-01 4.549000e-01 4.088000e-01 8.564000e-01 4.991000e-01 9.237000e-01 3.774000e-01 8.868000e-01 4.258000e-01 1.034300e+00 4.400000e-01 8.127000e-01 2.453000e-01 4.791000e-01 + 2.755000e-01 5.100000e-02 5.709000e-01 7.640000e-02 6.809000e-01 8.630000e-02 1.552000e+00 2.105000e-01 1.029000e-01 -1.244000e-01 4.730000e-02 -5.740000e-02 3.188000e-01 4.080000e-02 + 1.982000e-01 4.741000e-01 5.539000e-01 4.219000e-01 5.941000e-01 5.179000e-01 1.645400e+00 9.840000e-02 -2.010000e-02 2.023500e+00 2.800000e-03 -4.000000e-03 2.789000e-01 2.966000e-01 + 5.457000e-01 5.523000e-01 2.855000e-01 1.444900e+00 2.507000e-01 9.536000e-01 2.293000e-01 1.511200e+00 1.937000e-01 9.833000e-01 2.684000e-01 1.465500e+00 2.168000e-01 6.102000e-01 +-1.760000e-02 7.540000e-01 2.606000e-01 7.783000e-01 3.203000e-01 8.502000e-01 3.800000e-02 1.688500e+00 -8.700000e-03 2.010800e+00 1.600000e-03 -5.200000e-03 1.225000e-01 4.917000e-01 + 4.159000e-01 3.145000e-01 5.517000e-01 6.108000e-01 6.129000e-01 7.097000e-01 5.831000e-01 5.751000e-01 5.852000e-01 7.686000e-01 6.010000e-01 5.550000e-01 3.130000e-01 3.593000e-01 + 9.423000e-01 8.490000e-02 1.512500e+00 9.520000e-02 1.424700e+00 4.486000e-01 1.395900e+00 2.259000e-01 1.504800e+00 3.934000e-01 1.365400e+00 2.641000e-01 7.831000e-01 1.587000e-01 + 4.414000e-01 3.166000e-01 7.875000e-01 2.755000e-01 8.969000e-01 2.846000e-01 1.744600e+00 -1.050000e-02 2.500000e-02 1.969800e+00 6.880000e-02 -8.310000e-02 4.572000e-01 1.650000e-01 + 4.897000e-01 6.166000e-01 7.778000e-01 7.313000e-01 9.998000e-01 6.125000e-01 1.702800e+00 3.800000e-02 1.101000e-01 1.871500e+00 6.670000e-02 1.654300e+00 4.621000e-01 4.153000e-01 + 3.592000e-01 1.203000e-01 5.896000e-01 1.628000e-01 7.070000e-01 1.390000e-01 6.235000e-01 1.224000e-01 6.721000e-01 1.972000e-01 6.086000e-01 1.439000e-01 3.612000e-01 6.870000e-02 + 1.177800e+00 -8.270000e-02 1.425000e+00 2.890000e-02 1.075300e+00 -1.025000e-01 1.426900e+00 3.190000e-02 9.603000e-01 8.000000e-04 1.419000e+00 4.170000e-02 7.187000e-01 1.360000e-02 + 2.602000e-01 7.164000e-01 7.322000e-01 5.280000e-01 7.810000e-01 6.129000e-01 1.754400e+00 -2.570000e-02 -4.170000e-02 2.048600e+00 -7.420000e-02 8.850000e-02 3.622000e-01 3.902000e-01 + 9.853000e-01 1.293000e-01 4.345000e-01 2.152000e-01 3.437000e-01 1.954000e-01 4.099000e-01 2.471000e-01 3.284000e-01 2.032000e-01 4.200000e-01 2.330000e-01 3.041000e-01 1.577000e-01 + 9.117000e-01 1.948000e-01 1.158700e+00 4.074000e-01 1.335200e+00 3.364000e-01 1.714900e+00 1.760000e-02 2.016600e+00 -2.550000e-02 -1.041000e-01 1.856400e+00 7.200000e-01 1.807000e-01 + 1.140600e+00 -3.580000e-02 6.300000e-01 1.801000e-01 3.542000e-01 3.308000e-01 5.145000e-01 3.221000e-01 4.868000e-01 1.562000e-01 5.023000e-01 3.346000e-01 4.021000e-01 1.433000e-01 + 6.267000e-01 4.142000e-01 9.552000e-01 6.795000e-01 1.028300e+00 8.437000e-01 9.689000e-01 6.655000e-01 1.223900e+00 6.474000e-01 9.202000e-01 7.226000e-01 5.495000e-01 3.955000e-01 + 2.915000e-01 2.898000e-01 3.733000e-01 5.469000e-01 3.424000e-01 7.258000e-01 3.200000e-01 6.140000e-01 4.248000e-01 6.449000e-01 2.811000e-01 6.647000e-01 1.979000e-01 3.389000e-01 + 7.741000e-01 1.946000e-01 1.068400e+00 4.755000e-01 1.394300e+00 3.300000e-01 1.170700e+00 3.506000e-01 1.343000e+00 4.205000e-01 1.218300e+00 2.957000e-01 6.731000e-01 2.077000e-01 +-3.680000e-02 4.017000e-01 6.590000e-02 6.122000e-01 2.108000e-01 5.843000e-01 -7.320000e-02 1.821100e+00 -7.040000e-02 8.380000e-02 -1.620000e-02 1.770000e-02 6.800000e-02 3.089000e-01 + 3.511000e-01 8.330000e-02 6.243000e-01 4.320000e-02 6.950000e-01 7.220000e-02 6.273000e-01 3.880000e-02 8.073000e-01 -5.250000e-02 6.866000e-01 -2.930000e-02 3.668000e-01 1.780000e-02 +-2.200000e-03 4.328000e-01 9.620000e-02 5.566000e-01 1.032000e-01 6.397000e-01 7.780000e-02 5.799000e-01 1.418000e-01 6.060000e-01 1.540000e-02 6.526000e-01 4.010000e-02 3.400000e-01 + 1.157000e-01 3.932000e-01 2.233000e-01 5.684000e-01 3.136000e-01 5.740000e-01 2.221000e-01 5.673000e-01 1.697000e-01 7.595000e-01 1.936000e-01 6.002000e-01 1.201000e-01 3.379000e-01 + 3.699000e-01 7.311000e-01 7.281000e-01 9.934000e-01 6.734000e-01 1.311100e+00 1.734800e+00 -5.500000e-03 1.186000e-01 1.858800e+00 -1.470000e-02 1.746300e+00 3.813000e-01 6.166000e-01 + 3.891000e-01 2.215000e-01 6.282000e-01 3.252000e-01 7.111000e-01 3.700000e-01 6.102000e-01 3.434000e-01 7.287000e-01 3.707000e-01 5.959000e-01 3.620000e-01 3.536000e-01 1.964000e-01 + 1.167600e+00 -7.010000e-02 3.770000e-01 1.359000e+00 2.343000e-01 1.332700e+00 4.709000e-01 1.250900e+00 2.915000e-01 1.174700e+00 3.580000e-01 1.382700e+00 3.080000e-01 6.191000e-01 + 4.259000e-01 4.327000e-01 6.630000e-01 6.835000e-01 6.497000e-01 9.032000e-01 6.208000e-01 7.378000e-01 6.525000e-01 9.307000e-01 6.688000e-01 6.760000e-01 3.576000e-01 4.249000e-01 + 4.507000e-01 5.850000e-01 7.612000e-01 8.484000e-01 8.878000e-01 9.469000e-01 7.216000e-01 8.978000e-01 7.942000e-01 1.079400e+00 7.377000e-01 8.773000e-01 4.067000e-01 5.303000e-01 + 1.700000e-01 9.436000e-01 1.486000e-01 1.091900e+00 5.420000e-02 8.650000e-01 2.837000e-01 9.306000e-01 1.378000e-01 7.436000e-01 1.412000e-01 1.099300e+00 1.027000e-01 5.879000e-01 + 3.313000e-01 2.213000e-01 5.529000e-01 3.276000e-01 7.630000e-01 2.247000e-01 1.775100e+00 -4.960000e-02 -1.123000e-01 2.130400e+00 3.450000e-02 -4.120000e-02 3.109000e-01 1.995000e-01 +-3.970000e-02 8.653000e-01 2.490000e-02 1.106400e+00 1.539000e-01 1.077600e+00 2.170000e-02 1.703100e+00 -3.390000e-02 2.043400e+00 4.890000e-02 -5.940000e-02 9.900000e-03 6.578000e-01 + 4.160000e-02 2.492000e-01 3.630000e-02 5.778000e-01 2.428000e-01 4.730000e-01 -2.500000e-03 1.733400e+00 -5.520000e-02 6.750000e-02 4.880000e-02 -5.870000e-02 4.570000e-02 2.934000e-01 + 1.156100e+00 -6.180000e-02 4.096000e-01 1.040000e+00 4.179000e-01 5.648000e-01 4.842000e-01 9.480000e-01 3.375000e-01 6.273000e-01 3.995000e-01 1.051100e+00 3.377000e-01 3.912000e-01 + 1.151400e+00 -5.380000e-02 5.146000e-01 1.212200e+00 4.147000e-01 8.332000e-01 5.307000e-01 1.194600e+00 3.870000e-01 8.223000e-01 5.730000e-01 1.144800e+00 3.725000e-01 4.698000e-01 + 8.854000e-01 2.118000e-01 2.810000e-01 5.846000e-01 3.241000e-01 3.542000e-01 3.327000e-01 5.222000e-01 3.007000e-01 3.679000e-01 4.131000e-01 4.262000e-01 2.879000e-01 2.674000e-01 + 3.638000e-01 3.804000e-01 7.445000e-01 2.988000e-01 8.495000e-01 3.202000e-01 1.906100e+00 -2.026000e-01 0.000000e+00 2.001200e+00 -8.700000e-03 9.600000e-03 3.821000e-01 2.389000e-01 + 2.077000e-01 4.487000e-01 2.482000e-01 7.953000e-01 3.955000e-01 7.727000e-01 3.535000e-01 6.678000e-01 4.896000e-01 6.782000e-01 3.969000e-01 6.183000e-01 2.022000e-01 3.902000e-01 + 2.329000e-01 6.285000e-01 5.094000e-01 8.113000e-01 6.033000e-01 8.956000e-01 5.496000e-01 7.599000e-01 6.561000e-01 8.529000e-01 5.272000e-01 7.862000e-01 3.007000e-01 4.596000e-01 + 2.791000e-01 3.898000e-01 3.058000e-01 7.622000e-01 5.019000e-01 6.861000e-01 4.306000e-01 6.145000e-01 4.562000e-01 7.582000e-01 4.978000e-01 5.388000e-01 2.469000e-01 3.571000e-01 + 7.123000e-01 4.088000e-01 4.206000e-01 5.596000e-01 2.809000e-01 4.953000e-01 3.921000e-01 5.953000e-01 3.269000e-01 4.263000e-01 3.688000e-01 6.182000e-01 2.817000e-01 3.309000e-01 + 4.560000e-01 6.668000e-01 9.033000e-01 7.476000e-01 1.013600e+00 7.590000e-01 1.754300e+00 -2.050000e-02 1.967200e+00 3.600000e-02 -5.360000e-02 1.796700e+00 5.007000e-01 4.429000e-01 + 8.890000e-02 1.325000e-01 4.074000e-01 1.244000e-01 5.965000e-01 4.450000e-02 1.626600e+00 1.235000e-01 -4.290000e-02 5.310000e-02 4.770000e-02 -5.700000e-02 2.280000e-01 6.230000e-02 + 1.146600e+00 -4.920000e-02 7.559000e-01 2.360000e-02 5.875000e-01 5.060000e-02 7.465000e-01 3.910000e-02 6.143000e-01 2.300000e-03 8.245000e-01 -5.390000e-02 5.154000e-01 7.700000e-03 + 8.170000e-02 2.001000e-01 1.828000e-01 2.480000e-01 5.410000e-02 4.648000e-01 1.310000e-01 3.071000e-01 6.150000e-02 4.636000e-01 1.292000e-01 3.150000e-01 6.720000e-02 1.890000e-01 + 3.180000e-02 6.007000e-01 9.280000e-02 8.930000e-01 6.660000e-02 1.058400e+00 1.871000e-01 7.811000e-01 1.862000e-01 9.383000e-01 1.827000e-01 7.888000e-01 4.380000e-02 5.275000e-01 + 4.072000e-01 3.659000e-01 5.085000e-01 7.296000e-01 5.936000e-01 8.117000e-01 6.028000e-01 6.189000e-01 5.598000e-01 8.765000e-01 6.076000e-01 6.148000e-01 3.187000e-01 3.925000e-01 + 2.243000e-01 6.124000e-01 4.186000e-01 7.552000e-01 5.787000e-01 7.057000e-01 1.668500e+00 7.700000e-02 -8.110000e-02 2.092600e+00 -5.970000e-02 7.010000e-02 2.660000e-01 4.185000e-01 + 1.119900e+00 -1.730000e-02 7.295000e-01 1.010000e+00 6.374000e-01 6.944000e-01 7.769000e-01 9.545000e-01 7.149000e-01 5.439000e-01 7.168000e-01 1.024200e+00 5.167000e-01 3.524000e-01 + 4.520000e-01 4.722000e-01 6.902000e-01 7.582000e-01 9.625000e-01 6.599000e-01 7.994000e-01 6.382000e-01 8.570000e-01 8.108000e-01 7.531000e-01 6.950000e-01 4.221000e-01 4.123000e-01 + 1.081400e+00 2.580000e-02 7.381000e-01 8.548000e-01 5.455000e-01 4.843000e-01 8.089000e-01 7.721000e-01 6.447000e-01 3.352000e-01 7.806000e-01 8.049000e-01 5.102000e-01 2.349000e-01 +-3.530000e-02 9.139000e-01 3.109000e-01 8.767000e-01 3.875000e-01 9.291000e-01 1.760000e-02 1.708500e+00 5.190000e-02 1.937300e+00 -2.270000e-02 2.460000e-02 1.649000e-01 5.366000e-01 + 9.807000e-01 1.179000e-01 4.309000e-01 2.824000e-01 3.117000e-01 2.823000e-01 4.382000e-01 2.777000e-01 3.246000e-01 2.548000e-01 4.180000e-01 3.002000e-01 3.007000e-01 1.943000e-01 + 3.250000e-01 2.790000e-01 7.360000e-01 1.607000e-01 7.521000e-01 2.878000e-01 1.784500e+00 -6.330000e-02 -7.240000e-02 2.082300e+00 3.470000e-02 -4.150000e-02 3.443000e-01 1.906000e-01 + 5.688000e-01 9.040000e-02 8.446000e-01 1.970000e-01 9.281000e-01 2.649000e-01 8.492000e-01 1.929000e-01 8.796000e-01 3.376000e-01 7.574000e-01 3.030000e-01 4.745000e-01 1.294000e-01 + 3.120000e-01 4.004000e-01 6.671000e-01 3.566000e-01 5.774000e-01 6.072000e-01 1.690600e+00 5.000000e-02 1.290000e-02 1.987400e+00 -1.380000e-02 1.650000e-02 3.382000e-01 2.667000e-01 + 1.663000e-01 6.186000e-01 1.562000e-01 1.084300e+00 1.519000e-01 1.267800e+00 1.862000e-01 1.049100e+00 3.050000e-01 1.105300e+00 1.669000e-01 1.073600e+00 1.202000e-01 5.912000e-01 + 3.376000e-01 5.206000e-01 4.022000e-01 9.626000e-01 6.526000e-01 8.635000e-01 5.593000e-01 7.763000e-01 5.713000e-01 9.856000e-01 6.018000e-01 7.242000e-01 3.129000e-01 4.596000e-01 + 4.579000e-01 6.553000e-01 2.936000e-01 8.654000e-01 2.920000e-01 5.680000e-01 2.777000e-01 8.882000e-01 2.360000e-01 6.118000e-01 2.499000e-01 9.187000e-01 2.159000e-01 4.540000e-01 + 3.084000e-01 5.384000e-01 6.887000e-01 4.607000e-01 7.591000e-01 5.193000e-01 1.674400e+00 6.190000e-02 -2.330000e-02 2.028300e+00 -1.730000e-02 2.100000e-02 3.601000e-01 3.235000e-01 + 2.808000e-01 2.563000e-01 4.488000e-01 3.873000e-01 5.796000e-01 3.615000e-01 4.806000e-01 3.485000e-01 5.999000e-01 3.518000e-01 4.774000e-01 3.545000e-01 2.802000e-01 1.986000e-01 + 8.380000e-02 1.481000e-01 1.604000e-01 1.943000e-01 2.514000e-01 1.399000e-01 2.079000e-01 1.384000e-01 2.309000e-01 1.706000e-01 1.796000e-01 1.728000e-01 1.045000e-01 9.790000e-02 + 5.270000e-02 4.620000e-01 3.844000e-01 4.432000e-01 5.097000e-01 4.337000e-01 -2.350000e-02 1.761600e+00 -9.900000e-03 1.100000e-02 3.140000e-02 -3.780000e-02 1.804000e-01 3.003000e-01 + 4.784000e-01 -3.000000e-03 7.456000e-01 2.300000e-03 7.309000e-01 1.364000e-01 6.966000e-01 5.930000e-02 7.600000e-01 1.187000e-01 7.205000e-01 3.540000e-02 4.033000e-01 3.320000e-02 + 3.585000e-01 6.260000e-01 7.743000e-01 5.065000e-01 7.985000e-01 6.149000e-01 1.737500e+00 -1.090000e-02 4.900000e-02 1.945400e+00 -8.970000e-02 1.065000e-01 4.380000e-01 3.180000e-01 + 5.082000e-01 5.910000e-01 7.890000e-01 7.576000e-01 8.673000e-01 8.025000e-01 1.677600e+00 6.540000e-02 -5.020000e-02 2.058300e+00 8.400000e-02 1.632800e+00 4.487000e-01 4.481000e-01 + 3.315000e-01 5.409000e-01 6.587000e-01 5.250000e-01 6.879000e-01 6.287000e-01 1.730300e+00 0.000000e+00 8.930000e-02 1.898600e+00 4.910000e-02 -5.990000e-02 3.776000e-01 3.192000e-01 + 7.500000e-03 7.326000e-01 2.217000e-01 8.010000e-01 2.055000e-01 9.643000e-01 6.180000e-02 1.656800e+00 -5.870000e-02 2.070400e+00 -2.560000e-02 3.100000e-02 4.460000e-02 5.755000e-01 +-4.240000e-02 7.591000e-01 3.428000e-01 6.647000e-01 3.545000e-01 7.933000e-01 5.000000e-02 1.675500e+00 -9.690000e-02 2.112200e+00 -1.800000e-03 2.400000e-03 1.535000e-01 4.441000e-01 + 3.393000e-01 5.055000e-01 6.030000e-01 7.080000e-01 7.056000e-01 7.814000e-01 6.102000e-01 7.022000e-01 7.245000e-01 7.864000e-01 5.652000e-01 7.547000e-01 3.327000e-01 4.272000e-01 + 6.824000e-01 4.344000e-01 1.050700e+00 6.378000e-01 1.118400e+00 7.004000e-01 1.775100e+00 -5.320000e-02 1.875000e+00 1.442000e-01 3.940000e-02 1.686500e+00 6.244000e-01 3.308000e-01 + 3.648000e-01 5.643000e-01 7.458000e-01 4.867000e-01 8.342000e-01 5.187000e-01 1.737500e+00 -6.500000e-03 -9.810000e-02 2.117800e+00 -5.920000e-02 7.120000e-02 3.983000e-01 3.318000e-01 + 1.129600e+00 -3.020000e-02 6.534000e-01 1.097600e+00 6.021000e-01 4.781000e-01 7.221000e-01 1.015500e+00 6.298000e-01 4.144000e-01 7.729000e-01 9.528000e-01 5.045000e-01 2.708000e-01 + 4.052000e-01 3.131000e-01 6.783000e-01 4.436000e-01 7.699000e-01 5.071000e-01 6.499000e-01 4.775000e-01 8.500000e-01 4.314000e-01 7.182000e-01 3.959000e-01 4.053000e-01 2.391000e-01 + 1.131800e+00 -3.150000e-02 6.668000e-01 5.677000e-01 5.198000e-01 3.855000e-01 6.295000e-01 6.142000e-01 4.733000e-01 4.171000e-01 7.019000e-01 5.303000e-01 4.325000e-01 2.588000e-01 +-1.350000e-02 9.692000e-01 6.860000e-02 1.201600e+00 2.007000e-01 1.184700e+00 -1.011000e-01 1.850400e+00 2.550000e-02 1.973700e+00 6.100000e-02 -7.320000e-02 5.340000e-02 6.952000e-01 + 1.009400e+00 1.134000e-01 4.213000e-01 8.066000e-01 3.547000e-01 5.525000e-01 4.474000e-01 7.713000e-01 4.188000e-01 4.510000e-01 5.455000e-01 6.543000e-01 3.627000e-01 3.198000e-01 + 4.146000e-01 4.600000e-02 7.304000e-01 4.780000e-02 8.895000e-01 -9.000000e-04 1.766900e+00 -4.180000e-02 1.940000e-02 -2.560000e-02 1.600000e-03 -2.100000e-03 4.215000e-01 2.020000e-02 + 4.200000e-03 7.406000e-01 4.619000e-01 5.687000e-01 5.006000e-01 6.651000e-01 1.647100e+00 9.630000e-02 -8.380000e-02 2.093600e+00 1.680000e-02 -2.180000e-02 2.127000e-01 3.993000e-01 + 8.960000e-01 2.122000e-01 1.236100e+00 2.560000e-01 1.376200e+00 2.300000e-01 1.731100e+00 5.000000e-04 1.994500e+00 4.600000e-03 -2.330000e-02 1.759100e+00 7.241000e-01 1.468000e-01 + 1.125000e+00 -2.640000e-02 1.185100e+00 5.672000e-01 9.300000e-01 2.285000e-01 1.218800e+00 5.273000e-01 8.381000e-01 2.953000e-01 1.288500e+00 4.426000e-01 7.039000e-01 1.022000e-01 + 9.570000e-02 2.974000e-01 4.062000e-01 1.635000e-01 4.607000e-01 1.812000e-01 3.996000e-01 1.672000e-01 4.821000e-01 1.683000e-01 3.296000e-01 2.546000e-01 2.160000e-01 1.141000e-01 + 1.066400e+00 5.060000e-02 4.352000e-01 1.309700e+00 3.315000e-01 1.332100e+00 4.690000e-01 1.277800e+00 4.950000e-01 1.028800e+00 4.517000e-01 1.293800e+00 3.870000e-01 5.589000e-01 + 9.807000e-01 1.493000e-01 1.107100e+00 6.114000e-01 7.685000e-01 2.679000e-01 9.559000e-01 7.915000e-01 7.832000e-01 2.189000e-01 1.040300e+00 6.888000e-01 6.295000e-01 1.212000e-01 + 4.976000e-01 3.970000e-01 6.387000e-01 7.825000e-01 8.251000e-01 7.792000e-01 7.457000e-01 6.585000e-01 7.633000e-01 8.793000e-01 7.247000e-01 6.854000e-01 3.927000e-01 4.256000e-01 + 7.080000e-02 8.437000e-01 1.565000e-01 1.267600e+00 2.215000e-01 1.388900e+00 2.746000e-01 1.129200e+00 2.840000e-01 1.339000e+00 1.668000e-01 1.250800e+00 1.165000e-01 7.000000e-01 + 2.704000e-01 7.288000e-01 6.289000e-01 6.757000e-01 7.274000e-01 7.037000e-01 1.694200e+00 4.620000e-02 5.750000e-02 1.930000e+00 -1.150000e-02 1.200000e-02 3.791000e-01 3.871000e-01 + 3.198000e-01 7.852000e-01 6.013000e-01 1.135000e+00 7.234000e-01 1.198300e+00 1.628900e+00 1.248000e-01 -4.420000e-02 2.053300e+00 8.170000e-02 1.635800e+00 3.488000e-01 6.410000e-01 + 8.978000e-01 1.226000e-01 1.111100e+00 2.376000e-01 1.333900e+00 1.199000e-01 1.906700e+00 -2.029000e-01 1.955700e+00 4.500000e-02 1.680000e-02 -1.970000e-02 7.193000e-01 6.620000e-02 + 3.376000e-01 4.681000e-01 7.722000e-01 3.205000e-01 7.227000e-01 5.211000e-01 1.563600e+00 1.941000e-01 5.300000e-03 1.993600e+00 -4.300000e-02 5.260000e-02 3.808000e-01 2.757000e-01 +-1.780000e-02 9.781000e-01 2.480000e-01 1.004500e+00 2.617000e-01 1.130300e+00 -5.760000e-02 1.800700e+00 3.090000e-02 1.965300e+00 -6.770000e-02 7.990000e-02 9.110000e-02 6.573000e-01 + 3.065000e-01 4.802000e-01 5.440000e-01 6.745000e-01 5.246000e-01 8.795000e-01 4.652000e-01 7.694000e-01 5.639000e-01 8.570000e-01 4.735000e-01 7.587000e-01 2.864000e-01 4.227000e-01 + 7.427000e-01 7.390000e-02 1.189300e+00 8.300000e-02 1.414300e+00 2.050000e-02 1.250600e+00 6.000000e-03 1.441500e+00 1.240000e-02 1.193500e+00 7.420000e-02 7.161000e-01 1.110000e-02 + 4.666000e-01 6.226000e-01 8.800000e-01 7.976000e-01 9.467000e-01 9.688000e-01 6.725000e-01 1.039400e+00 8.872000e-01 1.067700e+00 7.361000e-01 9.614000e-01 4.290000e-01 5.515000e-01 + 3.063000e-01 6.264000e-01 7.307000e-01 4.934000e-01 8.177000e-01 5.290000e-01 1.775700e+00 -4.860000e-02 -1.500000e-02 2.013600e+00 -5.090000e-02 6.170000e-02 4.033000e-01 3.219000e-01 +-2.300000e-03 9.057000e-01 2.291000e-01 9.915000e-01 4.589000e-01 8.647000e-01 5.000000e-04 1.731800e+00 4.500000e-03 1.990400e+00 -5.000000e-03 5.300000e-03 1.490000e-01 5.679000e-01 + 7.854000e-01 3.098000e-01 2.999000e-01 6.876000e-01 2.856000e-01 4.767000e-01 3.644000e-01 6.130000e-01 2.626000e-01 4.918000e-01 3.000000e-01 6.889000e-01 2.698000e-01 3.384000e-01 + 4.141000e-01 2.904000e-01 6.394000e-01 3.938000e-01 8.197000e-01 3.237000e-01 1.624800e+00 1.227000e-01 -1.530000e-02 2.018500e+00 -3.490000e-02 4.160000e-02 4.051000e-01 1.915000e-01 + 6.392000e-01 4.688000e-01 9.039000e-01 8.416000e-01 1.061400e+00 8.376000e-01 1.598700e+00 1.593000e-01 1.972200e+00 3.310000e-02 -9.980000e-02 1.852400e+00 5.675000e-01 4.143000e-01 + 6.436000e-01 2.330000e-02 1.019000e+00 -4.770000e-02 9.992000e-01 1.174000e-01 1.605600e+00 1.474000e-01 2.067800e+00 -8.150000e-02 5.890000e-02 -6.990000e-02 5.617000e-01 1.030000e-02 + 1.056800e+00 5.610000e-02 9.277000e-01 3.840000e-01 7.862000e-01 1.421000e-01 9.047000e-01 4.088000e-01 7.154000e-01 1.977000e-01 9.725000e-01 3.286000e-01 5.982000e-01 1.060000e-01 + 2.863000e-01 2.397000e-01 3.017000e-01 5.512000e-01 4.118000e-01 5.414000e-01 4.823000e-01 3.335000e-01 2.984000e-01 6.918000e-01 4.641000e-01 3.558000e-01 2.224000e-01 2.593000e-01 + 4.731000e-01 8.560000e-02 7.429000e-01 1.325000e-01 9.045000e-01 8.170000e-02 8.110000e-01 4.620000e-02 8.391000e-01 1.784000e-01 7.406000e-01 1.352000e-01 4.483000e-01 5.340000e-02 + 1.335000e-01 9.647000e-01 4.820000e-02 1.051100e+00 1.408000e-01 6.785000e-01 3.630000e-02 1.068100e+00 7.970000e-02 7.308000e-01 1.641000e-01 9.179000e-01 4.960000e-02 6.001000e-01 + 4.580000e-02 9.044000e-01 3.932000e-01 8.614000e-01 5.322000e-01 8.431000e-01 1.675300e+00 6.230000e-02 3.500000e-02 1.957700e+00 -3.200000e-03 3.000000e-03 2.104000e-01 5.312000e-01 + 1.039000e+00 7.570000e-02 7.307000e-01 1.009800e+00 5.982000e-01 1.377000e+00 7.363000e-01 1.004300e+00 6.714000e-01 1.031500e+00 8.457000e-01 8.775000e-01 5.253000e-01 4.539000e-01 + 5.764000e-01 3.032000e-01 9.577000e-01 4.120000e-01 9.673000e-01 6.153000e-01 9.352000e-01 4.392000e-01 9.752000e-01 6.314000e-01 7.572000e-01 6.480000e-01 5.075000e-01 2.932000e-01 + 1.140700e+00 -4.550000e-02 9.417000e-01 7.776000e-01 7.075000e-01 5.760000e-01 8.199000e-01 9.228000e-01 6.980000e-01 5.391000e-01 9.480000e-01 7.681000e-01 5.668000e-01 2.880000e-01 +-2.900000e-02 6.733000e-01 3.202000e-01 6.315000e-01 4.043000e-01 6.685000e-01 -7.370000e-02 1.813700e+00 1.229000e-01 1.851200e+00 4.170000e-02 -5.200000e-02 1.569000e-01 3.977000e-01 + 3.265000e-01 4.894000e-01 6.227000e-01 5.130000e-01 7.186000e-01 5.417000e-01 1.765500e+00 -3.930000e-02 -4.840000e-02 2.053500e+00 9.800000e-03 -1.190000e-02 3.285000e-01 3.432000e-01 + 1.117100e+00 -8.600000e-03 7.302000e-01 3.616000e-01 6.130000e-01 2.139000e-01 6.646000e-01 4.410000e-01 6.113000e-01 1.987000e-01 7.062000e-01 3.899000e-01 4.921000e-01 1.567000e-01 + 1.141900e+00 -4.700000e-02 6.574000e-01 7.773000e-01 5.479000e-01 4.382000e-01 7.244000e-01 6.954000e-01 5.710000e-01 3.823000e-01 7.053000e-01 7.203000e-01 4.940000e-01 2.317000e-01 + 4.965000e-01 1.738000e-01 7.885000e-01 2.035000e-01 9.662000e-01 1.357000e-01 1.835300e+00 -1.154000e-01 -1.560000e-02 2.021700e+00 -1.450000e-02 1.650000e-02 4.759000e-01 9.810000e-02 + 4.544000e-01 3.766000e-01 8.270000e-01 4.520000e-01 7.125000e-01 7.868000e-01 6.932000e-01 6.157000e-01 7.669000e-01 7.423000e-01 5.673000e-01 7.614000e-01 3.870000e-01 3.691000e-01 + 4.055000e-01 3.654000e-01 7.823000e-01 3.933000e-01 7.759000e-01 5.908000e-01 7.125000e-01 4.798000e-01 9.026000e-01 4.538000e-01 7.434000e-01 4.432000e-01 4.013000e-01 2.881000e-01 + 5.728000e-01 2.989000e-01 8.306000e-01 3.644000e-01 9.304000e-01 3.902000e-01 1.696000e+00 4.100000e-02 -8.430000e-02 2.101000e+00 -7.130000e-02 8.580000e-02 5.000000e-01 2.026000e-01 + 3.376000e-01 7.775000e-01 3.464000e-01 1.362100e+00 2.232000e-01 9.473000e-01 2.115000e-01 1.521300e+00 2.551000e-01 8.705000e-01 2.435000e-01 1.486300e+00 1.729000e-01 6.418000e-01 + 1.090600e+00 1.860000e-02 6.090000e-01 1.122400e+00 4.901000e-01 5.531000e-01 5.431000e-01 1.201300e+00 5.042000e-01 5.040000e-01 5.107000e-01 1.237100e+00 4.253000e-01 3.304000e-01 + 1.114300e+00 -1.080000e-02 5.238000e-01 4.260000e-01 3.534000e-01 4.049000e-01 5.256000e-01 4.235000e-01 4.289000e-01 2.981000e-01 5.392000e-01 4.056000e-01 3.375000e-01 2.647000e-01 + 4.647000e-01 9.610000e-02 8.450000e-01 1.090000e-02 9.376000e-01 3.980000e-02 7.987000e-01 6.400000e-02 8.026000e-01 2.185000e-01 7.818000e-01 8.880000e-02 4.603000e-01 3.840000e-02 + 1.721000e-01 2.183000e-01 2.819000e-01 3.280000e-01 3.557000e-01 3.364000e-01 2.338000e-01 3.847000e-01 3.266000e-01 3.775000e-01 2.295000e-01 3.909000e-01 1.535000e-01 2.003000e-01 + 1.860000e-02 2.330000e-01 8.280000e-02 2.981000e-01 6.380000e-02 3.789000e-01 -1.660000e-02 4.184000e-01 1.610000e-01 2.707000e-01 8.580000e-02 2.988000e-01 3.820000e-02 1.848000e-01 + 4.782000e-01 4.641000e-01 8.946000e-01 3.435000e-01 8.573000e-01 5.357000e-01 1.875700e+00 -1.691000e-01 -6.900000e-03 2.014400e+00 2.090000e-02 -2.310000e-02 4.434000e-01 3.007000e-01 + 3.309000e-01 4.250000e-01 4.928000e-01 6.981000e-01 6.051000e-01 7.404000e-01 6.155000e-01 5.528000e-01 6.382000e-01 7.205000e-01 5.233000e-01 6.605000e-01 3.175000e-01 3.624000e-01 + 2.536000e-01 1.080000e-01 5.554000e-01 1.227000e-01 7.170000e-01 7.060000e-02 1.696900e+00 4.170000e-02 -1.440000e-02 1.760000e-02 -1.016000e-01 1.216000e-01 3.275000e-01 4.920000e-02 + 3.390000e-01 3.020000e-01 5.153000e-01 4.942000e-01 5.936000e-01 5.545000e-01 5.314000e-01 4.704000e-01 6.225000e-01 5.370000e-01 5.093000e-01 4.972000e-01 3.172000e-01 2.615000e-01 + 1.167900e+00 -7.670000e-02 1.151900e+00 5.657000e-01 8.234000e-01 1.057900e+00 1.061100e+00 6.778000e-01 6.292000e-01 1.053300e+00 1.050300e+00 6.942000e-01 6.490000e-01 3.211000e-01 + 4.128000e-01 6.886000e-01 7.720000e-01 6.333000e-01 8.191000e-01 7.276000e-01 1.696900e+00 4.170000e-02 -1.400000e-03 1.999100e+00 -2.770000e-02 1.763800e+00 4.395000e-01 3.880000e-01 + 2.401000e-01 8.456000e-01 2.801000e-01 1.440500e+00 3.156000e-01 1.645200e+00 4.983000e-01 1.182000e+00 4.511000e-01 1.513000e+00 4.349000e-01 1.260300e+00 2.223000e-01 7.598000e-01 + 3.190000e-02 2.327000e-01 1.329000e-01 4.503000e-01 1.820000e-01 5.382000e-01 1.320000e-01 1.579700e+00 6.880000e-02 -8.450000e-02 -1.080000e-02 1.390000e-02 8.020000e-02 2.422000e-01 + 4.039000e-01 3.573000e-01 6.585000e-01 5.334000e-01 6.388000e-01 7.320000e-01 6.293000e-01 5.603000e-01 6.803000e-01 7.047000e-01 5.667000e-01 6.358000e-01 3.484000e-01 3.435000e-01 + 1.027700e+00 8.760000e-02 5.360000e-01 1.174600e+00 3.294000e-01 7.892000e-01 3.724000e-01 1.366100e+00 2.953000e-01 7.905000e-01 4.678000e-01 1.253200e+00 3.132000e-01 4.753000e-01 + 6.950000e-02 4.076000e-01 1.708000e-01 5.636000e-01 1.306000e-01 7.205000e-01 -1.290000e-02 7.861000e-01 1.415000e-01 7.203000e-01 6.850000e-02 6.867000e-01 5.100000e-02 3.835000e-01 + 5.578000e-01 4.956000e-01 8.414000e-01 5.372000e-01 9.265000e-01 5.695000e-01 1.720000e+00 1.960000e-02 7.740000e-02 1.906200e+00 -9.720000e-02 1.850300e+00 4.662000e-01 3.445000e-01 + 2.043000e-01 1.528000e-01 3.941000e-01 1.483000e-01 4.314000e-01 1.899000e-01 4.244000e-01 1.136000e-01 5.053000e-01 1.130000e-01 4.428000e-01 8.950000e-02 2.396000e-01 7.130000e-02 + 3.115000e-01 6.206000e-01 5.196000e-01 9.349000e-01 4.160000e-01 1.275600e+00 4.773000e-01 9.806000e-01 5.755000e-01 1.103300e+00 3.650000e-01 1.116200e+00 2.662000e-01 5.800000e-01 + 1.036700e+00 8.220000e-02 8.868000e-01 8.528000e-01 7.260000e-01 4.464000e-01 1.059900e+00 6.539000e-01 6.681000e-01 4.769000e-01 9.639000e-01 7.606000e-01 5.826000e-01 2.310000e-01 + 1.069800e+00 4.080000e-02 4.507000e-01 2.553000e-01 3.520000e-01 2.347000e-01 4.403000e-01 2.715000e-01 3.898000e-01 1.776000e-01 4.880000e-01 2.127000e-01 3.393000e-01 1.490000e-01 + 9.398000e-01 9.490000e-02 1.514900e+00 9.620000e-02 1.657200e+00 1.844000e-01 1.592300e+00 4.300000e-03 1.785300e+00 7.150000e-02 1.490300e+00 1.233000e-01 8.673000e-01 6.500000e-02 + 2.916000e-01 8.154000e-01 2.496000e-01 7.557000e-01 7.570000e-02 7.302000e-01 2.293000e-01 7.797000e-01 1.223000e-01 6.540000e-01 1.639000e-01 8.590000e-01 1.455000e-01 4.776000e-01 + 1.027000e-01 1.016100e+00 1.556000e-01 1.566700e+00 1.183000e-01 9.786000e-01 1.365000e-01 1.589500e+00 1.133000e-01 9.559000e-01 1.248000e-01 1.605400e+00 8.940000e-02 6.949000e-01 + 1.103500e+00 3.500000e-03 5.619000e-01 1.930000e-01 4.270000e-01 1.961000e-01 5.009000e-01 2.624000e-01 4.615000e-01 1.391000e-01 5.809000e-01 1.712000e-01 3.970000e-01 1.167000e-01 + 1.000000e-03 9.440000e-01 3.068000e-01 9.555000e-01 4.642000e-01 9.027000e-01 -1.152000e-01 1.865900e+00 -8.080000e-02 2.098800e+00 -8.500000e-03 1.020000e-02 1.939000e-01 5.414000e-01 + 4.455000e-01 6.606000e-01 2.377000e-01 6.792000e-01 2.448000e-01 4.713000e-01 2.568000e-01 6.523000e-01 3.088000e-01 3.794000e-01 2.070000e-01 7.139000e-01 1.991000e-01 3.831000e-01 + 3.431000e-01 3.872000e-01 5.262000e-01 6.181000e-01 5.925000e-01 7.169000e-01 6.176000e-01 5.152000e-01 6.126000e-01 7.107000e-01 4.996000e-01 6.554000e-01 3.275000e-01 3.312000e-01 + 3.390000e-02 6.709000e-01 4.034000e-01 6.058000e-01 4.849000e-01 6.530000e-01 1.696000e+00 4.940000e-02 -7.460000e-02 2.088600e+00 -6.100000e-03 7.900000e-03 2.012000e-01 3.928000e-01 +-3.340000e-02 1.136700e+00 8.860000e-02 1.330500e+00 2.922000e-01 1.235300e+00 1.830000e-02 1.712800e+00 -8.160000e-02 2.100200e+00 3.500000e-03 1.726300e+00 7.930000e-02 7.498000e-01 + 1.158900e+00 -6.460000e-02 4.903000e-01 1.257400e+00 5.872000e-01 4.286000e-01 5.796000e-01 1.149900e+00 5.293000e-01 4.681000e-01 5.713000e-01 1.158500e+00 4.171000e-01 3.362000e-01 + 1.061400e+00 5.460000e-02 4.624000e-01 4.138000e-01 4.538000e-01 2.395000e-01 6.261000e-01 2.194000e-01 4.383000e-01 2.373000e-01 5.559000e-01 2.980000e-01 3.813000e-01 1.803000e-01 +-6.870000e-02 1.177000e+00 3.070000e-01 1.094200e+00 3.066000e-01 1.239500e+00 8.420000e-02 1.636600e+00 -2.100000e-03 1.999800e+00 -3.140000e-02 1.768700e+00 1.517000e-01 6.749000e-01 + 4.358000e-01 5.687000e-01 7.750000e-01 5.402000e-01 9.538000e-01 4.687000e-01 1.750700e+00 -2.560000e-02 1.563000e-01 1.817000e+00 6.030000e-02 -7.160000e-02 4.426000e-01 3.335000e-01 + 3.493000e-01 3.879000e-01 4.468000e-01 7.277000e-01 5.066000e-01 8.343000e-01 6.364000e-01 5.005000e-01 6.528000e-01 6.812000e-01 4.985000e-01 6.694000e-01 3.007000e-01 3.705000e-01 + 3.451000e-01 3.304000e-01 4.110000e-01 6.678000e-01 4.958000e-01 7.328000e-01 4.974000e-01 5.712000e-01 4.849000e-01 7.611000e-01 3.385000e-01 7.543000e-01 2.568000e-01 3.634000e-01 + 6.628000e-01 4.250000e-01 2.748000e-01 2.046000e-01 1.555000e-01 2.684000e-01 2.439000e-01 2.413000e-01 2.985000e-01 8.700000e-02 2.819000e-01 1.962000e-01 2.311000e-01 1.262000e-01 + 5.381000e-01 2.008000e-01 9.185000e-01 2.281000e-01 1.066400e+00 2.308000e-01 9.062000e-01 2.368000e-01 9.814000e-01 3.535000e-01 8.856000e-01 2.679000e-01 5.252000e-01 1.371000e-01 + 1.112900e+00 -9.100000e-03 8.964000e-01 8.369000e-01 6.871000e-01 7.348000e-01 8.873000e-01 8.529000e-01 7.308000e-01 6.121000e-01 8.880000e-01 8.495000e-01 5.774000e-01 3.174000e-01 + 9.408000e-01 1.572000e-01 1.180300e+00 3.368000e-01 1.296200e+00 3.425000e-01 1.684700e+00 5.910000e-02 2.001600e+00 2.300000e-03 4.320000e-02 1.685900e+00 7.190000e-01 1.595000e-01 + 1.581000e-01 3.713000e-01 3.274000e-01 4.837000e-01 2.731000e-01 6.740000e-01 4.187000e-01 3.731000e-01 3.721000e-01 5.650000e-01 3.401000e-01 4.673000e-01 1.892000e-01 2.795000e-01 + 3.393000e-01 1.388000e-01 2.235000e-01 3.140000e-02 7.320000e-02 1.648000e-01 2.143000e-01 4.110000e-02 1.466000e-01 7.950000e-02 1.840000e-01 7.610000e-02 1.639000e-01 4.260000e-02 + 2.449000e-01 1.485000e-01 1.748000e-01 3.990000e-02 7.900000e-02 1.229000e-01 1.639000e-01 5.250000e-02 1.908000e-01 -1.550000e-02 7.440000e-02 1.631000e-01 1.260000e-01 5.300000e-02 + 3.539000e-01 7.224000e-01 4.408000e-01 1.274400e+00 6.796000e-01 1.237900e+00 4.989000e-01 1.199600e+00 6.023000e-01 1.362300e+00 4.924000e-01 1.209500e+00 2.847000e-01 6.989000e-01 + 4.341000e-01 4.468000e-01 6.116000e-01 7.835000e-01 7.314000e-01 8.503000e-01 6.383000e-01 7.511000e-01 8.338000e-01 7.523000e-01 6.846000e-01 6.988000e-01 3.918000e-01 4.062000e-01 + 5.380000e-01 4.697000e-01 7.407000e-01 8.628000e-01 7.885000e-01 1.048800e+00 7.124000e-01 8.969000e-01 9.390000e-01 9.041000e-01 7.319000e-01 8.754000e-01 4.184000e-01 5.099000e-01 + 2.920000e-01 8.900000e-02 4.256000e-01 1.747000e-01 4.704000e-01 2.201000e-01 4.925000e-01 9.680000e-02 4.691000e-01 2.292000e-01 4.426000e-01 1.539000e-01 2.753000e-01 6.600000e-02 + 4.122000e-01 8.940000e-02 5.834000e-01 2.571000e-01 6.954000e-01 2.638000e-01 1.677400e+00 6.480000e-02 -2.700000e-03 2.000000e-03 1.870000e-02 -2.020000e-02 3.243000e-01 1.595000e-01 +-5.000000e-04 1.120800e+00 6.350000e-02 1.665100e+00 -5.980000e-02 1.582700e+00 -7.100000e-03 1.746900e+00 8.840000e-02 1.332500e+00 1.032000e-01 1.617900e+00 3.190000e-02 8.864000e-01 + 2.250000e-02 4.186000e-01 2.250000e-02 2.158000e-01 4.480000e-02 1.556000e-01 6.030000e-02 1.744000e-01 6.030000e-02 1.389000e-01 -2.940000e-02 2.825000e-01 1.240000e-02 1.834000e-01 + 1.856000e-01 3.467000e-01 6.820000e-02 2.194000e-01 4.480000e-02 2.059000e-01 9.490000e-02 1.882000e-01 3.840000e-02 2.095000e-01 5.270000e-02 2.388000e-01 7.130000e-02 1.545000e-01 + 5.003000e-01 2.536000e-01 1.001900e+00 1.310000e-01 1.054300e+00 2.531000e-01 8.935000e-01 2.608000e-01 1.017600e+00 3.172000e-01 9.822000e-01 1.566000e-01 5.335000e-01 1.295000e-01 + 4.205000e-01 6.878000e-01 8.430000e-02 1.679500e+00 1.585000e-01 9.973000e-01 2.664000e-01 1.464800e+00 1.663000e-01 9.523000e-01 3.439000e-01 1.375300e+00 1.936000e-01 6.086000e-01 + 7.230000e-01 2.897000e-01 1.061700e+00 5.399000e-01 1.291800e+00 5.118000e-01 1.153700e+00 4.307000e-01 1.208600e+00 6.427000e-01 9.571000e-01 6.620000e-01 6.198000e-01 3.023000e-01 + 2.124000e-01 6.169000e-01 4.337000e-01 8.492000e-01 4.279000e-01 1.041800e+00 4.311000e-01 8.475000e-01 4.554000e-01 1.033700e+00 3.460000e-01 9.528000e-01 2.282000e-01 5.165000e-01 + 1.087000e+00 2.050000e-02 3.814000e-01 9.184000e-01 3.311000e-01 6.046000e-01 3.696000e-01 9.346000e-01 3.396000e-01 5.683000e-01 4.533000e-01 8.328000e-01 3.132000e-01 3.887000e-01 + 1.122500e+00 -2.000000e-02 7.035000e-01 1.028900e+00 5.001000e-01 6.363000e-01 7.524000e-01 9.642000e-01 5.688000e-01 5.181000e-01 7.309000e-01 9.958000e-01 4.889000e-01 3.028000e-01 + 1.122700e+00 -2.120000e-02 7.345000e-01 1.126000e-01 5.620000e-01 1.254000e-01 7.846000e-01 5.580000e-02 5.239000e-01 1.548000e-01 7.256000e-01 1.218000e-01 4.972000e-01 5.870000e-02 + 2.307000e-01 7.521000e-01 4.215000e-01 8.968000e-01 4.895000e-01 9.583000e-01 1.706200e+00 3.010000e-02 2.510000e-02 1.976100e+00 5.130000e-02 -5.990000e-02 2.203000e-01 5.587000e-01 + 3.168000e-01 1.169000e-01 7.908000e-01 -7.260000e-02 8.958000e-01 -5.690000e-02 1.596400e+00 1.594000e-01 -4.350000e-02 5.030000e-02 -4.990000e-02 5.970000e-02 3.999000e-01 1.530000e-02 + 1.098600e+00 1.060000e-02 1.170200e+00 5.680000e-01 8.732000e-01 2.272000e-01 1.142800e+00 5.946000e-01 8.219000e-01 2.528000e-01 1.195700e+00 5.319000e-01 6.724000e-01 1.097000e-01 + 1.823000e-01 6.265000e-01 1.940000e-01 1.089600e+00 2.839000e-01 1.166500e+00 2.785000e-01 9.865000e-01 3.379000e-01 1.124600e+00 2.203000e-01 1.059100e+00 1.262000e-01 6.122000e-01 + 4.150000e-02 7.618000e-01 2.864000e-01 8.324000e-01 4.110000e-01 8.252000e-01 4.500000e-03 1.725100e+00 -2.590000e-02 2.027700e+00 -2.960000e-02 3.420000e-02 1.374000e-01 5.252000e-01 + 5.909000e-01 4.002000e-01 8.241000e-01 7.434000e-01 1.137400e+00 6.116000e-01 9.519000e-01 5.897000e-01 1.061400e+00 7.307000e-01 8.974000e-01 6.602000e-01 5.347000e-01 3.598000e-01 + 7.907000e-01 2.081000e-01 1.288000e+00 2.727000e-01 1.441700e+00 3.342000e-01 1.260200e+00 3.046000e-01 1.501900e+00 2.953000e-01 1.235200e+00 3.337000e-01 7.069000e-01 1.993000e-01 + 5.221000e-01 5.633000e-01 9.389000e-01 7.340000e-01 9.714000e-01 9.561000e-01 9.044000e-01 7.730000e-01 1.116400e+00 8.080000e-01 9.204000e-01 7.541000e-01 5.285000e-01 4.408000e-01 + 1.143700e+00 -4.720000e-02 5.848000e-01 9.845000e-01 6.145000e-01 3.859000e-01 6.835000e-01 8.577000e-01 5.857000e-01 3.928000e-01 6.892000e-01 8.599000e-01 4.808000e-01 2.604000e-01 + 1.652000e-01 5.220000e-01 7.564000e-01 1.959000e-01 7.352000e-01 3.619000e-01 1.798300e+00 -8.000000e-02 -6.110000e-02 2.072900e+00 -3.170000e-02 3.590000e-02 3.185000e-01 2.556000e-01 + 2.429000e-01 8.192000e-01 5.522000e-01 8.254000e-01 6.584000e-01 8.397000e-01 1.758500e+00 -2.970000e-02 1.117000e-01 1.871200e+00 2.010000e-02 1.704900e+00 3.055000e-01 5.041000e-01 + 1.027500e+00 9.290000e-02 7.770000e-01 9.647000e-01 7.027000e-01 6.231000e-01 8.090000e-01 9.281000e-01 7.108000e-01 5.548000e-01 9.417000e-01 7.737000e-01 5.562000e-01 3.125000e-01 + 2.098000e-01 4.697000e-01 2.148000e-01 8.691000e-01 2.709000e-01 9.603000e-01 2.065000e-01 8.809000e-01 1.751000e-01 1.090800e+00 1.634000e-01 9.365000e-01 1.426000e-01 4.818000e-01 + 8.182000e-01 2.860000e-01 1.228700e+00 2.121000e-01 1.275100e+00 2.980000e-01 1.731500e+00 2.500000e-03 2.029300e+00 -3.240000e-02 2.870000e-02 1.699900e+00 6.670000e-01 1.885000e-01 + 1.416000e-01 9.664000e-01 7.030000e-02 8.976000e-01 1.443000e-01 5.991000e-01 4.980000e-02 9.250000e-01 1.007000e-01 6.328000e-01 6.130000e-02 9.041000e-01 7.600000e-02 5.271000e-01 + 6.420000e-01 1.712000e-01 8.587000e-01 4.432000e-01 9.461000e-01 5.424000e-01 1.049100e+00 2.220000e-01 1.132600e+00 3.475000e-01 9.113000e-01 3.815000e-01 5.211000e-01 2.263000e-01 diff --git a/sourcecodes/bnt-master/netlab3.3/oilTst.dat b/sourcecodes/bnt-master/netlab3.3/oilTst.dat new file mode 100644 index 00000000..0fc51cc9 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/oilTst.dat @@ -0,0 +1,503 @@ + nin 12 + nout 2 + ndata 500 + 6.448000e-01 4.650000e-01 3.659000e-01 8.761000e-01 2.214000e-01 7.023000e-01 3.702000e-01 8.750000e-01 2.981000e-01 5.870000e-01 3.130000e-01 9.374000e-01 2.574000e-01 4.349000e-01 + 5.013000e-01 9.710000e-02 7.749000e-01 1.599000e-01 8.716000e-01 1.946000e-01 7.986000e-01 1.304000e-01 8.874000e-01 1.974000e-01 6.991000e-01 2.499000e-01 4.547000e-01 8.450000e-02 + 3.908000e-01 4.842000e-01 5.148000e-01 8.738000e-01 6.925000e-01 8.681000e-01 5.985000e-01 7.700000e-01 5.325000e-01 1.079000e+00 6.069000e-01 7.554000e-01 3.284000e-01 4.656000e-01 + 1.038400e+00 8.240000e-02 1.578400e+00 1.819000e-01 1.242100e+00 5.329000e-01 1.780600e+00 -5.680000e-02 1.239700e+00 3.690000e-01 1.656900e+00 8.240000e-02 8.512000e-01 1.109000e-01 +-9.100000e-03 6.477000e-01 3.510000e-02 9.081000e-01 3.810000e-02 1.031800e+00 6.780000e-02 1.656900e+00 3.950000e-02 1.950300e+00 -2.490000e-02 3.090000e-02 7.000000e-03 5.461000e-01 + 6.458000e-01 2.965000e-01 9.246000e-01 3.379000e-01 1.054200e+00 3.263000e-01 1.751900e+00 -2.230000e-02 2.070800e+00 -7.860000e-02 -1.010000e-02 1.030000e-02 5.419000e-01 2.027000e-01 + 4.951000e-01 5.899000e-01 9.069000e-01 7.700000e-01 8.367000e-01 1.101400e+00 7.952000e-01 9.014000e-01 9.185000e-01 1.039600e+00 7.024000e-01 1.008900e+00 4.378000e-01 5.457000e-01 + 1.136300e+00 -3.530000e-02 3.838000e-01 2.079000e-01 3.385000e-01 1.536000e-01 4.569000e-01 1.199000e-01 2.972000e-01 1.929000e-01 3.505000e-01 2.441000e-01 3.074000e-01 1.153000e-01 + 1.646000e-01 5.148000e-01 4.558000e-01 5.405000e-01 5.855000e-01 5.258000e-01 1.727000e+00 4.400000e-03 -1.250000e-02 2.019000e+00 3.960000e-02 -4.630000e-02 2.334000e-01 3.525000e-01 + 2.028000e-01 3.866000e-01 2.659000e-01 6.712000e-01 3.368000e-01 7.250000e-01 3.495000e-01 5.704000e-01 3.358000e-01 7.372000e-01 2.509000e-01 6.834000e-01 1.608000e-01 3.771000e-01 + 6.930000e-01 1.407000e-01 1.015300e+00 3.020000e-01 1.114100e+00 3.990000e-01 1.070600e+00 2.370000e-01 1.106800e+00 4.251000e-01 1.041400e+00 2.751000e-01 6.056000e-01 1.516000e-01 + 5.753000e-01 5.257000e-01 8.693000e-01 5.906000e-01 8.992000e-01 6.983000e-01 1.670000e+00 7.510000e-02 1.942200e+00 6.400000e-02 -2.000000e-02 1.753100e+00 5.033000e-01 3.513000e-01 + 8.400000e-03 5.567000e-01 2.480000e-02 8.560000e-01 1.377000e-01 8.414000e-01 8.530000e-02 7.866000e-01 -6.210000e-02 1.098600e+00 9.400000e-02 7.736000e-01 1.660000e-02 4.914000e-01 + 1.122000e-01 2.212000e-01 2.767000e-01 2.258000e-01 4.065000e-01 1.461000e-01 2.236000e-01 2.891000e-01 2.271000e-01 3.709000e-01 2.522000e-01 2.523000e-01 1.470000e-01 1.456000e-01 + 2.765000e-01 8.222000e-01 6.055000e-01 9.075000e-01 7.480000e-01 8.832000e-01 1.749700e+00 -2.060000e-02 4.540000e-02 1.950100e+00 4.800000e-02 1.672300e+00 3.080000e-01 5.781000e-01 + 3.000000e-01 2.843000e-01 5.036000e-01 4.081000e-01 6.104000e-01 4.229000e-01 4.810000e-01 4.380000e-01 5.600000e-01 5.012000e-01 4.791000e-01 4.364000e-01 2.898000e-01 2.373000e-01 + 1.142700e+00 -4.400000e-02 8.870000e-01 1.008000e-01 7.355000e-01 2.940000e-02 9.660000e-01 8.200000e-03 6.765000e-01 7.800000e-02 1.012500e+00 -4.360000e-02 5.768000e-01 3.400000e-02 + 2.788000e-01 1.947000e-01 4.153000e-01 3.330000e-01 4.493000e-01 4.033000e-01 3.633000e-01 3.906000e-01 4.544000e-01 4.106000e-01 3.182000e-01 4.397000e-01 2.318000e-01 1.993000e-01 + 4.448000e-01 6.422000e-01 8.192000e-01 5.634000e-01 8.853000e-01 6.292000e-01 1.754600e+00 -2.460000e-02 2.870000e-02 1.963900e+00 -1.710000e-02 1.754600e+00 4.858000e-01 3.275000e-01 + 1.169600e+00 -7.950000e-02 6.595000e-01 3.301000e-01 5.647000e-01 2.009000e-01 7.108000e-01 2.644000e-01 4.990000e-01 2.600000e-01 6.660000e-01 3.246000e-01 4.636000e-01 1.476000e-01 + 2.800000e-02 4.749000e-01 3.082000e-01 5.165000e-01 4.995000e-01 4.292000e-01 -6.680000e-02 1.809500e+00 -5.310000e-02 6.340000e-02 7.020000e-02 -8.250000e-02 1.794000e-01 2.888000e-01 +-2.050000e-02 1.080400e+00 1.990000e-01 1.171300e+00 2.044000e-01 1.301100e+00 -6.300000e-02 1.807000e+00 -1.090000e-01 2.131100e+00 9.110000e-02 1.625400e+00 1.084000e-01 6.975000e-01 +-1.990000e-02 1.129400e+00 6.400000e-03 1.475300e+00 1.195000e-01 1.479400e+00 -8.570000e-02 1.833300e+00 -5.250000e-02 2.061500e+00 8.600000e-02 1.630200e+00 1.760000e-02 8.479000e-01 + 7.734000e-01 3.108000e-01 1.078300e+00 6.418000e-01 1.267100e+00 6.887000e-01 1.212800e+00 4.844000e-01 1.260900e+00 7.241000e-01 1.197800e+00 5.027000e-01 6.789000e-01 3.063000e-01 + 4.740000e-01 6.379000e-01 2.710000e-01 7.720000e-01 2.148000e-01 5.921000e-01 2.197000e-01 8.294000e-01 2.198000e-01 5.664000e-01 2.387000e-01 8.073000e-01 2.159000e-01 4.153000e-01 + 1.412000e-01 9.814000e-01 1.210000e-01 5.891000e-01 5.280000e-02 5.381000e-01 1.400000e-01 5.603000e-01 1.442000e-01 4.171000e-01 1.087000e-01 6.031000e-01 1.059000e-01 3.814000e-01 + 8.093000e-01 1.447000e-01 1.019800e+00 2.690000e-01 1.202300e+00 2.014000e-01 1.642000e+00 1.033000e-01 2.117700e+00 -1.391000e-01 -8.490000e-02 1.015000e-01 6.179000e-01 1.411000e-01 + 6.051000e-01 3.129000e-01 1.013600e+00 4.145000e-01 1.062800e+00 5.765000e-01 9.327000e-01 5.089000e-01 1.094600e+00 5.687000e-01 9.348000e-01 5.066000e-01 5.441000e-01 2.873000e-01 + 1.044400e+00 7.250000e-02 7.841000e-01 6.701000e-01 5.830000e-01 4.092000e-01 7.340000e-01 7.407000e-01 4.950000e-01 4.886000e-01 7.533000e-01 7.148000e-01 4.912000e-01 2.421000e-01 + 6.847000e-01 3.897000e-01 9.817000e-01 4.101000e-01 1.045500e+00 4.734000e-01 1.734700e+00 -3.000000e-04 1.986600e+00 1.320000e-02 5.550000e-02 1.665800e+00 5.296000e-01 2.952000e-01 + 1.740000e-02 8.344000e-01 -7.070000e-02 1.251500e+00 4.490000e-02 1.250200e+00 -8.910000e-02 1.834800e+00 3.770000e-02 1.952000e+00 1.850000e-02 -2.100000e-02 2.190000e-02 6.670000e-01 + 2.622000e-01 8.393000e-01 1.888000e-01 7.998000e-01 2.062000e-01 5.559000e-01 2.330000e-01 7.465000e-01 1.065000e-01 6.558000e-01 1.107000e-01 8.962000e-01 1.222000e-01 4.926000e-01 + 6.600000e-03 1.097000e+00 4.751000e-01 9.095000e-01 5.517000e-01 9.640000e-01 1.777900e+00 -5.290000e-02 -2.350000e-02 2.028300e+00 5.520000e-02 1.666900e+00 2.132000e-01 6.103000e-01 + 4.735000e-01 2.950000e-02 6.149000e-01 2.003000e-01 7.815000e-01 1.288000e-01 6.322000e-01 1.794000e-01 6.874000e-01 2.575000e-01 6.046000e-01 2.099000e-01 3.779000e-01 8.710000e-02 + 6.480000e-02 5.514000e-01 3.730000e-02 9.392000e-01 1.505000e-01 9.401000e-01 2.350000e-02 9.564000e-01 1.424000e-01 9.696000e-01 3.140000e-02 9.441000e-01 3.910000e-02 5.217000e-01 + 1.271900e+00 -1.940000e-01 4.195000e-01 1.021300e+00 3.379000e-01 6.556000e-01 4.200000e-01 1.027200e+00 3.894000e-01 5.664000e-01 4.430000e-01 1.001100e+00 3.273000e-01 4.017000e-01 + 3.084000e-01 7.897000e-01 1.522000e-01 1.576900e+00 7.180000e-02 9.782000e-01 2.220000e-01 1.498500e+00 4.430000e-02 9.810000e-01 9.340000e-02 1.646600e+00 1.257000e-01 6.302000e-01 + 5.818000e-01 2.661000e-01 8.678000e-01 4.688000e-01 9.546000e-01 5.728000e-01 7.304000e-01 6.334000e-01 8.540000e-01 7.134000e-01 8.607000e-01 4.792000e-01 5.011000e-01 2.696000e-01 + 5.940000e-02 7.090000e-01 4.563000e-01 6.108000e-01 5.679000e-01 6.220000e-01 1.778700e+00 -5.690000e-02 3.970000e-02 1.952300e+00 3.590000e-02 -4.270000e-02 2.095000e-01 4.262000e-01 + 5.933000e-01 2.304000e-01 9.529000e-01 3.319000e-01 1.151400e+00 2.991000e-01 9.822000e-01 3.042000e-01 1.184000e+00 2.901000e-01 9.725000e-01 3.120000e-01 5.705000e-01 1.696000e-01 + 8.719000e-01 2.312000e-01 4.310000e-01 5.507000e-01 3.146000e-01 4.567000e-01 4.258000e-01 5.563000e-01 3.740000e-01 3.700000e-01 3.330000e-01 6.646000e-01 2.901000e-01 3.226000e-01 + 1.360000e-02 9.961000e-01 2.466000e-01 1.077500e+00 3.379000e-01 1.112000e+00 5.620000e-02 1.662300e+00 -4.080000e-02 2.046100e+00 -1.250000e-02 1.610000e-02 1.257000e-01 6.558000e-01 + 1.218000e+00 -1.338000e-01 6.016000e-01 1.136800e+00 4.434000e-01 7.737000e-01 6.493000e-01 1.074600e+00 5.333000e-01 6.233000e-01 5.916000e-01 1.146600e+00 4.410000e-01 3.835000e-01 + 7.636000e-01 1.110000e-01 1.172100e+00 4.000000e-04 1.307500e+00 -1.810000e-02 1.756100e+00 -3.150000e-02 1.876900e+00 1.464000e-01 4.910000e-02 -5.820000e-02 6.767000e-01 1.780000e-02 + 4.733000e-01 5.533000e-01 7.878000e-01 5.479000e-01 9.136000e-01 5.419000e-01 1.809900e+00 -8.920000e-02 -7.870000e-02 2.088800e+00 2.570000e-02 -3.140000e-02 4.780000e-01 3.072000e-01 + 2.005000e-01 5.829000e-01 6.213000e-01 4.544000e-01 6.185000e-01 6.031000e-01 1.674800e+00 6.370000e-02 4.690000e-02 1.944500e+00 4.540000e-02 -5.330000e-02 2.894000e-01 3.546000e-01 + 2.873000e-01 3.904000e-01 5.542000e-01 4.862000e-01 6.085000e-01 5.771000e-01 6.228000e-01 4.063000e-01 6.241000e-01 5.769000e-01 5.590000e-01 4.789000e-01 3.226000e-01 2.772000e-01 + 1.065800e+00 4.890000e-02 9.754000e-01 2.339000e-01 7.418000e-01 1.505000e-01 8.248000e-01 4.087000e-01 6.896000e-01 1.874000e-01 8.415000e-01 3.902000e-01 5.896000e-01 9.380000e-02 + 2.749000e-01 8.322000e-01 7.550000e-02 8.470000e-01 1.179000e-01 6.081000e-01 2.079000e-01 6.938000e-01 1.133000e-01 5.986000e-01 1.856000e-01 7.187000e-01 1.310000e-01 4.502000e-01 + 1.160400e+00 -7.070000e-02 3.873000e-01 4.790000e-01 3.702000e-01 3.135000e-01 3.731000e-01 4.934000e-01 4.204000e-01 2.415000e-01 3.768000e-01 4.884000e-01 3.431000e-01 2.132000e-01 + 7.850000e-01 3.097000e-01 1.039200e+00 6.899000e-01 1.007400e+00 9.179000e-01 1.606700e+00 1.494000e-01 2.023600e+00 -2.750000e-02 7.740000e-02 1.643000e+00 6.075000e-01 3.769000e-01 + 4.767000e-01 6.200000e-01 8.124000e-01 6.201000e-01 8.201000e-01 7.476000e-01 1.762100e+00 -3.200000e-02 2.850000e-02 1.962500e+00 -2.400000e-03 1.735200e+00 4.170000e-01 4.287000e-01 + 7.000000e-02 5.804000e-01 3.603000e-01 6.051000e-01 5.032000e-01 5.804000e-01 1.808000e+00 -9.050000e-02 2.450000e-02 1.971800e+00 5.380000e-02 -6.290000e-02 2.108000e-01 3.495000e-01 + 2.398000e-01 5.381000e-01 4.280000e-01 7.827000e-01 4.594000e-01 9.249000e-01 4.543000e-01 7.498000e-01 4.837000e-01 9.178000e-01 3.558000e-01 8.688000e-01 2.354000e-01 4.663000e-01 + 6.772000e-01 4.153000e-01 8.702000e-01 6.673000e-01 1.097600e+00 5.371000e-01 1.671400e+00 7.230000e-02 1.913400e+00 1.031000e-01 4.360000e-02 1.682100e+00 5.430000e-01 3.427000e-01 + 5.771000e-01 2.690000e-01 9.626000e-01 1.865000e-01 9.748000e-01 3.090000e-01 1.720800e+00 1.200000e-02 1.904000e+00 1.157000e-01 4.180000e-02 -5.090000e-02 5.076000e-01 1.776000e-01 + 9.678000e-01 1.205000e-01 4.169000e-01 7.365000e-01 2.957000e-01 5.802000e-01 4.317000e-01 7.145000e-01 3.142000e-01 5.355000e-01 3.243000e-01 8.496000e-01 2.935000e-01 3.748000e-01 + 3.039000e-01 8.250000e-02 5.032000e-01 1.001000e-01 6.313000e-01 4.200000e-02 5.443000e-01 4.810000e-02 6.187000e-01 7.040000e-02 5.802000e-01 5.800000e-03 3.000000e-01 4.560000e-02 + 1.066000e+00 4.740000e-02 8.390000e-01 3.301000e-01 6.501000e-01 2.295000e-01 8.427000e-01 3.239000e-01 6.770000e-01 1.715000e-01 7.632000e-01 4.213000e-01 5.597000e-01 1.096000e-01 + 4.445000e-01 4.328000e-01 7.354000e-01 6.306000e-01 7.351000e-01 8.423000e-01 7.142000e-01 6.597000e-01 8.077000e-01 7.814000e-01 6.867000e-01 6.933000e-01 3.992000e-01 3.965000e-01 + 1.113800e+00 -1.200000e-02 4.509000e-01 1.285400e+00 4.255000e-01 1.115600e+00 4.581000e-01 1.271400e+00 3.918000e-01 1.065600e+00 4.900000e-01 1.234000e+00 3.392000e-01 5.883000e-01 + 3.929000e-01 1.168000e-01 6.146000e-01 2.238000e-01 8.554000e-01 8.000000e-02 1.677700e+00 6.240000e-02 -6.710000e-02 7.830000e-02 3.840000e-02 -4.460000e-02 3.705000e-01 1.066000e-01 + 1.174900e+00 -7.820000e-02 5.318000e-01 1.208200e+00 4.090000e-01 8.032000e-01 6.121000e-01 1.106000e+00 4.824000e-01 6.767000e-01 5.083000e-01 1.234000e+00 3.942000e-01 4.315000e-01 + 1.112500e+00 -1.900000e-03 7.953000e-01 4.182000e-01 5.772000e-01 3.317000e-01 8.666000e-01 3.376000e-01 6.822000e-01 1.837000e-01 7.594000e-01 4.607000e-01 5.409000e-01 1.411000e-01 + 1.064300e+00 4.830000e-02 4.031000e-01 4.715000e-01 4.217000e-01 2.610000e-01 5.285000e-01 3.166000e-01 4.123000e-01 2.549000e-01 4.966000e-01 3.544000e-01 3.839000e-01 1.711000e-01 + 1.462000e-01 1.741000e-01 3.355000e-01 3.210000e-01 5.901000e-01 1.645000e-01 1.819900e+00 -1.075000e-01 4.480000e-02 -5.060000e-02 -1.540000e-02 1.900000e-02 1.997000e-01 1.640000e-01 + 1.082600e+00 2.900000e-02 5.759000e-01 1.143600e+00 3.587000e-01 1.001100e+00 3.984000e-01 1.351900e+00 3.697000e-01 9.311000e-01 4.609000e-01 1.280500e+00 3.813000e-01 4.929000e-01 + 1.749000e-01 9.337000e-01 4.971000e-01 9.799000e-01 5.158000e-01 1.106500e+00 1.858000e+00 -1.477000e-01 5.880000e-02 1.932100e+00 -9.480000e-02 1.840300e+00 2.799000e-01 5.831000e-01 + 2.320000e-02 4.472000e-01 2.880000e-02 7.578000e-01 1.640000e-02 8.930000e-01 -2.840000e-02 1.768200e+00 -4.300000e-03 5.000000e-03 1.630000e-02 -1.920000e-02 1.400000e-03 4.502000e-01 +-1.890000e-02 9.049000e-01 2.733000e-01 9.240000e-01 4.054000e-01 9.117000e-01 -1.630000e-02 1.749400e+00 8.600000e-03 1.989800e+00 1.650000e-02 -1.970000e-02 1.497000e-01 5.558000e-01 + 3.017000e-01 1.810000e-02 5.904000e-01 5.250000e-02 8.312000e-01 -9.490000e-02 1.793900e+00 -7.140000e-02 2.200000e-03 -2.400000e-03 1.700000e-03 -1.600000e-03 3.347000e-01 2.090000e-02 + 3.039000e-01 3.121000e-01 6.046000e-01 3.240000e-01 8.146000e-01 2.188000e-01 1.698900e+00 4.070000e-02 -9.370000e-02 2.104700e+00 -1.610000e-02 2.060000e-02 3.003000e-01 2.485000e-01 + 1.249000e-01 9.827000e-01 6.260000e-02 1.675200e+00 8.900000e-02 9.563000e-01 1.085000e-01 1.615700e+00 5.530000e-02 9.614000e-01 3.070000e-02 1.713800e+00 7.130000e-02 6.857000e-01 + 1.155700e+00 -5.860000e-02 3.913000e-01 1.361500e+00 4.194000e-01 8.176000e-01 4.429000e-01 1.295800e+00 4.218000e-01 7.674000e-01 4.723000e-01 1.270600e+00 3.738000e-01 4.626000e-01 + 2.459000e-01 7.784000e-01 5.481000e-01 7.902000e-01 6.902000e-01 7.715000e-01 1.765200e+00 -3.940000e-02 -1.560000e-02 2.020000e+00 4.010000e-02 -4.900000e-02 3.023000e-01 4.882000e-01 + 6.220000e-01 4.951000e-01 3.017000e-01 7.115000e-01 2.143000e-01 5.816000e-01 2.724000e-01 7.436000e-01 2.240000e-01 5.504000e-01 3.288000e-01 6.809000e-01 2.659000e-01 3.523000e-01 + 3.249000e-01 2.344000e-01 5.291000e-01 3.440000e-01 5.390000e-01 4.644000e-01 6.531000e-01 1.941000e-01 5.935000e-01 4.141000e-01 5.112000e-01 3.632000e-01 3.074000e-01 1.955000e-01 + 3.186000e-01 6.189000e-01 7.353000e-01 5.008000e-01 7.807000e-01 5.874000e-01 1.759200e+00 -3.670000e-02 1.700000e-02 1.983200e+00 -3.570000e-02 4.160000e-02 3.922000e-01 3.411000e-01 + 8.110000e-02 5.730000e-01 -5.400000e-03 1.059500e+00 4.620000e-02 1.141800e+00 9.510000e-02 9.377000e-01 4.330000e-02 1.157600e+00 4.870000e-02 9.916000e-01 3.510000e-02 5.644000e-01 + 2.594000e-01 3.892000e-01 5.220000e-01 4.511000e-01 6.600000e-01 4.274000e-01 1.716700e+00 1.820000e-02 -2.340000e-02 2.023900e+00 2.910000e-02 -3.230000e-02 3.178000e-01 2.422000e-01 + 5.516000e-01 4.323000e-01 9.537000e-01 3.289000e-01 1.019900e+00 3.933000e-01 1.765300e+00 -3.410000e-02 2.026400e+00 -2.980000e-02 1.780000e-02 -2.150000e-02 5.026000e-01 2.628000e-01 + 1.111500e+00 -6.900000e-03 5.701000e-01 1.157500e+00 4.381000e-01 1.535200e+00 4.560000e-01 1.290200e+00 4.360000e-01 1.382700e+00 5.761000e-01 1.156000e+00 4.032000e-01 5.885000e-01 + 4.961000e-01 1.890000e-01 7.077000e-01 3.780000e-01 8.067000e-01 4.309000e-01 7.211000e-01 3.645000e-01 8.028000e-01 4.556000e-01 7.352000e-01 3.442000e-01 3.970000e-01 2.335000e-01 + 1.133100e+00 -3.640000e-02 4.325000e-01 4.307000e-01 3.780000e-01 3.109000e-01 4.364000e-01 4.285000e-01 3.326000e-01 3.505000e-01 4.787000e-01 3.815000e-01 3.200000e-01 2.438000e-01 + 4.649000e-01 6.439000e-01 7.775000e-01 6.826000e-01 9.982000e-01 5.638000e-01 1.757300e+00 -2.250000e-02 -1.580000e-02 2.011300e+00 4.970000e-02 1.671300e+00 4.476000e-01 4.080000e-01 + 5.782000e-01 6.300000e-03 9.617000e-01 -6.070000e-02 1.034800e+00 4.300000e-03 9.053000e-01 7.600000e-03 1.051300e+00 2.400000e-03 1.007700e+00 -1.158000e-01 5.201000e-01 7.200000e-03 + 7.918000e-01 3.077000e-01 5.134000e-01 9.919000e-01 2.700000e-01 7.473000e-01 4.521000e-01 1.058100e+00 2.535000e-01 7.352000e-01 4.208000e-01 1.096100e+00 2.795000e-01 4.604000e-01 + 5.928000e-01 5.222000e-01 1.003600e+00 4.579000e-01 1.078100e+00 5.070000e-01 1.647000e+00 9.860000e-02 2.106100e+00 -1.167000e-01 3.710000e-02 1.683300e+00 5.517000e-01 3.079000e-01 + 5.960000e-02 7.629000e-01 3.239000e-01 8.181000e-01 4.259000e-01 8.440000e-01 1.670200e+00 7.190000e-02 -2.630000e-02 2.035400e+00 9.080000e-02 -1.085000e-01 1.982000e-01 4.708000e-01 + 1.049800e+00 6.410000e-02 5.919000e-01 2.174000e-01 3.979000e-01 2.718000e-01 4.788000e-01 3.486000e-01 4.119000e-01 2.361000e-01 5.073000e-01 3.169000e-01 3.661000e-01 1.799000e-01 + 1.155500e+00 -5.990000e-02 1.190600e+00 2.981000e-01 9.963000e-01 -1.430000e-02 1.078900e+00 4.294000e-01 8.709000e-01 1.026000e-01 1.261800e+00 2.111000e-01 6.923000e-01 4.150000e-02 + 6.215000e-01 2.120000e-01 1.045100e+00 2.494000e-01 1.139400e+00 3.451000e-01 1.084900e+00 1.967000e-01 1.175900e+00 3.235000e-01 1.139200e+00 1.358000e-01 6.034000e-01 1.435000e-01 + 2.349000e-01 1.442000e-01 2.265000e-01 3.929000e-01 2.736000e-01 4.296000e-01 2.931000e-01 3.112000e-01 2.363000e-01 4.802000e-01 2.454000e-01 3.726000e-01 1.406000e-01 2.151000e-01 + 3.019000e-01 8.005000e-01 2.399000e-01 1.477400e+00 1.681000e-01 1.178700e+00 1.256000e-01 1.612900e+00 1.998000e-01 1.085000e+00 1.193000e-01 1.621700e+00 1.418000e-01 7.335000e-01 + 1.147600e+00 -4.770000e-02 6.603000e-01 5.889000e-01 4.982000e-01 4.222000e-01 6.575000e-01 5.907000e-01 5.041000e-01 3.874000e-01 5.851000e-01 6.789000e-01 4.479000e-01 2.445000e-01 + 1.101200e+00 7.300000e-03 8.426000e-01 1.042000e-01 6.891000e-01 5.540000e-02 9.469000e-01 -1.900000e-02 7.682000e-01 -6.170000e-02 8.742000e-01 6.580000e-02 5.727000e-01 2.010000e-02 +-4.210000e-02 6.206000e-01 -8.900000e-03 9.080000e-01 4.440000e-02 9.645000e-01 2.560000e-02 8.678000e-01 3.390000e-02 9.927000e-01 1.381000e-01 7.345000e-01 9.100000e-03 5.054000e-01 + 3.173000e-01 1.439000e-01 4.652000e-01 2.672000e-01 4.718000e-01 3.732000e-01 4.224000e-01 3.186000e-01 5.422000e-01 2.998000e-01 4.661000e-01 2.660000e-01 2.607000e-01 1.635000e-01 + 3.098000e-01 5.266000e-01 7.139000e-01 4.210000e-01 7.526000e-01 5.123000e-01 1.844200e+00 -1.276000e-01 4.160000e-02 1.950200e+00 3.560000e-02 -4.230000e-02 3.870000e-01 2.836000e-01 + 1.113300e+00 -1.020000e-02 6.586000e-01 3.186000e-01 5.528000e-01 2.098000e-01 6.548000e-01 3.203000e-01 4.692000e-01 2.883000e-01 5.417000e-01 4.562000e-01 4.591000e-01 1.485000e-01 + 1.120500e+00 -1.250000e-02 4.273000e-01 1.305000e+00 4.001000e-01 1.569000e+00 5.607000e-01 1.152400e+00 4.132000e-01 1.340800e+00 3.841000e-01 1.354200e+00 3.303000e-01 6.559000e-01 + 6.990000e-01 3.968000e-01 1.746000e-01 6.364000e-01 2.241000e-01 4.211000e-01 3.194000e-01 4.643000e-01 2.155000e-01 4.168000e-01 4.041000e-01 3.630000e-01 2.536000e-01 2.708000e-01 + 4.087000e-01 1.144000e-01 6.518000e-01 1.959000e-01 7.540000e-01 2.240000e-01 1.739300e+00 -8.900000e-03 4.300000e-03 -5.200000e-03 -2.820000e-02 3.580000e-02 3.589000e-01 1.307000e-01 + 8.990000e-02 4.962000e-01 1.670000e-01 7.478000e-01 1.831000e-01 8.579000e-01 1.603000e-01 7.559000e-01 1.481000e-01 9.170000e-01 1.539000e-01 7.605000e-01 6.710000e-02 4.654000e-01 + 7.275000e-01 8.800000e-02 1.026300e+00 2.773000e-01 1.203800e+00 2.712000e-01 1.026200e+00 2.689000e-01 1.253600e+00 2.343000e-01 9.783000e-01 3.308000e-01 5.823000e-01 1.706000e-01 + 5.495000e-01 5.437000e-01 2.686000e-01 7.107000e-01 1.793000e-01 5.921000e-01 2.232000e-01 7.645000e-01 2.558000e-01 4.846000e-01 3.518000e-01 6.074000e-01 2.065000e-01 4.018000e-01 + 5.979000e-01 2.631000e-01 9.246000e-01 2.454000e-01 1.040700e+00 2.497000e-01 1.743200e+00 -1.210000e-02 2.035700e+00 -4.280000e-02 5.510000e-02 -6.860000e-02 5.446000e-01 1.450000e-01 + 4.053000e-01 3.193000e-01 7.128000e-01 4.065000e-01 6.600000e-01 6.450000e-01 6.435000e-01 4.922000e-01 7.326000e-01 5.784000e-01 6.610000e-01 4.686000e-01 3.683000e-01 2.864000e-01 + 3.101000e-01 7.882000e-01 2.058000e-01 6.524000e-01 1.578000e-01 5.337000e-01 1.578000e-01 7.100000e-01 1.407000e-01 5.390000e-01 1.659000e-01 7.029000e-01 1.362000e-01 4.268000e-01 + 3.448000e-01 7.733000e-01 2.226000e-01 3.745000e-01 1.856000e-01 3.102000e-01 2.002000e-01 3.992000e-01 2.583000e-01 2.151000e-01 1.775000e-01 4.266000e-01 1.738000e-01 2.538000e-01 + 1.163000e-01 6.281000e-01 6.240000e-02 1.128500e+00 1.577000e-01 1.181700e+00 8.570000e-02 1.103700e+00 1.171000e-01 1.252400e+00 2.270000e-01 9.335000e-01 7.730000e-02 6.036000e-01 + 1.841000e-01 9.089000e-01 9.620000e-02 1.631700e+00 3.060000e-02 1.153200e+00 4.420000e-02 1.694200e+00 1.440000e-02 1.136600e+00 9.730000e-02 1.630200e+00 6.780000e-02 7.479000e-01 + 2.871000e-01 4.391000e-01 5.545000e-01 5.586000e-01 5.610000e-01 7.182000e-01 6.155000e-01 4.853000e-01 6.982000e-01 5.763000e-01 5.284000e-01 5.929000e-01 3.306000e-01 3.102000e-01 + 6.266000e-01 4.869000e-01 8.659000e-01 8.900000e-01 1.030200e+00 9.553000e-01 1.955700e+00 -2.583000e-01 1.954300e+00 5.250000e-02 1.870000e-02 1.709700e+00 5.739000e-01 4.255000e-01 + 2.521000e-01 5.208000e-01 4.874000e-01 6.131000e-01 7.096000e-01 4.927000e-01 1.693000e+00 4.670000e-02 2.250000e-02 1.967700e+00 1.410000e-02 -1.790000e-02 2.880000e-01 3.549000e-01 + 1.115700e+00 -1.130000e-02 5.825000e-01 6.135000e-01 6.044000e-01 2.598000e-01 5.369000e-01 6.677000e-01 5.063000e-01 3.521000e-01 5.327000e-01 6.701000e-01 4.238000e-01 2.538000e-01 + 4.343000e-01 2.892000e-01 6.943000e-01 4.379000e-01 8.366000e-01 4.458000e-01 8.259000e-01 2.838000e-01 8.670000e-01 4.263000e-01 7.693000e-01 3.494000e-01 4.452000e-01 2.014000e-01 + 3.767000e-01 6.610000e-01 7.348000e-01 6.040000e-01 9.230000e-01 5.251000e-01 1.740300e+00 -4.500000e-03 -4.600000e-02 2.054200e+00 2.160000e-02 -2.470000e-02 4.012000e-01 3.929000e-01 + 1.776000e-01 9.447000e-01 6.292000e-01 9.101000e-01 7.055000e-01 9.575000e-01 1.797300e+00 -8.050000e-02 -8.980000e-02 2.111000e+00 -2.000000e-03 1.733600e+00 3.322000e-01 5.643000e-01 + 2.923000e-01 5.579000e-01 3.704000e-01 9.829000e-01 3.558000e-01 1.191800e+00 3.351000e-01 1.025100e+00 3.176000e-01 1.266500e+00 3.619000e-01 9.876000e-01 1.767000e-01 6.106000e-01 + 1.100500e+00 6.600000e-03 4.421000e-01 8.560000e-01 3.823000e-01 5.487000e-01 4.909000e-01 7.966000e-01 3.241000e-01 5.907000e-01 5.004000e-01 7.838000e-01 3.299000e-01 3.734000e-01 + 1.048000e+00 6.760000e-02 7.788000e-01 9.475000e-01 6.592000e-01 1.156800e+00 7.752000e-01 9.544000e-01 6.585000e-01 9.674000e-01 7.535000e-01 9.787000e-01 5.198000e-01 4.468000e-01 + 6.970000e-02 9.352000e-01 2.593000e-01 1.061000e+00 3.419000e-01 1.108000e+00 7.300000e-03 1.719100e+00 -7.600000e-03 2.004800e+00 3.520000e-02 -4.400000e-02 1.164000e-01 6.692000e-01 + 3.260000e-01 2.160000e-01 7.528000e-01 7.900000e-02 7.924000e-01 1.742000e-01 1.687000e+00 5.020000e-02 -3.940000e-02 4.670000e-02 -6.030000e-02 7.090000e-02 4.082000e-01 7.520000e-02 + 1.050000e-01 6.663000e-01 3.348000e-01 7.601000e-01 5.875000e-01 6.110000e-01 1.713200e+00 2.610000e-02 1.228000e-01 1.855500e+00 -2.320000e-02 2.860000e-02 2.201000e-01 4.189000e-01 + 1.002700e+00 1.252000e-01 6.166000e-01 4.814000e-01 4.798000e-01 3.617000e-01 7.088000e-01 3.755000e-01 5.636000e-01 2.377000e-01 6.239000e-01 4.772000e-01 4.345000e-01 2.159000e-01 + 4.496000e-01 2.169000e-01 6.392000e-01 3.630000e-01 8.200000e-01 2.899000e-01 1.770400e+00 -4.520000e-02 -1.100000e-02 2.011200e+00 1.760000e-02 -1.910000e-02 4.239000e-01 1.501000e-01 + 4.926000e-01 6.137000e-01 3.186000e-01 1.404600e+00 2.477000e-01 1.260500e+00 1.928000e-01 1.552100e+00 2.102000e-01 1.225200e+00 2.542000e-01 1.481800e+00 2.139000e-01 7.059000e-01 + 1.138700e+00 -4.270000e-02 8.879000e-01 1.301000e-01 6.693000e-01 1.269000e-01 8.105000e-01 2.219000e-01 6.306000e-01 1.497000e-01 8.711000e-01 1.517000e-01 5.312000e-01 9.750000e-02 + 9.711000e-01 1.022000e-01 1.494600e+00 2.016000e-01 1.763500e+00 1.583000e-01 1.554200e+00 1.290000e-01 1.834300e+00 1.014000e-01 1.488900e+00 2.066000e-01 8.892000e-01 8.380000e-02 + 2.105000e-01 1.522000e-01 5.753000e-01 9.360000e-02 6.696000e-01 1.234000e-01 1.763000e+00 -3.660000e-02 4.100000e-03 -4.000000e-03 -1.860000e-02 2.320000e-02 3.220000e-01 5.130000e-02 + 5.364000e-01 5.697000e-01 2.754000e-01 1.459900e+00 2.340000e-01 9.770000e-01 2.480000e-01 1.489200e+00 3.099000e-01 8.484000e-01 2.989000e-01 1.428300e+00 2.200000e-01 6.076000e-01 + 1.165300e+00 -7.410000e-02 4.147000e-01 6.831000e-01 3.322000e-01 5.073000e-01 4.404000e-01 6.515000e-01 3.882000e-01 4.203000e-01 3.910000e-01 7.064000e-01 3.222000e-01 3.256000e-01 + 5.929000e-01 4.753000e-01 7.672000e-01 9.424000e-01 8.572000e-01 1.086400e+00 7.978000e-01 8.958000e-01 9.469000e-01 1.005600e+00 7.537000e-01 9.506000e-01 4.475000e-01 5.341000e-01 + 1.982000e-01 7.611000e-01 4.901000e-01 7.844000e-01 6.065000e-01 7.851000e-01 1.734700e+00 -4.100000e-03 9.350000e-02 1.890800e+00 -3.790000e-02 4.530000e-02 2.528000e-01 5.018000e-01 + 1.546000e-01 4.171000e-01 2.119000e-01 6.868000e-01 2.017000e-01 8.355000e-01 3.120000e-01 5.704000e-01 2.865000e-01 7.473000e-01 2.101000e-01 6.932000e-01 1.401000e-01 3.778000e-01 + 5.030000e-02 4.674000e-01 7.340000e-02 7.643000e-01 2.045000e-01 7.558000e-01 -4.600000e-03 1.736500e+00 -6.460000e-02 7.550000e-02 -1.480000e-02 1.930000e-02 5.150000e-02 4.315000e-01 + 1.041000e-01 6.762000e-01 3.926000e-01 7.050000e-01 5.157000e-01 7.068000e-01 1.774500e+00 -5.130000e-02 3.810000e-02 1.958000e+00 -2.210000e-02 2.530000e-02 2.124000e-01 4.345000e-01 + 6.700000e-03 5.981000e-01 -6.650000e-02 1.033500e+00 -2.740000e-02 1.112100e+00 3.610000e-02 9.109000e-01 1.080000e-02 1.085600e+00 -6.430000e-02 1.026400e+00 6.900000e-03 5.412000e-01 + 2.601000e-01 8.426000e-01 1.748000e-01 5.536000e-01 1.488000e-01 4.436000e-01 2.177000e-01 4.987000e-01 1.135000e-01 4.778000e-01 1.067000e-01 6.321000e-01 1.217000e-01 3.789000e-01 + 7.495000e-01 3.638000e-01 3.087000e-01 7.132000e-01 2.289000e-01 5.704000e-01 4.013000e-01 5.987000e-01 4.147000e-01 3.338000e-01 3.795000e-01 6.300000e-01 2.800000e-01 3.398000e-01 + 2.536000e-01 3.032000e-01 3.890000e-01 4.836000e-01 4.285000e-01 5.635000e-01 4.099000e-01 4.558000e-01 5.144000e-01 4.786000e-01 3.665000e-01 5.107000e-01 2.354000e-01 2.655000e-01 + 1.108700e+00 -1.400000e-03 6.231000e-01 1.103400e+00 5.175000e-01 5.217000e-01 5.151000e-01 1.232300e+00 5.308000e-01 4.732000e-01 4.814000e-01 1.269600e+00 4.136000e-01 3.438000e-01 + 1.243000e-01 8.552000e-01 2.607000e-01 1.254300e+00 2.224000e-01 1.517600e+00 2.060000e-01 1.322800e+00 2.795000e-01 1.476000e+00 3.078000e-01 1.201100e+00 1.504000e-01 7.250000e-01 + 9.213000e-01 1.830000e-01 1.271100e+00 2.182000e-01 1.436000e+00 1.685000e-01 1.739100e+00 -6.500000e-03 2.118200e+00 -1.382000e-01 2.510000e-02 1.703800e+00 7.304000e-01 1.416000e-01 + 4.906000e-01 2.284000e-01 7.607000e-01 3.695000e-01 7.548000e-01 5.566000e-01 8.102000e-01 3.068000e-01 8.265000e-01 4.896000e-01 7.111000e-01 4.283000e-01 4.367000e-01 2.166000e-01 + 5.023000e-01 2.296000e-01 8.095000e-01 3.391000e-01 8.645000e-01 4.546000e-01 7.044000e-01 4.620000e-01 8.431000e-01 5.025000e-01 7.605000e-01 3.984000e-01 4.166000e-01 2.559000e-01 + 1.156500e+00 -5.760000e-02 6.427000e-01 1.085200e+00 4.490000e-01 9.378000e-01 5.651000e-01 1.172700e+00 4.125000e-01 9.192000e-01 6.713000e-01 1.048000e+00 4.359000e-01 4.478000e-01 + 3.471000e-01 4.969000e-01 5.672000e-01 6.026000e-01 6.693000e-01 6.248000e-01 1.642200e+00 1.036000e-01 2.180000e-02 1.976600e+00 -6.900000e-03 6.900000e-03 3.297000e-01 3.596000e-01 + 1.772000e-01 7.112000e-01 3.103000e-01 1.079600e+00 2.551000e-01 1.339400e+00 2.338000e-01 1.168600e+00 3.075000e-01 1.305000e+00 2.437000e-01 1.153100e+00 1.468000e-01 6.600000e-01 + 4.819000e-01 5.060000e-01 7.012000e-01 8.559000e-01 8.285000e-01 9.376000e-01 6.944000e-01 8.615000e-01 7.004000e-01 1.112700e+00 7.276000e-01 8.193000e-01 4.284000e-01 4.650000e-01 + 4.597000e-01 2.749000e-01 7.482000e-01 3.990000e-01 9.198000e-01 3.746000e-01 7.880000e-01 3.529000e-01 8.253000e-01 5.067000e-01 7.385000e-01 4.140000e-01 4.344000e-01 2.271000e-01 + 4.523000e-01 6.470000e-01 2.413000e-01 1.312200e+00 1.894000e-01 8.252000e-01 1.803000e-01 1.379300e+00 1.961000e-01 7.919000e-01 1.998000e-01 1.361800e+00 1.884000e-01 5.538000e-01 + 4.270000e-02 6.164000e-01 1.516000e-01 8.657000e-01 8.410000e-02 1.095600e+00 1.261000e-01 8.992000e-01 1.549000e-01 1.029100e+00 7.630000e-02 9.625000e-01 3.020000e-02 5.699000e-01 + 4.810000e-02 6.301000e-01 3.641000e-01 6.274000e-01 4.213000e-01 6.975000e-01 4.650000e-02 1.677500e+00 -2.970000e-02 2.037900e+00 1.500000e-03 -2.200000e-03 1.792000e-01 4.041000e-01 +-5.410000e-02 8.424000e-01 2.830000e-01 8.134000e-01 4.024000e-01 8.135000e-01 6.600000e-03 1.724300e+00 -8.130000e-02 2.099400e+00 -3.590000e-02 4.370000e-02 1.669000e-01 4.733000e-01 + 1.492000e-01 9.433000e-01 4.581000e-01 1.263300e+00 4.340000e-01 1.517700e+00 1.692600e+00 4.630000e-02 -5.270000e-02 2.056400e+00 -1.043000e-01 1.855000e+00 2.145000e-01 7.789000e-01 + 5.950000e-01 5.037000e-01 8.418000e-01 6.593000e-01 1.035600e+00 5.684000e-01 1.736200e+00 -4.000000e-03 1.998700e+00 2.900000e-03 7.450000e-02 1.641700e+00 5.081000e-01 3.627000e-01 + 1.041200e+00 7.690000e-02 3.429000e-01 5.131000e-01 4.083000e-01 2.563000e-01 4.457000e-01 3.872000e-01 3.686000e-01 2.875000e-01 4.201000e-01 4.196000e-01 3.165000e-01 2.346000e-01 + 5.750000e-01 9.830000e-02 7.359000e-01 3.524000e-01 9.226000e-01 3.027000e-01 7.922000e-01 2.849000e-01 9.758000e-01 2.588000e-01 8.202000e-01 2.514000e-01 4.557000e-01 1.661000e-01 + 2.356000e-01 2.608000e-01 3.491000e-01 4.312000e-01 5.015000e-01 3.656000e-01 3.259000e-01 4.589000e-01 3.623000e-01 5.480000e-01 3.415000e-01 4.391000e-01 2.121000e-01 2.362000e-01 +-4.400000e-03 5.750000e-02 2.999000e-01 5.590000e-02 3.436000e-01 1.495000e-01 6.220000e-02 -7.260000e-02 3.770000e-02 -4.620000e-02 -1.930000e-02 2.460000e-02 1.388000e-01 5.610000e-02 + 3.853000e-01 2.266000e-01 6.398000e-01 2.940000e-01 7.465000e-01 3.086000e-01 1.889800e+00 -1.804000e-01 2.080000e-02 1.977300e+00 -1.660000e-02 1.840000e-02 3.421000e-01 2.053000e-01 + 5.732000e-01 9.820000e-02 9.495000e-01 2.150000e-02 1.010600e+00 9.040000e-02 1.689700e+00 5.020000e-02 1.855100e+00 1.663000e-01 -1.440000e-02 1.560000e-02 5.282000e-01 4.370000e-02 + 5.975000e-01 9.590000e-02 8.709000e-01 2.277000e-01 9.448000e-01 3.096000e-01 8.984000e-01 1.909000e-01 1.040400e+00 2.144000e-01 8.695000e-01 2.280000e-01 4.989000e-01 1.341000e-01 + 1.161700e+00 -6.800000e-02 5.103000e-01 1.213400e+00 4.322000e-01 6.767000e-01 5.040000e-01 1.219600e+00 3.956000e-01 6.859000e-01 5.204000e-01 1.201400e+00 3.472000e-01 4.431000e-01 + 1.021700e+00 9.880000e-02 4.318000e-01 4.948000e-01 3.271000e-01 4.096000e-01 4.788000e-01 4.344000e-01 3.361000e-01 3.875000e-01 4.107000e-01 5.175000e-01 3.394000e-01 2.462000e-01 + 9.910000e-02 6.097000e-01 1.928000e-01 9.126000e-01 2.849000e-01 9.650000e-01 1.819000e-01 9.313000e-01 2.587000e-01 1.014800e+00 2.217000e-01 8.827000e-01 1.360000e-01 4.989000e-01 + 4.196000e-01 4.070000e-01 8.509000e-01 2.705000e-01 8.090000e-01 4.584000e-01 1.633500e+00 1.120000e-01 -6.650000e-02 2.080700e+00 1.660000e-02 -1.950000e-02 4.122000e-01 2.621000e-01 + 6.442000e-01 4.657000e-01 2.805000e-01 9.211000e-01 3.062000e-01 5.754000e-01 3.203000e-01 8.736000e-01 3.428000e-01 5.107000e-01 2.195000e-01 9.937000e-01 2.591000e-01 4.182000e-01 + 1.083000e-01 3.704000e-01 2.198000e-01 5.239000e-01 1.973000e-01 6.579000e-01 1.038000e-01 6.641000e-01 2.724000e-01 5.815000e-01 2.600000e-01 4.734000e-01 9.710000e-02 3.372000e-01 + 1.143400e+00 -4.700000e-02 6.394000e-01 1.084000e+00 5.072000e-01 8.219000e-01 7.274000e-01 9.770000e-01 4.117000e-01 8.792000e-01 6.133000e-01 1.115600e+00 4.261000e-01 4.439000e-01 + 4.894000e-01 2.322000e-01 9.444000e-01 1.599000e-01 1.136500e+00 1.100000e-01 9.970000e-01 9.400000e-02 1.083900e+00 1.925000e-01 9.986000e-01 9.700000e-02 5.573000e-01 7.900000e-02 + 6.940000e-02 4.496000e-01 3.861000e-01 4.431000e-01 5.083000e-01 4.420000e-01 1.722500e+00 8.000000e-03 -2.180000e-02 2.580000e-02 -1.260000e-02 1.530000e-02 2.012000e-01 2.778000e-01 + 4.709000e-01 1.637000e-01 7.540000e-01 2.368000e-01 9.168000e-01 1.987000e-01 7.055000e-01 2.919000e-01 8.799000e-01 2.597000e-01 6.879000e-01 3.119000e-01 4.364000e-01 1.345000e-01 + 3.695000e-01 7.475000e-01 7.178000e-01 7.092000e-01 8.749000e-01 6.676000e-01 1.720000e+00 6.400000e-03 -9.800000e-03 2.015000e+00 5.230000e-02 1.671600e+00 4.293000e-01 4.062000e-01 + 2.833000e-01 6.929000e-01 3.343000e-01 1.207300e+00 4.848000e-01 1.259600e+00 4.201000e-01 1.108400e+00 4.528000e-01 1.313000e+00 3.483000e-01 1.200200e+00 2.231000e-01 6.640000e-01 + 3.249000e-01 2.663000e-01 2.135000e-01 8.380000e-02 2.176000e-01 3.230000e-02 1.633000e-01 1.440000e-01 1.643000e-01 8.870000e-02 2.186000e-01 7.850000e-02 1.557000e-01 8.340000e-02 + 2.509000e-01 4.415000e-01 3.253000e-01 7.774000e-01 3.237000e-01 9.379000e-01 2.093000e-01 9.146000e-01 3.470000e-01 9.313000e-01 2.318000e-01 8.854000e-01 1.571000e-01 4.836000e-01 + 4.089000e-01 3.129000e-01 5.581000e-01 5.068000e-01 8.138000e-01 3.462000e-01 1.677900e+00 6.840000e-02 2.080000e-02 1.974000e+00 -2.290000e-02 2.710000e-02 3.926000e-01 2.173000e-01 + 6.643000e-01 4.440000e-01 4.709000e-01 3.021000e-01 3.044000e-01 3.373000e-01 3.318000e-01 4.683000e-01 2.892000e-01 3.421000e-01 3.202000e-01 4.828000e-01 2.603000e-01 2.726000e-01 + 1.089800e+00 1.820000e-02 4.522000e-01 1.280700e+00 2.793000e-01 9.139000e-01 4.674000e-01 1.263400e+00 4.811000e-01 6.379000e-01 5.040000e-01 1.218100e+00 3.527000e-01 4.602000e-01 + 7.746000e-01 2.773000e-01 1.262500e+00 3.768000e-01 1.368200e+00 5.105000e-01 1.176300e+00 4.776000e-01 1.362200e+00 5.502000e-01 1.244100e+00 4.004000e-01 7.037000e-01 2.478000e-01 + 1.214400e+00 -1.345000e-01 5.728000e-01 8.045000e-01 4.732000e-01 4.932000e-01 5.432000e-01 8.459000e-01 4.365000e-01 5.079000e-01 6.445000e-01 7.233000e-01 4.171000e-01 3.019000e-01 + 1.020300e+00 1.023000e-01 1.339900e+00 3.863000e-01 8.918000e-01 4.010000e-01 1.400000e+00 3.113000e-01 9.264000e-01 3.066000e-01 1.256800e+00 4.880000e-01 7.044000e-01 1.491000e-01 +-8.000000e-04 8.211000e-01 2.361000e-01 9.075000e-01 3.880000e-01 8.679000e-01 -2.060000e-02 1.757600e+00 -1.680000e-02 2.020600e+00 -5.400000e-02 6.580000e-02 1.555000e-01 5.119000e-01 + 8.590000e-02 5.093000e-01 1.094000e-01 8.284000e-01 1.888000e-01 8.672000e-01 2.009000e-01 7.242000e-01 3.171000e-01 7.291000e-01 2.109000e-01 7.078000e-01 1.055000e-01 4.283000e-01 + 5.585000e-01 4.104000e-01 9.091000e-01 6.040000e-01 9.648000e-01 7.640000e-01 8.639000e-01 6.560000e-01 9.755000e-01 7.808000e-01 9.222000e-01 5.776000e-01 4.978000e-01 3.786000e-01 + 6.952000e-01 9.790000e-02 1.119200e+00 1.273000e-01 1.299400e+00 1.131000e-01 1.206300e+00 2.280000e-02 1.367300e+00 5.920000e-02 1.061200e+00 1.961000e-01 6.713000e-01 4.250000e-02 +-2.900000e-02 6.100000e-01 1.768000e-01 7.117000e-01 2.572000e-01 7.548000e-01 -4.300000e-02 1.783800e+00 -1.134000e-01 2.131000e+00 2.780000e-02 -3.580000e-02 9.120000e-02 4.239000e-01 + 6.690000e-02 1.033000e+00 5.470000e-02 4.185000e-01 6.780000e-02 3.254000e-01 1.660000e-02 4.632000e-01 3.370000e-02 3.636000e-01 6.200000e-03 4.806000e-01 1.390000e-02 3.445000e-01 + 4.392000e-01 1.730000e-01 7.467000e-01 1.972000e-01 7.673000e-01 3.279000e-01 8.360000e-01 9.120000e-02 8.220000e-01 2.830000e-01 7.616000e-01 1.845000e-01 4.333000e-01 1.134000e-01 + 1.127900e+00 -2.750000e-02 1.157800e+00 5.609000e-01 8.199000e-01 7.000000e-01 1.071100e+00 6.639000e-01 8.206000e-01 6.127000e-01 1.003100e+00 7.467000e-01 6.514000e-01 2.694000e-01 + 8.172000e-01 2.907000e-01 3.781000e-01 1.356400e+00 3.831000e-01 1.575800e+00 5.063000e-01 1.203100e+00 3.232000e-01 1.481600e+00 5.606000e-01 1.139900e+00 2.864000e-01 7.047000e-01 + 3.522000e-01 3.816000e-01 6.513000e-01 4.030000e-01 8.796000e-01 2.741000e-01 1.853000e+00 -1.437000e-01 2.020000e-02 1.981500e+00 4.100000e-02 -4.790000e-02 4.054000e-01 2.034000e-01 + 6.243000e-01 3.136000e-01 9.404000e-01 5.338000e-01 1.054800e+00 6.177000e-01 1.003000e+00 4.583000e-01 1.125700e+00 5.655000e-01 8.917000e-01 5.895000e-01 5.389000e-01 3.107000e-01 + 1.032000e-01 2.145000e-01 2.228000e-01 2.618000e-01 2.334000e-01 3.197000e-01 2.795000e-01 1.928000e-01 2.514000e-01 3.102000e-01 2.510000e-01 2.285000e-01 1.353000e-01 1.438000e-01 + 9.120000e-02 4.763000e-01 8.260000e-02 8.195000e-01 1.323000e-01 8.828000e-01 1.208000e-01 7.711000e-01 1.502000e-01 8.749000e-01 8.230000e-02 8.164000e-01 4.590000e-02 4.733000e-01 + 8.400000e-03 5.233000e-01 3.114000e-01 5.303000e-01 3.548000e-01 6.212000e-01 -7.810000e-02 1.824600e+00 -1.500000e-02 1.630000e-02 1.900000e-02 -2.190000e-02 1.548000e-01 3.324000e-01 + 3.621000e-01 3.948000e-01 5.814000e-01 5.091000e-01 7.371000e-01 4.622000e-01 1.827200e+00 -1.137000e-01 2.880000e-02 1.964000e+00 -9.400000e-03 1.210000e-02 3.026000e-01 3.417000e-01 + 3.580000e-01 2.763000e-01 4.854000e-01 5.267000e-01 6.305000e-01 5.067000e-01 6.041000e-01 3.864000e-01 6.265000e-01 5.287000e-01 5.911000e-01 4.011000e-01 3.291000e-01 2.458000e-01 + 3.187000e-01 7.551000e-01 4.209000e-01 1.268700e+00 6.258000e-01 1.271600e+00 6.451000e-01 1.001000e+00 5.475000e-01 1.395100e+00 5.579000e-01 1.113300e+00 3.066000e-01 6.586000e-01 + 1.185000e-01 7.796000e-01 2.708000e-01 1.118500e+00 3.414000e-01 1.232300e+00 2.990000e-01 1.083900e+00 4.309000e-01 1.150900e+00 3.358000e-01 1.039400e+00 1.932000e-01 6.014000e-01 + 1.116200e+00 -1.300000e-02 5.206000e-01 2.610000e-02 4.651000e-01 -1.630000e-02 5.007000e-01 4.660000e-02 4.346000e-01 1.190000e-02 4.827000e-01 6.850000e-02 3.726000e-01 2.650000e-02 + 8.306000e-01 9.530000e-02 1.104800e+00 3.818000e-01 1.345200e+00 3.360000e-01 1.102500e+00 3.846000e-01 1.274300e+00 4.425000e-01 1.127500e+00 3.547000e-01 6.521000e-01 2.045000e-01 + 1.156300e+00 -6.370000e-02 9.582000e-01 7.839000e-01 8.017000e-01 5.624000e-01 1.087200e+00 6.377000e-01 7.981000e-01 5.080000e-01 9.966000e-01 7.431000e-01 6.323000e-01 2.479000e-01 + 1.113700e+00 -1.060000e-02 5.751000e-01 8.128000e-01 5.425000e-01 4.148000e-01 5.360000e-01 8.606000e-01 6.221000e-01 2.954000e-01 6.010000e-01 7.833000e-01 4.364000e-01 2.829000e-01 +-7.000000e-04 1.000200e+00 6.240000e-02 1.491200e+00 3.840000e-02 1.734700e+00 1.108000e-01 1.435400e+00 1.197000e-01 1.670000e+00 -2.700000e-03 1.567500e+00 3.070000e-02 8.685000e-01 + 1.240000e-01 9.872000e-01 1.695000e-01 5.126000e-01 1.271000e-01 4.343000e-01 1.380000e-02 6.953000e-01 2.270000e-02 5.462000e-01 1.374000e-01 5.482000e-01 7.740000e-02 4.041000e-01 + 4.141000e-01 6.719000e-01 2.314000e-01 1.497700e+00 2.280000e-01 9.137000e-01 2.178000e-01 1.504500e+00 1.092000e-01 1.019000e+00 1.247000e-01 1.614500e+00 1.445000e-01 6.616000e-01 + 6.177000e-01 1.427000e-01 9.623000e-01 1.017000e-01 1.103000e+00 7.800000e-02 1.600300e+00 1.562000e-01 1.871500e+00 1.531000e-01 2.210000e-02 -2.600000e-02 5.527000e-01 7.560000e-02 + 1.059300e+00 5.470000e-02 9.245000e-01 3.887000e-01 7.576000e-01 1.769000e-01 9.578000e-01 3.412000e-01 7.952000e-01 1.095000e-01 1.041500e+00 2.450000e-01 6.124000e-01 9.040000e-02 + 9.449000e-01 1.353000e-01 3.213000e-01 1.421500e+00 3.429000e-01 7.263000e-01 3.767000e-01 1.351400e+00 3.432000e-01 6.875000e-01 3.776000e-01 1.354700e+00 2.832000e-01 4.873000e-01 + 6.301000e-01 4.748000e-01 4.008000e-01 9.958000e-01 2.741000e-01 6.979000e-01 3.320000e-01 1.073400e+00 3.438000e-01 5.908000e-01 3.196000e-01 1.086800e+00 2.474000e-01 4.766000e-01 + 1.978000e-01 9.140000e-01 1.938000e-01 4.248000e-01 6.020000e-02 4.722000e-01 9.960000e-02 5.354000e-01 7.210000e-02 4.447000e-01 1.717000e-01 4.529000e-01 1.206000e-01 3.249000e-01 + 2.309000e-01 2.064000e-01 6.680000e-01 5.680000e-02 6.932000e-01 1.725000e-01 1.859700e+00 -1.525000e-01 -3.130000e-02 3.630000e-02 5.910000e-02 -7.020000e-02 3.368000e-01 8.280000e-02 + 2.211000e-01 7.943000e-01 6.555000e-01 6.537000e-01 6.771000e-01 7.673000e-01 1.808700e+00 -8.840000e-02 7.200000e-03 1.994200e+00 -1.680000e-02 1.970000e-02 3.111000e-01 4.701000e-01 + 2.125000e-01 6.560000e-01 4.932000e-01 6.953000e-01 6.084000e-01 6.982000e-01 1.701900e+00 3.800000e-02 -5.100000e-02 2.060000e+00 -4.300000e-03 2.300000e-03 2.587000e-01 4.444000e-01 + 5.390000e-02 1.055300e+00 1.167000e-01 5.124000e-01 -7.110000e-02 6.230000e-01 1.115000e-01 5.170000e-01 3.800000e-02 4.834000e-01 6.440000e-02 5.757000e-01 3.960000e-02 4.136000e-01 + 2.916000e-01 2.618000e-01 3.233000e-01 5.671000e-01 5.434000e-01 4.386000e-01 4.561000e-01 4.079000e-01 4.168000e-01 6.065000e-01 3.949000e-01 4.845000e-01 2.435000e-01 2.603000e-01 + 3.046000e-01 1.198000e-01 5.786000e-01 1.665000e-01 7.735000e-01 8.260000e-02 1.760100e+00 -3.140000e-02 2.880000e-02 -3.630000e-02 -2.370000e-02 2.720000e-02 3.292000e-01 9.370000e-02 + 1.812000e-01 5.192000e-01 4.632000e-01 5.597000e-01 6.344000e-01 4.976000e-01 1.709800e+00 2.520000e-02 -7.980000e-02 2.089700e+00 -4.440000e-02 5.320000e-02 2.372000e-01 3.645000e-01 + 3.179000e-01 4.891000e-01 7.155000e-01 3.883000e-01 7.770000e-01 4.547000e-01 1.566300e+00 1.949000e-01 5.830000e-02 1.927600e+00 3.480000e-02 -4.360000e-02 3.791000e-01 2.744000e-01 + 1.477000e-01 4.547000e-01 2.569000e-01 6.831000e-01 2.450000e-01 8.360000e-01 3.765000e-01 5.412000e-01 2.740000e-01 8.186000e-01 1.857000e-01 7.708000e-01 1.411000e-01 4.037000e-01 + 2.131000e-01 6.178000e-01 6.259000e-01 4.974000e-01 7.543000e-01 4.947000e-01 1.708600e+00 2.940000e-02 4.010000e-02 1.945600e+00 2.130000e-02 -2.470000e-02 3.206000e-01 3.490000e-01 + 1.050800e+00 6.520000e-02 4.232000e-01 8.030000e-02 3.003000e-01 1.362000e-01 3.865000e-01 1.220000e-01 3.645000e-01 4.690000e-02 4.060000e-01 9.850000e-02 3.132000e-01 6.030000e-02 + 2.090000e-02 7.060000e-01 2.470000e-01 8.011000e-01 3.740000e-01 7.966000e-01 -3.690000e-02 1.777300e+00 4.610000e-02 1.946200e+00 -2.460000e-02 2.910000e-02 1.503000e-01 4.630000e-01 + 4.137000e-01 2.892000e-01 4.493000e-01 6.976000e-01 7.307000e-01 5.271000e-01 5.331000e-01 5.940000e-01 5.688000e-01 7.422000e-01 5.595000e-01 5.605000e-01 3.348000e-01 3.115000e-01 + 5.298000e-01 6.420000e-02 6.824000e-01 2.788000e-01 7.870000e-01 3.054000e-01 6.489000e-01 3.206000e-01 8.537000e-01 2.453000e-01 7.167000e-01 2.387000e-01 4.121000e-01 1.402000e-01 + 4.221000e-01 3.664000e-01 7.573000e-01 4.597000e-01 8.641000e-01 5.222000e-01 7.522000e-01 4.671000e-01 8.513000e-01 5.595000e-01 7.478000e-01 4.688000e-01 4.424000e-01 2.608000e-01 + 4.622000e-01 1.290000e-01 7.078000e-01 2.222000e-01 7.751000e-01 2.817000e-01 7.012000e-01 2.305000e-01 8.362000e-01 2.308000e-01 6.095000e-01 3.373000e-01 3.959000e-01 1.415000e-01 + 6.361000e-01 8.160000e-02 9.489000e-01 1.881000e-01 1.033400e+00 2.630000e-01 9.601000e-01 1.715000e-01 1.003700e+00 3.191000e-01 1.146200e+00 -4.830000e-02 5.663000e-01 8.560000e-02 + 2.930000e-01 5.022000e-01 7.152000e-01 3.739000e-01 7.363000e-01 4.900000e-01 1.759500e+00 -3.130000e-02 -6.460000e-02 2.074000e+00 4.020000e-02 -4.750000e-02 3.524000e-01 2.979000e-01 + 4.311000e-01 1.810000e-01 8.205000e-01 9.240000e-02 1.006300e+00 1.160000e-02 1.695800e+00 3.640000e-02 -8.400000e-03 2.010900e+00 -5.130000e-02 6.280000e-02 4.713000e-01 5.760000e-02 + 3.465000e-01 1.600000e-01 6.633000e-01 1.569000e-01 7.640000e-01 1.793000e-01 1.622700e+00 1.281000e-01 1.860000e-02 -2.340000e-02 -2.300000e-03 2.000000e-03 3.697000e-01 1.015000e-01 + 9.872000e-01 -1.126000e-01 1.332000e+00 7.640000e-02 1.548500e+00 5.010000e-02 1.269100e+00 1.510000e-01 1.585700e+00 2.980000e-02 1.368100e+00 3.380000e-02 7.755000e-01 3.590000e-02 + 2.610000e-01 7.540000e-01 6.275000e-01 6.880000e-01 6.739000e-01 7.751000e-01 1.638600e+00 1.087000e-01 5.130000e-02 1.938700e+00 4.940000e-02 -5.850000e-02 3.284000e-01 4.529000e-01 + 2.109000e-01 6.045000e-01 2.995000e-01 9.821000e-01 5.221000e-01 9.051000e-01 3.581000e-01 9.091000e-01 4.162000e-01 1.046800e+00 3.561000e-01 9.154000e-01 1.993000e-01 5.354000e-01 + 5.272000e-01 1.323000e-01 1.047100e+00 -1.195000e-01 1.095700e+00 -3.110000e-02 1.817900e+00 -1.026000e-01 1.931500e+00 7.750000e-02 6.700000e-03 -8.200000e-03 5.379000e-01 1.850000e-02 + 2.795000e-01 5.918000e-01 4.382000e-01 9.261000e-01 5.119000e-01 1.035300e+00 4.693000e-01 8.914000e-01 5.577000e-01 1.006300e+00 3.827000e-01 9.897000e-01 2.396000e-01 5.500000e-01 + 7.918000e-01 -6.760000e-02 9.720000e-01 9.020000e-02 1.069100e+00 1.199000e-01 1.781500e+00 -5.860000e-02 1.962800e+00 4.390000e-02 2.400000e-02 -2.710000e-02 5.785000e-01 4.470000e-02 + 1.210900e+00 -1.261000e-01 5.953000e-01 1.149400e+00 5.685000e-01 9.512000e-01 6.746000e-01 1.050000e+00 5.015000e-01 9.448000e-01 6.418000e-01 1.093700e+00 4.564000e-01 4.655000e-01 + 4.166000e-01 -2.610000e-02 7.158000e-01 -8.600000e-03 8.091000e-01 2.500000e-02 1.774000e+00 -4.670000e-02 -6.200000e-03 6.800000e-03 -6.670000e-02 7.770000e-02 3.702000e-01 3.560000e-02 + 1.066400e+00 4.440000e-02 3.890000e-01 1.361000e+00 4.381000e-01 6.264000e-01 5.386000e-01 1.188200e+00 4.047000e-01 6.327000e-01 3.706000e-01 1.379200e+00 3.626000e-01 4.052000e-01 + 3.560000e-01 5.282000e-01 6.139000e-01 5.976000e-01 7.348000e-01 5.920000e-01 1.600500e+00 1.565000e-01 -3.680000e-02 2.045900e+00 2.100000e-03 -9.000000e-04 3.704000e-01 3.378000e-01 + 2.368000e-01 4.550000e-01 4.686000e-01 5.905000e-01 5.762000e-01 6.224000e-01 4.614000e-01 5.998000e-01 5.424000e-01 6.833000e-01 5.509000e-01 4.959000e-01 2.914000e-01 3.168000e-01 + 1.412000e-01 5.766000e-01 2.682000e-01 7.994000e-01 4.284000e-01 7.535000e-01 2.010000e-02 1.708900e+00 -5.190000e-02 2.063600e+00 -2.990000e-02 3.540000e-02 1.924000e-01 4.226000e-01 + 4.028000e-01 3.229000e-01 5.538000e-01 5.916000e-01 6.593000e-01 6.407000e-01 5.754000e-01 5.726000e-01 6.679000e-01 6.512000e-01 4.964000e-01 6.656000e-01 3.227000e-01 3.403000e-01 + 1.014100e+00 1.139000e-01 6.173000e-01 1.123200e+00 5.625000e-01 5.521000e-01 7.679000e-01 9.384000e-01 5.604000e-01 5.170000e-01 5.513000e-01 1.193400e+00 4.537000e-01 3.371000e-01 + 2.231000e-01 2.160000e-01 3.652000e-01 3.177000e-01 3.538000e-01 4.355000e-01 3.714000e-01 3.082000e-01 3.945000e-01 4.011000e-01 3.866000e-01 2.949000e-01 2.034000e-01 1.923000e-01 + 3.449000e-01 7.698000e-01 2.342000e-01 1.963000e-01 1.374000e-01 2.406000e-01 2.404000e-01 1.906000e-01 1.173000e-01 2.574000e-01 2.592000e-01 1.675000e-01 1.858000e-01 1.419000e-01 + 3.156000e-01 1.832000e-01 3.843000e-01 4.186000e-01 5.150000e-01 3.846000e-01 4.053000e-01 3.939000e-01 4.708000e-01 4.471000e-01 4.462000e-01 3.420000e-01 2.460000e-01 2.126000e-01 + 6.092000e-01 1.414000e-01 8.663000e-01 3.277000e-01 8.921000e-01 4.823000e-01 8.205000e-01 3.809000e-01 1.009700e+00 3.674000e-01 9.123000e-01 2.730000e-01 5.124000e-01 1.745000e-01 + 8.463000e-01 1.475000e-01 1.383300e+00 1.659000e-01 1.409400e+00 3.871000e-01 1.236000e+00 3.424000e-01 1.439400e+00 3.775000e-01 1.240400e+00 3.315000e-01 7.584000e-01 1.434000e-01 + 1.186000e-01 5.166000e-01 2.754000e-01 7.017000e-01 3.716000e-01 7.266000e-01 1.927000e-01 8.017000e-01 2.624000e-01 8.739000e-01 3.585000e-01 6.027000e-01 1.595000e-01 4.034000e-01 +-7.150000e-02 1.194100e+00 1.877000e-01 1.441100e+00 3.201000e-01 1.431200e+00 4.790000e-02 1.673200e+00 -5.740000e-02 2.074100e+00 1.340000e-02 1.714800e+00 9.210000e-02 8.434000e-01 + 1.061600e+00 5.140000e-02 8.015000e-01 9.388000e-01 6.061000e-01 6.110000e-01 9.143000e-01 7.975000e-01 6.790000e-01 4.857000e-01 7.059000e-01 1.043000e+00 5.455000e-01 2.809000e-01 + 7.813000e-01 2.707000e-01 1.166100e+00 1.809000e-01 1.340200e+00 1.161000e-01 1.784300e+00 -6.440000e-02 2.014600e+00 -1.000000e-02 4.940000e-02 -5.860000e-02 6.698000e-01 1.289000e-01 +-2.000000e-02 8.819000e-01 2.728000e-01 8.867000e-01 2.715000e-01 1.033800e+00 1.349000e-01 1.575400e+00 2.450000e-02 1.974500e+00 2.900000e-02 -3.400000e-02 1.103000e-01 5.812000e-01 + 1.124500e+00 -2.180000e-02 4.186000e-01 1.652000e-01 3.845000e-01 9.650000e-02 4.261000e-01 1.539000e-01 3.543000e-01 1.256000e-01 4.996000e-01 6.970000e-02 3.512000e-01 6.390000e-02 +-6.500000e-03 7.993000e-01 3.267000e-01 7.767000e-01 4.522000e-01 7.680000e-01 -2.820000e-02 1.766800e+00 4.970000e-02 1.942100e+00 -5.330000e-02 6.490000e-02 1.645000e-01 4.867000e-01 + 6.900000e-02 7.249000e-01 9.240000e-02 1.152900e+00 4.710000e-02 1.384400e+00 6.110000e-02 1.191900e+00 4.000000e-02 1.400800e+00 1.907000e-01 1.034100e+00 6.170000e-02 6.550000e-01 + 1.428000e-01 5.786000e-01 4.101000e-01 6.276000e-01 4.227000e-01 7.570000e-01 1.796700e+00 -7.820000e-02 1.590000e-02 1.975800e+00 3.080000e-02 -3.800000e-02 2.194000e-01 3.917000e-01 + 4.230000e-01 6.180000e-01 6.667000e-01 9.600000e-01 8.423000e-01 9.991000e-01 6.821000e-01 9.383000e-01 7.610000e-01 1.118300e+00 6.837000e-01 9.371000e-01 3.987000e-01 5.377000e-01 + 1.123200e+00 -1.980000e-02 4.842000e-01 1.256400e+00 4.733000e-01 5.613000e-01 6.713000e-01 1.041900e+00 4.869000e-01 5.181000e-01 6.174000e-01 1.101800e+00 3.947000e-01 3.597000e-01 + 3.235000e-01 4.008000e-01 6.078000e-01 4.450000e-01 7.643000e-01 3.984000e-01 1.743500e+00 -1.240000e-02 7.710000e-02 1.904800e+00 -4.670000e-02 5.660000e-02 3.336000e-01 2.813000e-01 + 8.635000e-01 2.416000e-01 3.268000e-01 7.437000e-01 3.194000e-01 4.958000e-01 4.484000e-01 5.974000e-01 3.312000e-01 4.618000e-01 3.342000e-01 7.332000e-01 2.896000e-01 3.473000e-01 + 3.989000e-01 3.064000e-01 7.602000e-01 3.139000e-01 7.549000e-01 4.913000e-01 6.895000e-01 3.998000e-01 6.527000e-01 6.348000e-01 6.298000e-01 4.730000e-01 3.961000e-01 2.339000e-01 +-5.620000e-02 6.196000e-01 6.960000e-02 8.052000e-01 1.929000e-01 8.025000e-01 2.680000e-02 1.702400e+00 5.080000e-02 1.938200e+00 3.790000e-02 -4.510000e-02 7.300000e-02 4.286000e-01 + 7.670000e-02 5.532000e-01 3.697000e-01 5.783000e-01 5.229000e-01 5.426000e-01 1.730300e+00 2.500000e-03 -5.700000e-03 2.006900e+00 4.780000e-02 -5.690000e-02 2.083000e-01 3.421000e-01 + 1.222200e+00 -1.385000e-01 5.873000e-01 1.141900e+00 4.907000e-01 8.950000e-01 4.724000e-01 1.276000e+00 4.236000e-01 9.068000e-01 5.406000e-01 1.195200e+00 4.016000e-01 4.845000e-01 + 2.563000e-01 6.222000e-01 3.700000e-01 1.011000e+00 4.776000e-01 1.086300e+00 4.206000e-01 9.524000e-01 4.446000e-01 1.143800e+00 3.690000e-01 1.014000e+00 2.464000e-01 5.458000e-01 + 1.840000e-01 6.729000e-01 4.951000e-01 6.792000e-01 5.736000e-01 7.248000e-01 1.651000e+00 9.460000e-02 -1.130000e-02 2.015000e+00 1.500000e-03 -2.300000e-03 2.611000e-01 4.318000e-01 + 1.102100e+00 3.900000e-03 1.025300e+00 7.218000e-01 8.517000e-01 8.106000e-01 1.152900e+00 5.722000e-01 9.200000e-01 6.121000e-01 1.224200e+00 4.924000e-01 6.570000e-01 2.914000e-01 + 4.650000e-01 1.452000e-01 7.837000e-01 1.378000e-01 9.267000e-01 1.065000e-01 1.763500e+00 -3.920000e-02 1.258000e-01 1.849400e+00 -1.710000e-02 2.010000e-02 4.704000e-01 5.990000e-02 + 1.125400e+00 -1.860000e-02 4.118000e-01 1.341000e+00 5.358000e-01 4.982000e-01 5.828000e-01 1.137600e+00 4.601000e-01 5.488000e-01 5.933000e-01 1.129500e+00 3.844000e-01 3.734000e-01 + 1.075400e+00 3.380000e-02 7.599000e-01 9.807000e-01 6.185000e-01 7.503000e-01 8.009000e-01 9.346000e-01 6.739000e-01 6.285000e-01 8.539000e-01 8.723000e-01 5.453000e-01 3.332000e-01 + 3.525000e-01 3.345000e-01 5.116000e-01 5.722000e-01 5.640000e-01 6.692000e-01 4.808000e-01 6.056000e-01 5.902000e-01 6.561000e-01 5.130000e-01 5.635000e-01 3.051000e-01 3.173000e-01 + 5.190000e-01 5.924000e-01 8.648000e-01 7.441000e-01 1.038200e+00 6.802000e-01 1.758000e+00 -3.610000e-02 1.986400e+00 1.070000e-02 -4.070000e-02 1.777700e+00 5.055000e-01 4.177000e-01 + 5.158000e-01 1.822000e-01 8.558000e-01 1.480000e-01 9.620000e-01 1.680000e-01 1.790400e+00 -7.240000e-02 -3.050000e-02 2.032800e+00 7.580000e-02 -9.170000e-02 4.772000e-01 1.143000e-01 + 5.610000e-01 2.925000e-01 7.260000e-01 4.631000e-01 9.444000e-01 3.525000e-01 1.723300e+00 7.400000e-03 3.060000e-02 1.958500e+00 3.750000e-02 -4.560000e-02 4.457000e-01 2.517000e-01 + 5.050000e-01 6.070000e-01 8.627000e-01 8.609000e-01 8.843000e-01 1.080400e+00 1.662900e+00 7.810000e-02 9.900000e-02 1.886300e+00 -5.900000e-02 1.799200e+00 4.807000e-01 5.152000e-01 + 4.083000e-01 6.277000e-01 5.078000e-01 1.137300e+00 5.682000e-01 1.305400e+00 6.229000e-01 1.002100e+00 6.980000e-01 1.180200e+00 6.487000e-01 9.710000e-01 3.243000e-01 6.188000e-01 + 5.610000e-01 1.063000e-01 7.032000e-01 3.773000e-01 7.404000e-01 4.993000e-01 7.263000e-01 3.493000e-01 8.452000e-01 3.924000e-01 6.268000e-01 4.698000e-01 4.199000e-01 2.013000e-01 +-4.600000e-03 4.540000e-01 1.915000e-01 5.706000e-01 3.021000e-01 5.829000e-01 -6.530000e-02 1.807800e+00 -4.130000e-02 4.920000e-02 9.000000e-03 -1.080000e-02 1.095000e-01 3.238000e-01 + 4.558000e-01 6.417000e-01 7.385000e-01 7.060000e-01 9.107000e-01 6.386000e-01 1.640500e+00 1.095000e-01 -1.430000e-02 2.021500e+00 -6.380000e-02 1.806700e+00 4.203000e-01 4.233000e-01 + 3.749000e-01 3.478000e-01 6.614000e-01 3.763000e-01 8.617000e-01 2.874000e-01 1.680900e+00 6.080000e-02 5.110000e-02 1.937900e+00 -4.430000e-02 5.130000e-02 3.498000e-01 2.637000e-01 + 1.172000e+00 -7.900000e-02 5.569000e-01 2.045000e-01 5.080000e-01 1.028000e-01 5.461000e-01 2.194000e-01 4.582000e-01 1.502000e-01 6.272000e-01 1.214000e-01 4.147000e-01 9.840000e-02 + 1.114400e+00 -1.480000e-02 6.321000e-01 8.080000e-02 5.397000e-01 4.140000e-02 5.880000e-01 1.360000e-01 4.192000e-01 1.711000e-01 5.523000e-01 1.767000e-01 4.362000e-01 5.740000e-02 + 4.172000e-01 4.878000e-01 7.567000e-01 4.570000e-01 8.219000e-01 5.137000e-01 1.698300e+00 4.280000e-02 -2.360000e-02 2.029800e+00 -3.000000e-04 2.000000e-03 4.055000e-01 3.136000e-01 + 3.340000e-02 9.060000e-02 1.528000e-01 2.821000e-01 2.778000e-01 2.702000e-01 3.840000e-02 1.689500e+00 8.330000e-02 -1.008000e-01 7.790000e-02 -9.420000e-02 6.010000e-02 1.798000e-01 + 1.125000e+00 -1.910000e-02 6.254000e-01 1.100600e+00 3.906000e-01 8.229000e-01 6.111000e-01 1.120500e+00 4.252000e-01 7.403000e-01 6.368000e-01 1.083800e+00 4.206000e-01 4.015000e-01 + 6.356000e-01 4.044000e-01 8.279000e-01 8.315000e-01 9.620000e-01 9.261000e-01 8.210000e-01 8.397000e-01 1.001200e+00 9.082000e-01 8.058000e-01 8.573000e-01 4.849000e-01 4.709000e-01 + 1.061500e+00 5.180000e-02 7.158000e-01 1.021300e+00 6.573000e-01 4.111000e-01 7.008000e-01 1.036400e+00 5.019000e-01 5.624000e-01 7.263000e-01 1.009700e+00 4.987000e-01 2.754000e-01 + 1.755000e-01 3.300000e-01 3.664000e-01 4.088000e-01 3.820000e-01 5.115000e-01 3.903000e-01 3.836000e-01 4.519000e-01 4.405000e-01 2.869000e-01 5.086000e-01 1.966000e-01 2.555000e-01 + 2.700000e-02 1.430000e-02 3.190000e-02 3.198000e-01 3.130000e-02 4.564000e-01 -1.280000e-02 1.590000e-02 -6.610000e-02 7.780000e-02 -8.400000e-03 8.100000e-03 1.590000e-02 1.756000e-01 + 4.663000e-01 -4.600000e-03 7.408000e-01 4.170000e-02 9.221000e-01 -2.920000e-02 1.735200e+00 -1.600000e-03 7.200000e-02 -8.600000e-02 3.510000e-02 -4.170000e-02 4.131000e-01 3.500000e-02 + 3.971000e-01 1.871000e-01 6.050000e-01 3.133000e-01 6.799000e-01 3.637000e-01 5.564000e-01 3.703000e-01 7.048000e-01 3.569000e-01 5.925000e-01 3.298000e-01 3.250000e-01 2.095000e-01 + 1.881000e-01 2.852000e-01 4.351000e-01 3.640000e-01 5.826000e-01 3.299000e-01 1.650900e+00 9.930000e-02 -7.760000e-02 9.370000e-02 3.610000e-02 -4.430000e-02 2.443000e-01 2.118000e-01 + 7.071000e-01 4.006000e-01 3.163000e-01 1.421300e+00 2.512000e-01 1.208400e+00 3.451000e-01 1.385900e+00 3.214000e-01 1.050100e+00 3.445000e-01 1.383400e+00 2.696000e-01 6.342000e-01 + 6.183000e-01 4.862000e-01 8.977000e-01 8.166000e-01 1.012400e+00 8.138000e-01 1.728200e+00 4.700000e-03 2.084100e+00 -9.930000e-02 -2.800000e-03 1.733200e+00 5.319000e-01 4.307000e-01 + 7.480000e-01 2.723000e-01 1.088900e+00 5.234000e-01 1.335700e+00 4.807000e-01 1.197500e+00 3.954000e-01 1.335600e+00 5.135000e-01 1.092100e+00 5.162000e-01 6.428000e-01 2.840000e-01 + 1.220000e-02 4.463000e-01 3.817000e-01 3.752000e-01 4.660000e-01 4.161000e-01 -3.900000e-02 1.775400e+00 2.450000e-02 -2.800000e-02 -2.000000e-02 2.500000e-02 1.909000e-01 2.436000e-01 + 4.191000e-01 7.018000e-01 7.914000e-01 6.675000e-01 1.036300e+00 5.181000e-01 1.751200e+00 -2.240000e-02 -1.037000e-01 2.116700e+00 0.000000e+00 1.728200e+00 4.685000e-01 3.836000e-01 + 5.673000e-01 5.564000e-01 3.051000e-01 8.024000e-01 2.870000e-01 5.490000e-01 3.614000e-01 7.325000e-01 2.592000e-01 5.638000e-01 3.773000e-01 7.175000e-01 2.555000e-01 3.958000e-01 + 5.016000e-01 1.766000e-01 7.704000e-01 2.323000e-01 9.845000e-01 1.148000e-01 1.770000e+00 -4.560000e-02 -1.452000e-01 2.174300e+00 2.120000e-02 -2.570000e-02 4.619000e-01 1.186000e-01 + 3.840000e-02 4.844000e-01 2.104000e-01 6.335000e-01 3.165000e-01 6.526000e-01 7.100000e-03 1.721900e+00 2.400000e-02 -3.070000e-02 -3.000000e-04 -1.300000e-03 1.120000e-01 3.751000e-01 + 6.022000e-01 4.851000e-01 9.271000e-01 4.647000e-01 1.181800e+00 3.103000e-01 1.728700e+00 6.000000e-04 2.143800e+00 -1.692000e-01 2.100000e-02 1.706200e+00 6.078000e-01 2.020000e-01 + 4.198000e-01 1.360000e-01 7.111000e-01 1.498000e-01 7.691000e-01 2.240000e-01 7.126000e-01 1.501000e-01 8.195000e-01 1.747000e-01 7.925000e-01 5.360000e-02 4.255000e-01 6.960000e-02 + 4.734000e-01 3.226000e-01 7.321000e-01 5.214000e-01 6.302000e-01 8.344000e-01 7.167000e-01 5.399000e-01 8.025000e-01 6.486000e-01 7.317000e-01 5.186000e-01 3.959000e-01 3.324000e-01 + 1.976000e-01 2.874000e-01 5.407000e-01 2.456000e-01 6.124000e-01 3.106000e-01 1.765600e+00 -4.640000e-02 -8.500000e-03 1.090000e-02 -3.050000e-02 3.470000e-02 2.834000e-01 1.711000e-01 + 2.472000e-01 3.830000e-01 4.762000e-01 4.897000e-01 5.658000e-01 5.331000e-01 4.445000e-01 5.271000e-01 4.262000e-01 7.066000e-01 5.670000e-01 3.801000e-01 2.861000e-01 2.700000e-01 + 5.640000e-02 8.770000e-01 4.094000e-01 8.304000e-01 4.598000e-01 9.081000e-01 1.685500e+00 5.740000e-02 -1.160000e-02 2.014600e+00 3.930000e-02 -4.600000e-02 2.180000e-01 5.141000e-01 + 5.933000e-01 5.124000e-01 3.009000e-01 4.766000e-01 2.757000e-01 3.484000e-01 2.430000e-01 5.403000e-01 2.414000e-01 3.768000e-01 3.809000e-01 3.783000e-01 2.467000e-01 2.736000e-01 + 4.738000e-01 1.604000e-01 6.897000e-01 3.143000e-01 7.454000e-01 4.065000e-01 7.160000e-01 2.829000e-01 7.983000e-01 3.611000e-01 6.483000e-01 3.632000e-01 4.115000e-01 1.662000e-01 + 3.263000e-01 2.228000e-01 5.195000e-01 3.371000e-01 6.499000e-01 3.145000e-01 6.013000e-01 2.391000e-01 5.882000e-01 3.997000e-01 4.504000e-01 4.183000e-01 2.802000e-01 2.177000e-01 + 2.952000e-01 3.011000e-01 3.907000e-01 5.610000e-01 5.623000e-01 4.951000e-01 3.930000e-01 5.548000e-01 5.071000e-01 5.777000e-01 3.762000e-01 5.773000e-01 2.368000e-01 3.101000e-01 + 8.571000e-01 2.373000e-01 3.476000e-01 9.596000e-01 2.949000e-01 6.430000e-01 3.605000e-01 9.372000e-01 2.396000e-01 6.787000e-01 4.688000e-01 8.115000e-01 2.814000e-01 4.219000e-01 +-8.300000e-02 7.175000e-01 2.840000e-02 8.961000e-01 4.980000e-02 1.014500e+00 -5.960000e-02 1.804600e+00 -2.070000e-02 2.025200e+00 1.300000e-03 -1.400000e-03 2.410000e-02 5.176000e-01 + 4.093000e-01 4.796000e-01 7.052000e-01 4.992000e-01 8.137000e-01 5.116000e-01 1.559200e+00 2.063000e-01 -9.400000e-03 2.012900e+00 -4.380000e-02 5.310000e-02 3.694000e-01 3.477000e-01 +-4.070000e-02 3.429000e-01 1.610000e-01 4.522000e-01 3.372000e-01 3.839000e-01 -1.471000e-01 1.905600e+00 2.170000e-02 -2.650000e-02 1.330000e-02 -1.820000e-02 1.011000e-01 2.359000e-01 + 4.655000e-01 3.234000e-01 6.666000e-01 5.896000e-01 7.814000e-01 6.414000e-01 7.481000e-01 4.871000e-01 8.391000e-01 5.939000e-01 7.590000e-01 4.738000e-01 4.179000e-01 2.995000e-01 + 3.631000e-01 3.763000e-01 5.393000e-01 6.265000e-01 6.059000e-01 7.227000e-01 5.750000e-01 5.816000e-01 5.516000e-01 8.031000e-01 5.457000e-01 6.142000e-01 3.099000e-01 3.612000e-01 + 4.637000e-01 4.033000e-01 7.539000e-01 5.969000e-01 7.829000e-01 7.668000e-01 6.929000e-01 6.707000e-01 9.046000e-01 6.521000e-01 7.372000e-01 6.145000e-01 4.322000e-01 3.487000e-01 + 1.090400e+00 1.660000e-02 6.835000e-01 4.016000e-01 5.539000e-01 2.774000e-01 6.936000e-01 3.948000e-01 5.649000e-01 2.433000e-01 7.215000e-01 3.609000e-01 4.784000e-01 1.675000e-01 + 1.758000e-01 4.663000e-01 4.616000e-01 5.046000e-01 5.843000e-01 4.959000e-01 1.676900e+00 5.550000e-02 1.330000e-02 1.984500e+00 2.270000e-02 -2.750000e-02 2.619000e-01 2.967000e-01 + 1.214600e+00 -1.319000e-01 5.935000e-01 5.697000e-01 5.128000e-01 3.593000e-01 5.345000e-01 6.402000e-01 4.997000e-01 3.476000e-01 5.162000e-01 6.670000e-01 4.160000e-01 2.552000e-01 + 1.620000e-02 -1.880000e-02 2.976000e-01 1.450000e-02 4.584000e-01 -3.510000e-02 -5.870000e-02 6.980000e-02 9.900000e-03 -9.800000e-03 -5.190000e-02 6.120000e-02 1.608000e-01 6.200000e-03 + 9.487000e-01 1.556000e-01 4.250000e-01 9.013000e-01 2.160000e-01 7.500000e-01 5.109000e-01 8.025000e-01 3.473000e-01 5.742000e-01 4.527000e-01 8.721000e-01 2.995000e-01 4.109000e-01 + 4.073000e-01 6.996000e-01 6.978000e-01 8.732000e-01 8.389000e-01 8.391000e-01 1.652000e+00 9.270000e-02 -9.850000e-02 2.120300e+00 4.640000e-02 1.671500e+00 4.081000e-01 4.970000e-01 + 1.179000e-01 3.772000e-01 1.543000e-01 6.233000e-01 2.983000e-01 5.636000e-01 2.670000e-01 4.901000e-01 2.800000e-01 6.010000e-01 1.791000e-01 5.917000e-01 1.150000e-01 3.288000e-01 + 3.808000e-01 3.059000e-01 7.125000e-01 2.866000e-01 8.360000e-01 2.762000e-01 1.762600e+00 -4.020000e-02 -1.060000e-02 2.014400e+00 -2.190000e-02 2.570000e-02 3.894000e-01 1.950000e-01 + 1.100700e+00 6.400000e-03 6.214000e-01 1.107200e+00 5.279000e-01 5.033000e-01 6.244000e-01 1.102300e+00 4.872000e-01 5.243000e-01 4.857000e-01 1.269100e+00 4.358000e-01 3.183000e-01 + 4.452000e-01 4.062000e-01 7.953000e-01 5.247000e-01 7.304000e-01 8.046000e-01 7.100000e-01 6.177000e-01 8.594000e-01 6.745000e-01 7.237000e-01 6.055000e-01 4.072000e-01 3.626000e-01 +-3.360000e-02 7.815000e-01 3.460000e-01 6.911000e-01 3.249000e-01 8.536000e-01 -7.280000e-02 1.825300e+00 1.340000e-02 1.986900e+00 1.620000e-02 -2.070000e-02 1.240000e-01 4.968000e-01 + 4.713000e-01 4.386000e-01 7.293000e-01 6.936000e-01 7.792000e-01 8.545000e-01 6.979000e-01 7.343000e-01 8.918000e-01 7.419000e-01 6.975000e-01 7.380000e-01 4.071000e-01 4.185000e-01 + 4.175000e-01 3.384000e-01 7.303000e-01 4.443000e-01 7.458000e-01 6.023000e-01 6.859000e-01 4.919000e-01 8.668000e-01 4.794000e-01 7.331000e-01 4.360000e-01 4.239000e-01 2.520000e-01 + 5.340000e-02 9.828000e-01 2.123000e-01 1.128600e+00 2.351000e-01 1.247100e+00 -7.900000e-02 1.824500e+00 -3.620000e-02 2.039300e+00 -1.320000e-02 1.450000e-02 8.350000e-02 7.143000e-01 + 3.388000e-01 3.895000e-01 4.675000e-01 6.855000e-01 4.922000e-01 8.274000e-01 5.061000e-01 6.373000e-01 6.460000e-01 6.687000e-01 4.679000e-01 6.876000e-01 2.774000e-01 3.879000e-01 + 3.247000e-01 5.262000e-01 6.185000e-01 5.524000e-01 7.928000e-01 4.811000e-01 1.705800e+00 3.450000e-02 -1.000000e-04 1.999800e+00 -2.430000e-02 2.690000e-02 3.642000e-01 3.209000e-01 + 4.667000e-01 2.944000e-01 8.114000e-01 3.655000e-01 8.536000e-01 4.975000e-01 7.835000e-01 3.983000e-01 9.240000e-01 4.428000e-01 7.512000e-01 4.366000e-01 4.402000e-01 2.459000e-01 + 1.345000e-01 6.435000e-01 3.909000e-01 7.931000e-01 5.475000e-01 7.823000e-01 3.952000e-01 7.846000e-01 3.832000e-01 9.931000e-01 3.469000e-01 8.406000e-01 2.234000e-01 4.592000e-01 + 1.940000e-01 4.096000e-01 2.225000e-01 7.397000e-01 2.921000e-01 7.992000e-01 3.331000e-01 6.114000e-01 3.428000e-01 7.537000e-01 2.425000e-01 7.173000e-01 1.560000e-01 3.954000e-01 + 5.959000e-01 1.939000e-01 9.863000e-01 2.432000e-01 1.230900e+00 1.453000e-01 1.001400e+00 2.181000e-01 1.243900e+00 1.572000e-01 9.501000e-01 2.823000e-01 5.876000e-01 1.168000e-01 + 3.336000e-01 3.599000e-01 5.074000e-01 5.793000e-01 6.561000e-01 5.657000e-01 6.266000e-01 4.343000e-01 6.604000e-01 5.796000e-01 6.106000e-01 4.549000e-01 3.467000e-01 2.701000e-01 + 4.059000e-01 3.798000e-01 7.394000e-01 3.555000e-01 8.615000e-01 3.500000e-01 1.760300e+00 -3.010000e-02 -1.800000e-02 2.019900e+00 -1.730000e-02 2.140000e-02 4.065000e-01 2.399000e-01 + 3.310000e-01 6.894000e-01 7.105000e-01 6.104000e-01 8.226000e-01 6.172000e-01 1.725000e+00 9.400000e-03 1.055000e-01 1.875200e+00 3.200000e-03 -3.000000e-03 4.023000e-01 3.763000e-01 + 7.199000e-01 3.784000e-01 2.954000e-01 7.870000e-01 2.970000e-01 5.207000e-01 2.539000e-01 8.341000e-01 2.433000e-01 5.660000e-01 3.386000e-01 7.365000e-01 2.611000e-01 3.805000e-01 + 3.351000e-01 4.388000e-01 4.206000e-01 8.098000e-01 5.920000e-01 7.924000e-01 4.709000e-01 7.497000e-01 6.010000e-01 8.065000e-01 4.283000e-01 8.027000e-01 2.862000e-01 4.179000e-01 + 2.682000e-01 2.294000e-01 5.970000e-01 1.434000e-01 7.460000e-01 8.930000e-02 5.686000e-01 1.835000e-01 6.593000e-01 2.042000e-01 6.265000e-01 1.115000e-01 3.488000e-01 8.000000e-02 + 1.968000e-01 5.983000e-01 4.334000e-01 6.884000e-01 6.950000e-01 5.214000e-01 1.668500e+00 7.710000e-02 -1.500000e-02 2.017100e+00 -9.400000e-03 1.200000e-02 2.899000e-01 3.599000e-01 + 2.743000e-01 2.707000e-01 5.182000e-01 3.226000e-01 6.576000e-01 2.817000e-01 4.334000e-01 4.222000e-01 5.786000e-01 3.969000e-01 5.792000e-01 2.490000e-01 3.026000e-01 1.813000e-01 + 1.044000e-01 8.482000e-01 2.786000e-01 1.196100e+00 3.187000e-01 1.361100e+00 2.881000e-01 1.186100e+00 2.551000e-01 1.462000e+00 2.166000e-01 1.271600e+00 1.215000e-01 7.374000e-01 + 7.940000e-02 7.657000e-01 3.944000e-01 7.673000e-01 4.722000e-01 8.130000e-01 1.623500e+00 1.296000e-01 1.566000e-01 1.815600e+00 8.910000e-02 -1.058000e-01 2.025000e-01 4.817000e-01 + 5.802000e-01 5.219000e-01 2.736000e-01 3.596000e-01 3.377000e-01 1.663000e-01 3.450000e-01 2.768000e-01 2.607000e-01 2.504000e-01 2.996000e-01 3.324000e-01 2.310000e-01 2.168000e-01 + 3.435000e-01 7.501000e-01 1.504000e-01 4.932000e-01 6.410000e-02 4.826000e-01 2.269000e-01 4.038000e-01 1.557000e-01 3.627000e-01 6.050000e-02 6.040000e-01 1.334000e-01 3.200000e-01 + 2.296000e-01 8.800000e-02 3.187000e-01 1.855000e-01 3.660000e-01 2.098000e-01 2.541000e-01 2.652000e-01 3.659000e-01 2.184000e-01 1.360000e-01 4.045000e-01 1.647000e-01 1.304000e-01 + 3.472000e-01 5.533000e-01 7.213000e-01 6.622000e-01 7.495000e-01 8.366000e-01 6.911000e-01 7.010000e-01 7.428000e-01 8.703000e-01 6.704000e-01 7.270000e-01 3.956000e-01 4.075000e-01 + 1.127300e+00 -2.910000e-02 5.484000e-01 3.926000e-01 4.000000e-01 3.540000e-01 4.938000e-01 4.571000e-01 3.574000e-01 3.831000e-01 3.600000e-01 6.188000e-01 3.658000e-01 2.323000e-01 + 3.116000e-01 2.006000e-01 6.378000e-01 1.844000e-01 8.251000e-01 1.021000e-01 1.731600e+00 -2.000000e-03 -5.400000e-03 5.500000e-03 6.160000e-02 -7.480000e-02 3.810000e-01 8.600000e-02 + 5.224000e-01 9.700000e-03 7.253000e-01 1.418000e-01 8.516000e-01 1.318000e-01 1.666600e+00 7.340000e-02 1.910000e-02 -1.910000e-02 -1.140000e-02 1.270000e-02 4.417000e-01 5.310000e-02 + 5.163000e-01 2.570000e-01 7.918000e-01 4.214000e-01 9.877000e-01 3.764000e-01 8.280000e-01 3.769000e-01 9.526000e-01 4.407000e-01 8.246000e-01 3.858000e-01 4.710000e-01 2.260000e-01 + 5.571000e-01 5.544000e-01 2.239000e-01 9.428000e-01 2.880000e-01 5.743000e-01 3.653000e-01 7.806000e-01 2.938000e-01 5.459000e-01 3.241000e-01 8.265000e-01 2.381000e-01 4.292000e-01 + 2.273000e-01 4.148000e-01 3.511000e-01 6.521000e-01 3.822000e-01 7.671000e-01 4.646000e-01 5.182000e-01 4.037000e-01 7.567000e-01 3.498000e-01 6.586000e-01 2.074000e-01 3.724000e-01 + 4.839000e-01 4.570000e-02 1.840000e-01 1.079000e-01 2.019000e-01 4.250000e-02 3.259000e-01 -6.090000e-02 2.704000e-01 -4.380000e-02 2.482000e-01 3.690000e-02 2.045000e-01 1.880000e-02 + 1.147700e+00 -5.040000e-02 1.158300e+00 1.214000e-01 9.293000e-01 -1.200000e-02 1.340100e+00 -9.310000e-02 8.479000e-01 5.720000e-02 1.220500e+00 5.150000e-02 6.836000e-01 1.350000e-02 + 6.989000e-01 3.994000e-01 1.083100e+00 3.312000e-01 1.074000e+00 4.911000e-01 1.831800e+00 -1.164000e-01 2.058700e+00 -6.400000e-02 3.980000e-02 1.682900e+00 5.699000e-01 2.715000e-01 + 3.769000e-01 4.515000e-01 1.789000e-01 1.933000e-01 1.166000e-01 2.087000e-01 1.752000e-01 1.987000e-01 1.617000e-01 1.490000e-01 1.450000e-01 2.362000e-01 1.476000e-01 1.407000e-01 + 4.429000e-01 2.770000e-01 6.559000e-01 4.770000e-01 7.533000e-01 5.333000e-01 6.636000e-01 4.679000e-01 9.272000e-01 3.484000e-01 6.942000e-01 4.315000e-01 4.358000e-01 2.080000e-01 + 2.678000e-01 5.396000e-01 5.706000e-01 5.481000e-01 7.094000e-01 5.272000e-01 1.753200e+00 -2.780000e-02 1.525000e-01 1.819500e+00 -8.100000e-03 9.300000e-03 3.374000e-01 3.192000e-01 + 5.565000e-01 2.985000e-01 9.732000e-01 1.751000e-01 1.013600e+00 2.741000e-01 1.663900e+00 7.960000e-02 1.972300e+00 3.250000e-02 -3.070000e-02 3.620000e-02 5.264000e-01 1.595000e-01 + 7.532000e-01 3.612000e-01 3.635000e-01 1.018900e+00 3.685000e-01 5.868000e-01 3.342000e-01 1.066600e+00 2.891000e-01 6.552000e-01 3.743000e-01 1.018400e+00 2.811000e-01 4.401000e-01 + 1.108600e+00 -5.300000e-03 4.635000e-01 1.283100e+00 4.084000e-01 1.442300e+00 4.793000e-01 1.265200e+00 4.490000e-01 1.193800e+00 6.262000e-01 1.087300e+00 3.890000e-01 5.791000e-01 + 4.608000e-01 2.314000e-01 8.328000e-01 2.275000e-01 8.639000e-01 3.633000e-01 8.046000e-01 2.602000e-01 8.485000e-01 3.997000e-01 7.342000e-01 3.473000e-01 4.069000e-01 2.202000e-01 + 4.410000e-02 5.566000e-01 2.636000e-01 6.394000e-01 3.755000e-01 6.360000e-01 2.398000e-01 6.638000e-01 2.537000e-01 7.974000e-01 2.936000e-01 6.030000e-01 1.496000e-01 3.718000e-01 + 5.005000e-01 3.034000e-01 7.548000e-01 5.121000e-01 1.041600e+00 3.657000e-01 8.597000e-01 3.826000e-01 1.028000e+00 4.066000e-01 7.794000e-01 4.771000e-01 4.800000e-01 2.417000e-01 + 3.198000e-01 7.901000e-01 1.934000e-01 7.084000e-01 1.126000e-01 6.119000e-01 1.950000e-01 7.055000e-01 1.826000e-01 5.124000e-01 2.665000e-01 6.217000e-01 1.639000e-01 4.130000e-01 + 5.197000e-01 2.097000e-01 8.513000e-01 1.889000e-01 9.604000e-01 2.034000e-01 1.725800e+00 3.100000e-03 -2.900000e-02 2.034500e+00 -4.180000e-02 4.840000e-02 4.906000e-01 1.207000e-01 + 2.273000e-01 7.301000e-01 3.601000e-01 1.140200e+00 3.853000e-01 1.328900e+00 3.072000e-01 1.205700e+00 4.551000e-01 1.270000e+00 3.043000e-01 1.206400e+00 2.025000e-01 6.645000e-01 +-4.580000e-02 5.081000e-01 -4.810000e-02 8.260000e-01 -1.860000e-02 9.120000e-01 4.690000e-02 1.675900e+00 -5.220000e-02 6.230000e-02 -4.100000e-03 6.400000e-03 2.600000e-03 4.344000e-01 + 1.116200e+00 -1.530000e-02 5.599000e-01 5.398000e-01 3.945000e-01 4.545000e-01 5.338000e-01 5.660000e-01 4.524000e-01 3.615000e-01 5.239000e-01 5.846000e-01 3.863000e-01 2.650000e-01 + 6.918000e-01 3.518000e-01 1.041500e+00 3.028000e-01 1.083500e+00 3.950000e-01 1.622800e+00 1.234000e-01 2.023100e+00 -2.900000e-02 6.140000e-02 -7.670000e-02 6.017000e-01 1.932000e-01 + 7.017000e-01 3.974000e-01 1.022200e+00 5.651000e-01 1.199500e+00 5.025000e-01 1.701000e+00 3.670000e-02 2.096700e+00 -1.106000e-01 6.400000e-03 1.724200e+00 5.794000e-01 3.400000e-01 + 3.025000e-01 4.500000e-01 6.046000e-01 4.666000e-01 7.225000e-01 4.639000e-01 1.802600e+00 -8.280000e-02 -4.240000e-02 2.049500e+00 -3.420000e-02 4.300000e-02 3.911000e-01 2.265000e-01 + 2.405000e-01 4.124000e-01 3.764000e-01 6.427000e-01 4.999000e-01 6.475000e-01 4.117000e-01 5.978000e-01 4.728000e-01 6.944000e-01 4.349000e-01 5.751000e-01 2.347000e-01 3.495000e-01 + 3.042000e-01 4.975000e-01 5.813000e-01 5.410000e-01 5.880000e-01 6.772000e-01 1.773000e+00 -4.970000e-02 4.010000e-02 1.949800e+00 5.640000e-02 -6.680000e-02 3.048000e-01 3.605000e-01 + 5.140000e-02 6.044000e-01 1.610000e-01 8.481000e-01 1.999000e-01 9.469000e-01 1.133000e-01 9.042000e-01 1.455000e-01 1.025900e+00 1.506000e-01 8.629000e-01 8.570000e-02 4.987000e-01 + 7.159000e-01 3.887000e-01 3.221000e-01 2.428000e-01 3.203000e-01 1.404000e-01 3.725000e-01 1.797000e-01 2.981000e-01 1.584000e-01 2.153000e-01 3.692000e-01 2.653000e-01 1.412000e-01 + 1.038500e+00 8.070000e-02 6.368000e-01 2.733000e-01 5.443000e-01 1.728000e-01 5.774000e-01 3.499000e-01 5.029000e-01 2.067000e-01 6.043000e-01 3.120000e-01 4.286000e-01 1.561000e-01 + 3.442000e-01 6.581000e-01 6.663000e-01 8.721000e-01 5.898000e-01 1.199100e+00 6.507000e-01 8.913000e-01 5.997000e-01 1.210200e+00 5.873000e-01 9.664000e-01 3.414000e-01 5.564000e-01 + 4.650000e-02 1.068000e+00 1.044000e-01 4.719000e-01 1.267000e-01 3.475000e-01 2.050000e-02 5.726000e-01 7.500000e-03 4.792000e-01 5.820000e-02 5.241000e-01 4.720000e-02 3.740000e-01 + 5.320000e-02 3.194000e-01 1.602000e-01 4.124000e-01 1.449000e-01 5.132000e-01 1.762000e-01 3.928000e-01 2.111000e-01 4.435000e-01 1.781000e-01 3.896000e-01 1.014000e-01 2.271000e-01 + 1.134400e+00 -3.280000e-02 3.935000e-01 4.392000e-01 3.808000e-01 2.835000e-01 4.294000e-01 3.987000e-01 3.470000e-01 3.056000e-01 3.493000e-01 4.929000e-01 3.117000e-01 2.354000e-01 + 3.821000e-01 6.817000e-01 6.292000e-01 7.613000e-01 8.703000e-01 6.141000e-01 1.673400e+00 6.200000e-02 5.200000e-03 1.990900e+00 -8.500000e-02 1.832600e+00 4.190000e-01 3.872000e-01 + 3.013000e-01 6.641000e-01 5.217000e-01 9.826000e-01 5.665000e-01 1.154200e+00 6.320000e-01 8.539000e-01 7.991000e-01 9.070000e-01 5.886000e-01 9.031000e-01 3.344000e-01 5.291000e-01 + 1.506000e-01 7.948000e-01 1.765000e-01 1.315600e+00 2.341000e-01 1.461900e+00 2.085000e-01 1.275200e+00 2.402000e-01 1.474500e+00 1.973000e-01 1.294000e+00 1.203000e-01 7.379000e-01 + 3.685000e-01 6.324000e-01 7.840000e-01 5.156000e-01 8.288000e-01 6.059000e-01 1.723100e+00 1.340000e-02 -3.700000e-02 2.044700e+00 -1.219000e-01 1.448000e-01 4.090000e-01 3.624000e-01 + 4.675000e-01 3.226000e-01 7.764000e-01 4.498000e-01 8.159000e-01 5.908000e-01 6.720000e-01 5.726000e-01 8.785000e-01 5.377000e-01 5.608000e-01 7.012000e-01 4.061000e-01 3.096000e-01 + 1.032100e+00 8.920000e-02 7.001000e-01 1.020800e+00 4.296000e-01 1.249300e+00 5.637000e-01 1.175500e+00 5.356000e-01 1.006600e+00 5.957000e-01 1.141200e+00 4.473000e-01 5.009000e-01 + 3.537000e-01 3.750000e-01 5.846000e-01 5.609000e-01 6.021000e-01 7.155000e-01 5.300000e-01 6.204000e-01 6.224000e-01 7.085000e-01 5.285000e-01 6.264000e-01 3.126000e-01 3.522000e-01 + 1.028400e+00 5.230000e-02 3.442000e-01 1.396500e+00 2.839000e-01 1.006800e+00 3.878000e-01 1.345500e+00 4.144000e-01 8.010000e-01 3.794000e-01 1.354100e+00 2.956000e-01 5.589000e-01 + 1.255500e+00 -1.783000e-01 9.172000e-01 1.170000e-02 6.671000e-01 7.930000e-02 8.712000e-01 6.710000e-02 6.812000e-01 4.210000e-02 9.521000e-01 -2.890000e-02 5.712000e-01 2.040000e-02 + 3.895000e-01 2.033000e-01 5.606000e-01 3.758000e-01 7.402000e-01 3.101000e-01 5.242000e-01 4.181000e-01 7.393000e-01 3.271000e-01 5.231000e-01 4.183000e-01 3.386000e-01 1.988000e-01 + 3.526000e-01 7.464000e-01 6.101000e-01 9.113000e-01 7.630000e-01 8.749000e-01 1.689900e+00 4.920000e-02 1.480000e-02 1.983400e+00 -1.200000e-02 1.745600e+00 3.587000e-01 5.248000e-01 +-1.300000e-03 6.041000e-01 4.347000e-01 4.496000e-01 4.004000e-01 6.332000e-01 2.100000e-03 1.727800e+00 -8.340000e-02 2.097300e+00 1.580000e-02 -1.760000e-02 1.446000e-01 3.869000e-01 + 3.791000e-01 2.530000e-01 8.544000e-01 6.310000e-02 8.729000e-01 1.838000e-01 1.780600e+00 -5.310000e-02 7.710000e-02 1.912700e+00 -3.610000e-02 4.320000e-02 4.118000e-01 1.349000e-01 + 1.102700e+00 1.900000e-03 4.964000e-01 5.009000e-01 3.975000e-01 3.810000e-01 5.711000e-01 4.126000e-01 3.578000e-01 4.157000e-01 6.194000e-01 3.533000e-01 3.989000e-01 2.126000e-01 + 3.882000e-01 4.172000e-01 6.311000e-01 6.264000e-01 8.014000e-01 6.151000e-01 7.290000e-01 5.097000e-01 8.256000e-01 6.037000e-01 7.664000e-01 4.626000e-01 4.112000e-01 3.055000e-01 + 1.168300e+00 -7.610000e-02 5.486000e-01 1.192800e+00 5.699000e-01 4.532000e-01 5.990000e-01 1.130500e+00 4.773000e-01 5.330000e-01 5.430000e-01 1.196900e+00 4.272000e-01 3.263000e-01 + 1.946000e-01 8.463000e-01 3.935000e-01 9.886000e-01 5.201000e-01 9.817000e-01 1.797000e+00 -7.150000e-02 -9.460000e-02 2.110900e+00 3.840000e-02 1.683800e+00 2.347000e-01 5.724000e-01 + 2.387000e-01 5.287000e-01 5.081000e-01 5.803000e-01 6.225000e-01 5.861000e-01 1.602700e+00 1.486000e-01 -3.100000e-03 2.002900e+00 -3.110000e-02 3.750000e-02 2.908000e-01 3.475000e-01 + 3.340000e-02 8.543000e-01 9.320000e-02 1.295600e+00 1.707000e-01 1.390800e+00 3.800000e-02 1.361300e+00 1.285000e-01 1.471100e+00 7.590000e-02 1.317000e+00 5.550000e-02 7.455000e-01 + 2.603000e-01 6.276000e-01 6.728000e-01 5.071000e-01 7.549000e-01 5.508000e-01 1.824600e+00 -1.058000e-01 -1.900000e-02 2.019000e+00 -7.000000e-04 4.000000e-04 3.338000e-01 3.718000e-01 + 5.957000e-01 4.387000e-01 8.091000e-01 8.394000e-01 9.161000e-01 9.635000e-01 7.718000e-01 8.814000e-01 1.003700e+00 8.898000e-01 7.043000e-01 9.604000e-01 4.712000e-01 4.792000e-01 + 1.171000e+00 -7.670000e-02 5.022000e-01 3.121000e-01 4.242000e-01 2.324000e-01 4.959000e-01 3.193000e-01 4.355000e-01 2.043000e-01 5.301000e-01 2.812000e-01 3.576000e-01 1.846000e-01 + 1.121500e+00 -2.200000e-02 5.811000e-01 1.162100e+00 5.564000e-01 5.510000e-01 6.498000e-01 1.077300e+00 4.685000e-01 6.165000e-01 5.802000e-01 1.157100e+00 4.429000e-01 3.451000e-01 + 1.084900e+00 2.450000e-02 3.846000e-01 1.362700e+00 4.707000e-01 7.384000e-01 4.256000e-01 1.313500e+00 4.017000e-01 7.778000e-01 4.385000e-01 1.299300e+00 3.552000e-01 4.773000e-01 + 1.153600e+00 -5.470000e-02 4.680000e-01 3.564000e-01 3.592000e-01 3.113000e-01 3.693000e-01 4.764000e-01 3.756000e-01 2.744000e-01 4.279000e-01 4.029000e-01 3.265000e-01 2.203000e-01 + 3.748000e-01 2.994000e-01 6.536000e-01 3.408000e-01 8.457000e-01 2.527000e-01 1.739800e+00 -9.200000e-03 -6.410000e-02 2.075800e+00 3.580000e-02 -4.420000e-02 4.033000e-01 1.715000e-01 + 4.363000e-01 6.118000e-01 7.460000e-01 6.139000e-01 8.352000e-01 6.512000e-01 1.741400e+00 -1.040000e-02 -3.700000e-02 2.039400e+00 4.330000e-02 -5.150000e-02 4.164000e-01 3.860000e-01 + 1.798000e-01 2.871000e-01 3.085000e-01 4.197000e-01 3.603000e-01 4.645000e-01 2.507000e-01 4.863000e-01 3.400000e-01 5.030000e-01 3.426000e-01 3.776000e-01 1.842000e-01 2.344000e-01 + 6.403000e-01 5.420000e-02 1.025400e+00 6.470000e-02 1.219600e+00 9.200000e-03 9.108000e-01 1.963000e-01 1.133800e+00 1.292000e-01 9.212000e-01 1.857000e-01 5.768000e-01 5.450000e-02 + 4.600000e-02 8.990000e-01 1.115000e-01 1.363600e+00 1.098000e-01 1.570600e+00 1.085000e-01 1.371400e+00 2.028000e-01 1.486500e+00 4.920000e-02 1.434900e+00 6.760000e-02 7.827000e-01 + 3.339000e-01 7.763000e-01 7.458000e-01 7.458000e-01 7.757000e-01 8.486000e-01 1.733500e+00 7.000000e-04 -8.480000e-02 2.099900e+00 -2.780000e-02 1.761200e+00 3.877000e-01 4.879000e-01 + 2.704000e-01 8.443000e-01 6.167000e-01 8.324000e-01 7.813000e-01 7.781000e-01 1.633800e+00 1.169000e-01 -2.190000e-02 2.026000e+00 -8.870000e-02 1.835800e+00 3.523000e-01 4.940000e-01 + 3.663000e-01 5.339000e-01 4.716000e-01 9.602000e-01 5.079000e-01 1.127100e+00 5.363000e-01 8.867000e-01 5.595000e-01 1.094100e+00 5.606000e-01 8.596000e-01 3.032000e-01 5.188000e-01 + 6.235000e-01 1.920000e-01 1.090900e+00 1.685000e-01 1.179400e+00 2.634000e-01 1.007500e+00 2.645000e-01 1.161100e+00 3.117000e-01 1.011000e+00 2.615000e-01 5.957000e-01 1.379000e-01 +-1.810000e-02 6.916000e-01 2.449000e-01 7.359000e-01 4.121000e-01 6.803000e-01 -6.350000e-02 1.805700e+00 7.400000e-02 1.915100e+00 2.490000e-02 -2.860000e-02 1.347000e-01 4.371000e-01 + 5.528000e-01 5.532000e-01 3.238000e-01 7.494000e-01 2.123000e-01 6.204000e-01 2.494000e-01 8.322000e-01 2.191000e-01 5.926000e-01 2.836000e-01 7.916000e-01 2.339000e-01 4.084000e-01 + 7.864000e-01 -2.360000e-02 1.092900e+00 1.320000e-01 1.351800e+00 2.660000e-02 1.164200e+00 4.800000e-02 1.409700e+00 -1.860000e-02 1.181300e+00 2.760000e-02 6.641000e-01 3.780000e-02 + 4.138000e-01 5.768000e-01 7.693000e-01 5.243000e-01 9.692000e-01 4.379000e-01 1.803000e+00 -8.240000e-02 6.050000e-02 1.930600e+00 -1.340000e-02 1.410000e-02 4.108000e-01 3.600000e-01 + 1.148100e+00 -5.410000e-02 7.842000e-01 2.590000e-01 6.844000e-01 1.110000e-01 8.148000e-01 2.220000e-01 7.089000e-01 6.410000e-02 8.862000e-01 1.437000e-01 5.566000e-01 7.150000e-02 + 2.915000e-01 6.952000e-01 5.862000e-01 7.160000e-01 8.323000e-01 5.673000e-01 1.671200e+00 7.010000e-02 1.290000e-02 1.985300e+00 7.770000e-02 -9.110000e-02 3.706000e-01 3.907000e-01 + 1.131700e+00 -3.170000e-02 9.419000e-01 7.896000e-01 7.905000e-01 4.900000e-01 9.017000e-01 8.375000e-01 7.738000e-01 4.528000e-01 9.162000e-01 8.146000e-01 5.810000e-01 2.747000e-01 + 3.090000e-02 9.172000e-01 3.080000e-02 1.457100e+00 1.922000e-01 1.471700e+00 4.110000e-02 1.445200e+00 1.063000e-01 1.599500e+00 7.490000e-02 1.411300e+00 2.740000e-02 8.309000e-01 + 6.018000e-01 2.622000e-01 8.873000e-01 2.966000e-01 1.037300e+00 2.592000e-01 1.817400e+00 -9.650000e-02 2.013200e+00 -1.730000e-02 9.150000e-02 -1.097000e-01 5.341000e-01 1.592000e-01 + 3.616000e-01 5.395000e-01 7.610000e-01 4.352000e-01 8.018000e-01 5.294000e-01 1.749900e+00 -2.180000e-02 6.930000e-02 1.918400e+00 -5.740000e-02 6.910000e-02 4.071000e-01 3.045000e-01 + 3.879000e-01 5.093000e-01 6.455000e-01 7.533000e-01 7.869000e-01 7.986000e-01 5.291000e-01 8.947000e-01 7.951000e-01 8.101000e-01 6.323000e-01 7.764000e-01 3.686000e-01 4.411000e-01 + 8.884000e-01 2.158000e-01 1.187600e+00 5.403000e-01 1.283500e+00 5.980000e-01 1.680100e+00 5.840000e-02 2.025300e+00 -3.460000e-02 4.970000e-02 1.670900e+00 6.889000e-01 2.914000e-01 + 1.108700e+00 2.100000e-03 1.015600e+00 7.153000e-01 7.320000e-01 1.086200e+00 9.141000e-01 8.361000e-01 8.051000e-01 8.140000e-01 9.557000e-01 7.831000e-01 6.136000e-01 3.523000e-01 + 2.780000e-02 6.130000e-02 2.001000e-01 2.144000e-01 3.815000e-01 1.446000e-01 1.435000e-01 1.563000e+00 -2.340000e-02 2.930000e-02 -1.340000e-02 1.620000e-02 1.320000e-01 8.730000e-02 + 1.968000e-01 8.896000e-01 4.827000e-01 9.247000e-01 6.167000e-01 9.017000e-01 1.671100e+00 6.720000e-02 -1.130000e-02 2.014000e+00 5.820000e-02 1.666900e+00 2.653000e-01 5.603000e-01 + 5.201000e-01 5.841000e-01 3.171000e-01 7.668000e-01 3.882000e-01 4.175000e-01 2.912000e-01 8.033000e-01 1.717000e-01 6.541000e-01 2.294000e-01 8.757000e-01 2.216000e-01 4.260000e-01 + 5.786000e-01 4.950000e-02 8.336000e-01 1.201000e-01 9.328000e-01 1.453000e-01 1.627900e+00 1.184000e-01 1.938700e+00 6.730000e-02 2.700000e-03 -2.800000e-03 5.267000e-01 1.920000e-02 + 2.881000e-01 5.880000e-01 6.085000e-01 5.808000e-01 7.821000e-01 5.208000e-01 1.687500e+00 5.650000e-02 -1.156000e-01 2.135100e+00 -2.450000e-02 2.660000e-02 3.434000e-01 3.586000e-01 + 3.163000e-01 4.248000e-01 6.331000e-01 4.197000e-01 7.968000e-01 3.696000e-01 1.673900e+00 6.850000e-02 4.840000e-02 1.946100e+00 1.170000e-02 -1.470000e-02 3.255000e-01 2.971000e-01 + 1.184100e+00 -9.100000e-02 5.882000e-01 6.830000e-01 4.091000e-01 5.274000e-01 6.264000e-01 6.381000e-01 4.739000e-01 4.277000e-01 5.825000e-01 6.909000e-01 3.942000e-01 3.051000e-01 + 2.379000e-01 4.085000e-01 3.863000e-01 6.242000e-01 3.834000e-01 7.741000e-01 3.794000e-01 6.333000e-01 3.210000e-01 8.707000e-01 3.651000e-01 6.482000e-01 2.093000e-01 3.762000e-01 + 1.124500e+00 -2.220000e-02 3.990000e-01 1.027600e+00 2.620000e-01 7.330000e-01 3.240000e-01 1.115300e+00 3.763000e-01 5.714000e-01 4.534000e-01 9.653000e-01 3.213000e-01 4.047000e-01 + 1.099400e+00 8.000000e-03 8.668000e-01 8.454000e-01 5.698000e-01 1.414000e+00 6.727000e-01 1.078300e+00 5.144000e-01 1.395700e+00 6.686000e-01 1.077000e+00 5.120000e-01 4.853000e-01 + 3.698000e-01 1.077000e-01 5.224000e-01 2.341000e-01 5.438000e-01 3.333000e-01 5.149000e-01 2.459000e-01 5.892000e-01 2.902000e-01 5.628000e-01 1.885000e-01 3.137000e-01 1.224000e-01 + 1.164700e+00 -6.970000e-02 3.553000e-01 1.395300e+00 3.349000e-01 1.135600e+00 4.702000e-01 1.253800e+00 3.487000e-01 1.043500e+00 4.655000e-01 1.262100e+00 3.359000e-01 5.706000e-01 + 3.183000e-01 3.051000e-01 7.323000e-01 2.050000e-01 6.892000e-01 4.029000e-01 4.636000e-01 5.233000e-01 6.157000e-01 5.086000e-01 5.282000e-01 4.456000e-01 3.188000e-01 2.398000e-01 + 3.521000e-01 3.844000e-01 5.984000e-01 5.504000e-01 6.438000e-01 6.688000e-01 6.586000e-01 4.784000e-01 6.964000e-01 6.264000e-01 5.053000e-01 6.583000e-01 3.198000e-01 3.473000e-01 + 5.951000e-01 4.877000e-01 8.186000e-01 8.973000e-01 8.961000e-01 1.057700e+00 6.305000e-01 1.118100e+00 8.322000e-01 1.168700e+00 7.579000e-01 9.702000e-01 4.368000e-01 5.585000e-01 + 3.389000e-01 -2.500000e-03 6.462000e-01 1.600000e-03 8.299000e-01 -7.180000e-02 1.751500e+00 -2.660000e-02 6.930000e-02 -8.170000e-02 -3.820000e-02 4.560000e-02 3.626000e-01 1.000000e-03 + 5.361000e-01 6.900000e-03 8.957000e-01 -5.900000e-02 8.035000e-01 1.913000e-01 6.853000e-01 1.933000e-01 8.260000e-01 1.836000e-01 8.115000e-01 4.050000e-02 4.454000e-01 5.170000e-02 + 1.890000e-02 9.513000e-01 1.228000e-01 1.177100e+00 3.517000e-01 1.045600e+00 6.800000e-03 1.724800e+00 -4.210000e-02 2.044000e+00 6.100000e-03 -7.700000e-03 1.020000e-01 6.578000e-01 + 9.760000e-02 1.367000e-01 1.787000e-01 1.816000e-01 9.870000e-02 3.317000e-01 7.080000e-02 3.092000e-01 1.655000e-01 2.588000e-01 1.027000e-01 2.720000e-01 6.920000e-02 1.454000e-01 + 8.847000e-01 -4.760000e-02 1.263500e+00 7.040000e-02 1.474600e+00 3.880000e-02 1.329900e+00 -7.300000e-03 1.572400e+00 -5.190000e-02 1.251700e+00 8.340000e-02 7.533000e-01 1.260000e-02 + 4.102000e-01 6.910000e-01 7.255000e-01 8.216000e-01 8.329000e-01 8.367000e-01 1.705600e+00 3.830000e-02 6.300000e-02 1.926100e+00 -7.250000e-02 1.816300e+00 3.804000e-01 5.212000e-01 + 1.200200e+00 -1.123000e-01 7.355000e-01 6.722000e-01 6.381000e-01 3.311000e-01 7.752000e-01 6.237000e-01 5.633000e-01 3.925000e-01 7.433000e-01 6.622000e-01 5.182000e-01 2.040000e-01 + 1.303000e-01 5.144000e-01 2.051000e-01 8.057000e-01 2.318000e-01 9.195000e-01 1.189000e-01 9.103000e-01 2.451000e-01 9.204000e-01 6.870000e-02 9.648000e-01 9.940000e-02 4.873000e-01 +-1.940000e-02 1.128400e+00 9.260000e-02 8.642000e-01 -7.870000e-02 8.619000e-01 1.940000e-02 9.523000e-01 -3.220000e-02 7.856000e-01 1.277000e-01 8.247000e-01 2.400000e-03 6.047000e-01 + 4.836000e-01 3.774000e-01 8.733000e-01 2.825000e-01 9.310000e-01 3.550000e-01 1.847400e+00 -1.350000e-01 -6.380000e-02 2.077500e+00 -3.200000e-03 5.300000e-03 4.776000e-01 2.108000e-01 + 3.880000e-01 7.093000e-01 2.357000e-01 1.496200e+00 1.546000e-01 1.279100e+00 2.154000e-01 1.517400e+00 1.865000e-01 1.174800e+00 2.110000e-01 1.518400e+00 1.589000e-01 7.401000e-01 + 9.033000e-01 1.708000e-01 1.453700e+00 2.186000e-01 1.625800e+00 2.814000e-01 1.394600e+00 2.859000e-01 1.638100e+00 3.015000e-01 1.507100e+00 1.582000e-01 8.207000e-01 1.479000e-01 + 7.640000e-02 9.569000e-01 1.070000e-02 1.348700e+00 -2.890000e-02 1.525800e+00 -6.690000e-02 1.814900e+00 5.600000e-02 1.930900e+00 -7.500000e-03 8.600000e-03 1.070000e-02 7.913000e-01 + 6.920000e-01 1.085000e-01 1.126700e+00 1.214000e-01 1.123300e+00 3.258000e-01 9.782000e-01 2.989000e-01 1.109100e+00 3.648000e-01 9.312000e-01 3.528000e-01 5.654000e-01 1.710000e-01 + 1.140900e+00 -4.250000e-02 5.855000e-01 1.156300e+00 5.839000e-01 5.531000e-01 6.872000e-01 1.039200e+00 5.730000e-01 5.251000e-01 6.184000e-01 1.117100e+00 4.544000e-01 3.476000e-01 + 2.265000e-01 6.849000e-01 5.588000e-01 6.636000e-01 5.984000e-01 7.636000e-01 1.767000e+00 -3.880000e-02 4.490000e-02 1.949400e+00 -5.390000e-02 6.500000e-02 3.104000e-01 4.117000e-01 + 1.136600e+00 -3.860000e-02 7.026000e-01 6.384000e-01 4.141000e-01 5.617000e-01 6.621000e-01 6.825000e-01 6.124000e-01 3.018000e-01 7.418000e-01 5.917000e-01 4.552000e-01 2.588000e-01 + 1.151400e+00 -5.220000e-02 7.812000e-01 9.582000e-01 6.172000e-01 8.571000e-01 8.638000e-01 8.509000e-01 6.028000e-01 8.008000e-01 8.650000e-01 8.518000e-01 5.288000e-01 3.810000e-01 + 3.512000e-01 7.155000e-01 4.879000e-01 1.195400e+00 6.452000e-01 1.257600e+00 5.178000e-01 1.157700e+00 5.995000e-01 1.341400e+00 4.382000e-01 1.253600e+00 2.947000e-01 6.734000e-01 + 3.928000e-01 2.647000e-01 5.870000e-01 4.481000e-01 6.574000e-01 5.253000e-01 6.692000e-01 3.489000e-01 7.849000e-01 3.914000e-01 6.414000e-01 3.851000e-01 3.603000e-01 2.330000e-01 + 9.810000e-02 5.790000e-01 2.610000e-02 1.055300e+00 1.204000e-01 1.094800e+00 1.193000e-01 9.473000e-01 1.539000e-01 1.071500e+00 1.487000e-01 9.104000e-01 5.580000e-02 5.622000e-01 + 3.960000e-01 5.710000e-01 6.352000e-01 8.757000e-01 6.999000e-01 1.032200e+00 6.124000e-01 9.060000e-01 7.728000e-01 9.632000e-01 6.477000e-01 8.664000e-01 3.643000e-01 5.100000e-01 + 1.056200e+00 5.920000e-02 5.754000e-01 5.725000e-01 3.995000e-01 4.736000e-01 4.606000e-01 7.063000e-01 4.667000e-01 3.719000e-01 5.426000e-01 6.059000e-01 3.744000e-01 2.928000e-01 + 1.544000e-01 3.905000e-01 1.708000e-01 6.948000e-01 1.166000e-01 8.799000e-01 1.731000e-01 6.905000e-01 2.379000e-01 7.522000e-01 1.968000e-01 6.632000e-01 1.038000e-01 3.948000e-01 + 7.474000e-01 3.670000e-01 4.144000e-01 1.308300e+00 3.011000e-01 7.628000e-01 4.681000e-01 1.247800e+00 2.852000e-01 7.465000e-01 2.409000e-01 1.516200e+00 2.805000e-01 4.849000e-01 + 5.245000e-01 1.796000e-01 8.253000e-01 2.766000e-01 9.779000e-01 2.720000e-01 8.928000e-01 1.989000e-01 1.056100e+00 1.957000e-01 8.944000e-01 1.973000e-01 4.657000e-01 1.730000e-01 + 2.587000e-01 3.300000e-01 3.558000e-01 5.741000e-01 4.228000e-01 6.357000e-01 3.226000e-01 6.190000e-01 4.366000e-01 6.308000e-01 3.088000e-01 6.295000e-01 2.096000e-01 3.275000e-01 + 4.381000e-01 3.545000e-01 7.058000e-01 5.334000e-01 7.713000e-01 6.482000e-01 6.476000e-01 6.045000e-01 8.761000e-01 5.459000e-01 7.537000e-01 4.779000e-01 4.071000e-01 3.084000e-01 + 1.175000e-01 6.417000e-01 2.160000e-02 3.414000e-01 -4.500000e-03 3.157000e-01 4.350000e-02 3.126000e-01 8.800000e-02 2.014000e-01 8.650000e-02 2.635000e-01 5.010000e-02 2.254000e-01 + 1.150900e+00 -5.330000e-02 8.148000e-01 1.712000e-01 6.775000e-01 8.840000e-02 8.537000e-01 1.243000e-01 6.137000e-01 1.453000e-01 7.788000e-01 2.154000e-01 5.315000e-01 8.070000e-02 +-5.630000e-02 9.848000e-01 1.351000e-01 1.084000e+00 6.650000e-02 1.297100e+00 8.910000e-02 1.623000e+00 6.430000e-02 1.925100e+00 4.000000e-04 -6.000000e-04 4.630000e-02 6.800000e-01 + 1.074000e+00 4.070000e-02 4.828000e-01 1.259400e+00 3.677000e-01 1.607300e+00 4.678000e-01 1.268900e+00 5.239000e-01 1.195600e+00 4.787000e-01 1.261600e+00 3.748000e-01 6.080000e-01 + 3.009000e-01 2.211000e-01 5.387000e-01 3.073000e-01 6.518000e-01 3.179000e-01 1.888000e+00 -1.786000e-01 1.250000e-02 -1.310000e-02 4.310000e-02 -5.170000e-02 3.051000e-01 1.811000e-01 + 4.833000e-01 5.855000e-01 6.725000e-01 1.015700e+00 8.943000e-01 1.002800e+00 7.816000e-01 8.851000e-01 9.070000e-01 1.016200e+00 7.665000e-01 9.063000e-01 4.491000e-01 5.144000e-01 + 2.202000e-01 1.536000e-01 3.610000e-01 2.209000e-01 5.806000e-01 5.000000e-02 3.611000e-01 2.241000e-01 4.184000e-01 2.550000e-01 4.442000e-01 1.251000e-01 2.334000e-01 9.850000e-02 + 5.666000e-01 5.338000e-01 2.767000e-01 1.301500e+00 2.398000e-01 7.728000e-01 2.772000e-01 1.298300e+00 2.390000e-01 7.463000e-01 3.174000e-01 1.256700e+00 2.205000e-01 5.236000e-01 + 3.376000e-01 7.419000e-01 4.848000e-01 1.221000e+00 6.335000e-01 1.291000e+00 5.921000e-01 1.088100e+00 6.010000e-01 1.356400e+00 4.958000e-01 1.201200e+00 3.003000e-01 6.781000e-01 + 9.463000e-01 2.020000e-02 1.380900e+00 1.522000e-01 1.676300e+00 5.780000e-02 1.385100e+00 1.532000e-01 1.648500e+00 1.135000e-01 1.339300e+00 2.017000e-01 8.296000e-01 5.060000e-02 + 3.907000e-01 6.294000e-01 5.879000e-01 7.718000e-01 8.172000e-01 6.377000e-01 1.697300e+00 4.520000e-02 1.263000e-01 1.845900e+00 -1.900000e-03 1.300000e-03 4.017000e-01 3.842000e-01 + 1.724000e-01 3.817000e-01 1.702000e-01 7.203000e-01 2.873000e-01 7.049000e-01 2.921000e-01 5.731000e-01 3.515000e-01 6.442000e-01 2.986000e-01 5.659000e-01 1.455000e-01 3.580000e-01 + 1.312000e-01 7.093000e-01 3.948000e-01 7.673000e-01 5.272000e-01 7.554000e-01 1.743300e+00 -1.250000e-02 2.350000e-02 1.975500e+00 2.500000e-03 -3.900000e-03 2.266000e-01 4.576000e-01 + 1.263000e-01 6.105000e-01 2.655000e-01 8.748000e-01 2.336000e-01 1.076800e+00 1.833000e-01 9.726000e-01 3.004000e-01 1.016500e+00 1.962000e-01 9.532000e-01 1.264000e-01 5.363000e-01 + 6.043000e-01 4.914000e-01 1.090000e+00 5.961000e-01 1.056900e+00 8.966000e-01 1.027900e+00 6.694000e-01 1.212700e+00 7.549000e-01 1.065400e+00 6.278000e-01 6.046000e-01 3.749000e-01 + 1.028400e+00 8.930000e-02 4.097000e-01 9.375000e-01 2.813000e-01 6.865000e-01 3.827000e-01 9.620000e-01 2.830000e-01 6.586000e-01 3.508000e-01 1.008600e+00 3.087000e-01 4.039000e-01 +-1.970000e-02 7.719000e-01 4.634000e-01 5.700000e-01 5.143000e-01 6.520000e-01 -2.580000e-02 1.762600e+00 -1.730000e-02 2.023200e+00 2.500000e-02 -2.650000e-02 1.832000e-01 4.379000e-01 diff --git a/sourcecodes/bnt-master/netlab3.3/olgd.m b/sourcecodes/bnt-master/netlab3.3/olgd.m new file mode 100644 index 00000000..20681cf0 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/olgd.m @@ -0,0 +1,181 @@ +function [net, options, errlog, pointlog] = olgd(net, options, x, t) +%OLGD On-line gradient descent optimization. +% +% Description +% [NET, OPTIONS, ERRLOG, POINTLOG] = OLGD(NET, OPTIONS, X, T) uses on- +% line gradient descent to find a local minimum of the error function +% for the network NET computed on the input data X and target values T. +% A log of the error values after each cycle is (optionally) returned +% in ERRLOG, and a log of the points visited is (optionally) returned +% in POINTLOG. Because the gradient is computed on-line (i.e. after +% each pattern) this can be quite inefficient in Matlab. +% +% The error function value at final weight vector is returned in +% OPTIONS(8). +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG, and the points visited in the +% return argument POINTSLOG. If OPTIONS(1) is set to 0, then only +% warning messages are displayed. If OPTIONS(1) is -1, then nothing is +% displayed. +% +% OPTIONS(2) is the precision required for the value of X at the +% solution. If the absolute difference between the values of X between +% two successive steps is less than OPTIONS(2), then this condition is +% satisfied. +% +% OPTIONS(3) is the precision required of the objective function at the +% solution. If the absolute difference between the error functions +% between two successive steps is less than OPTIONS(3), then this +% condition is satisfied. Both this and the previous condition must be +% satisfied for termination. Note that testing the function value at +% each iteration roughly halves the speed of the algorithm. +% +% OPTIONS(5) determines whether the patterns are sampled randomly with +% replacement. If it is 0 (the default), then patterns are sampled in +% order. +% +% OPTIONS(6) determines if the learning rate decays. If it is 1 then +% the learning rate decays at a rate of 1/T. If it is 0 (the default) +% then the learning rate is constant. +% +% OPTIONS(9) should be set to 1 to check the user defined gradient +% function. +% +% OPTIONS(10) returns the total number of function evaluations +% (including those in any line searches). +% +% OPTIONS(11) returns the total number of gradient evaluations. +% +% OPTIONS(14) is the maximum number of iterations (passes through the +% complete pattern set); default 100. +% +% OPTIONS(17) is the momentum; default 0.5. +% +% OPTIONS(18) is the learning rate; default 0.01. +% +% See also +% GRADDESC +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set up the options. +if length(options) < 18 + error('Options vector too short') +end + +if (options(14)) + niters = options(14); +else + niters = 100; +end + +% Learning rate: must be positive +if (options(18) > 0) + eta = options(18); +else + eta = 0.01; +end +% Save initial learning rate for annealing +lr = eta; +% Momentum term: allow zero momentum +if (options(17) >= 0) + mu = options(17); +else + mu = 0.5; +end + +pakstr = [net.type, 'pak']; +unpakstr = [net.type, 'unpak']; + +% Extract initial weights from the network +w = feval(pakstr, net); + +display = options(1); + +% Work out if we need to compute f at each iteration. +% Needed if display results or if termination +% criterion requires it. +fcneval = (display | options(3)); + +% Check gradients +if (options(9)) + feval('gradchek', w, 'neterr', 'netgrad', net, x, t); +end + +dwold = zeros(1, length(w)); +fold = 0; % Must be initialised so that termination test can be performed +ndata = size(x, 1); + +if fcneval + fnew = neterr(w, net, x, t); + options(10) = options(10) + 1; + fold = fnew; +end + +j = 1; +if nargout >= 3 + errlog(j, :) = fnew; + if nargout == 4 + pointlog(j, :) = w; + end +end + +% Main optimization loop. +while j <= niters + wold = w; + if options(5) + % Randomise order of pattern presentation: with replacement + pnum = ceil(rand(ndata, 1).*ndata); + else + pnum = 1:ndata; + end + for k = 1:ndata + grad = netgrad(w, net, x(pnum(k),:), t(pnum(k),:)); + if options(6) + % Let learning rate decrease as 1/t + lr = eta/((j-1)*ndata + k); + end + dw = mu*dwold - lr*grad; + w = w + dw; + dwold = dw; + end + options(11) = options(11) + 1; % Increment gradient evaluation count + if fcneval + fold = fnew; + fnew = neterr(w, net, x, t); + options(10) = options(10) + 1; + end + if display + fprintf(1, 'Iteration %5d Error %11.8f\n', j, fnew); + end + j = j + 1; + if nargout >= 3 + errlog(j) = fnew; + if nargout == 4 + pointlog(j, :) = w; + end + end + if (max(abs(w - wold)) < options(2) & abs(fnew - fold) < options(3)) + % Termination criteria are met + options(8) = fnew; + net = feval(unpakstr, net, w); + return; + end +end + +if fcneval + options(8) = fnew; +else + % Return error on entire dataset + options(8) = neterr(w, net, x, t); + options(10) = options(10) + 1; +end +if (options(1) >= 0) + disp(maxitmess); +end + +net = feval(unpakstr, net, w); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/pca.m b/sourcecodes/bnt-master/netlab3.3/pca.m new file mode 100644 index 00000000..4b7063d6 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/pca.m @@ -0,0 +1,42 @@ +function [PCcoeff, PCvec] = pca(data, N) +%PCA Principal Components Analysis +% +% Description +% PCCOEFF = PCA(DATA) computes the eigenvalues of the covariance +% matrix of the dataset DATA and returns them as PCCOEFF. These +% coefficients give the variance of DATA along the corresponding +% principal components. +% +% PCCOEFF = PCA(DATA, N) returns the largest N eigenvalues. +% +% [PCCOEFF, PCVEC] = PCA(DATA) returns the principal components as well +% as the coefficients. This is considerably more computationally +% demanding than just computing the eigenvalues. +% +% See also +% EIGDEC, GTMINIT, PPCA +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargin == 1 + N = size(data, 2); +end + +if nargout == 1 + evals_only = logical(1); +else + evals_only = logical(0); +end + +if N ~= round(N) | N < 1 | N > size(data, 2) + error('Number of PCs must be integer, >0, < dim'); +end + +% Find the sorted eigenvalues of the data covariance matrix +if evals_only + PCcoeff = eigdec(cov(data), N); +else + [PCcoeff, PCvec] = eigdec(cov(data), N); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/plotmat.m b/sourcecodes/bnt-master/netlab3.3/plotmat.m new file mode 100644 index 00000000..64f82c2c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/plotmat.m @@ -0,0 +1,44 @@ +function plotmat(matrix, textcolour, gridcolour, fontsize) +%PLOTMAT Display a matrix. +% +% Description +% PLOTMAT(MATRIX, TEXTCOLOUR, GRIDCOLOUR, FONTSIZE) displays the matrix +% MATRIX on the current figure. The TEXTCOLOUR and GRIDCOLOUR +% arguments control the colours of the numbers and grid labels +% respectively and should follow the usual Matlab specification. The +% parameter FONTSIZE should be an integer. +% +% See also +% CONFFIG, DEMMLP2 +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[m,n]=size(matrix); +for rowCnt=1:m, + for colCnt=1:n, + numberString=num2str(matrix(rowCnt,colCnt)); + text(colCnt-.5,m-rowCnt+.5,numberString, ... + 'HorizontalAlignment','center', ... + 'Color', textcolour, ... + 'FontWeight','bold', ... + 'FontSize', fontsize); + end; +end; + +set(gca,'Box','on', ... + 'Visible','on', ... + 'xLim',[0 n], ... + 'xGrid','on', ... + 'xTickLabel',[], ... + 'xTick',0:n, ... + 'yGrid','on', ... + 'yLim',[0 m], ... + 'yTickLabel',[], ... + 'yTick',0:m, ... + 'DataAspectRatio',[1, 1, 1], ... + 'GridLineStyle',':', ... + 'LineWidth',3, ... + 'XColor',gridcolour, ... + 'YColor',gridcolour); + diff --git a/sourcecodes/bnt-master/netlab3.3/ppca.m b/sourcecodes/bnt-master/netlab3.3/ppca.m new file mode 100644 index 00000000..fd125b02 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/ppca.m @@ -0,0 +1,52 @@ +function [var, U, lambda] = ppca(x, ppca_dim) +%PPCA Probabilistic Principal Components Analysis +% +% Description +% [VAR, U, LAMBDA] = PPCA(X, PPCA_DIM) computes the principal +% component subspace U of dimension PPCA_DIM using a centred covariance +% matrix X. The variable VAR contains the off-subspace variance (which +% is assumed to be spherical), while the vector LAMBDA contains the +% variances of each of the principal components. This is computed +% using the eigenvalue and eigenvector decomposition of X. +% +% See also +% EIGDEC, PCA +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +if ppca_dim ~= round(ppca_dim) | ppca_dim < 1 | ppca_dim > size(x, 2) + error('Number of PCs must be integer, >0, < dim'); +end + +[ndata, data_dim] = size(x); +% Assumes that x is centred and responsibility weighted +% covariance matrix +[l Utemp] = eigdec(x, data_dim); +% Zero any negative eigenvalues (caused by rounding) +l(l<0) = 0; +% Now compute the sigma squared values for all possible values +% of q +s2_temp = cumsum(l(end:-1:1))./[1:data_dim]'; +% If necessary, reduce the value of q so that var is at least +% eps * largest eigenvalue +q_temp = min([ppca_dim; data_dim-min(find(s2_temp/l(1) > eps))]); +if q_temp ~= ppca_dim + wstringpart = 'Covariance matrix ill-conditioned: extracted'; + wstring = sprintf('%s %d/%d PCs', ... + wstringpart, q_temp, ppca_dim); + warning(wstring); +end +if q_temp == 0 + % All the latent dimensions have disappeared, so we are + % just left with the noise model + var = l(1)/data_dim; + lambda = var*ones(1, ppca_dim); +else + var = mean(l(q_temp+1:end)); +end +U = Utemp(:, 1:q_temp); +lambda(1:q_temp) = l(1:q_temp); + + diff --git a/sourcecodes/bnt-master/netlab3.3/quasinew.m b/sourcecodes/bnt-master/netlab3.3/quasinew.m new file mode 100644 index 00000000..680e7d15 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/quasinew.m @@ -0,0 +1,176 @@ +function [x, options, flog, pointlog] = quasinew(f, x, options, gradf, ... + varargin) +%QUASINEW Quasi-Newton optimization. +% +% Description +% [X, OPTIONS, FLOG, POINTLOG] = QUASINEW(F, X, OPTIONS, GRADF) uses a +% quasi-Newton algorithm to find a local minimum of the function F(X) +% whose gradient is given by GRADF(X). Here X is a row vector and F +% returns a scalar value. The point at which F has a local minimum is +% returned as X. The function value at that point is returned in +% OPTIONS(8). A log of the function values after each cycle is +% (optionally) returned in FLOG, and a log of the points visited is +% (optionally) returned in POINTLOG. +% +% QUASINEW(F, X, OPTIONS, GRADF, P1, P2, ...) allows additional +% arguments to be passed to F() and GRADF(). +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG, and the points visited in the +% return argument POINTSLOG. If OPTIONS(1) is set to 0, then only +% warning messages are displayed. If OPTIONS(1) is -1, then nothing is +% displayed. +% +% OPTIONS(2) is a measure of the absolute precision required for the +% value of X at the solution. If the absolute difference between the +% values of X between two successive steps is less than OPTIONS(2), +% then this condition is satisfied. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. If the absolute difference between the +% objective function values between two successive steps is less than +% OPTIONS(3), then this condition is satisfied. Both this and the +% previous condition must be satisfied for termination. +% +% OPTIONS(9) should be set to 1 to check the user defined gradient +% function. +% +% OPTIONS(10) returns the total number of function evaluations +% (including those in any line searches). +% +% OPTIONS(11) returns the total number of gradient evaluations. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% OPTIONS(15) is the precision in parameter space of the line search; +% default 1E-2. +% +% See also +% CONJGRAD, GRADDESC, LINEMIN, MINBRACK, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set up the options. +if length(options) < 18 + error('Options vector too short') +end + +if(options(14)) + niters = options(14); +else + niters = 100; +end + +% Set up options for line search +line_options = foptions; +% Don't need a very precise line search +if options(15) > 0 + line_options(2) = options(15); +else + line_options(2) = 1e-2; % Default +end +% Minimal fractional change in f from Newton step: otherwise do a line search +min_frac_change = 1e-4; + +display = options(1); + +% Next two lines allow quasinew to work with expression strings +f = fcnchk(f, length(varargin)); +gradf = fcnchk(gradf, length(varargin)); + +% Check gradients +if (options(9)) + feval('gradchek', x, f, gradf, varargin{:}); +end + +nparams = length(x); +fnew = feval(f, x, varargin{:}); +options(10) = options(10) + 1; +gradnew = feval(gradf, x, varargin{:}); +options(11) = options(11) + 1; +p = -gradnew; % Search direction +hessinv = eye(nparams); % Initialise inverse Hessian to be identity matrix +j = 1; +if nargout >= 3 + flog(j, :) = fnew; + if nargout == 4 + pointlog(j, :) = x; + end +end + +while (j <= niters) + + xold = x; + fold = fnew; + gradold = gradnew; + + x = xold + p; + fnew = feval(f, x, varargin{:}); + options(10) = options(10) + 1; + + % This shouldn't occur, but rest of code depends on sd being downhill + if (gradnew*p' >= 0) + p = -p; + if options(1) >= 0 + warning('search direction uphill in quasinew'); + end + end + + % Does the Newton step reduce the function value sufficiently? + if (fnew >= fold + min_frac_change * (gradnew*p')) + % No it doesn't + % Minimize along current search direction: must be less than Newton step + [lmin, line_options] = feval('linemin', f, xold, p, fold, ... + line_options, varargin{:}); + options(10) = options(10) + line_options(10); + options(11) = options(11) + line_options(11); + % Correct x and fnew to be the actual search point we have found + x = xold + lmin * p; + p = x - xold; + fnew = line_options(8); + end + + % Check for termination + if (max(abs(x - xold)) < options(2) & max(abs(fnew - fold)) < options(3)) + options(8) = fnew; + return; + end + gradnew = feval(gradf, x, varargin{:}); + options(11) = options(11) + 1; + v = gradnew - gradold; + vdotp = v*p'; + + % Skip update to inverse Hessian if fac not sufficiently positive + if (vdotp*vdotp > eps*sum(v.^2)*sum(p.^2)) + Gv = (hessinv*v')'; + vGv = sum(v.*Gv); + u = p./vdotp - Gv./vGv; + % Use BFGS update rule + hessinv = hessinv + (p'*p)/vdotp - (Gv'*Gv)/vGv + vGv*(u'*u); + end + + p = -(hessinv * gradnew')'; + + if (display > 0) + fprintf(1, 'Cycle %4d Function %11.6f\n', j, fnew); + end + + j = j + 1; + if nargout >= 3 + flog(j, :) = fnew; + if nargout == 4 + pointlog(j, :) = x; + end + end +end + +% If we get here, then we haven't terminated in the given number of +% iterations. + +options(8) = fold; +if (options(1) >= 0) + disp(maxitmess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/rbf.m b/sourcecodes/bnt-master/netlab3.3/rbf.m new file mode 100644 index 00000000..5c3b3096 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbf.m @@ -0,0 +1,123 @@ +function net = rbf(nin, nhidden, nout, rbfunc, outfunc, prior, beta) +%RBF Creates an RBF network with specified architecture +% +% Description +% NET = RBF(NIN, NHIDDEN, NOUT, RBFUNC) constructs and initialises a +% radial basis function network returning a data structure NET. The +% weights are all initialised with a zero mean, unit variance normal +% distribution, with the exception of the variances, which are set to +% one. This makes use of the Matlab function RANDN and so the seed for +% the random weight initialization can be set using RANDN('STATE', S) +% where S is the seed value. The activation functions are defined in +% terms of the distance between the data point and the corresponding +% centre. Note that the functions are computed to a convenient +% constant multiple: for example, the Gaussian is not normalised. +% (Normalisation is not needed as the function outputs are linearly +% combined in the next layer.) +% +% The fields in NET are +% type = 'rbf' +% nin = number of inputs +% nhidden = number of hidden units +% nout = number of outputs +% nwts = total number of weights and biases +% actfn = string defining hidden unit activation function: +% 'gaussian' for a radially symmetric Gaussian function. +% 'tps' for r^2 log r, the thin plate spline function. +% 'r4logr' for r^4 log r. +% outfn = string defining output error function: +% 'linear' for linear outputs (default) and SoS error. +% 'neuroscale' for Sammon stress measure. +% c = centres +% wi = squared widths (null for rlogr and tps) +% w2 = second layer weight matrix +% b2 = second layer bias vector +% +% NET = RBF(NIN, NHIDDEN, NOUT, RBFUND, OUTFUNC) allows the user to +% specify the type of error function to be used. The field OUTFN is +% set to the value of this string. Linear outputs (for regression +% problems) and Neuroscale outputs (for topographic mappings) are +% supported. +% +% NET = RBF(NIN, NHIDDEN, NOUT, RBFUNC, OUTFUNC, PRIOR, BETA), in which +% PRIOR is a scalar, allows the field NET.ALPHA in the data structure +% NET to be set, corresponding to a zero-mean isotropic Gaussian prior +% with inverse variance with value PRIOR. Alternatively, PRIOR can +% consist of a data structure with fields ALPHA and INDEX, allowing +% individual Gaussian priors to be set over groups of weights in the +% network. Here ALPHA is a column vector in which each element +% corresponds to a separate group of weights, which need not be +% mutually exclusive. The membership of the groups is defined by the +% matrix INDX in which the columns correspond to the elements of ALPHA. +% Each column has one element for each weight in the matrix, in the +% order defined by the function RBFPAK, and each element is 1 or 0 +% according to whether the weight is a member of the corresponding +% group or not. A utility function RBFPRIOR is provided to help in +% setting up the PRIOR data structure. +% +% NET = RBF(NIN, NHIDDEN, NOUT, FUNC, PRIOR, BETA) also sets the +% additional field NET.BETA in the data structure NET, where beta +% corresponds to the inverse noise variance. +% +% See also +% RBFERR, RBFFWD, RBFGRAD, RBFPAK, RBFTRAIN, RBFUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.type = 'rbf'; +net.nin = nin; +net.nhidden = nhidden; +net.nout = nout; + +% Check that function is an allowed type +actfns = {'gaussian', 'tps', 'r4logr'}; +outfns = {'linear', 'neuroscale'}; +if (strcmp(rbfunc, actfns)) == 0 + error('Undefined activation function.') +else + net.actfn = rbfunc; +end +if nargin <= 4 + net.outfn = outfns{1}; +elseif (strcmp(outfunc, outfns) == 0) + error('Undefined output function.') +else + net.outfn = outfunc; + end + +% Assume each function has a centre and a single width parameter, and that +% hidden layer to output weights include a bias. Only the Gaussian function +% requires a width +net.nwts = nin*nhidden + (nhidden + 1)*nout; +if strcmp(rbfunc, 'gaussian') + % Extra weights for width parameters + net.nwts = net.nwts + nhidden; +end + +if nargin > 5 + if isstruct(prior) + net.alpha = prior.alpha; + net.index = prior.index; + elseif size(prior) == [1 1] + net.alpha = prior; + else + error('prior must be a scalar or a structure'); + end + if nargin > 6 + net.beta = beta; + end +end + +w = randn(1, net.nwts); +net = rbfunpak(net, w); + +% Make widths equal to one +if strcmp(rbfunc, 'gaussian') + net.wi = ones(1, nhidden); +end + +if strcmp(net.outfn, 'neuroscale') + net.mask = rbfprior(rbfunc, nin, nhidden, nout); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/rbfbkp.m b/sourcecodes/bnt-master/netlab3.3/rbfbkp.m new file mode 100644 index 00000000..2d7ffe54 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfbkp.m @@ -0,0 +1,72 @@ +function g = rbfbkp(net, x, z, n2, deltas) +%RBFBKP Backpropagate gradient of error function for RBF network. +% +% Description +% G = RBFBKP(NET, X, Z, N2, DELTAS) takes a network data structure NET +% together with a matrix X of input vectors, a matrix Z of hidden unit +% activations, a matrix N2 of the squared distances between centres and +% inputs, and a matrix DELTAS of the gradient of the error function +% with respect to the values of the output units (i.e. the summed +% inputs to the output units, before the activation function is +% applied). The return value is the gradient G of the error function +% with respect to the network weights. Each row of X corresponds to one +% input vector. +% +% This function is provided so that the common backpropagation +% algorithm can be used by RBF network models to compute gradients for +% the output values (in RBFDERIV) as well as standard error functions. +% +% See also +% RBF, RBFGRAD, RBFDERIV +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Evaluate second-layer gradients. +gw2 = z'*deltas; +gb2 = sum(deltas); + +% Evaluate hidden unit gradients +delhid = deltas*net.w2'; + +gc = zeros(net.nhidden, net.nin); +ndata = size(x, 1); +t1 = ones(ndata, 1); +t2 = ones(1, net.nin); +% Switch on activation function type +switch net.actfn + +case 'gaussian' % Gaussian + delhid = (delhid.*z); + % A loop seems essential, so do it with the shortest index vector + if (net.nin < net.nhidden) + for i = 1:net.nin + gc(:,i) = (sum(((x(:,i)*ones(1, net.nhidden)) - ... + (ones(ndata, 1)*(net.c(:,i)'))).*delhid, 1)./net.wi)'; + end + else + for i = 1:net.nhidden + gc(i,:) = sum((x - (t1*(net.c(i,:)))./net.wi(i)).*(delhid(:,i)*t2), 1); + end + end + gwi = sum((n2.*delhid)./(2.*(ones(ndata, 1)*(net.wi.^2))), 1); + +case 'tps' % Thin plate spline activation function + delhid = delhid.*(1+log(n2+(n2==0))); + for i = 1:net.nhidden + gc(i,:) = sum(2.*((t1*(net.c(i,:)) - x)).*(delhid(:,i)*t2), 1); + end + % widths are not adjustable in this model + gwi = []; +case 'r4logr' % r^4 log r activation function + delhid = delhid.*(n2.*(1+2.*log(n2+(n2==0)))); + for i = 1:net.nhidden + gc(i,:) = sum(2.*((t1*(net.c(i,:)) - x)).*(delhid(:,i)*t2), 1); + end + % widths are not adjustable in this model + gwi = []; +otherwise + error('Unknown activation function in rbfgrad') +end + +g = [gc(:)', gwi, gw2(:)', gb2]; diff --git a/sourcecodes/bnt-master/netlab3.3/rbfderiv.m b/sourcecodes/bnt-master/netlab3.3/rbfderiv.m new file mode 100644 index 00000000..ac644aaa --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfderiv.m @@ -0,0 +1,53 @@ +function g = rbfderiv(net, x) +%RBFDERIV Evaluate derivatives of RBF network outputs with respect to weights. +% +% Description +% G = RBFDERIV(NET, X) takes a network data structure NET and a matrix +% of input vectors X and returns a three-index matrix G whose I, J, K +% element contains the derivative of network output K with respect to +% weight or bias parameter J for input pattern I. The ordering of the +% weight and bias parameters is defined by RBFUNPAK. This function +% also takes into account any mask in the network data structure. +% +% See also +% RBF, RBFPAK, RBFGRAD, RBFBKP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'rbf', x); +if ~isempty(errstring); + error(errstring); +end + +if ~strcmp(net.outfn, 'linear') + error('Function only implemented for linear outputs') +end + +[y, z, n2] = rbffwd(net, x); +ndata = size(x, 1); + +if isfield(net, 'mask') + nwts = size(find(net.mask), 1); + temp = zeros(1, net.nwts); +else + nwts = net.nwts; +end + +g = zeros(ndata, nwts, net.nout); +for k = 1 : net.nout + delta = zeros(1, net.nout); + delta(1, k) = 1; + for n = 1 : ndata + if isfield(net, 'mask') + temp = rbfbkp(net, x(n, :), z(n, :), n2(n, :), delta); + g(n, :, k) = temp(logical(net.mask)); + else + g(n, :, k) = rbfbkp(net, x(n, :), z(n, :), n2(n, :),... + delta); + end + end +end + + \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbferr.m b/sourcecodes/bnt-master/netlab3.3/rbferr.m new file mode 100644 index 00000000..8f5b6a39 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbferr.m @@ -0,0 +1,50 @@ +function [e, edata, eprior] = rbferr(net, x, t) +%RBFERR Evaluate error function for RBF network. +% +% Description +% E = RBFERR(NET, X, T) takes a network data structure NET together +% with a matrix X of input vectors and a matrix T of target vectors, +% and evaluates the appropriate error function E depending on +% NET.OUTFN. Each row of X corresponds to one input vector and each +% row of T contains the corresponding target vector. +% +% [E, EDATA, EPRIOR] = RBFERR(NET, X, T) additionally returns the data +% and prior components of the error, assuming a zero mean Gaussian +% prior on the weights with inverse variance parameters ALPHA and BETA +% taken from the network data structure NET. +% +% See also +% RBF, RBFFWD, RBFGRAD, RBFPAK, RBFTRAIN, RBFUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +switch net.outfn +case 'linear' + errstring = consist(net, 'rbf', x, t); +case 'neuroscale' + errstring = consist(net, 'rbf', x); +otherwise + error(['Unknown output function ', net.outfn]); +end +if ~isempty(errstring); + error(errstring); +end + +switch net.outfn +case 'linear' + y = rbffwd(net, x); + edata = 0.5*sum(sum((y - t).^2)); +case 'neuroscale' + y = rbffwd(net, x); + y_dist = sqrt(dist2(y, y)); + % Take t as target distance matrix + edata = 0.5.*(sum(sum((t-y_dist).^2))); +otherwise + error(['Unknown output function ', net.outfn]); +end + +% Compute Bayesian regularised error +[e, edata, eprior] = errbayes(net, edata); + diff --git a/sourcecodes/bnt-master/netlab3.3/rbfevfwd.m b/sourcecodes/bnt-master/netlab3.3/rbfevfwd.m new file mode 100644 index 00000000..5bafc732 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfevfwd.m @@ -0,0 +1,30 @@ +function [y, extra, invhess] = rbfevfwd(net, x, t, x_test, invhess) +%RBFEVFWD Forward propagation with evidence for RBF +% +% Description +% Y = RBFEVFWD(NET, X, T, X_TEST) takes a network data structure NET +% together with the input X and target T training data and input test +% data X_TEST. It returns the normal forward propagation through the +% network Y together with a matrix EXTRA which consists of error bars +% (variance) for a regression problem or moderated outputs for a +% classification problem. +% +% The optional argument (and return value) INVHESS is the inverse of +% the network Hessian computed on the training data inputs and targets. +% Passing it in avoids recomputing it, which can be a significant +% saving for large training sets. +% +% See also +% FEVBAYES +% + +% Copyright (c) Ian T Nabney (1996-2001) + +y = rbffwd(net, x_test); +% RBF outputs must be linear, so just pass them twice (second copy is +% not used +if nargin == 4 + [extra, invhess] = fevbayes(net, y, y, x, t, x_test); +else + [extra, invhess] = fevbayes(net, y, y, x, t, x_test, invhess); +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbffwd.m b/sourcecodes/bnt-master/netlab3.3/rbffwd.m new file mode 100644 index 00000000..af8a63e8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbffwd.m @@ -0,0 +1,55 @@ +function [a, z, n2] = rbffwd(net, x) +%RBFFWD Forward propagation through RBF network with linear outputs. +% +% Description +% A = RBFFWD(NET, X) takes a network data structure NET and a matrix X +% of input vectors and forward propagates the inputs through the +% network to generate a matrix A of output vectors. Each row of X +% corresponds to one input vector and each row of A contains the +% corresponding output vector. The activation function that is used is +% determined by NET.ACTFN. +% +% [A, Z, N2] = RBFFWD(NET, X) also generates a matrix Z of the hidden +% unit activations where each row corresponds to one pattern. These +% hidden unit activations represent the design matrix for the RBF. The +% matrix N2 is the squared distances between each basis function centre +% and each pattern in which each row corresponds to a data point. +% +% See also +% RBF, RBFERR, RBFGRAD, RBFPAK, RBFTRAIN, RBFUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'rbf', x); +if ~isempty(errstring); + error(errstring); +end + +[ndata, data_dim] = size(x); + +% Calculate squared norm matrix, of dimension (ndata, ncentres) +n2 = dist2(x, net.c); + +% Switch on activation function type +switch net.actfn + + case 'gaussian' % Gaussian + % Calculate width factors: net.wi contains squared widths + wi2 = ones(ndata, 1) * (2 .* net.wi); + + % Now compute the activations + z = exp(-(n2./wi2)); + + case 'tps' % Thin plate spline + z = n2.*log(n2+(n2==0)); + + case 'r4logr' % r^4 log r + z = n2.*n2.*log(n2+(n2==0)); + + otherwise + error('Unknown activation function in rbffwd') +end + +a = z*net.w2 + ones(ndata, 1)*net.b2; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbfgrad.m b/sourcecodes/bnt-master/netlab3.3/rbfgrad.m new file mode 100644 index 00000000..af6767ac --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfgrad.m @@ -0,0 +1,65 @@ +function [g, gdata, gprior] = rbfgrad(net, x, t) +%RBFGRAD Evaluate gradient of error function for RBF network. +% +% Description +% G = RBFGRAD(NET, X, T) takes a network data structure NET together +% with a matrix X of input vectors and a matrix T of target vectors, +% and evaluates the gradient G of the error function with respect to +% the network weights (i.e. including the hidden unit parameters). The +% error function is sum of squares. Each row of X corresponds to one +% input vector and each row of T contains the corresponding target +% vector. If the output function is 'NEUROSCALE' then the gradient is +% only computed for the output layer weights and biases. +% +% [G, GDATA, GPRIOR] = RBFGRAD(NET, X, T) also returns separately the +% data and prior contributions to the gradient. In the case of multiple +% groups in the prior, GPRIOR is a matrix with a row for each group and +% a column for each weight parameter. +% +% See also +% RBF, RBFFWD, RBFERR, RBFPAK, RBFUNPAK, RBFBKP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +switch net.outfn +case 'linear' + errstring = consist(net, 'rbf', x, t); +case 'neuroscale' + errstring = consist(net, 'rbf', x); +otherwise + error(['Unknown output function ', net.outfn]); +end +if ~isempty(errstring); + error(errstring); +end + +ndata = size(x, 1); + +[y, z, n2] = rbffwd(net, x); + +switch net.outfn +case 'linear' + + % Sum squared error at output units + delout = y - t; + + gdata = rbfbkp(net, x, z, n2, delout); + [g, gdata, gprior] = gbayes(net, gdata); + +case 'neuroscale' + % Compute the error gradient with respect to outputs + y_dist = sqrt(dist2(y, y)); + D = (t - y_dist)./(y_dist+diag(ones(ndata, 1))); + temp = y'; + gradient = 2.*sum(kron(D, ones(1, net.nout)) .* ... + (repmat(y, 1, ndata) - repmat((temp(:))', ndata, 1)), 1); + gradient = (reshape(gradient, net.nout, ndata))'; + % Compute the error gradient + gdata = rbfbkp(net, x, z, n2, gradient); + [g, gdata, gprior] = gbayes(net, gdata); +otherwise + error(['Unknown output function ', net.outfn]); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/rbfhess.m b/sourcecodes/bnt-master/netlab3.3/rbfhess.m new file mode 100644 index 00000000..440d3ac0 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfhess.m @@ -0,0 +1,91 @@ +function [h, hdata] = rbfhess(net, x, t, hdata) +%RBFHESS Evaluate the Hessian matrix for RBF network. +% +% Description +% H = RBFHESS(NET, X, T) takes an RBF network data structure NET, a +% matrix X of input values, and a matrix T of target values and returns +% the full Hessian matrix H corresponding to the second derivatives of +% the negative log posterior distribution, evaluated for the current +% weight and bias values as defined by NET. Currently, the +% implementation only computes the Hessian for the output layer +% weights. +% +% [H, HDATA] = RBFHESS(NET, X, T) returns both the Hessian matrix H and +% the contribution HDATA arising from the data dependent term in the +% Hessian. +% +% H = RBFHESS(NET, X, T, HDATA) takes a network data structure NET, a +% matrix X of input values, and a matrix T of target values, together +% with the contribution HDATA arising from the data dependent term in +% the Hessian, and returns the full Hessian matrix H corresponding to +% the second derivatives of the negative log posterior distribution. +% This version saves computation time if HDATA has already been +% evaluated for the current weight and bias values. +% +% See also +% MLPHESS, HESSCHEK, EVIDENCE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'rbf', x, t); +if ~isempty(errstring); + error(errstring); +end + +if nargin == 3 + % Data term in Hessian needs to be computed + [a, z] = rbffwd(net, x); + hdata = datahess(net, z, t); +end + +% Add in effect of regularisation +[h, hdata] = hbayes(net, hdata); + +% Sub-function to compute data part of Hessian +function hdata = datahess(net, z, t) + +% Only works for output layer Hessian currently +if (isfield(net, 'mask') & ~any(net.mask(... + 1:(net.nwts - net.nout*(net.nhidden+1))))) + hdata = zeros(net.nwts); + ndata = size(z, 1); + out_hess = [z ones(ndata, 1)]'*[z ones(ndata, 1)]; + for j = 1:net.nout + hdata = rearrange_hess(net, j, out_hess, hdata); + end +else + error('Output layer Hessian only.'); +end +return + +% Sub-function to rearrange Hessian matrix +function hdata = rearrange_hess(net, j, out_hess, hdata) + +% Because all the biases come after all the input weights, +% we have to rearrange the blocks that make up the network Hessian. +% This function assumes that we are on the jth output and that all outputs +% are independent. + +% Start of bias weights block +bb_start = net.nwts - net.nout + 1; +% Start of weight block for jth output +ob_start = net.nwts - net.nout*(net.nhidden+1) + (j-1)*net.nhidden... + + 1; +% End of weight block for jth output +ob_end = ob_start + net.nhidden - 1; +% Index of bias weight +b_index = bb_start+(j-1); +% Put input weight block in right place +hdata(ob_start:ob_end, ob_start:ob_end) = out_hess(1:net.nhidden, ... + 1:net.nhidden); +% Put second derivative of bias weight in right place +hdata(b_index, b_index) = out_hess(net.nhidden+1, net.nhidden+1); +% Put cross terms (input weight v bias weight) in right place +hdata(b_index, ob_start:ob_end) = out_hess(net.nhidden+1, ... + 1:net.nhidden); +hdata(ob_start:ob_end, b_index) = out_hess(1:net.nhidden, ... + net.nhidden+1); + +return \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbfjacob.m b/sourcecodes/bnt-master/netlab3.3/rbfjacob.m new file mode 100644 index 00000000..6a6ec73a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfjacob.m @@ -0,0 +1,49 @@ +function jac = rbfjacob(net, x) +%RBFJACOB Evaluate derivatives of RBF network outputs with respect to inputs. +% +% Description +% G = RBFJACOB(NET, X) takes a network data structure NET and a matrix +% of input vectors X and returns a three-index matrix G whose I, J, K +% element contains the derivative of network output K with respect to +% input parameter J for input pattern I. +% +% See also +% RBF, RBFGRAD, RBFBKP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'rbf', x); +if ~isempty(errstring); + error(errstring); +end + +if ~strcmp(net.outfn, 'linear') + error('Function only implemented for linear outputs') +end + +[y, z, n2] = rbffwd(net, x); + +ndata = size(x, 1); +jac = zeros(ndata, net.nin, net.nout); +Psi = zeros(net.nin, net.nhidden); +% Calculate derivative of activations wrt n2 +switch net.actfn +case 'gaussian' + dz = -z./(ones(ndata, 1)*net.wi); +case 'tps' + dz = 2*(1 + log(n2+(n2==0))); +case 'r4logr' + dz = 2*(n2.*(1+2.*log(n2+(n2==0)))); +otherwise + error(['Unknown activation function ', net.actfn]); +end + +% Ignore biases as they cannot affect Jacobian +for n = 1:ndata + Psi = (ones(net.nin, 1)*dz(n, :)).* ... + (x(n, :)'*ones(1, net.nhidden) - net.c'); + % Now compute the Jacobian + jac(n, :, :) = Psi * net.w2; +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbfpak.m b/sourcecodes/bnt-master/netlab3.3/rbfpak.m new file mode 100644 index 00000000..a482f99b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfpak.m @@ -0,0 +1,19 @@ +function w = rbfpak(net) +%RBFPAK Combines all the parameters in an RBF network into one weights vector. +% +% Description +% W = RBFPAK(NET) takes a network data structure NET and combines the +% component parameter matrices into a single row vector W. +% +% See also +% RBFUNPAK, RBF +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'rbf'); +if ~errstring + error(errstring); +end + +w = [net.c(:)', net.wi, net.w2(:)', net.b2]; diff --git a/sourcecodes/bnt-master/netlab3.3/rbfprior.m b/sourcecodes/bnt-master/netlab3.3/rbfprior.m new file mode 100644 index 00000000..c2dfaeb7 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfprior.m @@ -0,0 +1,54 @@ +function [mask, prior] = rbfprior(rbfunc, nin, nhidden, nout, aw2, ab2) +%RBFPRIOR Create Gaussian prior and output layer mask for RBF. +% +% Description +% [MASK, PRIOR] = RBFPRIOR(RBFUNC, NIN, NHIDDEN, NOUT, AW2, AB2) +% generates a vector MASK that selects only the output layer weights. +% This is because most uses of RBF networks in a Bayesian context have +% fixed basis functions with the output layer as the only adjustable +% parameters. In particular, the Neuroscale output error function is +% designed to work only with this mask. +% +% The return value PRIOR is a data structure, with fields PRIOR.ALPHA +% and PRIOR.INDEX, which specifies a Gaussian prior distribution for +% the network weights in an RBF network. The parameters AW2 and AB2 are +% all scalars and represent the regularization coefficients for two +% groups of parameters in the network corresponding to second-layer +% weights, and second-layer biases respectively. Then PRIOR.ALPHA +% represents a column vector of length 2 containing the parameters, and +% PRIOR.INDEX is a matrix specifying which weights belong in each +% group. Each column has one element for each weight in the matrix, +% using the standard ordering as defined in RBFPAK, and each element is +% 1 or 0 according to whether the weight is a member of the +% corresponding group or not. +% +% See also +% RBF, RBFERR, RBFGRAD, EVIDENCE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +nwts_layer2 = nout + (nhidden *nout); +switch rbfunc +case 'gaussian' + nwts_layer1 = nin*nhidden + nhidden; +case {'tps', 'r4logr'} + nwts_layer1 = nin*nhidden; +otherwise + error('Undefined activation function'); +end +nwts = nwts_layer1 + nwts_layer2; + +% Make a mask only for output layer +mask = [zeros(nwts_layer1, 1); ones(nwts_layer2, 1)]; + +if nargout > 1 + % Construct prior + indx = zeros(nwts, 2); + mark2 = nwts_layer1 + (nhidden * nout); + indx(nwts_layer1 + 1:mark2, 1) = ones(nhidden * nout, 1); + indx(mark2 + 1:nwts, 2) = ones(nout, 1); + + prior.index = indx; + prior.alpha = [aw2, ab2]'; +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbfsetbf.m b/sourcecodes/bnt-master/netlab3.3/rbfsetbf.m new file mode 100644 index 00000000..9e8c3ca4 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfsetbf.m @@ -0,0 +1,39 @@ +function net = rbfsetbf(net, options, x) +%RBFSETBF Set basis functions of RBF from data. +% +% Description +% NET = RBFSETBF(NET, OPTIONS, X) sets the basis functions of the RBF +% network NET so that they model the unconditional density of the +% dataset X. This is done by training a GMM with spherical covariances +% using GMMEM. The OPTIONS vector is passed to GMMEM. The widths of +% the functions are set by a call to RBFSETFW. +% +% See also +% RBFTRAIN, RBFSETFW, GMMEM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'rbf', x); +if ~isempty(errstring) + error(errstring); +end + +% Create a spherical Gaussian mixture model +mix = gmm(net.nin, net.nhidden, 'spherical'); + +% Initialise the parameters from the input data +% Just use a small number of k means iterations +kmoptions = zeros(1, 18); +kmoptions(1) = -1; % Turn off warnings +kmoptions(14) = 5; % Just 5 iterations to get centres roughly right +mix = gmminit(mix, x, kmoptions); + +% Train mixture model using EM algorithm +[mix, options] = gmmem(mix, x, options); + +% Now set the centres of the RBF from the centres of the mixture model +net.c = mix.centres; + +% options(7) gives scale of function widths +net = rbfsetfw(net, options(7)); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbfsetfw.m b/sourcecodes/bnt-master/netlab3.3/rbfsetfw.m new file mode 100644 index 00000000..8ed27605 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfsetfw.m @@ -0,0 +1,30 @@ +function net = rbfsetfw(net, scale) +%RBFSETFW Set basis function widths of RBF. +% +% Description +% NET = RBFSETFW(NET, SCALE) sets the widths of the basis functions of +% the RBF network NET. If Gaussian basis functions are used, then the +% variances are set to the largest squared distance between centres if +% SCALE is non-positive and SCALE times the mean distance of each +% centre to its nearest neighbour if SCALE is positive. Non-Gaussian +% basis functions do not have a width. +% +% See also +% RBFTRAIN, RBFSETBF, GMMEM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set the variances to be the largest squared distance between centres +if strcmp(net.actfn, 'gaussian') + cdist = dist2(net.c, net.c); + if scale > 0.0 + % Set variance of basis to be scale times average + % distance to nearest neighbour + cdist = cdist + realmax*eye(net.nhidden); + widths = scale*mean(min(cdist)); + else + widths = max(max(cdist)); + end + net.wi = widths * ones(size(net.wi)); +end diff --git a/sourcecodes/bnt-master/netlab3.3/rbftrain.m b/sourcecodes/bnt-master/netlab3.3/rbftrain.m new file mode 100644 index 00000000..f2b68422 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbftrain.m @@ -0,0 +1,203 @@ +function [net, options] = rbftrain(net, options, x, t) +%RBFTRAIN Two stage training of RBF network. +% +% Description +% NET = RBFTRAIN(NET, OPTIONS, X, T) uses a two stage training +% algorithm to set the weights in the RBF model structure NET. Each row +% of X corresponds to one input vector and each row of T contains the +% corresponding target vector. The centres are determined by fitting a +% Gaussian mixture model with circular covariances using the EM +% algorithm through a call to RBFSETBF. (The mixture model is +% initialised using a small number of iterations of the K-means +% algorithm.) If the activation functions are Gaussians, then the basis +% function widths are then set to the maximum inter-centre squared +% distance. +% +% For linear outputs, the hidden to output weights that give rise to +% the least squares solution can then be determined using the pseudo- +% inverse. For neuroscale outputs, the hidden to output weights are +% determined using the iterative shadow targets algorithm. Although +% this two stage procedure may not give solutions with as low an error +% as using general purpose non-linear optimisers, it is much faster. +% +% The options vector may have two rows: if this is the case, then the +% second row is passed to RBFSETBF, which allows the user to specify a +% different number iterations for RBF and GMM training. The optional +% parameters to RBFTRAIN have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values during EM training. +% +% OPTIONS(2) is a measure of the precision required for the value of +% the weights W at the solution. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. Both this and the previous condition must +% be satisfied for termination. +% +% OPTIONS(5) is set to 1 if the basis functions parameters should +% remain unchanged; default 0. +% +% OPTIONS(6) is set to 1 if the output layer weights should be should +% set using PCA. This is only relevant for Neuroscale outputs; default +% 0. +% +% OPTIONS(14) is the maximum number of iterations for the shadow +% targets algorithm; default 100. +% +% See also +% RBF, RBFERR, RBFFWD, RBFGRAD, RBFPAK, RBFUNPAK, RBFSETBF +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +switch net.outfn +case 'linear' + errstring = consist(net, 'rbf', x, t); +case 'neuroscale' + errstring = consist(net, 'rbf', x); +otherwise + error(['Unknown output function ', net.outfn]); +end +if ~isempty(errstring) + error(errstring); +end + +% Allow options to have two rows: if this is the case, then the second row +% is passed to rbfsetbf +if size(options, 1) == 2 + setbfoptions = options(2, :); + options = options(1, :); +else + setbfoptions = options; +end + +if(~options(14)) + options(14) = 100; +end +% Do we need to test for termination? +test = (options(2) | options(3)); + +% Set up the basis function parameters to model the input data density +% unless options(5) is set. +if ~(logical(options(5))) + net = rbfsetbf(net, setbfoptions, x); +end + +% Compute the design (or activations) matrix +[y, act] = rbffwd(net, x); +ndata = size(x, 1); + +if strcmp(net.outfn, 'neuroscale') & options(6) + % Initialise output layer weights by projecting data with PCA + mu = mean(x); + [pcvals, pcvecs] = pca(x, net.nout); + xproj = (x - ones(ndata, 1)*mu)*pcvecs; + % Now use projected data as targets to compute output layer weights + temp = pinv([act ones(ndata, 1)]) * xproj; + net.w2 = temp(1:net.nhidden, :); + net.b2 = temp(net.nhidden+1, :); + % Propagate again to compute revised outputs + [y, act] = rbffwd(net, x); +end + +switch net.outfn +case 'linear' + % Sum of squares error function in regression model + % Solve for the weights and biases using pseudo-inverse from activations + Phi = [act ones(ndata, 1)]; + if ~isfield(net, 'alpha') + % Solve for the weights and biases using left matrix divide + temp = pinv(Phi)*t; + elseif size(net.alpha == [1 1]) + % Use normal form equation + hessian = Phi'*Phi + net.alpha*eye(net.nhidden+1); + temp = pinv(hessian)*(Phi'*t); + else + error('Only scalar alpha allowed'); + end + net.w2 = temp(1:net.nhidden, :); + net.b2 = temp(net.nhidden+1, :); + +case 'neuroscale' + % Use the shadow targets training algorithm + if nargin < 4 + % If optional input distances not passed in, then use + % Euclidean distance + x_dist = sqrt(dist2(x, x)); + else + x_dist = t; + end + Phi = [act, ones(ndata, 1)]; + % Compute the pseudo-inverse of Phi + PhiDag = pinv(Phi); + % Compute y_dist, distances between image points + y_dist = sqrt(dist2(y, y)); + + % Save old weights so that we can check the termination criterion + wold = netpak(net); + % Compute initial error (stress) value + errold = 0.5*(sum(sum((x_dist - y_dist).^2))); + + % Initial value for eta + eta = 0.1; + k_up = 1.2; + k_down = 0.1; + success = 1; % Force initial gradient calculation + + for j = 1:options(14) + if success + % Compute the negative error gradient with respect to network outputs + D = (x_dist - y_dist)./(y_dist+(y_dist==0)); + temp = y'; + neg_gradient = -2.*sum(kron(D, ones(1, net.nout)) .* ... + (repmat(y, 1, ndata) - repmat((temp(:))', ndata, 1)), 1); + neg_gradient = (reshape(neg_gradient, net.nout, ndata))'; + end + % Compute the shadow targets + t = y + eta*neg_gradient; + % Solve for the weights and biases + temp = PhiDag * t; + net.w2 = temp(1:net.nhidden, :); + net.b2 = temp(net.nhidden+1, :); + + % Do housekeeping and test for convergence + ynew = rbffwd(net, x); + y_distnew = sqrt(dist2(ynew, ynew)); + err = 0.5.*(sum(sum((x_dist-y_distnew).^2))); + if err > errold + success = 0; + % Restore previous weights + net = netunpak(net, wold); + err = errold; + eta = eta * k_down; + else + success = 1; + eta = eta * k_up; + errold = err; + y = ynew; + y_dist = y_distnew; + if test & j > 1 + w = netpak(net); + if (max(abs(w - wold)) < options(2) & abs(err-errold) < options(3)) + options(8) = err; + return; + end + end + wold = netpak(net); + end + if options(1) + fprintf(1, 'Cycle %4d Error %11.6f\n', j, err) + end + if nargout >= 3 + errlog(j) = err; + end + end + options(8) = errold; + if (options(1) >= 0) + disp('Warning: Maximum number of iterations has been exceeded'); + end +otherwise + error(['Unknown output function ', net.outfn]); + +end diff --git a/sourcecodes/bnt-master/netlab3.3/rbfunpak.m b/sourcecodes/bnt-master/netlab3.3/rbfunpak.m new file mode 100644 index 00000000..64b15a9a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfunpak.m @@ -0,0 +1,43 @@ +function net = rbfunpak(net, w) +%RBFUNPAK Separates a vector of RBF weights into its components. +% +% Description +% NET = RBFUNPAK(NET, W) takes an RBF network data structure NET and a +% weight vector W, and returns a network data structure identical to +% the input network, except that the centres C, the widths WI, the +% second-layer weight matrix W2 and the second-layer bias vector B2 +% have all been set to the corresponding elements of W. +% +% See also +% RBFPAK, RBF +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'rbf'); +if ~errstring + error(errstring); +end + +if net.nwts ~= length(w) + error('Invalid length of weight vector') +end + +nin = net.nin; +nhidden = net.nhidden; +nout = net.nout; + +mark1 = nin*nhidden; +net.c = reshape(w(1:mark1), nhidden, nin); +if strcmp(net.actfn, 'gaussian') + mark2 = mark1 + nhidden; + net.wi = reshape(w(mark1+1:mark2), 1, nhidden); +else + mark2 = mark1; + net.wi = []; +end +mark3 = mark2 + nhidden*nout; +net.w2 = reshape(w(mark2+1:mark3), nhidden, nout); +mark4 = mark3 + nout; +net.b2 = reshape(w(mark3+1:mark4), 1, nout); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rosegrad.m b/sourcecodes/bnt-master/netlab3.3/rosegrad.m new file mode 100644 index 00000000..898e5d80 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rosegrad.m @@ -0,0 +1,20 @@ +function g = rosegrad(x) +%ROSEGRAD Calculate gradient of Rosenbrock's function. +% +% Description +% G = ROSEGRAD(X) computes the gradient of Rosenbrock's function at +% each row of X, which should have two columns. +% +% See also +% DEMOPT1, ROSEN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Return gradient of Rosenbrock's test function + +nrows = size(x, 1); +g = zeros(nrows,2); + +g(:,1) = -400 * (x(:,2) - x(:,1).^2) * x(:,1) - 2 * (1 - x(:,1)); +g(:,2) = 200 * (x(:,2) - x(:,1).^2); diff --git a/sourcecodes/bnt-master/netlab3.3/rosen.m b/sourcecodes/bnt-master/netlab3.3/rosen.m new file mode 100644 index 00000000..89332a52 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rosen.m @@ -0,0 +1,16 @@ +function y = rosen(x) +%ROSEN Calculate Rosenbrock's function. +% +% Description +% Y = ROSEN(X) computes the value of Rosenbrock's function at each row +% of X, which should have two columns. +% +% See also +% DEMOPT1, ROSEGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Calculate value of Rosenbrock's function: x should be nrows by 2 columns + +y = 100 * ((x(:,2) - x(:,1).^2).^2) + (1.0 - x(:,1)).^2; diff --git a/sourcecodes/bnt-master/netlab3.3/scg.m b/sourcecodes/bnt-master/netlab3.3/scg.m new file mode 100644 index 00000000..c248d9e4 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/scg.m @@ -0,0 +1,208 @@ +function [x, options, flog, pointlog, scalelog] = scg(f, x, options, gradf, varargin) +%SCG Scaled conjugate gradient optimization. +% +% Description +% [X, OPTIONS] = SCG(F, X, OPTIONS, GRADF) uses a scaled conjugate +% gradients algorithm to find a local minimum of the function F(X) +% whose gradient is given by GRADF(X). Here X is a row vector and F +% returns a scalar value. The point at which F has a local minimum is +% returned as X. The function value at that point is returned in +% OPTIONS(8). +% +% [X, OPTIONS, FLOG, POINTLOG, SCALELOG] = SCG(F, X, OPTIONS, GRADF) +% also returns (optionally) a log of the function values after each +% cycle in FLOG, a log of the points visited in POINTLOG, and a log of +% the scale values in the algorithm in SCALELOG. +% +% SCG(F, X, OPTIONS, GRADF, P1, P2, ...) allows additional arguments to +% be passed to F() and GRADF(). The optional parameters have the +% following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG, and the points visited in the +% return argument POINTSLOG. If OPTIONS(1) is set to 0, then only +% warning messages are displayed. If OPTIONS(1) is -1, then nothing is +% displayed. +% +% OPTIONS(2) is a measure of the absolute precision required for the +% value of X at the solution. If the absolute difference between the +% values of X between two successive steps is less than OPTIONS(2), +% then this condition is satisfied. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. If the absolute difference between the +% objective function values between two successive steps is less than +% OPTIONS(3), then this condition is satisfied. Both this and the +% previous condition must be satisfied for termination. +% +% OPTIONS(9) is set to 1 to check the user defined gradient function. +% +% OPTIONS(10) returns the total number of function evaluations +% (including those in any line searches). +% +% OPTIONS(11) returns the total number of gradient evaluations. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% See also +% CONJGRAD, QUASINEW +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set up the options. +if length(options) < 18 + error('Options vector too short') +end + +if(options(14)) + niters = options(14); +else + niters = 100; +end + +display = options(1); +gradcheck = options(9); + +% Set up strings for evaluating function and gradient +f = fcnchk(f, length(varargin)); +gradf = fcnchk(gradf, length(varargin)); + +nparams = length(x); + +% Check gradients +if (gradcheck) + feval('gradchek', x, f, gradf, varargin{:}); +end + +sigma0 = 1.0e-4; +fold = feval(f, x, varargin{:}); % Initial function value. +fnow = fold; +options(10) = options(10) + 1; % Increment function evaluation counter. +gradnew = feval(gradf, x, varargin{:}); % Initial gradient. +gradold = gradnew; +options(11) = options(11) + 1; % Increment gradient evaluation counter. +d = -gradnew; % Initial search direction. +success = 1; % Force calculation of directional derivs. +nsuccess = 0; % nsuccess counts number of successes. +beta = 1.0; % Initial scale parameter. +betamin = 1.0e-15; % Lower bound on scale. +betamax = 1.0e100; % Upper bound on scale. +j = 1; % j counts number of iterations. +if nargout >= 3 + flog(j, :) = fold; + if nargout == 4 + pointlog(j, :) = x; + end +end + +% Main optimization loop. +while (j <= niters) + + % Calculate first and second directional derivatives. + if (success == 1) + mu = d*gradnew'; + if (mu >= 0) + d = - gradnew; + mu = d*gradnew'; + end + kappa = d*d'; + if kappa < eps + options(8) = fnow; + return + end + sigma = sigma0/sqrt(kappa); + xplus = x + sigma*d; + gplus = feval(gradf, xplus, varargin{:}); + options(11) = options(11) + 1; + theta = (d*(gplus' - gradnew'))/sigma; + end + + % Increase effective curvature and evaluate step size alpha. + delta = theta + beta*kappa; + if (delta <= 0) + delta = beta*kappa; + beta = beta - theta/kappa; + end + alpha = - mu/delta; + + % Calculate the comparison ratio. + xnew = x + alpha*d; + fnew = feval(f, xnew, varargin{:}); + options(10) = options(10) + 1; + Delta = 2*(fnew - fold)/(alpha*mu); + if (Delta >= 0) + success = 1; + nsuccess = nsuccess + 1; + x = xnew; + fnow = fnew; + else + success = 0; + fnow = fold; + end + + if nargout >= 3 + % Store relevant variables + flog(j) = fnow; % Current function value + if nargout >= 4 + pointlog(j,:) = x; % Current position + if nargout >= 5 + scalelog(j) = beta; % Current scale parameter + end + end + end + if display > 0 + fprintf(1, 'Cycle %4d Error %11.6f Scale %e\n', j, fnow, beta); + end + + if (success == 1) + % Test for termination + + if (max(abs(alpha*d)) < options(2) & max(abs(fnew-fold)) < options(3)) + options(8) = fnew; + return; + + else + % Update variables for new position + fold = fnew; + gradold = gradnew; + gradnew = feval(gradf, x, varargin{:}); + options(11) = options(11) + 1; + % If the gradient is zero then we are done. + if (gradnew*gradnew' == 0) + options(8) = fnew; + return; + end + end + end + + % Adjust beta according to comparison ratio. + if (Delta < 0.25) + beta = min(4.0*beta, betamax); + end + if (Delta > 0.75) + beta = max(0.5*beta, betamin); + end + + % Update search direction using Polak-Ribiere formula, or re-start + % in direction of negative gradient after nparams steps. + if (nsuccess == nparams) + d = -gradnew; + nsuccess = 0; + else + if (success == 1) + gamma = (gradold - gradnew)*gradnew'/(mu); + d = gamma*d - gradnew; + end + end + j = j + 1; +end + +% If we get here, then we haven't terminated in the given number of +% iterations. + +options(8) = fold; +if (options(1) >= 0) + disp(maxitmess); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/som.m b/sourcecodes/bnt-master/netlab3.3/som.m new file mode 100644 index 00000000..4c975c2f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/som.m @@ -0,0 +1,55 @@ +function net = som(nin, map_size) +%SOM Creates a Self-Organising Map. +% +% Description +% NET = SOM(NIN, MAP_SIZE) creates a SOM NET with input dimension (i.e. +% data dimension) NIN and map dimensions MAP_SIZE. Only two- +% dimensional maps are currently implemented. +% +% The fields in NET are +% type = 'som' +% nin = number of inputs +% map_dim = dimension of map (constrained to be 2) +% map_size = grid size: number of nodes in each dimension +% num_nodes = number of nodes: the product of values in map_size +% map = map_dim+1 dimensional array containing nodes +% inode_dist = map of inter-node distances using Manhatten metric +% +% The map contains the node vectors arranged column-wise in the first +% dimension of the array. +% +% See also +% KMEANS, SOMFWD, SOMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.type = 'som'; +net.nin = nin; + +% Create Map of nodes +if round(map_size) ~= map_size | (map_size < 1) + error('SOM specification must contain positive integers'); +end + +net.map_dim = length(map_size); +if net.map_dim ~= 2 + error('SOM is a 2 dimensional map'); +end +net.num_nodes = prod(map_size); +% Centres are stored by column as first index of multi-dimensional array. +% This makes extracting them later more easy. +% Initialise with rand to create square grid +net.map = rand([nin, map_size]); +net.map_size = map_size; + +% Crude function to compute inter-node distances +net.inode_dist = zeros([map_size, net.num_nodes]); +for m = 1:net.num_nodes + node_loc = [1+fix((m-1)/map_size(2)), 1+rem((m-1),map_size(2))]; + for k = 1:map_size(1) + for l = 1:map_size(2) + net.inode_dist(k, l, m) = round(max(abs([k l] - node_loc))); + end + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/somfwd.m b/sourcecodes/bnt-master/netlab3.3/somfwd.m new file mode 100644 index 00000000..0e980284 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/somfwd.m @@ -0,0 +1,30 @@ +function [d2, win_nodes] = somfwd(net, x) +%SOMFWD Forward propagation through a Self-Organising Map. +% +% Description +% D2 = SOMFWD(NET, X) propagates the data matrix X through a SOM NET, +% returning the squared distance matrix D2 with dimension NIN by +% NUM_NODES. The $i$th row represents the squared Euclidean distance +% to each of the nodes of the SOM. +% +% [D2, WIN_NODES] = SOMFWD(NET, X) also returns the indices of the +% winning nodes for each pattern. +% +% See also +% SOM, SOMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check for consistency +errstring = consist(net, 'som', x); +if ~isempty(errstring) + error(errstring); +end + +% Turn nodes into matrix of centres +nodes = (reshape(net.map, net.nin, net.num_nodes))'; +% Compute squared distance matrix +d2 = dist2(x, nodes); +% Find winning node for each pattern: minimum value in each row +[w, win_nodes] = min(d2, [], 2); diff --git a/sourcecodes/bnt-master/netlab3.3/sompak.m b/sourcecodes/bnt-master/netlab3.3/sompak.m new file mode 100644 index 00000000..a1d7653a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/sompak.m @@ -0,0 +1,23 @@ +function [c] = sompak(net) +%SOMPAK Combines node weights into one weights matrix. +% +% Description +% C = SOMPAK(NET) takes a SOM data structure NET and combines the node +% weights into a matrix of centres C where each row represents the node +% vector. +% +% The ordering of the parameters in W is defined by the indexing of the +% multi-dimensional array NET.MAP. +% +% See also +% SOM, SOMUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'som'); +if ~isempty(errstring) + error(errstring); +end +% Returns map as a sequence of row vectors +c = (reshape(net.map, net.nin, net.num_nodes))'; diff --git a/sourcecodes/bnt-master/netlab3.3/somtrain.m b/sourcecodes/bnt-master/netlab3.3/somtrain.m new file mode 100644 index 00000000..47df11af --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/somtrain.m @@ -0,0 +1,157 @@ +function net = somtrain(net, options, x) +%SOMTRAIN Kohonen training algorithm for SOM. +% +% Description +% NET = SOMTRAIN{NET, OPTIONS, X) uses Kohonen's algorithm to train a +% SOM. Both on-line and batch algorithms are implemented. The learning +% rate (for on-line) and neighbourhood size decay linearly. There is no +% error function minimised during training (so there is no termination +% criterion other than the number of epochs), but the sum-of-squares +% is computed and returned in OPTIONS(8). +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs learning +% rate ALPHA and neighbourhood size NSIZE. Otherwise nothing is +% displayed. +% +% OPTIONS(5) determines whether the patterns are sampled randomly with +% replacement. If it is 0 (the default), then patterns are sampled in +% order. This is only relevant to the on-line algorithm. +% +% OPTIONS(6) determines if the on-line or batch algorithm is used. If +% it is 1 then the batch algorithm is used. If it is 0 (the default) +% then the on-line algorithm is used. +% +% OPTIONS(14) is the maximum number of iterations (passes through the +% complete pattern set); default 100. +% +% OPTIONS(15) is the final neighbourhood size; default value is the +% same as the initial neighbourhood size. +% +% OPTIONS(16) is the final learning rate; default value is the same as +% the initial learning rate. +% +% OPTIONS(17) is the initial neighbourhood size; default 0.5*maximum +% map size. +% +% OPTIONS(18) is the initial learning rate; default 0.9. This +% parameter must be positive. +% +% See also +% KMEANS, SOM, SOMFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'som', x); +if ~isempty(errstring) + error(errstring); +end + +% Set number of iterations in convergence phase +if (~options(14)) + options(14) = 100; +end +niters = options(14); + +% Learning rate must be positive +if (options(18) > 0) + alpha_first = options(18); +else + alpha_first = 0.9; +end +% Final learning rate must be no greater than initial learning rate +if (options(16) > alpha_first | options(16) < 0) + alpha_last = alpha_first; +else + alpha_last = options(16); +end + +% Neighbourhood size +if (options(17) >= 0) + nsize_first = options(17); +else + nsize_first = max(net.map_dim)/2; +end +% Final neighbourhood size must be no greater than initial size +if (options(15) > nsize_first | options(15) < 0) + nsize_last = nsize_first; +else + nsize_last = options(15); +end + +ndata = size(x, 1); + +if options(6) + % Batch algorithm + H = zeros(ndata, net.num_nodes); +end +% Put weights into matrix form +tempw = sompak(net); + +% Then carry out training +j = 1; +while j <= niters + if options(6) + % Batch version of algorithm + alpha = 0.0; + frac_done = (niters - j)/niters; + % Compute neighbourhood + nsize = round((nsize_first - nsize_last)*frac_done + nsize_last); + + % Find winning node: put weights back into net so that we can + % call somunpak + net = somunpak(net, tempw); + [temp, bnode] = somfwd(net, x); + for k = 1:ndata + H(k, :) = reshape(net.inode_dist(:, :, bnode(k))<=nsize, ... + 1, net.num_nodes); + end + s = sum(H, 1); + for k = 1:net.num_nodes + if s(k) > 0 + tempw(k, :) = sum((H(:, k)*ones(1, net.nin)).*x, 1)/ ... + s(k); + end + end + else + % On-line version of algorithm + if options(5) + % Randomise order of pattern presentation: with replacement + pnum = ceil(rand(ndata, 1).*ndata); + else + pnum = 1:ndata; + end + % Cycle through dataset + for k = 1:ndata + % Fraction done + frac_done = (((niters+1)*ndata)-(j*ndata + k))/((niters+1)*ndata); + % Compute learning rate + alpha = (alpha_first - alpha_last)*frac_done + alpha_last; + % Compute neighbourhood + nsize = round((nsize_first - nsize_last)*frac_done + nsize_last); + % Find best node + pat_diff = ones(net.num_nodes, 1)*x(pnum(k), :) - tempw; + [temp, bnode] = min(sum(abs(pat_diff), 2)); + + % Now update neighbourhood + neighbourhood = (net.inode_dist(:, :, bnode) <= nsize); + tempw = tempw + ... + ((alpha*(neighbourhood(:)))*ones(1, net.nin)).*pat_diff; + end + end + if options(1) + % Print iteration information + fprintf(1, 'Iteration %d; alpha = %f, nsize = %f. ', j, alpha, ... + nsize); + % Print sum squared error to nearest node + d2 = dist2(tempw, x); + fprintf(1, 'Error = %f\n', sum(min(d2))); + end + j = j + 1; +end + +net = somunpak(net, tempw); +options(8) = sum(min(dist2(tempw, x))); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/somunpak.m b/sourcecodes/bnt-master/netlab3.3/somunpak.m new file mode 100644 index 00000000..e243820a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/somunpak.m @@ -0,0 +1,23 @@ +function net = somunpak(net, w) +%SOMUNPAK Replaces node weights in SOM. +% +% Description +% NET = SOMUNPAK(NET, W) takes a SOM data structure NET and weight +% matrix W (each node represented by a row) and puts the nodes back +% into the multi-dimensional array NET.MAP. +% +% The ordering of the parameters in W is defined by the indexing of the +% multi-dimensional array NET.MAP. +% +% See also +% SOM, SOMPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'som'); +if ~isempty(errstring) + error(errstring); +end +% Put weights back into network data structure +net.map = reshape(w', [net.nin net.map_size]); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/xor.dat b/sourcecodes/bnt-master/netlab3.3/xor.dat new file mode 100644 index 00000000..f0885cee --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/xor.dat @@ -0,0 +1,15 @@ +nin 2 +nout 1 +ndata 12 +1 0 1 +0 1 1 +0 0 0 +1 1 0 +1 0 1 +0 1 1 +0 0 0 +1 1 0 +1 0 1 +0 1 1 +0 0 0 +1 1 0 |