diff options
| author | ziejd2 | 2018-03-14 23:23:33 -0500 |
|---|---|---|
| committer | GitHub | 2018-03-14 23:23:33 -0500 |
| commit | 1ff6baa44e22b91eefb48aea6f3befa078c0489b (patch) | |
| tree | e0fd79d2e32fd2aedda2eadaed0f19af3514c520 /sourcecodes/bnt-master/netlab3.3 | |
| parent | 6882395afdadf4e982b25b5215071a0932730950 (diff) | |
| parent | c80226899f5cdd9f11c163817d59445213f5bef0 (diff) | |
| download | BNW-1ff6baa44e22b91eefb48aea6f3befa078c0489b.tar.gz | |
Merge pull request #1 from ziejd2/octave_php_separate
Octave php separate
Diffstat (limited to 'sourcecodes/bnt-master/netlab3.3')
182 files changed, 16107 insertions, 0 deletions
diff --git a/sourcecodes/bnt-master/netlab3.3/CVS/Entries b/sourcecodes/bnt-master/netlab3.3/CVS/Entries new file mode 100644 index 00000000..5a92ce95 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/CVS/Entries @@ -0,0 +1,180 @@ +/Contents.m/1.1.1.1/Wed Apr 27 17:59:04 2005// +/LICENSE/1.1.1.1/Wed Apr 27 17:59:04 2005// +/conffig.m/1.1.1.1/Wed Apr 27 17:59:04 2005// +/confmat.m/1.1.1.1/Wed Apr 27 17:59:04 2005// +/conjgrad.m/1.1.1.1/Wed Apr 27 17:59:04 2005// +/consist.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/convertoldnet.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/datread.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/datwrite.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/dem2ddat.m/1.1.1.1/Tue May 10 01:32:18 2005// +/demard.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demev1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demev2.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demev3.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgauss.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demglm1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demglm2.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgmm1.m/1.1.1.1/Sun May 8 17:51:14 2005// +/demgmm2.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgmm3.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgmm4.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgmm5.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgp.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgpard.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgpot.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgtm1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demgtm2.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demhint.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demhmc1.m/1.1.1.1/Tue May 24 00:03:34 2005// +/demhmc2.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demhmc3.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demkmn1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demknn1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demmdn1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demmet1.m/1.1.1.1/Mon May 23 06:23:08 2005// +/demmlp1.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demmlp2.m/1.1.1.1/Wed Apr 27 17:59:06 2005// +/demnlab.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demns1.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demolgd1.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demopt1.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/dempot.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demprgp.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demprior.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demrbf1.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demsom1.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/demtrain.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/dist2.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/eigdec.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/errbayes.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/evidence.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/fevbayes.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gauss.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gbayes.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glm.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmderiv.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmerr.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmevfwd.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmfwd.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmgrad.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmhess.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glminit.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmpak.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmtrain.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/glmunpak.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gmm.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gmmactiv.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gmmem.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gmminit.m/1.1.1.1/Mon May 23 23:44:46 2005// +/gmmpak.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gmmpost.m/1.1.1.1/Wed Apr 27 17:59:08 2005// +/gmmprob.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gmmsamp.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gmmunpak.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gp.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpcovar.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpcovarf.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpcovarp.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gperr.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpfwd.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpgrad.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpinit.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gppak.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gpunpak.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gradchek.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/graddesc.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gsamp.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtm.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmem.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmfwd.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtminit.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmlmean.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmlmode.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmmag.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmpost.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/gtmprob.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/hbayes.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/hesschek.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/hintmat.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/hinton.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/histp.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/hmc.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/kmeansNetlab.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/knn.m/1.1.1.1/Wed Apr 27 17:59:10 2005// +/knnfwd.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/linef.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/linemin.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/maxitmess.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdn.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdn2gmm.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdndist2.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnerr.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnfwd.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdngrad.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdninit.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnnet.mat/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnpak.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnpost.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnprob.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mdnunpak.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/metrop.m/1.1.1.1/Mon May 23 23:07:40 2005// +/minbrack.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlp.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpbkp.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpderiv.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlperr.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpevfwd.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpfwd.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpgrad.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlphdotv.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlphess.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlphint.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpinit.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlppak.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpprior.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlptrain.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/mlpunpak.m/1.1.1.1/Wed Apr 27 17:59:12 2005// +/netderiv.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/neterr.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netevfwd.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netgrad.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/nethess.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netinit.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netlab3.3.zip/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netlogo.mat/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netopt.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netpak.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/netunpak.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/oilTrn.dat/1.1.1.1/Wed Apr 27 17:59:14 2005// +/oilTst.dat/1.1.1.1/Wed Apr 27 17:59:14 2005// +/olgd.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/pca.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/plotmat.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/ppca.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/quasinew.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbf.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfbkp.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfderiv.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbferr.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfevfwd.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbffwd.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfgrad.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfhess.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfjacob.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfpak.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfprior.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfsetbf.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbfsetfw.m/1.1.1.1/Wed Apr 27 17:59:14 2005// +/rbftrain.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/rbfunpak.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/rosegrad.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/rosen.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/scg.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/som.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/somfwd.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/sompak.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/somtrain.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/somunpak.m/1.1.1.1/Wed Apr 27 17:59:16 2005// +/xor.dat/1.1.1.1/Wed Apr 27 17:59:16 2005// +D diff --git a/sourcecodes/bnt-master/netlab3.3/CVS/Repository b/sourcecodes/bnt-master/netlab3.3/CVS/Repository new file mode 100644 index 00000000..297cfa8a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/CVS/Repository @@ -0,0 +1 @@ +FullBNT/netlab3.3 diff --git a/sourcecodes/bnt-master/netlab3.3/CVS/Root b/sourcecodes/bnt-master/netlab3.3/CVS/Root new file mode 100644 index 00000000..f3bd14a6 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/CVS/Root @@ -0,0 +1 @@ +:ext:nsaunier@bnt.cvs.sourceforge.net:/cvsroot/bnt diff --git a/sourcecodes/bnt-master/netlab3.3/Contents.m b/sourcecodes/bnt-master/netlab3.3/Contents.m new file mode 100644 index 00000000..aaf32e1f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/Contents.m @@ -0,0 +1,176 @@ +% Netlab Toolbox +% Version 3.3.1 18-Jun-2004 +% +% conffig - Display a confusion matrix. +% confmat - Compute a confusion matrix. +% conjgrad - Conjugate gradients optimization. +% consist - Check that arguments are consistent. +% convertoldnet- Convert pre-2.3 release MLP and MDN nets to new format +% datread - Read data from an ascii file. +% datwrite - Write data to ascii file. +% dem2ddat - Generates two dimensional data for demos. +% demard - Automatic relevance determination using the MLP. +% demev1 - Demonstrate Bayesian regression for the MLP. +% demev2 - Demonstrate Bayesian classification for the MLP. +% demev3 - Demonstrate Bayesian regression for the RBF. +% demgauss - Demonstrate sampling from Gaussian distributions. +% demglm1 - Demonstrate simple classification using a generalized linear model. +% demglm2 - Demonstrate simple classification using a generalized linear model. +% demgmm1 - Demonstrate density modelling with a Gaussian mixture model. +% demgmm3 - Demonstrate density modelling with a Gaussian mixture model. +% demgmm4 - Demonstrate density modelling with a Gaussian mixture model. +% demgmm5 - Demonstrate density modelling with a PPCA mixture model. +% demgp - Demonstrate simple regression using a Gaussian Process. +% demgpard - Demonstrate ARD using a Gaussian Process. +% demgpot - Computes the gradient of the negative log likelihood for a mixture model. +% demgtm1 - Demonstrate EM for GTM. +% demgtm2 - Demonstrate GTM for visualisation. +% demhint - Demonstration of Hinton diagram for 2-layer feed-forward network. +% demhmc1 - Demonstrate Hybrid Monte Carlo sampling on mixture of two Gaussians. +% demhmc2 - Demonstrate Bayesian regression with Hybrid Monte Carlo sampling. +% demhmc3 - Demonstrate Bayesian regression with Hybrid Monte Carlo sampling. +% demkmean - Demonstrate simple clustering model trained with K-means. +% demknn1 - Demonstrate nearest neighbour classifier. +% demmdn1 - Demonstrate fitting a multi-valued function using a Mixture Density Network. +% demmet1 - Demonstrate Markov Chain Monte Carlo sampling on a Gaussian. +% demmlp1 - Demonstrate simple regression using a multi-layer perceptron +% demmlp2 - Demonstrate simple classification using a multi-layer perceptron +% demnlab - A front-end Graphical User Interface to the demos +% demns1 - Demonstrate Neuroscale for visualisation. +% demolgd1 - Demonstrate simple MLP optimisation with on-line gradient descent +% demopt1 - Demonstrate different optimisers on Rosenbrock's function. +% dempot - Computes the negative log likelihood for a mixture model. +% demprgp - Demonstrate sampling from a Gaussian Process prior. +% demprior - Demonstrate sampling from a multi-parameter Gaussian prior. +% demrbf1 - Demonstrate simple regression using a radial basis function network. +% demsom1 - Demonstrate SOM for visualisation. +% demtrain - Demonstrate training of MLP network. +% dist2 - Calculates squared distance between two sets of points. +% eigdec - Sorted eigendecomposition +% errbayes - Evaluate Bayesian error function for network. +% evidence - Re-estimate hyperparameters using evidence approximation. +% fevbayes - Evaluate Bayesian regularisation for network forward propagation. +% gauss - Evaluate a Gaussian distribution. +% gbayes - Evaluate gradient of Bayesian error function for network. +% glm - Create a generalized linear model. +% glmderiv - Evaluate derivatives of GLM outputs with respect to weights. +% glmerr - Evaluate error function for generalized linear model. +% glmevfwd - Forward propagation with evidence for GLM +% glmfwd - Forward propagation through generalized linear model. +% glmgrad - Evaluate gradient of error function for generalized linear model. +% glmhess - Evaluate the Hessian matrix for a generalised linear model. +% glminit - Initialise the weights in a generalized linear model. +% glmpak - Combines weights and biases into one weights vector. +% glmtrain - Specialised training of generalized linear model +% glmunpak - Separates weights vector into weight and bias matrices. +% gmm - Creates a Gaussian mixture model with specified architecture. +% gmmactiv - Computes the activations of a Gaussian mixture model. +% gmmem - EM algorithm for Gaussian mixture model. +% gmminit - Initialises Gaussian mixture model from data +% gmmpak - Combines all the parameters in a Gaussian mixture model into one vector. +% gmmpost - Computes the class posterior probabilities of a Gaussian mixture model. +% gmmprob - Computes the data probability for a Gaussian mixture model. +% gmmsamp - Sample from a Gaussian mixture distribution. +% gmmunpak - Separates a vector of Gaussian mixture model parameters into its components. +% gp - Create a Gaussian Process. +% gpcovar - Calculate the covariance for a Gaussian Process. +% gpcovarf - Calculate the covariance function for a Gaussian Process. +% gpcovarp - Calculate the prior covariance for a Gaussian Process. +% gperr - Evaluate error function for Gaussian Process. +% gpfwd - Forward propagation through Gaussian Process. +% gpgrad - Evaluate error gradient for Gaussian Process. +% gpinit - Initialise Gaussian Process model. +% gppak - Combines GP hyperparameters into one vector. +% gpunpak - Separates hyperparameter vector into components. +% gradchek - Checks a user-defined gradient function using finite differences. +% graddesc - Gradient descent optimization. +% gsamp - Sample from a Gaussian distribution. +% gtm - Create a Generative Topographic Map. +% gtmem - EM algorithm for Generative Topographic Mapping. +% gtmfwd - Forward propagation through GTM. +% gtminit - Initialise the weights and latent sample in a GTM. +% gtmlmean - Mean responsibility for data in a GTM. +% gtmlmode - Mode responsibility for data in a GTM. +% gtmmag - Magnification factors for a GTM +% gtmpost - Latent space responsibility for data in a GTM. +% gtmprob - Probability for data under a GTM. +% hbayes - Evaluate Hessian of Bayesian error function for network. +% hesschek - Use central differences to confirm correct evaluation of Hessian matrix. +% hintmat - Evaluates the coordinates of the patches for a Hinton diagram. +% hinton - Plot Hinton diagram for a weight matrix. +% histp - Histogram estimate of 1-dimensional probability distribution. +% hmc - Hybrid Monte Carlo sampling. +% kmeans - Trains a k means cluster model. +% knn - Creates a K-nearest-neighbour classifier. +% knnfwd - Forward propagation through a K-nearest-neighbour classifier. +% linef - Calculate function value along a line. +% linemin - One dimensional minimization. +% maxitmess- Create a standard error message when training reaches max. iterations. +% mdn - Creates a Mixture Density Network with specified architecture. +% mdn2gmm - Converts an MDN mixture data structure to array of GMMs. +% mdndist2 - Calculates squared distance between centres of Gaussian kernels and data +% mdnerr - Evaluate error function for Mixture Density Network. +% mdnfwd - Forward propagation through Mixture Density Network. +% mdngrad - Evaluate gradient of error function for Mixture Density Network. +% mdninit - Initialise the weights in a Mixture Density Network. +% mdnpak - Combines weights and biases into one weights vector. +% mdnpost - Computes the posterior probability for each MDN mixture component. +% mdnprob - Computes the data probability likelihood for an MDN mixture structure. +% mdnunpak - Separates weights vector into weight and bias matrices. +% metrop - Markov Chain Monte Carlo sampling with Metropolis algorithm. +% minbrack - Bracket a minimum of a function of one variable. +% mlp - Create a 2-layer feedforward network. +% mlpbkp - Backpropagate gradient of error function for 2-layer network. +% mlpderiv - Evaluate derivatives of network outputs with respect to weights. +% mlperr - Evaluate error function for 2-layer network. +% mlpevfwd - Forward propagation with evidence for MLP +% mlpfwd - Forward propagation through 2-layer network. +% mlpgrad - Evaluate gradient of error function for 2-layer network. +% mlphdotv - Evaluate the product of the data Hessian with a vector. +% mlphess - Evaluate the Hessian matrix for a multi-layer perceptron network. +% mlphint - Plot Hinton diagram for 2-layer feed-forward network. +% mlpinit - Initialise the weights in a 2-layer feedforward network. +% mlppak - Combines weights and biases into one weights vector. +% mlpprior - Create Gaussian prior for mlp. +% mlptrain - Utility to train an MLP network for demtrain +% mlpunpak - Separates weights vector into weight and bias matrices. +% netderiv - Evaluate derivatives of network outputs by weights generically. +% neterr - Evaluate network error function for generic optimizers +% netevfwd - Generic forward propagation with evidence for network +% netgrad - Evaluate network error gradient for generic optimizers +% nethess - Evaluate network Hessian +% netinit - Initialise the weights in a network. +% netopt - Optimize the weights in a network model. +% netpak - Combines weights and biases into one weights vector. +% netunpak - Separates weights vector into weight and bias matrices. +% olgd - On-line gradient descent optimization. +% pca - Principal Components Analysis +% plotmat - Display a matrix. +% ppca - Probabilistic Principal Components Analysis +% quasinew - Quasi-Newton optimization. +% rbf - Creates an RBF network with specified architecture +% rbfbkp - Backpropagate gradient of error function for RBF network. +% rbfderiv - Evaluate derivatives of RBF network outputs with respect to weights. +% rbferr - Evaluate error function for RBF network. +% rbfevfwd - Forward propagation with evidence for RBF +% rbffwd - Forward propagation through RBF network with linear outputs. +% rbfgrad - Evaluate gradient of error function for RBF network. +% rbfhess - Evaluate the Hessian matrix for RBF network. +% rbfjacob - Evaluate derivatives of RBF network outputs with respect to inputs. +% rbfpak - Combines all the parameters in an RBF network into one weights vector. +% rbfprior - Create Gaussian prior and output layer mask for RBF. +% rbfsetbf - Set basis functions of RBF from data. +% rbfsetfw - Set basis function widths of RBF. +% rbftrain - Two stage training of RBF network. +% rbfunpak - Separates a vector of RBF weights into its components. +% rosegrad - Calculate gradient of Rosenbrock's function. +% rosen - Calculate Rosenbrock's function. +% scg - Scaled conjugate gradient optimization. +% som - Creates a Self-Organising Map. +% somfwd - Forward propagation through a Self-Organising Map. +% sompak - Combines node weights into one weights matrix. +% somtrain - Kohonen training algorithm for SOM. +% somunpak - Replaces node weights in SOM. +% +% Copyright (c) Ian T Nabney (1996-2001) +% diff --git a/sourcecodes/bnt-master/netlab3.3/LICENSE b/sourcecodes/bnt-master/netlab3.3/LICENSE new file mode 100644 index 00000000..0dd4a362 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/LICENSE @@ -0,0 +1,42 @@ + Copyright (c) 1996-2001, Ian T. Nabney + All rights reserved. + + Redistribution and use in source and binary + forms, with or without modification, are + permitted provided that the following + conditions are met: + + * Redistributions of source code must + retain the above copyright notice, this + list of conditions and the following + disclaimer. + * Redistributions in binary form must + reproduce the above copyright notice, + this list of conditions and the + following disclaimer in the + documentation and/or other materials + provided with the distribution. + * Neither the name of the Aston University, Birmingham, U.K. + nor the names of its contributors may be + used to endorse or promote products + derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT + HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT + NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT + OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. diff --git a/sourcecodes/bnt-master/netlab3.3/conffig.m b/sourcecodes/bnt-master/netlab3.3/conffig.m new file mode 100644 index 00000000..23d774e0 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/conffig.m @@ -0,0 +1,29 @@ +function fh=conffig(y, t) +%CONFFIG Display a confusion matrix. +% +% Description +% CONFFIG(Y, T) displays the confusion matrix and classification +% performance for the predictions mat{y} compared with the targets T. +% The data is assumed to be in a 1-of-N encoding, unless there is just +% one column, when it is assumed to be a 2 class problem with a 0-1 +% encoding. Each row of Y and T corresponds to a single example. +% +% In the confusion matrix, the rows represent the true classes and the +% columns the predicted classes. +% +% FH = CONFFIG(Y, T) also returns the figure handle FH which can be +% used, for instance, to delete the figure when it is no longer needed. +% +% See also +% CONFMAT, DEMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[C, rate] = confmat(y, t); + +fh = figure('Name', 'Confusion matrix', ... + 'NumberTitle', 'off'); + +plotmat(C, 'k', 'k', 14); +title(['Classification rate: ' num2str(rate(1)) '%'], 'FontSize', 14); diff --git a/sourcecodes/bnt-master/netlab3.3/confmat.m b/sourcecodes/bnt-master/netlab3.3/confmat.m new file mode 100644 index 00000000..e03315bb --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/confmat.m @@ -0,0 +1,56 @@ +function [C,rate]=confmat(Y,T) +%CONFMAT Compute a confusion matrix. +% +% Description +% [C, RATE] = CONFMAT(Y, T) computes the confusion matrix C and +% classification performance RATE for the predictions mat{y} compared +% with the targets T. The data is assumed to be in a 1-of-N encoding, +% unless there is just one column, when it is assumed to be a 2 class +% problem with a 0-1 encoding. Each row of Y and T corresponds to a +% single example. +% +% In the confusion matrix, the rows represent the true classes and the +% columns the predicted classes. The vector RATE has two entries: the +% percentage of correct classifications and the total number of correct +% classifications. +% +% See also +% CONFFIG, DEMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[n c]=size(Y); +[n2 c2]=size(T); + +if n~=n2 | c~=c2 + error('Outputs and targets are different sizes') +end + +if c > 1 + % Find the winning class assuming 1-of-N encoding + [maximum Yclass] = max(Y', [], 1); + + TL=[1:c]*T'; +else + % Assume two classes with 0-1 encoding + c = 2; + class2 = find(T > 0.5); + TL = ones(n, 1); + TL(class2) = 2; + class2 = find(Y > 0.5); + Yclass = ones(n, 1); + Yclass(class2) = 2; +end + +% Compute +correct = (Yclass==TL); +total=sum(sum(correct)); +rate=[total*100/n total]; + +C=zeros(c,c); +for i=1:c + for j=1:c + C(i,j) = sum((Yclass==j).*(TL==i)); + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/conjgrad.m b/sourcecodes/bnt-master/netlab3.3/conjgrad.m new file mode 100644 index 00000000..d459af2d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/conjgrad.m @@ -0,0 +1,168 @@ +function [x, options, flog, pointlog] = conjgrad(f, x, options, gradf, ... + varargin) +%CONJGRAD Conjugate gradients optimization. +% +% Description +% [X, OPTIONS, FLOG, POINTLOG] = CONJGRAD(F, X, OPTIONS, GRADF) uses a +% conjugate gradients algorithm to find the minimum of the function +% F(X) whose gradient is given by GRADF(X). Here X is a row vector and +% F returns a scalar value. The point at which F has a local minimum +% is returned as X. The function value at that point is returned in +% OPTIONS(8). A log of the function values after each cycle is +% (optionally) returned in FLOG, and a log of the points visited is +% (optionally) returned in POINTLOG. +% +% CONJGRAD(F, X, OPTIONS, GRADF, P1, P2, ...) allows additional +% arguments to be passed to F() and GRADF(). +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG, and the points visited in the +% return argument POINTSLOG. If OPTIONS(1) is set to 0, then only +% warning messages are displayed. If OPTIONS(1) is -1, then nothing is +% displayed. +% +% OPTIONS(2) is a measure of the absolute precision required for the +% value of X at the solution. If the absolute difference between the +% values of X between two successive steps is less than OPTIONS(2), +% then this condition is satisfied. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. If the absolute difference between the +% objective function values between two successive steps is less than +% OPTIONS(3), then this condition is satisfied. Both this and the +% previous condition must be satisfied for termination. +% +% OPTIONS(9) is set to 1 to check the user defined gradient function. +% +% OPTIONS(10) returns the total number of function evaluations +% (including those in any line searches). +% +% OPTIONS(11) returns the total number of gradient evaluations. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% OPTIONS(15) is the precision in parameter space of the line search; +% default 1E-4. +% +% See also +% GRADDESC, LINEMIN, MINBRACK, QUASINEW, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set up the options. +if length(options) < 18 + error('Options vector too short') +end + +if(options(14)) + niters = options(14); +else + niters = 100; +end + +% Set up options for line search +line_options = foptions; +% Need a precise line search for success +if options(15) > 0 + line_options(2) = options(15); +else + line_options(2) = 1e-4; +end + +display = options(1); + +% Next two lines allow conjgrad to work with expression strings +f = fcnchk(f, length(varargin)); +gradf = fcnchk(gradf, length(varargin)); + +% Check gradients +if (options(9)) + feval('gradchek', x, f, gradf, varargin{:}); +end + +options(10) = 0; +options(11) = 0; +nparams = length(x); +fnew = feval(f, x, varargin{:}); +options(10) = options(10) + 1; +gradnew = feval(gradf, x, varargin{:}); +options(11) = options(11) + 1; +d = -gradnew; % Initial search direction +br_min = 0; +br_max = 1.0; % Initial value for maximum distance to search along +tol = sqrt(eps); + +j = 1; +if nargout >= 3 + flog(j, :) = fnew; + if nargout == 4 + pointlog(j, :) = x; + end +end + +while (j <= niters) + + xold = x; + fold = fnew; + gradold = gradnew; + + gg = gradold*gradold'; + if (gg == 0.0) + % If the gradient is zero then we are done. + options(8) = fnew; + return; + end + + % This shouldn't occur, but rest of code depends on d being downhill + if (gradnew*d' > 0) + d = -d; + if options(1) >= 0 + warning('search direction uphill in conjgrad'); + end + end + + line_sd = d./norm(d); + [lmin, line_options] = feval('linemin', f, xold, line_sd, fold, ... + line_options, varargin{:}); + options(10) = options(10) + line_options(10); + options(11) = options(11) + line_options(11); + % Set x and fnew to be the actual search point we have found + x = xold + lmin * line_sd; + fnew = line_options(8); + + % Check for termination + if (max(abs(x - xold)) < options(2) & max(abs(fnew - fold)) < options(3)) + options(8) = fnew; + return; + end + + gradnew = feval(gradf, x, varargin{:}); + options(11) = options(11) + 1; + + % Use Polak-Ribiere formula to update search direction + gamma = ((gradnew - gradold)*(gradnew)')/gg; + d = (d .* gamma) - gradnew; + + if (display > 0) + fprintf(1, 'Cycle %4d Function %11.6f\n', j, line_options(8)); + end + + j = j + 1; + if nargout >= 3 + flog(j, :) = fnew; + if nargout == 4 + pointlog(j, :) = x; + end + end +end + +% If we get here, then we haven't terminated in the given number of +% iterations. + +options(8) = fold; +if (options(1) >= 0) + disp(maxitmess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/consist.m b/sourcecodes/bnt-master/netlab3.3/consist.m new file mode 100644 index 00000000..9305b265 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/consist.m @@ -0,0 +1,87 @@ +function errstring = consist(model, type, inputs, outputs) +%CONSIST Check that arguments are consistent. +% +% Description +% +% ERRSTRING = CONSIST(NET, TYPE, INPUTS) takes a network data structure +% NET together with a string TYPE containing the correct network type, +% a matrix INPUTS of input vectors and checks that the data structure +% is consistent with the other arguments. An empty string is returned +% if there is no error, otherwise the string contains the relevant +% error message. If the TYPE string is empty, then any type of network +% is allowed. +% +% ERRSTRING = CONSIST(NET, TYPE) takes a network data structure NET +% together with a string TYPE containing the correct network type, and +% checks that the two types match. +% +% ERRSTRING = CONSIST(NET, TYPE, INPUTS, OUTPUTS) also checks that the +% network has the correct number of outputs, and that the number of +% patterns in the INPUTS and OUTPUTS is the same. The fields in NET +% that are used are +% type +% nin +% nout +% +% See also +% MLPFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Assume that all is OK as default +errstring = ''; + +% If type string is not empty +if ~isempty(type) + % First check that model has type field + if ~isfield(model, 'type') + errstring = 'Data structure does not contain type field'; + return + end + % Check that model has the correct type + s = model.type; + if ~strcmp(s, type) + errstring = ['Model type ''', s, ''' does not match expected type ''',... + type, '''']; + return + end +end + +% If inputs are present, check that they have correct dimension +if nargin > 2 + if ~isfield(model, 'nin') + errstring = 'Data structure does not contain nin field'; + return + end + + data_nin = size(inputs, 2); + if model.nin ~= data_nin + errstring = ['Dimension of inputs ', num2str(data_nin), ... + ' does not match number of model inputs ', num2str(model.nin)]; + return + end +end + +% If outputs are present, check that they have correct dimension +if nargin > 3 + if ~isfield(model, 'nout') + errstring = 'Data structure does not conatin nout field'; + return + end + data_nout = size(outputs, 2); + if model.nout ~= data_nout + errstring = ['Dimension of outputs ', num2str(data_nout), ... + ' does not match number of model outputs ', num2str(model.nout)]; + return + end + +% Also check that number of data points in inputs and outputs is the same + num_in = size(inputs, 1); + num_out = size(outputs, 1); + if num_in ~= num_out + errstring = ['Number of input patterns ', num2str(num_in), ... + ' does not match number of output patterns ', num2str(num_out)]; + return + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/convertoldnet.m b/sourcecodes/bnt-master/netlab3.3/convertoldnet.m new file mode 100644 index 00000000..bf281c83 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/convertoldnet.m @@ -0,0 +1,26 @@ +function net = convertoldnet(net) +%CONVERTOLDNET Convert pre-2.3 release MLP and MDN nets to new format +% +% Description +% NET = CONVERTOLDNET(NET) takes a network NET and, if appropriate, +% converts it from pre-2.3 to the current format. The difference is +% simply that in MLPs and the MLP sub-net of MDNs the field ACTFN has +% been renamed OUTFN to make it consistent with GLM and RBF networks. +% If the network is not old-format or an MLP or MDN it is left +% unchanged. +% +% See also +% MLP, MDN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +switch net.type + case 'mlp' + if (isfield(net, 'actfn')) + net.outfn = net.actfn; + net = rmfield(net, 'actfn'); + end + case 'mdn' + net.mlp = convertoldnet(net.mlp); +end diff --git a/sourcecodes/bnt-master/netlab3.3/datread.m b/sourcecodes/bnt-master/netlab3.3/datread.m new file mode 100644 index 00000000..f729ef53 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/datread.m @@ -0,0 +1,99 @@ +function [x, t, nin, nout, ndata] = datread(filename) +%DATREAD Read data from an ascii file. +% +% Description +% +% [X, T, NIN, NOUT, NDATA] = DATREAD(FILENAME) reads from the file +% FILENAME and returns a matrix X of input vectors, a matrix T of +% target vectors, and integers NIN, NOUT and NDATA specifying the +% number of inputs, the number of outputs and the number of data points +% respectively. +% +% The format of the data file is as follows: the first row contains the +% string NIN followed by the number of inputs, the second row contains +% the string NOUT followed by the number of outputs, and the third row +% contains the string NDATA followed by the number of data vectors. +% Subsequent lines each contain one input vector followed by one output +% vector, with individual values separated by spaces. +% +% See also +% nin 2 nout 1 ndata 4 0.000000e+00 0.000000e+00 +% 1.000000e+00 0.000000e+00 1.000000e+00 0.000000e+00 +% 1.000000e+00 0.000000e+00 0.000000e+00 1.000000e+00 +% 1.000000e+00 1.000000e+00 See Also +% DATWRITE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +fid = fopen(filename, 'rt'); +if fid == -1 + error('Failed to open file.') +end + +% Read number of inputs +s1 = fscanf(fid, '%s', 1); +if ~strcmp(s1, 'nin') + fclose(fid); + error('String ''nin'' not found') +end +nin = fscanf(fid, '%d\n', 1); +if ~isnumeric(nin) + fclose(fid); + error('No number for nin') +end +if nin < 0 | round(nin) ~= nin + fclose(fid); + error('nin must be a non-negative integer') +end + +% Read number of outputs +s2 = fscanf(fid, '%s', 1); +if ~strcmp(s2, 'nout') + fclose(fid); + error('String ''nout'' not found') +end +nout = fscanf(fid, '%d\n', 1); +if ~isnumeric(nout) + fclose(fid); + error('No number for nout') +end +if nout < 0 | round(nout) ~= nout + fclose(fid); + error('nout must be a non-negative integer') +end + +% Read number of data values +s3 = fscanf(fid, '%s', 1); +if ~strcmp(s3, 'ndata') + fclose(fid); + error('String ''ndata'' not found') +end +ndata = fscanf(fid, '%d\n', 1); +if ~isnumeric(ndata) + fclose(fid); + error('No number for ndata') +end +if ndata < 0 | round(ndata) ~= ndata + fclose(fid); + error('ndata must be a non-negative integer') +end + +% The following line reads all of the remaining data to the end of file. +temp = fscanf(fid, '%f', inf); + +% Check that size of temp is correct +if size(temp, 1) * size(temp,2) ~= (nin+nout) * ndata + fclose(fid); + error('Incorrect number of elements in file') +end + +temp = reshape(temp, nin + nout, ndata)'; +x = temp(:, 1:nin); +t = temp(:, nin + 1 : nin + nout); + +flag = fclose(fid); +if flag == -1 + error('Failed to close file.') +end + diff --git a/sourcecodes/bnt-master/netlab3.3/datwrite.m b/sourcecodes/bnt-master/netlab3.3/datwrite.m new file mode 100644 index 00000000..75df14d1 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/datwrite.m @@ -0,0 +1,44 @@ +function datwrite(filename, x, t) +%DATWRITE Write data to ascii file. +% +% Description +% +% DATWRITE(FILENAME, X, T) takes a matrix X of input vectors and a +% matrix T of target vectors and writes them to an ascii file named +% FILENAME. The file format is as follows: the first row contains the +% string NIN followed by the number of inputs, the second row contains +% the string NOUT followed by the number of outputs, and the third row +% contains the string NDATA followed by the number of data vectors. +% Subsequent lines each contain one input vector followed by one output +% vector, with individual values separated by spaces. +% +% See also +% DATREAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +nin = size(x, 2); +nout = size(t, 2); +ndata = size(x, 1); + +fid = fopen(filename, 'wt'); +if fid == -1 + error('Failed to open file.') +end + +if size(t, 1) ~= ndata + error('x and t must have same number of rows.'); +end + +fprintf(fid, ' nin %d\n nout %d\n ndata %d\n', nin , nout, ndata); +for i = 1 : ndata + fprintf(fid, '%13e ', x(i,:), t(i,:)); + fprintf(fid, '\n'); +end + +flag = fclose(fid); +if flag == -1 + error('Failed to close file.') +end + diff --git a/sourcecodes/bnt-master/netlab3.3/dem2ddat.m b/sourcecodes/bnt-master/netlab3.3/dem2ddat.m new file mode 100644 index 00000000..abdb7cc7 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/dem2ddat.m @@ -0,0 +1,48 @@ +function [data, c, prior, sd] = dem2ddat(ndata) +%DEM2DDAT Generates two dimensional data for demos. +% +% Description +% The data is drawn from three spherical Gaussian distributions with +% priors 0.3, 0.5 and 0.2; centres (2, 3.5), (0, 0) and (0,2); and +% standard deviations 0.2, 0.5 and 1.0. DATA = DEM2DDAT(NDATA) +% generates NDATA points. +% +% [DATA, C] = DEM2DDAT(NDATA) also returns a matrix containing the +% centres of the Gaussian distributions. +% +% See also +% DEMGMM1, DEMKMEAN, DEMKNN1 +% + +% Copyright (c) Ian T Nabney (1996-2001) + +input_dim = 2; + +% Fix seed for reproducible results +randn('state', 42); + +% Generate mixture of three Gaussians in two dimensional space +data = randn(ndata, input_dim); + +% Priors for the three clusters +prior(1) = 0.3; +prior(2) = 0.5; +prior(3) = 0.2; + +% Cluster centres +c = [2.0, 3.5; 0.0, 0.0; 0.0, 2.0]; + +% Cluster standard deviations +sd = [0.2 0.5 1.0]; + +% Put first cluster at (2, 3.5) +data(1:prior(1)*ndata, 1) = data(1:prior(1)*ndata, 1) * 0.2 + c(1,1); +data(1:prior(1)*ndata, 2) = data(1:prior(1)*ndata, 2) * 0.2 + c(1,2); + +% Leave second cluster at (0,0) +data((prior(1)*ndata + 1):(prior(2)+prior(1))*ndata, :) = ... + data((prior(1)*ndata + 1):(prior(2)+prior(1))*ndata, :) * 0.5; + +% Put third cluster at (0,2) +data((prior(1)+prior(2))*ndata +1:ndata, 2) = ... + data((prior(1)+prior(2))*ndata+1:ndata, 2) + c(3, 2); diff --git a/sourcecodes/bnt-master/netlab3.3/demard.m b/sourcecodes/bnt-master/netlab3.3/demard.m new file mode 100644 index 00000000..83478305 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demard.m @@ -0,0 +1,157 @@ +%DEMARD Automatic relevance determination using the MLP. +% +% Description +% This script demonstrates the technique of automatic relevance +% determination (ARD) using a synthetic problem having three input +% variables: X1 is sampled uniformly from the range (0,1) and has a low +% level of added Gaussian noise, X2 is a copy of X1 with a higher level +% of added noise, and X3 is sampled randomly from a Gaussian +% distribution. The single target variable is determined by +% SIN(2*PI*X1) with additive Gaussian noise. Thus X1 is very relevant +% for determining the target value, X2 is of some relevance, while X3 +% is irrelevant. The prior over weights is given by the ARD Gaussian +% prior with a separate hyper-parameter for the group of weights +% associated with each input. A multi-layer perceptron is trained on +% this data, with re-estimation of the hyper-parameters using EVIDENCE. +% The final values for the hyper-parameters reflect the relative +% importance of the three inputs. +% +% See also +% DEMMLP1, DEMEV1, MLP, EVIDENCE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc; +disp('This demonstration illustrates the technique of automatic relevance') +disp('determination (ARD) using a multi-layer perceptron.') +disp(' '); +disp('First, we set up a synthetic data set involving three input variables:') +disp('x1 is sampled uniformly from the range (0,1) and has a low level of') +disp('added Gaussian noise, x2 is a copy of x1 with a higher level of added') +disp('noise, and x3 is sampled randomly from a Gaussian distribution. The') +disp('single target variable is given by t = sin(2*pi*x1) with additive') +disp('Gaussian noise. Thus x1 is very relevant for determining the target') +disp('value, x2 is of some relevance, while x3 should in principle be') +disp('irrelevant.') +disp(' '); +disp('Press any key to see a plot of t against x1.') +pause; + +% Generate the data set. +randn('state', 0); +rand('state', 0); +ndata = 100; +noise = 0.05; +x1 = rand(ndata, 1) + 0.002*randn(ndata, 1); +x2 = x1 + 0.02*randn(ndata, 1); +x3 = 0.5 + 0.2*randn(ndata, 1); +x = [x1, x2, x3]; +t = sin(2*pi*x1) + noise*randn(ndata, 1); + +% Plot the data and the original function. +h = figure; +plotvals = linspace(0, 1, 200)'; +plot(x1, t, 'ob') +hold on +axis([0 1 -1.5 1.5]) +[fx, fy] = fplot('sin(2*pi*x)', [0 1]); +plot(fx, fy, '-g', 'LineWidth', 2); +legend('data', 'function'); + +disp(' '); +disp('Press any key to continue') +pause; clc; + +disp('The prior over weights is given by the ARD Gaussian prior with a') +disp('separate hyper-parameter for the group of weights associated with each') +disp('input. This prior is set up using the utility MLPPRIOR. The network is') +disp('trained by error minimization using scaled conjugate gradient function') +disp('SCG. There are two cycles of training, and at the end of each cycle') +disp('the hyper-parameters are re-estimated using EVIDENCE.') +disp(' '); +disp('Press any key to create and train the network.') +disp(' '); +pause; + +% Set up network parameters. +nin = 3; % Number of inputs. +nhidden = 2; % Number of hidden units. +nout = 1; % Number of outputs. +aw1 = 0.01*ones(1, nin); % First-layer ARD hyperparameters. +ab1 = 0.01; % Hyperparameter for hidden unit biases. +aw2 = 0.01; % Hyperparameter for second-layer weights. +ab2 = 0.01; % Hyperparameter for output unit biases. +beta = 50.0; % Coefficient of data error. + +% Create and initialize network. +prior = mlpprior(nin, nhidden, nout, aw1, ab1, aw2, ab2); +net = mlp(nin, nhidden, nout, 'linear', prior, beta); + +% Set up vector of options for the optimiser. +nouter = 2; % Number of outer loops +ninner = 10; % Number of inner loops +options = zeros(1,18); % Default options vector. +options(1) = 1; % This provides display of error values. +options(2) = 1.0e-7; % This ensures that convergence must occur +options(3) = 1.0e-7; +options(14) = 300; % Number of training cycles in inner loop. + +% Train using scaled conjugate gradients, re-estimating alpha and beta. +for k = 1:nouter + net = netopt(net, options, x, t, 'scg'); + [net, gamma] = evidence(net, x, t, ninner); + fprintf(1, '\n\nRe-estimation cycle %d:\n', k); + disp('The first three alphas are the hyperparameters for the corresponding'); + disp('input to hidden unit weights. The remainder are the hyperparameters'); + disp('for the hidden unit biases, second layer weights and output unit') + disp('biases, respectively.') + fprintf(1, ' alpha = %8.5f\n', net.alpha); + fprintf(1, ' beta = %8.5f\n', net.beta); + fprintf(1, ' gamma = %8.5f\n\n', gamma); + disp(' ') + disp('Press any key to continue.') + pause +end + +% Plot the function corresponding to the trained network. +figure(h); hold on; +[y, z] = mlpfwd(net, plotvals*ones(1,3)); +plot(plotvals, y, '-r', 'LineWidth', 2) +legend('data', 'function', 'network'); + +disp('Press any key to continue.'); +pause; clc; + +disp('We can now read off the hyperparameter values corresponding to the') +disp('three inputs x1, x2 and x3:') +disp(' '); +fprintf(1, ' alpha1: %8.5f\n', net.alpha(1)); +fprintf(1, ' alpha2: %8.5f\n', net.alpha(2)); +fprintf(1, ' alpha3: %8.5f\n', net.alpha(3)); +disp(' '); +disp('Since each alpha corresponds to an inverse variance, we see that the') +disp('posterior variance for weights associated with input x1 is large, that') +disp('of x2 has an intermediate value and the variance of weights associated') +disp('with x3 is small.') +disp(' ') +disp('Press any key to continue.') +disp(' ') +pause +disp('This is confirmed by looking at the corresponding weight values:') +disp(' '); +fprintf(1, ' %8.5f %8.5f\n', net.w1'); +disp(' '); +disp('where the three rows correspond to weights asssociated with x1, x2 and') +disp('x3 respectively. We see that the network is giving greatest emphasis') +disp('to x1 and least emphasis to x3, with intermediate emphasis on') +disp('x2. Since the target t is statistically independent of x3 we might') +disp('expect the weights associated with this input would go to') +disp('zero. However, for any finite data set there may be some chance') +disp('correlation between x3 and t, and so the corresponding alpha remains') +disp('finite.') + +disp(' '); +disp('Press any key to end.') +pause; clc; close(h); clear all + diff --git a/sourcecodes/bnt-master/netlab3.3/demev1.m b/sourcecodes/bnt-master/netlab3.3/demev1.m new file mode 100644 index 00000000..b8fd7464 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demev1.m @@ -0,0 +1,146 @@ +%DEMEV1 Demonstrate Bayesian regression for the MLP. +% +% Description +% The problem consists an input variable X which sampled from a +% Gaussian distribution, and a target variable T generated by computing +% SIN(2*PI*X) and adding Gaussian noise. A 2-layer network with linear +% outputs is trained by minimizing a sum-of-squares error function with +% isotropic Gaussian regularizer, using the scaled conjugate gradient +% optimizer. The hyperparameters ALPHA and BETA are re-estimated using +% the function EVIDENCE. A graph is plotted of the original function, +% the training data, the trained network function, and the error bars. +% +% See also +% EVIDENCE, MLP, SCG, DEMARD, DEMMLP1 +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc; +disp('This demonstration illustrates the application of Bayesian') +disp('re-estimation to determine the hyperparameters in a simple regression') +disp('problem. It is based on a local quadratic approximation to a mode of') +disp('the posterior distribution and the evidence maximization framework of') +disp('MacKay.') +disp(' ') +disp('First, we generate a synthetic data set consisting of a single input') +disp('variable x sampled from a Gaussian distribution, and a target variable') +disp('t obtained by evaluating sin(2*pi*x) and adding Gaussian noise.') +disp(' ') +disp('Press any key to see a plot of the data together with the sine function.') +pause; + +% Generate the matrix of inputs x and targets t. + +ndata = 16; % Number of data points. +noise = 0.1; % Standard deviation of noise distribution. +randn('state', 0); +x = 0.25 + 0.07*randn(ndata, 1); +t = sin(2*pi*x) + noise*randn(size(x)); + +% Plot the data and the original sine function. +h = figure; +nplot = 200; +plotvals = linspace(0, 1, nplot)'; +plot(x, t, 'ok') +xlabel('Input') +ylabel('Target') +hold on +axis([0 1 -1.5 1.5]) +fplot('sin(2*pi*x)', [0 1], '-g') +legend('data', 'function'); + +disp(' ') +disp('Press any key to continue') +pause; clc; + +disp('Next we create a two-layer MLP network having 3 hidden units and one') +disp('linear output. The model assumes Gaussian target noise governed by an') +disp('inverse variance hyperparmeter beta, and uses a simple Gaussian prior') +disp('distribution governed by an inverse variance hyperparameter alpha.') +disp(' '); +disp('The network weights and the hyperparameters are initialised and then') +disp('the weights are optimized with the scaled conjugate gradient') +disp('algorithm using the SCG function, with the hyperparameters kept') +disp('fixed. After a maximum of 500 iterations, the hyperparameters are') +disp('re-estimated using the EVIDENCE function. The process of optimizing') +disp('the weights with fixed hyperparameters and then re-estimating the') +disp('hyperparameters is repeated for a total of 3 cycles.') +disp(' ') +disp('Press any key to train the network and determine the hyperparameters.') +pause; + +% Set up network parameters. +nin = 1; % Number of inputs. +nhidden = 3; % Number of hidden units. +nout = 1; % Number of outputs. +alpha = 0.01; % Initial prior hyperparameter. +beta_init = 50.0; % Initial noise hyperparameter. + +% Create and initialize network weight vector. +net = mlp(nin, nhidden, nout, 'linear', alpha, beta_init); + +% Set up vector of options for the optimiser. +nouter = 3; % Number of outer loops. +ninner = 1; % Number of innter loops. +options = zeros(1,18); % Default options vector. +options(1) = 1; % This provides display of error values. +options(2) = 1.0e-7; % Absolute precision for weights. +options(3) = 1.0e-7; % Precision for objective function. +options(14) = 500; % Number of training cycles in inner loop. + +% Train using scaled conjugate gradients, re-estimating alpha and beta. +for k = 1:nouter + net = netopt(net, options, x, t, 'scg'); + [net, gamma] = evidence(net, x, t, ninner); + fprintf(1, '\nRe-estimation cycle %d:\n', k); + fprintf(1, ' alpha = %8.5f\n', net.alpha); + fprintf(1, ' beta = %8.5f\n', net.beta); + fprintf(1, ' gamma = %8.5f\n\n', gamma); + disp(' ') + disp('Press any key to continue.') + pause; +end + +fprintf(1, 'true beta: %f\n', 1/(noise*noise)); + +disp(' ') +disp('Network training and hyperparameter re-estimation are now complete.') +disp('Compare the final value for the hyperparameter beta with the true') +disp('value.') +disp(' ') +disp('Notice that the final error value is close to the number of data') +disp(['points (', num2str(ndata),') divided by two.']) +disp(' ') +disp('Press any key to continue.') +pause; clc; +disp('We can now plot the function represented by the trained network. This') +disp('corresponds to the mean of the predictive distribution. We can also') +disp('plot ''error bars'' representing one standard deviation of the') +disp('predictive distribution around the mean.') +disp(' ') +disp('Press any key to add the network function and error bars to the plot.') +pause; + +% Evaluate error bars. +[y, sig2] = netevfwd(mlppak(net), net, x, t, plotvals); +sig = sqrt(sig2); + +% Plot the data, the original function, and the trained network function. +[y, z] = mlpfwd(net, plotvals); +figure(h); hold on; +plot(plotvals, y, '-r') +xlabel('Input') +ylabel('Target') +plot(plotvals, y + sig, '-b'); +plot(plotvals, y - sig, '-b'); +legend('data', 'function', 'network', 'error bars'); + +disp(' ') +disp('Notice how the confidence interval spanned by the ''error bars'' is') +disp('smaller in the region of input space where the data density is high,') +disp('and becomes larger in regions away from the data.') +disp(' ') +disp('Press any key to end.') +pause; clc; close(h); +%clear all diff --git a/sourcecodes/bnt-master/netlab3.3/demev2.m b/sourcecodes/bnt-master/netlab3.3/demev2.m new file mode 100644 index 00000000..b479d88f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demev2.m @@ -0,0 +1,248 @@ +%DEMEV2 Demonstrate Bayesian classification for the MLP. +% +% Description +% A synthetic two class two-dimensional dataset X is sampled from a +% mixture of four Gaussians. Each class is associated with two of the +% Gaussians so that the optimal decision boundary is non-linear. A 2- +% layer network with logistic outputs is trained by minimizing the +% cross-entropy error function with isotroipc Gaussian regularizer (one +% hyperparameter for each of the four standard weight groups), using +% the scaled conjugate gradient optimizer. The hyperparameter vectors +% ALPHA and BETA are re-estimated using the function EVIDENCE. A graph +% is plotted of the optimal, regularised, and unregularised decision +% boundaries. A further plot of the moderated versus unmoderated +% contours is generated. +% +% See also +% EVIDENCE, MLP, SCG, DEMARD, DEMMLP2 +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +clc; + +disp('This program demonstrates the use of the evidence procedure on') +disp('a two-class problem. It also shows the improved generalisation') +disp('performance that can be achieved with moderated outputs; that is') +disp('predictions where an approximate integration over the true') +disp('posterior distribution is carried out.') +disp(' ') +disp('First we generate a synthetic dataset with two-dimensional input') +disp('sampled from a mixture of four Gaussians. Each class is') +disp('associated with two of the Gaussians so that the optimal decision') +disp('boundary is non-linear.') +disp(' ') +disp('Press any key to see a plot of the data.') +pause; + +% Generate the matrix of inputs x and targets t. + +rand('state', 423); +randn('state', 423); + +ClassSymbol1 = 'r.'; +ClassSymbol2 = 'y.'; +PointSize = 12; +titleSize = 10; + +fh1 = figure; +set(fh1, 'Name', 'True Data Distribution'); +whitebg(fh1, 'k'); + +% +% Generate the data +% +n=200; + +% Set up mixture model: 2d data with four centres +% Class 1 is first two centres, class 2 from the other two +mix = gmm(2, 4, 'full'); +mix.priors = [0.25 0.25 0.25 0.25]; +mix.centres = [0 -0.1; 1.5 0; 1 1; 1 -1]; +mix.covars(:,:,1) = [0.625 -0.2165; -0.2165 0.875]; +mix.covars(:,:,2) = [0.25 0; 0 0.25]; +mix.covars(:,:,3) = [0.2241 -0.1368; -0.1368 0.9759]; +mix.covars(:,:,4) = [0.2375 0.1516; 0.1516 0.4125]; + +[data, label] = gmmsamp(mix, n); + +% +% Calculate some useful axis limits +% +x0 = min(data(:,1)); +x1 = max(data(:,1)); +y0 = min(data(:,2)); +y1 = max(data(:,2)); +dx = x1-x0; +dy = y1-y0; +expand = 5/100; % Add on 5 percent each way +x0 = x0 - dx*expand; +x1 = x1 + dx*expand; +y0 = y0 - dy*expand; +y1 = y1 + dy*expand; +resolution = 100; +step = dx/resolution; +xrange = [x0:step:x1]; +yrange = [y0:step:y1]; +% +% Generate the grid +% +[X Y]=meshgrid([x0:step:x1],[y0:step:y1]); +% +% Calculate the class conditional densities, the unconditional densities and +% the posterior probabilities +% +px_j = gmmactiv(mix, [X(:) Y(:)]); +px = reshape(px_j*(mix.priors)',size(X)); +post = gmmpost(mix, [X(:) Y(:)]); +p1_x = reshape(post(:, 1) + post(:, 2), size(X)); +p2_x = reshape(post(:, 3) + post(:, 4), size(X)); + +plot(data((label<=2),1),data(label<=2,2),ClassSymbol1, 'MarkerSize', ... +PointSize) +hold on +axis([x0 x1 y0 y1]) +plot(data((label>2),1),data(label>2,2),ClassSymbol2, 'MarkerSize', ... + PointSize) + +% Convert targets to 0-1 encoding +target=[label<=2]; +disp(' ') +disp('Press any key to continue') +pause; clc; + +disp('Next we create a two-layer MLP network with 6 hidden units and') +disp('one logistic output. We use a separate inverse variance') +disp('hyperparameter for each group of weights (inputs, input bias,') +disp('outputs, output bias) and the weights are optimised with the') +disp('scaled conjugate gradient algorithm. After each 100 iterations') +disp('the hyperparameters are re-estimated twice. There are eight') +disp('cycles of the whole algorithm.') +disp(' ') +disp('Press any key to train the network and determine the hyperparameters.') +pause; + +% Set up network parameters. +nin = 2; % Number of inputs. +nhidden = 6; % Number of hidden units. +nout = 1; % Number of outputs. +alpha = 0.01; % Initial prior hyperparameter. +aw1 = 0.01; +ab1 = 0.01; +aw2 = 0.01; +ab2 = 0.01; + +% Create and initialize network weight vector. +prior = mlpprior(nin, nhidden, nout, aw1, ab1, aw2, ab2); +net = mlp(nin, nhidden, nout, 'logistic', prior); + +% Set up vector of options for the optimiser. +nouter = 8; % Number of outer loops. +ninner = 2; % Number of innter loops. +options = foptions; % Default options vector. +options(1) = 1; % This provides display of error values. +options(2) = 1.0e-5; % Absolute precision for weights. +options(3) = 1.0e-5; % Precision for objective function. +options(14) = 100; % Number of training cycles in inner loop. + +% Train using scaled conjugate gradients, re-estimating alpha and beta. +for k = 1:nouter + net = netopt(net, options, data, target, 'scg'); + [net, gamma] = evidence(net, data, target, ninner); + fprintf(1, '\nRe-estimation cycle %d:\n', k); + disp([' alpha = ', num2str(net.alpha')]); + fprintf(1, ' gamma = %8.5f\n\n', gamma); + disp(' ') + disp('Press any key to continue.') + pause; +end + +disp(' ') +disp('Network training and hyperparameter re-estimation are now complete.') +disp('Notice that the final error value is close to the number of data') +disp(['points (', num2str(n), ') divided by two.']) +disp('Also, the hyperparameter values differ, which suggests that a single') +disp('hyperparameter would not be so effective.') +disp(' ') +disp('First we train an MLP without Bayesian regularisation on the') +disp('same dataset using 400 iterations of scaled conjugate gradient') +disp(' ') +disp('Press any key to train the network by maximum likelihood.') +pause; +% Train standard network +net2 = mlp(nin, nhidden, nout, 'logistic'); +options(14) = 400; +net2 = netopt(net2, options, data, target, 'scg'); +y2g = mlpfwd(net2, [X(:), Y(:)]); +y2g = reshape(y2g(:, 1), size(X)); + +disp(' ') +disp('We can now plot the function represented by the trained networks.') +disp('We show the decision boundaries (output = 0.5) and the optimal') +disp('decision boundary given by applying Bayes'' theorem to the true') +disp('data model.') +disp(' ') +disp('Press any key to add the boundaries to the plot.') +pause; + +% Evaluate predictions. +[yg, ymodg] = mlpevfwd(net, data, target, [X(:) Y(:)]); +yg = reshape(yg(:,1),size(X)); +ymodg = reshape(ymodg(:,1),size(X)); + +% Bayesian decision boundary +[cB, hB] = contour(xrange,yrange,p1_x,[0.5 0.5],'b-'); +[cNb, hNb] = contour(xrange,yrange,yg,[0.5 0.5],'r-'); +[cN, hN] = contour(xrange,yrange,y2g,[0.5 0.5],'g-'); +set(hB, 'LineWidth', 2); +set(hNb, 'LineWidth', 2); +set(hN, 'LineWidth', 2); +Chandles = [hB(1) hNb(1) hN(1)]; +legend(Chandles, 'Bayes', ... + 'Reg. Network', 'Network', 3); + +disp(' ') +disp('Note how the regularised network predictions are closer to the') +disp('optimal decision boundary, while the unregularised network is') +disp('overtrained.') + +disp(' ') +disp('We will now compare moderated and unmoderated outputs for the'); +disp('regularised network by showing the contour plot of the posterior') +disp('probability estimates.') +disp(' ') +disp('The first plot shows the regularised (moderated) predictions') +disp('and the second shows the standard predictions from the same network.') +disp('These agree at the level 0.5.') +disp('Press any key to continue') +pause +levels = 0:0.1:1; +fh4 = figure; +set(fh4, 'Name', 'Moderated outputs'); +hold on +plot(data((label<=2),1),data(label<=2,2),'r.', 'MarkerSize', PointSize) +plot(data((label>2),1),data(label>2,2),'y.', 'MarkerSize', PointSize) + +[cNby, hNby] = contour(xrange, yrange, ymodg, levels, 'k-'); +set(hNby, 'LineWidth', 1); + +fh5 = figure; +set(fh5, 'Name', 'Unmoderated outputs'); +hold on +plot(data((label<=2),1),data(label<=2,2),'r.', 'MarkerSize', PointSize) +plot(data((label>2),1),data(label>2,2),'y.', 'MarkerSize', PointSize) + +[cNbm, hNbm] = contour(xrange, yrange, yg, levels, 'k-'); +set(hNbm, 'LineWidth', 1); + +disp(' ') +disp('Note how the moderated contours are more widely spaced. This shows') +disp('that there is a larger region where the outputs are close to 0.5') +disp('and a smaller region where the outputs are close to 0 or 1.') +disp(' ') +disp('Press any key to exit') +pause +close(fh1); +close(fh4); +close(fh5); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demev3.m b/sourcecodes/bnt-master/netlab3.3/demev3.m new file mode 100644 index 00000000..d288e037 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demev3.m @@ -0,0 +1,154 @@ +%DEMEV3 Demonstrate Bayesian regression for the RBF. +% +% Description +% The problem consists an input variable X which sampled from a +% Gaussian distribution, and a target variable T generated by computing +% SIN(2*PI*X) and adding Gaussian noise. An RBF network with linear +% outputs is trained by minimizing a sum-of-squares error function with +% isotropic Gaussian regularizer, using the scaled conjugate gradient +% optimizer. The hyperparameters ALPHA and BETA are re-estimated using +% the function EVIDENCE. A graph is plotted of the original function, +% the training data, the trained network function, and the error bars. +% +% See also +% DEMEV1, EVIDENCE, RBF, SCG, NETEVFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc; +disp('This demonstration illustrates the application of Bayesian') +disp('re-estimation to determine the hyperparameters in a simple regression') +disp('problem using an RBF netowk. It is based on a the fact that the') +disp('posterior distribution for the output weights of an RBF is Gaussian') +disp('and uses the evidence maximization framework of MacKay.') +disp(' ') +disp('First, we generate a synthetic data set consisting of a single input') +disp('variable x sampled from a Gaussian distribution, and a target variable') +disp('t obtained by evaluating sin(2*pi*x) and adding Gaussian noise.') +disp(' ') +disp('Press any key to see a plot of the data together with the sine function.') +pause; + +% Generate the matrix of inputs x and targets t. + +ndata = 16; % Number of data points. +noise = 0.1; % Standard deviation of noise distribution. +randn('state', 0); +rand('state', 0); +x = 0.25 + 0.07*randn(ndata, 1); +t = sin(2*pi*x) + noise*randn(size(x)); + +% Plot the data and the original sine function. +h = figure; +nplot = 200; +plotvals = linspace(0, 1, nplot)'; +plot(x, t, 'ok') +xlabel('Input') +ylabel('Target') +hold on +axis([0 1 -1.5 1.5]) +fplot('sin(2*pi*x)', [0 1], '-g') +legend('data', 'function'); + +disp(' ') +disp('Press any key to continue') +pause; clc; + +disp('Next we create a two-layer MLP network having 3 hidden units and one') +disp('linear output. The model assumes Gaussian target noise governed by an') +disp('inverse variance hyperparmeter beta, and uses a simple Gaussian prior') +disp('distribution governed by an inverse variance hyperparameter alpha.') +disp(' '); +disp('The network weights and the hyperparameters are initialised and then') +disp('the output layer weights are optimized with the scaled conjugate gradient') +disp('algorithm using the SCG function, with the hyperparameters kept') +disp('fixed. After a maximum of 50 iterations, the hyperparameters are') +disp('re-estimated using the EVIDENCE function. The process of optimizing') +disp('the weights with fixed hyperparameters and then re-estimating the') +disp('hyperparameters is repeated for a total of 3 cycles.') +disp(' ') +disp('Press any key to train the network and determine the hyperparameters.') +pause; + +% Set up network parameters. +nin = 1; % Number of inputs. +nhidden = 3; % Number of hidden units. +nout = 1; % Number of outputs. +alpha = 0.01; % Initial prior hyperparameter. +beta_init = 50.0; % Initial noise hyperparameter. + +% Create and initialize network weight vector. +net = rbf(nin, nhidden, nout, 'tps', 'linear', alpha, beta_init); +[net.mask, prior] = rbfprior('tps', nin, nhidden, nout, alpha, alpha); +net = netinit(net, prior); + +options = foptions; +options(14) = 5; % At most 5 EM iterations for basis functions +options(1) = -1; % Turn off all messages +net = rbfsetbf(net, options, x); % Initialise the basis functions + +% Now train the network +nouter = 5; +ninner = 2; +options = foptions; +options(1) = 1; +options(2) = 1.0e-5; % Absolute precision for weights. +options(3) = 1.0e-5; % Precision for objective function. +options(14) = 50; % Number of training cycles in inner loop. + +% Train using scaled conjugate gradients, re-estimating alpha and beta. +for k = 1:nouter + net = netopt(net, options, x, t, 'scg'); + [net, gamma] = evidence(net, x, t, ninner); + fprintf(1, '\nRe-estimation cycle %d:\n', k); + fprintf(1, ' alpha = %8.5f\n', net.alpha); + fprintf(1, ' beta = %8.5f\n', net.beta); + fprintf(1, ' gamma = %8.5f\n\n', gamma); + disp(' ') + disp('Press any key to continue.') + pause; +end + +fprintf(1, 'true beta: %f\n', 1/(noise*noise)); + +disp(' ') +disp('Network training and hyperparameter re-estimation are now complete.') +disp('Compare the final value for the hyperparameter beta with the true') +disp('value.') +disp(' ') +disp('Notice that the final error value is close to the number of data') +disp(['points (', num2str(ndata),') divided by two.']) +disp(' ') +disp('Press any key to continue.') +pause; clc; +disp('We can now plot the function represented by the trained network. This') +disp('corresponds to the mean of the predictive distribution. We can also') +disp('plot ''error bars'' representing one standard deviation of the') +disp('predictive distribution around the mean.') +disp(' ') +disp('Press any key to add the network function and error bars to the plot.') +pause; + +% Evaluate error bars. +[y, sig2] = netevfwd(netpak(net), net, x, t, plotvals); +sig = sqrt(sig2); + +% Plot the data, the original function, and the trained network function. +[y, z] = rbffwd(net, plotvals); +figure(h); hold on; +plot(plotvals, y, '-r') +xlabel('Input') +ylabel('Target') +plot(plotvals, y + sig, '-b'); +plot(plotvals, y - sig, '-b'); +legend('data', 'function', 'network', 'error bars'); + +disp(' ') +disp('Notice how the confidence interval spanned by the ''error bars'' is') +disp('smaller in the region of input space where the data density is high,') +disp('and becomes larger in regions away from the data.') +disp(' ') +disp('Press any key to end.') +pause; clc; close(h); + diff --git a/sourcecodes/bnt-master/netlab3.3/demgauss.m b/sourcecodes/bnt-master/netlab3.3/demgauss.m new file mode 100644 index 00000000..c9751ae9 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgauss.m @@ -0,0 +1,80 @@ +%DEMGAUSS Demonstrate sampling from Gaussian distributions. +% +% Description +% +% DEMGAUSS provides a simple illustration of the generation of data +% from Gaussian distributions. It first samples from a one-dimensional +% distribution using RANDN, and then plots a normalized histogram +% estimate of the distribution using HISTP together with the true +% density calculated using GAUSS. +% +% DEMGAUSS then demonstrates sampling from a Gaussian distribution in +% two dimensions. It creates a mean vector and a covariance matrix, and +% then plots contours of constant density using the function GAUSS. A +% sample of points drawn from this distribution, obtained using the +% function GSAMP, is then superimposed on the contours. +% +% See also +% GAUSS, GSAMP, HISTP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc +mean = 2; var = 5; nsamp = 3000; +xmin = -10; xmax = 10; nbins = 30; +disp('Demonstration of sampling from a uni-variate Gaussian with mean') +dstring = [num2str(mean), ' and variance ', num2str(var), '. ', ... + num2str(nsamp), ' samples are taken.']; +disp(dstring); +x = mean + sqrt(var)*randn(nsamp, 1); +fh1 = figure; +histp(x, xmin, xmax, nbins); +hold on; +axis([xmin xmax 0 0.2]); +plotvals = linspace(xmin, xmax, 200)'; +probs = gauss(mean, var, plotvals); +plot(plotvals, probs, '-r'); +xlabel('X') +ylabel('Density') + +disp(' ') +disp('Press any key to continue') +pause; +mu = [3 2]; +lam1 = 0.5; +lam2 = 5.0; +Sigma = lam1*[1,1]'*[1,1] + lam2*[1,-1]'*[1,-1]; +disp(' ') +disp('Demonstration of sampling from a bi-variate Gaussian. The mean is') +dstring = ['[', num2str(mu(1)), ', ', num2str(mu(2)), ... + '] and the covariance matrix is']; +disp(dstring) +disp(Sigma); +ngrid = 40; +cmin = -5; cmax = 10; +cvals = linspace(cmin, cmax, ngrid); +[X1, X2] = meshgrid(cvals, cvals); +XX = [X1(:), X2(:)]; +probs = gauss(mu, Sigma, XX); +probs = reshape(probs, ngrid, ngrid); + +fh2 = figure; +contour(X1, X2, probs, 'b'); +hold on + +nsamp = 300; +dstring = [num2str(nsamp), ' samples are generated.']; +disp('The plot shows the sampled data points with a contour plot of their density.') +samples = gsamp(mu, Sigma, nsamp); +plot(samples(:,1), samples(:,2), 'or'); +xlabel('X1') +ylabel('X2') +grid off; + +disp(' ') +disp('Press any key to end') +pause; +close(fh1); +close(fh2); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demglm1.m b/sourcecodes/bnt-master/netlab3.3/demglm1.m new file mode 100644 index 00000000..d6c54bac --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demglm1.m @@ -0,0 +1,95 @@ +%DEMGLM1 Demonstrate simple classification using a generalized linear model. +% +% Description +% The problem consists of a two dimensional input matrix DATA and a +% vector of classifications T. The data is generated from two +% Gaussian clusters, and a generalized linear model with logistic +% output is trained using iterative reweighted least squares. A plot of +% the data together with the 0.1, 0.5 and 0.9 contour lines of the +% conditional probability is generated. +% +% See also +% DEMGLM2, GLM, GLMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate data from two classes in 2d +input_dim = 2; + +% Fix seeds for reproducible results +randn('state', 42); +rand('state', 42); + +ndata = 100; +% Generate mixture of two Gaussians in two dimensional space +mix = gmm(2, 2, 'spherical'); +mix.priors = [0.4 0.6]; % Cluster priors +mix.centres = [2.0, 2.0; 0.0, 0.0]; % Cluster centres +mix.covars = [0.5, 1.0]; + +[data, label] = gmmsamp(mix, ndata); +targets = label - ones(ndata, 1); + +% Plot the result + +clc +disp('This demonstration illustrates the use of a generalized linear model') +disp('to classify data from two classes in a two-dimensional space. We') +disp('begin by generating and plotting the data.') +disp(' ') +disp('Press any key to continue.') +pause + +fh1 = figure; +plot(data(label==1,1), data(label==1,2), 'bo'); +hold on +axis([-4 5 -4 5]) +set(gca, 'box', 'on') +plot(data(label==2,1), data(label==2,2), 'rx') +title('Data') + +clc +disp('Now we fit a model consisting of a logistic sigmoid function of') +disp('a linear combination of the input variables.') +disp(' ') +disp('The model is trained using the IRLS algorithm for 5 iterations') +disp(' ') +disp('Press any key to continue.') +pause + +net = glm(input_dim, 1, 'logistic'); +options = foptions; +options(1) = 1; +options(14) = 5; +net = glmtrain(net, options, data, targets); + +disp(' ') +disp('We now plot some density contours given by this model.') +disp('The contour labelled 0.5 is the decision boundary.') +disp(' ') +disp('Press any key to continue.') +pause +x = -4.0:0.2:5.0; +y = -4.0:0.2:5.0; +[X, Y] = meshgrid(x,y); +X = X(:); +Y = Y(:); +grid = [X Y]; +Z = glmfwd(net, grid); +Z = reshape(Z, length(x), length(y)); +v = [0.1 0.5 0.9]; +[c, h] = contour(x, y, Z, v); +title('Generalized Linear Model') +set(h, 'linewidth', 3) +clabel(c, h); + +clc +disp('Note that the contours of constant density are straight lines.') +disp(' ') +disp('Press any key to end.') +pause +close(fh1); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demglm2.m b/sourcecodes/bnt-master/netlab3.3/demglm2.m new file mode 100644 index 00000000..ea3d6bde --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demglm2.m @@ -0,0 +1,103 @@ +%DEMGLM2 Demonstrate simple classification using a generalized linear model. +% +% Description +% The problem consists of a two dimensional input matrix DATA and a +% vector of classifications T. The data is generated from three +% Gaussian clusters, and a generalized linear model with softmax output +% is trained using iterative reweighted least squares. A plot of the +% data together with regions shaded by the classification given by the +% network is generated. +% +% See also +% DEMGLM1, GLM, GLMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate data from three classes in 2d +input_dim = 2; + +% Fix seeds for reproducible results +randn('state', 42); +rand('state', 42); + +ndata = 100; +% Generate mixture of three Gaussians in two dimensional space +mix = gmm(2, 3, 'spherical'); +mix.priors = [0.4 0.3 0.3]; % Cluster priors +mix.centres = [2, 2; 0.0, 0.0; 1, -1]; % Cluster centres +mix.covars = [0.5 1.0 0.6]; + +[data, label] = gmmsamp(mix, ndata); +id = eye(3); +targets = id(label,:); + +% Plot the result + +clc +disp('This demonstration illustrates the use of a generalized linear model') +disp('to classify data from three classes in a two-dimensional space. We') +disp('begin by generating and plotting the data.') +disp(' ') +disp('Press any key to continue.') +pause + +fh1 = figure; +plot(data(label==1,1), data(label==1,2), 'bo'); +hold on +axis([-4 5 -4 5]); +set(gca, 'Box', 'on') +plot(data(label==2,1), data(label==2,2), 'rx') +plot(data(label==3, 1), data(label==3, 2), 'go') +title('Data') + +clc +disp('Now we fit a model consisting of a softmax function of') +disp('a linear combination of the input variables.') +disp(' ') +disp('The model is trained using the IRLS algorithm for up to 10 iterations') +disp(' ') +disp('Press any key to continue.') +pause + +net = glm(input_dim, size(targets, 2), 'softmax'); +options = foptions; +options(1) = 1; +options(14) = 10; +net = glmtrain(net, options, data, targets); + +disp(' ') +disp('We now plot the decision regions given by this model.') +disp(' ') +disp('Press any key to continue.') +pause + +x = -4.0:0.2:5.0; +y = -4.0:0.2:5.0; +[X, Y] = meshgrid(x,y); +X = X(:); +Y = Y(:); +grid = [X Y]; +Z = glmfwd(net, grid); +[foo , class] = max(Z'); +class = class'; +colors = ['b.'; 'r.'; 'g.']; +for i = 1:3 + thisX = X(class == i); + thisY = Y(class == i); + h = plot(thisX, thisY, colors(i,:)); + set(h, 'MarkerSize', 8); +end +title('Plot of Decision regions') + +hold off + +clc +disp('Note that the boundaries of decision regions are straight lines.') +disp(' ') +disp('Press any key to end.') +pause +close(fh1); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demgmm1.m b/sourcecodes/bnt-master/netlab3.3/demgmm1.m new file mode 100644 index 00000000..e2d1ef4c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgmm1.m @@ -0,0 +1,173 @@ +%DEMGMM1 Demonstrate EM for Gaussian mixtures. +% +% Description +% This script demonstrates the use of the EM algorithm to fit a mixture +% of Gaussians to a set of data using maximum likelihood. A colour +% coding scheme is used to illustrate the evaluation of the posterior +% probabilities in the E-step of the EM algorithm. +% +% See also +% DEMGMM2, DEMGMM3, DEMGMM4, GMM, GMMEM, GMMPOST +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc; +disp('This demonstration illustrates the use of the EM (expectation-') +disp('maximization) algorithm for fitting of a mixture of Gaussians to a') +disp('data set by maximum likelihood.') +disp(' ') +disp('The data set consists of 40 data points in a 2-dimensional') +disp('space, generated by sampling from a mixture of 2 Gaussian') +disp('distributions.') +disp(' ') +disp('Press any key to see a plot of the data.') +pause; + +% Generate the data +randn('state', 0); rand('state', 0); +gmix = gmm(2, 2, 'spherical'); +ndat1 = 20; ndat2 = 20; ndata = ndat1+ndat2; +gmix.centres = [0.3 0.3; 0.7 0.7]; +gmix.covars = [0.01 0.01]; +x = gmmsamp(gmix, ndata); + +h = figure; +hd = plot(x(:, 1), x(:, 2), '.g', 'markersize', 30); +hold on; axis([0 1 0 1]); axis square; set(gca, 'box', 'on'); +ht = text(0.5, 1.05, 'Data', 'horizontalalignment', 'center'); +disp(' '); +disp('Press any key to continue.') +pause; clc; + +disp('We next create and initialize a mixture model consisting of a mixture') +disp('of 2 Gaussians having ''spherical'' covariance matrices, using the') +disp('function GMM. The Gaussian components can be displayed on the same') +disp('plot as the data by drawing a contour of constant probability density') +disp('for each component having radius equal to the corresponding standard') +disp('deviation. Component 1 is coloured red and component 2 is coloured') +disp('blue.') +disp(' ') +disp('Note that a particulary poor choice of initial parameters has been') +disp('made in order to illustrate more effectively the operation of the') +disp('EM algorithm.') +disp(' ') +disp('Press any key to see the initial configuration of the mixture model.') +pause; + +% Set up mixture model +ncentres = 2; input_dim = 2; +mix = gmm(input_dim, ncentres, 'spherical'); + +% Initialise the mixture model +mix.centres = [0.2 0.8; 0.8, 0.2]; +mix.covars = [0.01 0.01]; + +% Plot the initial model +ncirc = 30; theta = linspace(0, 2*pi, ncirc); +xs = cos(theta); ys = sin(theta); +xvals = mix.centres(:, 1)*ones(1,ncirc) + sqrt(mix.covars')*xs; +yvals = mix.centres(:, 2)*ones(1,ncirc) + sqrt(mix.covars')*ys; +hc(1)=line(xvals(1,:), yvals(1,:), 'color', 'r'); +hc(2)=line(xvals(2,:), yvals(2,:), 'color', 'b'); +set(ht, 'string', 'Initial Configuration'); +figure(h); +disp(' ') +disp('Press any key to continue'); +pause; clc; + +disp('Now we adapt the parameters of the mixture model iteratively using the') +disp('EM algorithm. Each cycle of the EM algorithm consists of an E-step') +disp('followed by an M-step. We start with the E-step, which involves the') +disp('evaluation of the posterior probabilities (responsibilities) which the') +disp('two components have for each of the data points.') +disp(' ') +disp('Since we have labelled the two components using the colours red and') +disp('blue, a convenient way to indicate the value of a posterior') +disp('probability for a given data point is to colour the point using a') +disp('scale ranging from pure red (corresponding to a posterior probability') +disp('of 1.0 for the red component and 0.0 for the blue component) through') +disp('to pure blue.') +disp(' ') +disp('Press any key to see the result of applying the first E-step.') +pause; + +% Initial E-step. +set(ht, 'string', 'E-step'); +post = gmmpost(mix, x); +dcols = [post(:,1), zeros(ndata, 1), post(:,2)]; +delete(hd); +for i = 1 : ndata + hd(i) = plot(x(i, 1), x(i, 2), 'color', dcols(i,:), ... + 'marker', '.', 'markersize', 30); +end +figure(h); + +disp(' '); +disp('Press any key to continue') +pause; clc; + +disp('Next we perform the corresponding M-step. This involves replacing the') +disp('centres of the component Gaussians by the corresponding weighted means') +disp('of the data. Thus the centre of the red component is replaced by the') +disp('mean of the data set, in which each data point is weighted according to') +disp('the amount of red ink (corresponding to the responsibility of') +disp('component 1 for explaining that data point). The variances and mixing') +disp('proportions of the two components are similarly re-estimated.') +disp(' ') +disp('Press any key to see the result of applying the first M-step.') +pause; + +% M-step. +set(ht, 'string', 'M-step'); +options = foptions; +options(14) = 1; % A single iteration +options(1) = -1; % Switch off all messages, including warning +mix = gmmem(mix, x, options); +delete(hc); +xvals = mix.centres(:, 1)*ones(1,ncirc) + sqrt(mix.covars')*xs; +yvals = mix.centres(:, 2)*ones(1,ncirc) + sqrt(mix.covars')*ys; +hc(1)=line(xvals(1,:), yvals(1,:), 'color', 'r'); +hc(2)=line(xvals(2,:), yvals(2,:), 'color', 'b'); +figure(h); +disp(' ') +disp('Press any key to continue') +pause; clc; + +disp('We can continue making alternate E and M steps until the changes in') +disp('the log likelihood at each cycle become sufficiently small.') +disp(' ') +disp('Press any key to see an animation of a further 9 EM cycles.') +pause; +figure(h); + +% Loop over EM iterations. +numiters = 9; +for n = 1 : numiters + + set(ht, 'string', 'E-step'); + post = gmmpost(mix, x); + dcols = [post(:,1), zeros(ndata, 1), post(:,2)]; + delete(hd); + for i = 1 : ndata + hd(i) = plot(x(i, 1), x(i, 2), 'color', dcols(i,:), ... + 'marker', '.', 'markersize', 30); + end + pause(1) + + set(ht, 'string', 'M-step'); + [mix, options] = gmmem(mix, x, options); + fprintf(1, 'Cycle %4d Error %11.6f\n', n, options(8)); + delete(hc); + xvals = mix.centres(:, 1)*ones(1,ncirc) + sqrt(mix.covars')*xs; + yvals = mix.centres(:, 2)*ones(1,ncirc) + sqrt(mix.covars')*ys; + hc(1)=line(xvals(1,:), yvals(1,:), 'color', 'r'); + hc(2)=line(xvals(2,:), yvals(2,:), 'color', 'b'); + pause(1) + +end + +disp(' '); +disp('Press any key to end.') +pause; clc; close(h); clear all + diff --git a/sourcecodes/bnt-master/netlab3.3/demgmm2.m b/sourcecodes/bnt-master/netlab3.3/demgmm2.m new file mode 100644 index 00000000..ae0e04fa --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgmm2.m @@ -0,0 +1,152 @@ +%DEMGMM1 Demonstrate density modelling with a Gaussian mixture model. +% +% Description +% The problem consists of modelling data generated by a mixture of +% three Gaussians in 2 dimensions. The priors are 0.3, 0.5 and 0.2; +% the centres are (2, 3.5), (0, 0) and (0,2); the variances are 0.2, +% 0.5 and 1.0. The first figure contains a scatter plot of the data. +% +% A Gaussian mixture model with three components is trained using EM. +% The parameter vector is printed before training and after training. +% The user should press any key to continue at these points. The +% parameter vector consists of priors (the column), centres (given as +% (x, y) pairs as the next two columns), and variances (the last +% column). +% +% The second figure is a 3 dimensional view of the density function, +% while the third shows the 1-standard deviation circles for the three +% components of the mixture model. +% +% See also +% GMM, GMMINIT, GMMEM, GMMPROB, GMMUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Generate the data +% Fix seeds for reproducible results +randn('state', 42); +rand('state', 42); + +ndata = 500; +[data, datac, datap, datasd] = dem2ddat(ndata); + +clc +disp('This demonstration illustrates the use of a Gaussian mixture model') +disp('to approximate the unconditional probability density of data in') +disp('a two-dimensional space. We begin by generating the data from') +disp('a mixture of three Gaussians and plotting it.') +disp(' ') +disp('Press any key to continue') +pause + +fh1 = figure; +plot(data(:, 1), data(:, 2), 'o') +set(gca, 'Box', 'on') +% Set up mixture model +ncentres = 3; +input_dim = 2; +mix = gmm(input_dim, ncentres, 'spherical'); + +options = foptions; +options(14) = 5; % Just use 5 iterations of k-means in initialisation +% Initialise the model parameters from the data +mix = gmminit(mix, data, options); + +clc +disp('The data is drawn from a mixture with parameters') +disp(' Priors Centres Variances') +disp([datap' datac (datasd.^2)']) +disp(' ') +disp('The mixture model has three components and spherical covariance') +disp('matrices. The model parameters after initialisation using the') +disp('k-means algorithm are as follows') +% Print out model +disp(' Priors Centres Variances') +disp([mix.priors' mix.centres mix.covars']) +disp('Press any key to continue') +pause + +% Set up vector of options for EM trainer +options = zeros(1, 18); +options(1) = 1; % Prints out error values. +options(14) = 10; % Max. Number of iterations. + +disp('We now train the model using the EM algorithm for 10 iterations') +disp(' ') +disp('Press any key to continue') +pause +[mix, options, errlog] = gmmem(mix, data, options); + +% Print out model +disp(' ') +disp('The trained model has parameters ') +disp(' Priors Centres Variances') +disp([mix.priors' mix.centres mix.covars']) +disp('Note the close correspondence between these parameters and those') +disp('of the distribution used to generate the data, which are repeated here.') +disp(' Priors Centres Variances') +disp([datap' datac (datasd.^2)']) +disp(' ') +disp('Press any key to continue') +pause + +clc +disp('We now plot the density given by the mixture model as a surface plot') +disp(' ') +disp('Press any key to continue') +pause +% Plot the result +x = -4.0:0.2:5.0; +y = -4.0:0.2:5.0; +[X, Y] = meshgrid(x,y); +X = X(:); +Y = Y(:); +grid = [X Y]; +Z = gmmprob(mix, grid); +Z = reshape(Z, length(x), length(y)); +c = mesh(x, y, Z); +hold on +title('Surface plot of probability density') +hold off + +clc +disp('The final plot shows the centres and widths, given by one standard') +disp('deviation, of the three components of the mixture model.') +disp(' ') +disp('Press any key to continue.') +pause +% Try to calculate a sensible position for the second figure, below the first +fig1_pos = get(fh1, 'Position'); +fig2_pos = fig1_pos; +fig2_pos(2) = fig2_pos(2) - fig1_pos(4); +fh2 = figure; +set(fh2, 'Position', fig2_pos) + +hp1 = plot(data(:, 1), data(:, 2), 'bo'); +axis('equal'); +hold on +hp2 = plot(mix.centres(:, 1), mix.centres(:,2), 'g+'); +set(hp2, 'MarkerSize', 10); +set(hp2, 'LineWidth', 3); + +title('Plot of data and mixture centres') +angles = 0:pi/30:2*pi; +for i = 1 : mix.ncentres + x_circle = mix.centres(i,1)*ones(1, length(angles)) + ... + sqrt(mix.covars(i))*cos(angles); + y_circle = mix.centres(i,2)*ones(1, length(angles)) + ... + sqrt(mix.covars(i))*sin(angles); + plot(x_circle, y_circle, 'r') +end +hold off +disp('Note how the data cluster positions and widths are captured by') +disp('the mixture model.') +disp(' ') +disp('Press any key to end.') +pause + +close(fh1); +close(fh2); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demgmm3.m b/sourcecodes/bnt-master/netlab3.3/demgmm3.m new file mode 100644 index 00000000..4e4210d1 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgmm3.m @@ -0,0 +1,192 @@ +%DEMGMM3 Demonstrate density modelling with a Gaussian mixture model. +% +% Description +% The problem consists of modelling data generated by a mixture of +% three Gaussians in 2 dimensions with a mixture model using diagonal +% covariance matrices. The priors are 0.3, 0.5 and 0.2; the centres +% are (2, 3.5), (0, 0) and (0,2); the covariances are all axis aligned +% (0.16, 0.64), (0.25, 1) and the identity matrix. The first figure +% contains a scatter plot of the data. +% +% A Gaussian mixture model with three components is trained using EM. +% The parameter vector is printed before training and after training. +% The user should press any key to continue at these points. The +% parameter vector consists of priors (the column), and centres (given +% as (x, y) pairs as the next two columns). The diagonal entries of +% the covariance matrices are printed separately. +% +% The second figure is a 3 dimensional view of the density function, +% while the third shows the axes of the 1-standard deviation circles +% for the three components of the mixture model. +% +% See also +% GMM, GMMINIT, GMMEM, GMMPROB, GMMUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Generate the data +ndata = 500; + +% Fix the seeds for reproducible results +randn('state', 42); +rand('state', 42); +data = randn(ndata, 2); +prior = [0.3 0.5 0.2]; +% Mixture model swaps clusters 1 and 3 +datap = [0.2 0.5 0.3]; +datac = [0 2; 0 0; 2 3.5]; +datacov = [1 1;1 0.25; 0.4*0.4 0.8*0.8]; +data1 = data(1:prior(1)*ndata,:); +data2 = data(prior(1)*ndata+1:(prior(2)+prior(1))*ndata, :); +data3 = data((prior(1)+prior(2))*ndata +1:ndata, :); + +% First cluster has axis aligned variance and centre (2, 3.5) +data1(:, 1) = data1(:, 1)*0.4 + 2.0; +data1(:, 2) = data1(:, 2)*0.8 + 3.5; + +% Second cluster has axis aligned variance and centre (0, 0) +data2(:,2) = data2(:, 2)*0.5; + +% Third cluster is at (0,2) with identity matrix for covariance +data3 = data3 + repmat([0 2], prior(3)*ndata, 1); + +% Put the dataset together again +data = [data1; data2; data3]; + +clc +disp('This demonstration illustrates the use of a Gaussian mixture model') +disp('with diagonal covariance matrices to approximate the unconditional') +disp('probability density of data in a two-dimensional space.') +disp('We begin by generating the data from a mixture of three Gaussians') +disp('with axis aligned covariance structure and plotting it.') +disp(' ') +disp('The first cluster has centre (0, 2).') +disp('The second cluster has centre (0, 0).') +disp('The third cluster has centre (2, 3.5).') +disp(' ') +disp('Press any key to continue') +pause + +fh1 = figure; +plot(data(:, 1), data(:, 2), 'o') +set(gca, 'Box', 'on') + +% Set up mixture model +ncentres = 3; +input_dim = 2; +mix = gmm(input_dim, ncentres, 'diag'); + +options = foptions; +options(14) = 5; % Just use 5 iterations of k-means in initialisation +% Initialise the model parameters from the data +mix = gmminit(mix, data, options); + +% Print out model +disp('The mixture model has three components and diagonal covariance') +disp('matrices. The model parameters after initialisation using the') +disp('k-means algorithm are as follows') +disp(' Priors Centres') +disp([mix.priors' mix.centres]) +disp('Covariance diagonals are') +disp(mix.covars) +disp('Press any key to continue.') +pause + +% Set up vector of options for EM trainer +options = zeros(1, 18); +options(1) = 1; % Prints out error values. +options(14) = 20; % Number of iterations. + +disp('We now train the model using the EM algorithm for 20 iterations.') +disp(' ') +disp('Press any key to continue.') +pause + +[mix, options, errlog] = gmmem(mix, data, options); + +% Print out model +disp(' ') +disp('The trained model has priors and centres:') +disp(' Priors Centres') +disp([mix.priors' mix.centres]) +disp('The data generator has priors and centres') +disp(' Priors Centres') +disp([datap' datac]) +disp('Model covariance diagonals are') +disp(mix.covars) +disp('Data generator covariance diagonals are') +disp(datacov) +disp('Note the close correspondence between these parameters and those') +disp('of the distribution used to generate the data.') +disp(' ') +disp('Press any key to continue.') +pause + +clc +disp('We now plot the density given by the mixture model as a surface plot.') +disp(' ') +disp('Press any key to continue.') +pause + +% Plot the result +x = -4.0:0.2:5.0; +y = -4.0:0.2:5.0; +[X, Y] = meshgrid(x,y); +X = X(:); +Y = Y(:); +grid = [X Y]; +Z = gmmprob(mix, grid); +Z = reshape(Z, length(x), length(y)); +c = mesh(x, y, Z); +hold on +title('Surface plot of probability density') +hold off +drawnow + +clc +disp('The final plot shows the centres and widths, given by one standard') +disp('deviation, of the three components of the mixture model. The axes') +disp('of the ellipses of constant density are shown.') +disp(' ') +disp('Press any key to continue.') +pause + +% Try to calculate a sensible position for the second figure, below the first +fig1_pos = get(fh1, 'Position'); +fig2_pos = fig1_pos; +fig2_pos(2) = fig2_pos(2) - fig1_pos(4); +fh2 = figure('Position', fig2_pos); + +h = plot(data(:, 1), data(:, 2), 'bo'); +hold on +axis('equal'); +title('Plot of data and covariances') +for i = 1:ncentres + v = [1 0]; + for j = 1:2 + start=mix.centres(i,:)-sqrt(mix.covars(i,:).*v); + endpt=mix.centres(i,:)+sqrt(mix.covars(i,:).*v); + linex = [start(1) endpt(1)]; + liney = [start(2) endpt(2)]; + line(linex, liney, 'Color', 'k', 'LineWidth', 3) + v = [0 1]; + end + % Plot ellipses of one standard deviation + theta = 0:0.02:2*pi; + x = sqrt(mix.covars(i,1))*cos(theta) + mix.centres(i,1); + y = sqrt(mix.covars(i,2))*sin(theta) + mix.centres(i,2); + plot(x, y, 'r-'); +end +hold off + +disp('Note how the data cluster positions and widths are captured by') +disp('the mixture model.') +disp(' ') +disp('Press any key to end.') +pause + +close(fh1); +close(fh2); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demgmm4.m b/sourcecodes/bnt-master/netlab3.3/demgmm4.m new file mode 100644 index 00000000..12426787 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgmm4.m @@ -0,0 +1,210 @@ +%DEMGMM4 Demonstrate density modelling with a Gaussian mixture model. +% +% Description +% The problem consists of modelling data generated by a mixture of +% three Gaussians in 2 dimensions with a mixture model using full +% covariance matrices. The priors are 0.3, 0.5 and 0.2; the centres +% are (2, 3.5), (0, 0) and (0,2); the variances are (0.16, 0.64) axis +% aligned, (0.25, 1) rotated by 30 degrees and the identity matrix. The +% first figure contains a scatter plot of the data. +% +% A Gaussian mixture model with three components is trained using EM. +% The parameter vector is printed before training and after training. +% The user should press any key to continue at these points. The +% parameter vector consists of priors (the column), and centres (given +% as (x, y) pairs as the next two columns). The covariance matrices +% are printed separately. +% +% The second figure is a 3 dimensional view of the density function, +% while the third shows the axes of the 1-standard deviation ellipses +% for the three components of the mixture model. +% +% See also +% GMM, GMMINIT, GMMEM, GMMPROB, GMMUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the data + +ndata = 500; + +% Fix the seeds for reproducible results +randn('state', 42); +rand('state', 42); +data = randn(ndata, 2); +prior = [0.3 0.5 0.2]; +% Mixture model swaps clusters 1 and 3 +datap = [0.2 0.5 0.3]; +datac = [0 2; 0 0; 2 3.5]; +datacov = repmat(eye(2), [1 1 3]); +data1 = data(1:prior(1)*ndata,:); +data2 = data(prior(1)*ndata+1:(prior(2)+prior(1))*ndata, :); +data3 = data((prior(1)+prior(2))*ndata +1:ndata, :); + +% First cluster has axis aligned variance and centre (2, 3.5) +data1(:, 1) = data1(:, 1)*0.4 + 2.0; +data1(:, 2) = data1(:, 2)*0.8 + 3.5; +datacov(:, :, 3) = [0.4*0.4 0; 0 0.8*0.8]; + +% Second cluster has variance axes rotated by 30 degrees and centre (0, 0) +rotn = [cos(pi/6) -sin(pi/6); sin(pi/6) cos(pi/6)]; +data2(:,1) = data2(:, 1)*0.5; +data2 = data2*rotn; +datacov(:, :, 2) = rotn' * [0.25 0; 0 1] * rotn; + +% Third cluster is at (0,2) +data3 = data3 + repmat([0 2], prior(3)*ndata, 1); + +% Put the dataset together again +data = [data1; data2; data3]; + +clc +disp('This demonstration illustrates the use of a Gaussian mixture model') +disp('with full covariance matrices to approximate the unconditional ') +disp('probability density of data in a two-dimensional space.') +disp('We begin by generating the data from a mixture of three Gaussians and') +disp('plotting it.') +disp(' ') +disp('The first cluster has axis aligned variance and centre (0, 2).') +disp('The second cluster has variance axes rotated by 30 degrees') +disp('and centre (0, 0). The third cluster has unit variance and centre') +disp('(2, 3.5).') +disp(' ') +disp('Press any key to continue.') +pause + +fh1 = figure; +plot(data(:, 1), data(:, 2), 'o') +set(gca, 'Box', 'on') + +% Set up mixture model +ncentres = 3; +input_dim = 2; +mix = gmm(input_dim, ncentres, 'full'); + +% Initialise the model parameters from the data +options = foptions; +options(14) = 5; % Just use 5 iterations of k-means in initialisation +mix = gmminit(mix, data, options); + +% Print out model +clc +disp('The mixture model has three components and full covariance') +disp('matrices. The model parameters after initialisation using the') +disp('k-means algorithm are as follows') +disp(' Priors Centres') +disp([mix.priors' mix.centres]) +disp('Covariance matrices are') +disp(mix.covars) +disp('Press any key to continue.') +pause + +% Set up vector of options for EM trainer +options = zeros(1, 18); +options(1) = 1; % Prints out error values. +options(14) = 50; % Number of iterations. + +disp('We now train the model using the EM algorithm for 50 iterations.') +disp(' ') +disp('Press any key to continue.') +pause +[mix, options, errlog] = gmmem(mix, data, options); + +% Print out model +disp(' ') +disp('The trained model has priors and centres:') +disp(' Priors Centres') +disp([mix.priors' mix.centres]) +disp('The data generator has priors and centres') +disp(' Priors Centres') +disp([datap' datac]) +disp('Model covariance matrices are') +disp(mix.covars(:, :, 1)) +disp(mix.covars(:, :, 2)) +disp(mix.covars(:, :, 3)) +disp('Data generator covariance matrices are') +disp(datacov(:, :, 1)) +disp(datacov(:, :, 2)) +disp(datacov(:, :, 3)) +disp('Note the close correspondence between these parameters and those') +disp('of the distribution used to generate the data. The match for') +disp('covariance matrices is not that close, but would be improved with') +disp('more iterations of the training algorithm.') +disp(' ') +disp('Press any key to continue.') +pause + +clc +disp('We now plot the density given by the mixture model as a surface plot.') +disp(' ') +disp('Press any key to continue.') +pause + +% Plot the result +x = -4.0:0.2:5.0; +y = -4.0:0.2:5.0; +[X, Y] = meshgrid(x,y); +X = X(:); +Y = Y(:); +grid = [X Y]; +Z = gmmprob(mix, grid); +Z = reshape(Z, length(x), length(y)); +c = mesh(x, y, Z); +hold on +title('Surface plot of probability density') +hold off +drawnow + +clc +disp('The final plot shows the centres and widths, given by one standard') +disp('deviation, of the three components of the mixture model. The axes') +disp('of the ellipses of constant density are shown.') +disp(' ') +disp('Press any key to continue.') +pause + +% Try to calculate a sensible position for the second figure, below the first +fig1_pos = get(fh1, 'Position'); +fig2_pos = fig1_pos; +fig2_pos(2) = fig2_pos(2) - fig1_pos(4) - 30; +fh2 = figure('Position', fig2_pos); + +h3 = plot(data(:, 1), data(:, 2), 'bo'); +axis equal; +hold on +title('Plot of data and covariances') +for i = 1:ncentres + [v,d] = eig(mix.covars(:,:,i)); + for j = 1:2 + % Ensure that eigenvector has unit length + v(:,j) = v(:,j)/norm(v(:,j)); + start=mix.centres(i,:)-sqrt(d(j,j))*(v(:,j)'); + endpt=mix.centres(i,:)+sqrt(d(j,j))*(v(:,j)'); + linex = [start(1) endpt(1)]; + liney = [start(2) endpt(2)]; + line(linex, liney, 'Color', 'k', 'LineWidth', 3) + end + % Plot ellipses of one standard deviation + theta = 0:0.02:2*pi; + x = sqrt(d(1,1))*cos(theta); + y = sqrt(d(2,2))*sin(theta); + % Rotate ellipse axes + ellipse = (v*([x; y]))'; + % Adjust centre + ellipse = ellipse + ones(length(theta), 1)*mix.centres(i,:); + plot(ellipse(:,1), ellipse(:,2), 'r-'); +end +hold off + +disp('Note how the data cluster positions and widths are captured by') +disp('the mixture model.') +disp(' ') +disp('Press any key to end.') +pause + +close(fh1); +close(fh2); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demgmm5.m b/sourcecodes/bnt-master/netlab3.3/demgmm5.m new file mode 100644 index 00000000..f265b35d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgmm5.m @@ -0,0 +1,147 @@ +%DEMGMM5 Demonstrate density modelling with a PPCA mixture model. +% +% Description +% The problem consists of modelling data generated by a mixture of +% three Gaussians in 2 dimensions with a mixture model using full +% covariance matrices. The priors are 0.3, 0.5 and 0.2; the centres +% are (2, 3.5), (0, 0) and (0,2); the variances are (0.16, 0.64) axis +% aligned, (0.25, 1) rotated by 30 degrees and the identity matrix. The +% first figure contains a scatter plot of the data. +% +% A mixture model with three one-dimensional PPCA components is trained +% using EM. The parameter vector is printed before training and after +% training. The parameter vector consists of priors (the column), and +% centres (given as (x, y) pairs as the next two columns). +% +% The second figure is a 3 dimensional view of the density function, +% while the third shows the axes of the 1-standard deviation ellipses +% for the three components of the mixture model together with the one +% standard deviation along the principal component of each mixture +% model component. +% +% See also +% GMM, GMMINIT, GMMEM, GMMPROB, PPCA +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +ndata = 500; +data = randn(ndata, 2); +prior = [0.3 0.5 0.2]; +% Mixture model swaps clusters 1 and 3 +datap = [0.2 0.5 0.3]; +datac = [0 2; 0 0; 2 3.5]; +datacov = repmat(eye(2), [1 1 3]); +data1 = data(1:prior(1)*ndata,:); +data2 = data(prior(1)*ndata+1:(prior(2)+prior(1))*ndata, :); +data3 = data((prior(1)+prior(2))*ndata +1:ndata, :); + +% First cluster has axis aligned variance and centre (2, 3.5) +data1(:, 1) = data1(:, 1)*0.1 + 2.0; +data1(:, 2) = data1(:, 2)*0.8 + 3.5; +datacov(:, :, 3) = [0.1*0.1 0; 0 0.8*0.8]; + +% Second cluster has variance axes rotated by 30 degrees and centre (0, 0) +rotn = [cos(pi/6) -sin(pi/6); sin(pi/6) cos(pi/6)]; +data2(:,1) = data2(:, 1)*0.2; +data2 = data2*rotn; +datacov(:, :, 2) = rotn' * [0.04 0; 0 1] * rotn; + +% Third cluster is at (0,2) +data3(:, 2) = data3(:, 2)*0.1; +data3 = data3 + repmat([0 2], prior(3)*ndata, 1); + +% Put the dataset together again +data = [data1; data2; data3]; + +ndata = 100; % Number of data points. +noise = 0.2; % Standard deviation of noise distribution. +x = [0:1/(2*(ndata - 1)):0.5]'; +randn('state', 1); +rand('state', 1); +t = sin(2*pi*x) + noise*randn(ndata, 1); + +% Fit three one-dimensional PPCA models +ncentres = 3; +ppca_dim = 1; + +clc +disp('This demonstration illustrates the use of a Gaussian mixture model') +disp('with a probabilistic PCA covariance structure to approximate the') +disp('unconditional probability density of data in a two-dimensional space.') +disp('We begin by generating the data from a mixture of three Gaussians and') +disp('plotting it.') +disp(' ') +disp('The first cluster has axis aligned variance and centre (0, 2).') +disp('The variance parallel to the x-axis is significantly greater') +disp('than that parallel to the y-axis.') +disp('The second cluster has variance axes rotated by 30 degrees') +disp('and centre (0, 0). The third cluster has significant variance') +disp('parallel to the y-axis and centre (2, 3.5).') +disp(' ') +disp('Press any key to continue.') +pause + +fh1 = figure; +plot(data(:, 1), data(:, 2), 'o') +set(gca, 'Box', 'on') +axis equal +hold on + +mix = gmm(2, ncentres, 'ppca', ppca_dim); +options = foptions; +options(14) = 10; +options(1) = -1; % Switch off all warnings + +% Just use 10 iterations of k-means in initialisation +% Initialise the model parameters from the data +mix = gmminit(mix, data, options); +disp('The mixture model has three components with 1-dimensional') +disp('PPCA subspaces. The model parameters after initialisation using') +disp('the k-means algorithm are as follows') +disp(' Priors Centres') +disp([mix.priors' mix.centres]) +disp(' ') +disp('Press any key to continue') +pause + +options(1) = 1; % Prints out error values. +options(14) = 30; % Number of iterations. + +disp('We now train the model using the EM algorithm for up to 30 iterations.') +disp(' ') +disp('Press any key to continue.') +pause + +[mix, options, errlog] = gmmem(mix, data, options); +disp('The trained model has priors and centres:') +disp(' Priors Centres') +disp([mix.priors' mix.centres]) + +% Now plot the result +for i = 1:ncentres + % Plot the PC vectors + v = mix.U(:,:,i); + start=mix.centres(i,:)-sqrt(mix.lambda(i))*(v'); + endpt=mix.centres(i,:)+sqrt(mix.lambda(i))*(v'); + linex = [start(1) endpt(1)]; + liney = [start(2) endpt(2)]; + line(linex, liney, 'Color', 'k', 'LineWidth', 3) + % Plot ellipses of one standard deviation + theta = 0:0.02:2*pi; + x = sqrt(mix.lambda(i))*cos(theta); + y = sqrt(mix.covars(i))*sin(theta); + % Rotate ellipse axes + rot_matrix = [v(1) -v(2); v(2) v(1)]; + ellipse = (rot_matrix*([x; y]))'; + % Adjust centre + ellipse = ellipse + ones(length(theta), 1)*mix.centres(i,:); + plot(ellipse(:,1), ellipse(:,2), 'r-') +end + +disp(' ') +disp('Press any key to exit') +pause +close (fh1); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demgp.m b/sourcecodes/bnt-master/netlab3.3/demgp.m new file mode 100644 index 00000000..65c587a3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgp.m @@ -0,0 +1,160 @@ +%DEMGP Demonstrate simple regression using a Gaussian Process. +% +% Description +% The problem consists of one input variable X and one target variable +% T. The values in X are chosen in two separated clusters and the +% target data is generated by computing SIN(2*PI*X) and adding Gaussian +% noise. Two Gaussian Processes, each with different covariance +% functions are trained by optimising the hyperparameters using the +% scaled conjugate gradient algorithm. The final predictions are +% plotted together with 2 standard deviation error bars. +% +% See also +% GP, GPERR, GPFWD, GPGRAD, GPINIT, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Find out if flops is available (i.e. pre-version 6 Matlab) +v = version; +if (str2num(strtok(v, '.')) >= 6) + flops_works = logical(0); +else + flops_works = logical(1); +end + +randn('state', 42); +x = [0.1 0.15 0.2 0.25 0.65 0.7 0.75 0.8 0.85 0.9]'; +ndata = length(x); +t = sin(2*pi*x) + 0.05*randn(ndata, 1); + +xtest = linspace(0, 1, 50)'; + +clc +disp('This demonstration illustrates the use of a Gaussian Process') +disp('model for regression problems. The data is generated from a noisy') +disp('sine function.') +disp(' ') +disp('Press any key to continue.') +pause + +flops(0); +% Initialise the parameters. +net = gp(1, 'sqexp'); +prior.pr_mean = 0; +prior.pr_var = 1; +net = gpinit(net, x, t, prior); + +clc +disp('The first GP uses the squared exponential covariance function.') +disp('The hyperparameters are initialised by sampling from a Gaussian with a') +disp(['mean of ', num2str(prior.pr_mean), ' and variance ', ... + num2str(prior.pr_var), '.']) +disp('After initializing the network, we train it using the scaled conjugate') +disp('gradients algorithm for 20 cycles.') +disp(' ') +disp('Press any key to continue') +pause + +% Now train to find the hyperparameters. +options = foptions; +options(1) = 1; % Display training error values +options(14) = 20; +flops(0) +[net, options] = netopt(net, options, x, t, 'scg'); +if flops_works + sflops = flops; +end + +disp('The second GP uses the rational quadratic covariance function.') +disp('The hyperparameters are initialised by sampling from a Gaussian with a') +disp(['mean of ', num2str(prior.pr_mean), ' and variance ', num2str(prior.pr_var)]) +disp('After initializing the network, we train it using the scaled conjugate') +disp('gradients algorithm for 20 cycles.') +disp(' ') +disp('Press any key to continue') +pause +flops(0) +net2 = gp(1, 'ratquad'); +net2 = gpinit(net2, x, t, prior); +flops(0) +[net2, options] = netopt(net2, options, x, t, 'scg'); +if flops_works + rflops = flops; +end + +disp(' ') +disp('Press any key to continue') +disp(' ') +pause +clc + +fprintf(1, 'For squared exponential covariance function,'); +if flops_works + fprintf(1, 'flops = %d', sflops); +end +fprintf(1, '\nfinal hyperparameters:\n') +format_string = strcat(' bias:\t\t\t%10.6f\n noise:\t\t%10.6f\n', ... + ' inverse lengthscale:\t%10.6f\n vertical scale:\t%10.6f\n'); +fprintf(1, format_string, ... + exp(net.bias), exp(net.noise), exp(net.inweights(1)), exp(net.fpar(1))); +fprintf(1, '\n\nFor rational quadratic covariance function,'); +if flops_works + fprintf(1, 'flops = %d', rflops); +end +fprintf(1, '\nfinal hyperparameters:\n') +format_string = [format_string ' cov decay order:\t%10.6f\n']; +fprintf(1, format_string, ... + exp(net2.bias), exp(net2.noise), exp(net2.inweights(1)), ... + exp(net2.fpar(1)), exp(net2.fpar(2))); +disp(' ') +disp('Press any key to continue') +pause + +disp(' ') +disp('Now we plot the data, underlying function, model outputs and two') +disp('standard deviation error bars on a single graph to compare the results.') +disp(' ') +disp('Press any key to continue.') +pause +cn = gpcovar(net, x); +cninv = inv(cn); +[ytest, sigsq] = gpfwd(net, xtest, cninv); +sig = sqrt(sigsq); + +fh1 = figure; +hold on +plot(x, t, 'ok'); +xlabel('Input') +ylabel('Target') +fplot('sin(2*pi*x)', [0 1], '--m'); +plot(xtest, ytest, '-k'); +plot(xtest, ytest+(2*sig), '-b', xtest, ytest-(2*sig), '-b'); +axis([0 1 -1.5 1.5]); +title('Squared exponential covariance function') +legend('data', 'function', 'GP', 'error bars'); +hold off + +cninv2 = inv(gpcovar(net2, x)); +[ytest2, sigsq2] = gpfwd(net2, xtest, cninv2); +sig2 = sqrt(sigsq2); +fh2 = figure; +hold on +plot(x, t, 'ok'); +xlabel('Input') +ylabel('Target') +fplot('sin(2*pi*x)', [0 1], '--m'); +plot(xtest, ytest2, '-k'); +plot(xtest, ytest2+(2*sig2), '-b', xtest, ytest2-(2*sig2), '-b'); +axis([0 1 -1.5 1.5]); +title('Rational quadratic covariance function') +legend('data', 'function', 'GP', 'error bars'); +hold off + +disp(' ') +disp('Press any key to end.') +pause +close(fh1); +close(fh2); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demgpard.m b/sourcecodes/bnt-master/netlab3.3/demgpard.m new file mode 100644 index 00000000..82e8d2a1 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgpard.m @@ -0,0 +1,126 @@ +%DEMGPARD Demonstrate ARD using a Gaussian Process. +% +% Description +% The data consists of three input variables X1, X2 and X3, and one +% target variable T. The target data is generated by computing +% SIN(2*PI*X1) and adding Gaussian noise, x2 is a copy of x1 with a +% higher level of added noise, and x3 is sampled randomly from a +% Gaussian distribution. A Gaussian Process, is trained by optimising +% the hyperparameters using the scaled conjugate gradient algorithm. +% The final values of the hyperparameters show that the model +% successfully identifies the importance of each input. +% +% See also +% DEMGP, GP, GPERR, GPFWD, GPGRAD, GPINIT, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc; +randn('state', 1729); +rand('state', 1729); +disp('This demonstration illustrates the technique of automatic relevance') +disp('determination (ARD) using a Gaussian Process.') +disp(' '); +disp('First, we set up a synthetic data set involving three input variables:') +disp('x1 is sampled uniformly from the range (0,1) and has a low level of') +disp('added Gaussian noise, x2 is a copy of x1 with a higher level of added') +disp('noise, and x3 is sampled randomly from a Gaussian distribution. The') +disp('single target variable is given by t = sin(2*pi*x1) with additive') +disp('Gaussian noise. Thus x1 is very relevant for determining the target') +disp('value, x2 is of some relevance, while x3 should in principle be') +disp('irrelevant.') +disp(' '); +disp('Press any key to see a plot of t against x1.') +pause; + +ndata = 100; +x1 = rand(ndata, 1); +x2 = x1 + 0.05*randn(ndata, 1); +x3 = 0.5 + 0.5*randn(ndata, 1); +x = [x1, x2, x3]; +t = sin(2*pi*x1) + 0.1*randn(ndata, 1); + +% Plot the data and the original function. +h = figure; +plotvals = linspace(0, 1, 200)'; +plot(x1, t, 'ob') +hold on +xlabel('Input x1') +ylabel('Target') +axis([0 1 -1.5 1.5]) +[fx, fy] = fplot('sin(2*pi*x)', [0 1]); +plot(fx, fy, '-g', 'LineWidth', 2); +legend('data', 'function'); + +disp(' '); +disp('Press any key to continue') +pause; clc; + +disp('The Gaussian Process has a separate hyperparameter for each input.') +disp('The hyperparameters are trained by error minimisation using the scaled.') +disp('conjugate gradient optimiser.') +disp(' '); +disp('Press any key to create and train the model.') +disp(' '); +pause; + +net = gp(3, 'sqexp'); +% Initialise the parameters. +prior.pr_mean = 0; +prior.pr_var = 0.1; +net = gpinit(net, x, t, prior); + +% Now train to find the hyperparameters. +options = foptions; +options(1) = 1; +options(14) = 30; + +[net, options] = netopt(net, options, x, t, 'scg'); + +rel = exp(net.inweights); + +fprintf(1, ... + '\nFinal hyperparameters:\n\n bias:\t\t%10.6f\n noise:\t%10.6f\n', ... + exp(net.bias), exp(net.noise)); +fprintf(1, ' Vertical scale: %8.6f\n', exp(net.fpar(1))); +fprintf(1, ' Input 1:\t%10.6f\n Input 2:\t%10.6f\n', ... + rel(1), rel(2)); +fprintf(1, ' Input 3:\t%10.6f\n\n', rel(3)); +disp(' '); +disp('We see that the inverse lengthscale associated with') +disp('input x1 is large, that of x2 has an intermediate value and the variance') +disp('of weights associated with x3 is small.') +disp(' '); +disp('This implies that the Gaussian Process is giving greatest emphasis') +disp('to x1 and least emphasis to x3, with intermediate emphasis on') +disp('x2 in the covariance function.') +disp(' ') +disp('Since the target t is statistically independent of x3 we might') +disp('expect the weights associated with this input would go to') +disp('zero. However, for any finite data set there may be some chance') +disp('correlation between x3 and t, and so the corresponding hyperparameter remains') +disp('finite.') +disp('Press any key to continue.') +pause + +disp('Finally, we plot the output of the Gaussian Process along the line') +disp('x1 = x2 = x3, together with the true underlying function.') +xt = linspace(0, 1, 50); +xtest = [xt', xt', xt']; + +cn = gpcovar(net, x); +cninv = inv(cn); +[ytest, sigsq] = gpfwd(net, xtest, cninv); +sig = sqrt(sigsq); + +figure(h); hold on; +plot(xt, ytest, '-k'); +plot(xt, ytest+(2*sig), '-b', xt, ytest-(2*sig), '-b'); +axis([0 1 -1.5 1.5]); +fplot('sin(2*pi*x)', [0 1], '--m'); + +disp(' '); +disp('Press any key to end.') +pause; clc; close(h); clear all + diff --git a/sourcecodes/bnt-master/netlab3.3/demgpot.m b/sourcecodes/bnt-master/netlab3.3/demgpot.m new file mode 100644 index 00000000..c859fd0b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgpot.m @@ -0,0 +1,29 @@ +function g = demgpot(x, mix) +%DEMGPOT Computes the gradient of the negative log likelihood for a mixture model. +% +% Description +% This function computes the gradient of the negative log of the +% unconditional data density P(X) with respect to the coefficients of +% the data vector X for a Gaussian mixture model. The data structure +% MIX defines the mixture model, while the matrix X contains the data +% vector as a row vector. Note the unusual order of the arguments: this +% is so that the function can be used in DEMHMC1 directly for sampling +% from the distribution P(X). +% +% See also +% DEMHMC1, DEMMET1, DEMPOT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Computes the potential gradient + +temp = (ones(mix.ncentres,1)*x)-mix.centres; +temp = temp.*(gmmactiv(mix,x)'*ones(1, mix.nin)); +% Assume spherical covariance structure +if ~strcmp(mix.covar_type, 'spherical') + error('Spherical covariance only.') +end +temp = temp./(mix.covars'*ones(1, mix.nin)); +temp = temp.*(mix.priors'*ones(1, mix.nin)); +g = sum(temp, 1)/gmmprob(mix, x); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demgtm1.m b/sourcecodes/bnt-master/netlab3.3/demgtm1.m new file mode 100644 index 00000000..a127617f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgtm1.m @@ -0,0 +1,147 @@ +%DEMGTM1 Demonstrate EM for GTM. +% +% Description +% This script demonstrates the use of the EM algorithm to fit a one- +% dimensional GTM to a two-dimensional set of data using maximum +% likelihood. The location and spread of the Gaussian kernels in the +% data space is shown during training. +% +% See also +% DEMGTM2, GTM, GTMEM, GTMPOST +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Demonstrates the GTM with a 2D target space and a 1D latent space. +% +% This script generates a simple data set in 2 dimensions, +% with an intrinsic dimensionality of 1, and trains a GTM +% with a 1-dimensional latent variable to model this data +% set, visually illustrating the training process +% +% Synopsis: gtm_demo + +% Generate and plot a 2D data set + +data_min = 0.15; +data_max = 3.05; +T = [data_min:0.05:data_max]'; +T = [T (T + 1.25*sin(2*T))]; +fh1 = figure; +plot(T(:,1), T(:,2), 'ro'); +axis([data_min-0.05 data_max+0.05 data_min-0.05 data_max+0.05]); +clc; +disp('This demonstration shows in detail how the EM algorithm works') +disp('for training a GTM with a one dimensional latent space.') +disp(' ') +fprintf([... +'The figure shows data generated by feeding a 1D uniform distribution\n', ... +'(on the X-axis) through a non-linear function (y = x + 1.25*sin(2*x))\n', ... +'\nPress any key to continue ...\n\n']); +pause; + +% Generate a unit circle figure, to be used for plotting +src = [0:(2*pi)/(20-1):2*pi]'; +unitC = [sin(src) cos(src)]; + +% Generate and plot (along with the data) an initial GTM model + +clc; +num_latent_points = 20; +num_rbf_centres = 5; + +net = gtm(1, num_latent_points, 2, num_rbf_centres, 'gaussian'); + +options = zeros(1, 18); +options(7) = 1; +net = gtminit(net, options, T, 'regular', num_latent_points, ... + num_rbf_centres); + +mix = gtmfwd(net); +% Replot the figure +hold off; +plot(mix.centres(:,1), mix.centres(:,2), 'g'); +hold on; +for i=1:num_latent_points + c = 2*unitC*sqrt(mix.covars(1)) + [ones(20,1)*mix.centres(i,1) ... + ones(num_latent_points,1)*mix.centres(i,2)]; + fill(c(:,1), c(:,2), [0.8 1 0.8]); +end +plot(T(:,1), T(:,2), 'ro'); +plot(mix.centres(:,1), mix.centres(:,2), 'g+'); +plot(mix.centres(:,1), mix.centres(:,2), 'g'); +axis([data_min-0.05 data_max+0.05 data_min-0.05 data_max+0.05]); +drawnow; +title('Initial configuration'); +disp(' ') +fprintf([... +'The figure shows the starting point for the GTM, before the training.\n', ... +'A discrete latent variable distribution of %d points in 1 dimension \n', ... +'is mapped to the 1st principal component of the target data by an RBF.\n', ... +'with %d basis functions. Each of the %d points defines the centre of\n', ... +'a Gaussian in a Gaussian mixture, marked by the green ''+''-signs. The\n', ... +'mixture components all have equal variance, illustrated by the filled\n', ... +'circle around each ''+''-sign, the radii corresponding to 2 standard\n', ... +'deviations. The ''+''-signs are connected with a line according to their\n', ... +'corresponding ordering in latent space.\n\n', ... +'Press any key to begin training ...\n\n'], num_latent_points, ... +num_rbf_centres, num_latent_points); +pause; + +figure(fh1); +%%%% Train the GTM and plot it (along with the data) as training proceeds %%%% +options = foptions; +options(1) = -1; % Turn off all warning messages +options(14) = 1; +for j = 1:15 + [net, options] = gtmem(net, T, options); + hold off; + mix = gtmfwd(net); + plot(mix.centres(:,1), mix.centres(:,2), 'g'); + hold on; + for i=1:20 + c = 2*unitC*sqrt(mix.covars(1)) + [ones(20,1)*mix.centres(i,1) ... + ones(20,1)*mix.centres(i,2)]; + fill(c(:,1), c(:,2), [0.8 1.0 0.8]); + end + plot(T(:,1), T(:,2), 'ro'); + plot(mix.centres(:,1), mix.centres(:,2), 'g+'); + plot(mix.centres(:,1), mix.centres(:,2), 'g'); + axis([0 3.5 0 3.5]); + title(['After ', int2str(j),' iterations of training.']); + drawnow; + if (j == 4) + fprintf([... +'The GTM initially adapts relatively quickly - already after \n', ... +'4 iterations of training, a rough fit is attained.\n\n', ... +'Press any key to continue training ...\n\n']); +pause; +figure(fh1); + elseif (j == 8) + fprintf([... +'After another 4 iterations of training: from now on further \n', ... +'training only makes small changes to the mapping, which combined with \n', ... +'decrements of the Gaussian mixture variance, optimize the fit in \n', ... +'terms of likelihood.\n\n', ... +'Press any key to continue training ...\n\n']); +pause; +figure(fh1); + else + pause(1); + end +end + +clc; +fprintf([... +'After 15 iterations of training the GTM can be regarded as converged. \n', ... +'Is has been adapted to fit the target data distribution as well \n', ... +'as possible, given prior smoothness constraints on the mapping. It \n', ... +'captures the fact that the probabilty density is higher at the two \n', ... +'bends of the curve, and lower towards its end points.\n\n']); +disp(' '); +disp('Press any key to exit.'); +pause; + +close(fh1); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demgtm2.m b/sourcecodes/bnt-master/netlab3.3/demgtm2.m new file mode 100644 index 00000000..2cfa907c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demgtm2.m @@ -0,0 +1,194 @@ +%DEMGTM2 Demonstrate GTM for visualisation. +% +% Description +% This script demonstrates the use of a GTM with a two-dimensional +% latent space to visualise data in a higher dimensional space. This is +% done through the use of the mean responsibility and magnification +% factors. +% +% See also +% DEMGTM1, GTM, GTMEM, GTMPOST +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Fix seeds for reproducible results +rand('state', 420); +randn('state', 420); + +ndata = 300 +clc; +disp('This demonstration shows how a Generative Topographic Mapping') +disp('can be used to model and visualise high dimensional data. The') +disp('data is generated from a mixture of two spherical Gaussians in') +dstring = ['four dimensional space. ', num2str(ndata), ... + ' data points are generated.']; +disp(dstring); +disp(' '); +disp('Press any key to continue.') +pause +% Create data +data_dim = 4; +latent_dim = 2; +mix = gmm(data_dim, 2, 'spherical'); +mix.centres = [1 1 1 1; 0 0 0 0]; +mix.priors = [0.5 0.5]; +mix.covars = [0.1 0.1]; + +[data, labels] = gmmsamp(mix, ndata); + +latent_shape = [15 15]; % Number of latent points in each dimension +nlatent = prod(latent_shape); % Number of latent points +num_rbf_centres = 16; + +clc; +dstring = ['Next we generate and initialise the GTM. There are ',... + num2str(nlatent), ' latent points']; +disp(dstring); +dstring = ['arranged in a square of ', num2str(latent_shape(1)), ... + ' points on a side. There are ', num2str(num_rbf_centres), ... + ' centres in the']; +disp(dstring); +disp('RBF model, which has Gaussian activation functions.') +disp(' ') +disp('Once the model is created, the latent data sample') +disp('and RBF centres are placed uniformly in the square [-1 1 -1 1].') +disp('The output weights of the RBF are computed to map the latent'); +disp('space to the two dimensional PCA subspace of the data.'); +disp(' ') +disp('Press any key to continue.'); +pause; + +% Create and initialise GTM model +net = gtm(latent_dim, nlatent, data_dim, num_rbf_centres, ... + 'gaussian', 0.1); + +options = foptions; +options(1) = -1; +options(7) = 1; % Set width factor of RBF +net = gtminit(net, options, data, 'regular', latent_shape, [4 4]); + +options = foptions; +options(14) = 30; +options(1) = 1; + +clc; +dstring = ['We now train the model with ', num2str(options(14)), ... + ' iterations of']; +disp(dstring) +disp('the EM algorithm for the GTM.') +disp(' ') +disp('Press any key to continue.') +pause; + +[net, options] = gtmem(net, data, options); + +disp(' ') +disp('Press any key to continue.') +pause; + +clc; +disp('We now visualise the data by plotting, for each data point,'); +disp('the posterior mean and mode (in latent space). These give'); +disp('a summary of the entire posterior distribution in latent space.') +disp('The corresponding values are joined by a line to aid the') +disp('interpretation.') +disp(' ') +disp('Press any key to continue.'); +pause; +% Plot posterior means +means = gtmlmean(net, data); +modes = gtmlmode(net, data); +PointSize = 12; +ClassSymbol1 = 'r.'; +ClassSymbol2 = 'b.'; +fh1 = figure; +hold on; +title('Visualisation in latent space') +plot(means((labels==1),1), means(labels==1,2), ... + ClassSymbol1, 'MarkerSize', PointSize) +plot(means((labels>1),1),means(labels>1,2),... + ClassSymbol2, 'MarkerSize', PointSize) + +ClassSymbol1 = 'ro'; +ClassSymbol2 = 'bo'; +plot(modes(labels==1,1), modes(labels==1,2), ... + ClassSymbol1) +plot(modes(labels>1,1),modes(labels>1,2),... + ClassSymbol2) + +% Join up means and modes +for n = 1:ndata + plot([means(n,1); modes(n,1)], [means(n,2); modes(n,2)], 'g-') +end +% Place legend outside data plot +legend('Mean (class 1)', 'Mean (class 2)', 'Mode (class 1)',... + 'Mode (class 2)', -1); + +% Display posterior for a data point +% Choose an interesting one with a large distance between mean and +% mode +[distance, point] = max(sum((means-modes).^2, 2)); +resp = gtmpost(net, data(point, :)); + +disp(' ') +disp('For more detailed information, the full posterior distribution') +disp('(or responsibility) can be plotted in latent space for a') +disp('single data point. This point has been chosen as the one') +disp('with the largest distance between mean and mode.') +disp(' ') +disp('Press any key to continue.'); +pause; + +R = reshape(resp, fliplr(latent_shape)); +XL = reshape(net.X(:,1), fliplr(latent_shape)); +YL = reshape(net.X(:,2), fliplr(latent_shape)); + +fh2 = figure; +imagesc(net.X(:, 1), net.X(:,2), R); +hold on; +tstr = ['Responsibility for point ', num2str(point)]; +title(tstr); +set(gca,'YDir','normal') +colormap(hot); +colorbar +disp(' '); +disp('Press any key to continue.') +pause + +clc +disp('Finally, we visualise the data with the posterior means in') +disp('latent space as before, but superimpose the magnification') +disp('factors to highlight the separation between clusters.') +disp(' ') +disp('Note the large magnitude factors down the centre of the') +disp('graph, showing that the manifold is stretched more in') +disp('this region than within each of the two clusters.') +ClassSymbol1 = 'g.'; +ClassSymbol2 = 'b.'; + +fh3 = figure; +mags = gtmmag(net, net.X); +% Reshape into grid form +Mags = reshape(mags, fliplr(latent_shape)); +imagesc(net.X(:, 1), net.X(:,2), Mags); +hold on +title('Dataset visualisation with magnification factors') +set(gca,'YDir','normal') +colormap(hot); +colorbar +hold on; % Else the magnification plot disappears +plot(means(labels==1,1), means(labels==1,2), ... + ClassSymbol1, 'MarkerSize', PointSize) +plot(means(labels>1,1), means(labels>1,2), ... + ClassSymbol2, 'MarkerSize', PointSize) + +disp(' ') +disp('Press any key to exit.') +pause + +close(fh1); +close(fh2); +close(fh3); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demhint.m b/sourcecodes/bnt-master/netlab3.3/demhint.m new file mode 100644 index 00000000..bd2e5025 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demhint.m @@ -0,0 +1,49 @@ +function demhint(nin, nhidden, nout) +%DEMHINT Demonstration of Hinton diagram for 2-layer feed-forward network. +% +% Description +% +% DEMHINT plots a Hinton diagram for a 2-layer feedforward network with +% 5 inputs, 4 hidden units and 3 outputs. The weight vector is chosen +% from a Gaussian distribution as described under MLP. +% +% DEMHINT(NIN, NHIDDEN, NOUT) allows the user to specify the number of +% inputs, hidden units and outputs. +% +% See also +% HINTON, HINTMAT, MLP, MLPPAK, MLPUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargin < 1 nin = 5; end +if nargin < 2 nhidden = 7; end +if nargin < 3 nout = 3; end + +% Fix the seed for reproducible results +randn('state', 42); +clc +disp('This demonstration illustrates the plotting of Hinton diagrams') +disp('for Multi-Layer Perceptron networks.') +disp(' ') +disp('Press any key to continue.') +pause +net = mlp(nin, nhidden, nout, 'linear'); + +[h1, h2] = mlphint(net); +clc +disp('The MLP has been created with') +disp([' ' int2str(nin) ' inputs']) +disp([' ' int2str(nhidden) ' hidden units']) +disp([' ' int2str(nout) ' outputs']) +disp(' ') +disp('One figure is produced for each layer of weights.') +disp('For each layer the fan-in weights are arranged in rows for each unit.') +disp('The bias weight is separated from the rest by a red vertical line.') +disp('The area of each box is proportional to the weight value: positive') +disp('values are white, and negative are black.') +disp(' ') +disp('Press any key to exit.'); +pause; +delete(h1); +delete(h2); diff --git a/sourcecodes/bnt-master/netlab3.3/demhmc1.m b/sourcecodes/bnt-master/netlab3.3/demhmc1.m new file mode 100644 index 00000000..69102f9e --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demhmc1.m @@ -0,0 +1,145 @@ +%DEMHMC1 Demonstrate Hybrid Monte Carlo sampling on mixture of two Gaussians. +% +% Description +% The problem consists of generating data from a mixture of two +% Gaussians in two dimensions using a hybrid Monte Carlo algorithm with +% persistence. A mixture model is then fitted to the sample to compare +% it with the true underlying generator. +% +% See also +% DEMHMC3, HMC, DEMPOT, DEMGPOT +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +dim = 2; % Data dimension +ncentres = 2; % Number of centres in mixture model + +seed = 42; % Seed for random weight initialization. +randn('state', seed); +rand('state', seed); + +clc +disp('This demonstration illustrates the use of the hybrid Monte Carlo') +disp('algorithm to sample from a mixture of two Gaussians.') +disp('The means of the two components are [0 0] and [2 2].') +disp(' ') +disp('First we set up the parameters of the mixture model we are sampling') +disp('from.') +disp(' ') +disp('Press any key to continue.') +pause + +% Set up mixture model to sample from +mix = gmm(dim, ncentres, 'spherical'); +mix.centres(1, :) = [0 0]; +mix.centres(2, :) = [2 2]; +x = [0 1]; % Start vector + +% Set up vector of options for hybrid Monte Carlo. + +nsamples = 160; % Number of retained samples. + +options = foptions; % Default options vector. +options(1) = 1; % Switch on diagnostics. +options(5) = 1; % Use persistence +options(7) = 50; % Number of steps in trajectory. +options(14) = nsamples; % Number of Monte Carlo samples returned. +options(15) = 30; % Number of samples omitted at start of chain. +options(18) = 0.02; + +clc +disp(['Next we take ', num2str(nsamples),' samples from the distribution.'... + , 'The first ', num2str(options(15))]) +disp('samples at the start of the chain are omitted. As persistence') +disp('is used, the momentum has a small random component added at each step.') +disp([num2str(options(7)), ... + ' iterations are used at each step and the step size is ',... + num2str(options(18))]) +disp('Sampling starts at the point [0 1].') +disp('The new state is accepted if the threshold value is greater than') +disp('a random number between 0 and 1.') +disp(' ') +disp('Negative step numbers indicate samples discarded from the start of the') +disp('chain.') +disp(' ') +disp('Press any key to continue.') +pause + +[samples, energies] = hmc('dempot', x, options, 'demgpot', mix); + +disp(' ') +disp('Press any key to continue.') +pause +clc +disp('The plot shows the samples generated by the HMC function.') +disp('The different colours are used to show how the samples move from') +disp('one component to the other over time.') +disp(' ') +disp('Press any key to continue.') +pause +probs = exp(-energies); +fh1 = figure; +% Plot data in 4 groups +ngroups = 4; +g1end = floor(nsamples/ngroups); +g2end = floor(2*nsamples/ngroups); +g3end = floor(3*nsamples/ngroups); +p1 = plot(samples(1:g1end,1), samples(1:g1end,2), 'k.', 'MarkerSize', 12); +hold on +lstrings = char(['Samples 1-' int2str(g1end)], ... + ['Samples ' int2str(g1end+1) '-' int2str(g2end)], ... + ['Samples ' int2str(g2end+1) '-' int2str(g3end)], ... + ['Samples ' int2str(g3end+1) '-' int2str(nsamples)]); +p2 = plot(samples(g1end+1:g2end,1), samples(g1end+1:g2end,2), ... + 'r.', 'MarkerSize', 12); +p3 = plot(samples(g2end+1:g3end,1), samples(g2end+1:g3end,2), ... + 'g.', 'MarkerSize', 12); +p4 = plot(samples(g3end+1:nsamples,1), samples(g3end+1:nsamples,2), ... + 'b.', 'MarkerSize', 12); +legend([p1 p2 p3 p4], lstrings, 2); + +clc +disp('We now fit a Gaussian mixture model to the sampled data.') +disp('The model has spherical covariance structure and the correct') +disp('number of components.') +disp(' ') +disp('Press any key to continue.') +pause +% Fit a mixture model to the sample +newmix = gmm(dim, ncentres, 'spherical'); +options = foptions; +options(1) = -1; % Switch off all diagnostics +options(14) = 5; % Just use 5 iterations of k-means in initialisation +% Initialise the model parameters from the samples +newmix = gmminit(newmix, samples, options); + +% Set up vector of options for EM trainer +options = zeros(1, 18); +options(1) = 1; % Prints out error values. +options(14) = 15; % Max. Number of iterations. + +disp('We now train the model using the EM algorithm for 15 iterations') +disp(' ') +disp('Press any key to continue') +pause +[newmix, options, errlog] = gmmem(newmix, samples, options); + +% Print out model +disp(' ') +disp('The trained model has parameters ') +disp(' Priors Centres Variances') +disp([newmix.priors' newmix.centres newmix.covars']) +disp('Note the close correspondence between these parameters and those') +disp('of the distribution used to generate the data') +disp(' ') +disp(' Priors Centres Variances') +disp([mix.priors' mix.centres mix.covars']) +disp(' ') +disp('Press any key to exit') +pause + +close(fh1); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demhmc2.m b/sourcecodes/bnt-master/netlab3.3/demhmc2.m new file mode 100644 index 00000000..d3df79d8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demhmc2.m @@ -0,0 +1,132 @@ +%DEMHMC2 Demonstrate Bayesian regression with Hybrid Monte Carlo sampling. +% +% Description +% The problem consists of one input variable X and one target variable +% T with data generated by sampling X at equal intervals and then +% generating target data by computing SIN(2*PI*X) and adding Gaussian +% noise. The model is a 2-layer network with linear outputs, and the +% hybrid Monte Carlo algorithm (without persistence) is used to sample +% from the posterior distribution of the weights. The graph shows the +% underlying function, 100 samples from the function given by the +% posterior distribution of the weights, and the average prediction +% (weighted by the posterior probabilities). +% +% See also +% DEMHMC3, HMC, MLP, MLPERR, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the matrix of inputs x and targets t. +ndata = 20; % Number of data points. +noise = 0.1; % Standard deviation of noise distribution. +nin = 1; % Number of inputs. +nout = 1; % Number of outputs. + +seed = 42; % Seed for random weight initialization. +randn('state', seed); +rand('state', seed); + +x = 0.25 + 0.1*randn(ndata, nin); +t = sin(2*pi*x) + noise*randn(size(x)); + +clc +disp('This demonstration illustrates the use of the hybrid Monte Carlo') +disp('algorithm to sample from the posterior weight distribution of a') +disp('multi-layer perceptron.') +disp(' ') +disp('A regression problem is used, with the one-dimensional data drawn') +disp('from a noisy sine function. The x values are sampled from a normal') +disp('distribution with mean 0.25 and variance 0.01.') +disp(' ') +disp('First we initialise the network.') +disp(' ') +disp('Press any key to continue.') +pause + +% Set up network parameters. +nhidden = 5; % Number of hidden units. +alpha = 0.001; % Coefficient of weight-decay prior. +beta = 100.0; % Coefficient of data error. + +% Create and initialize network model. +% Initialise weights reasonably close to 0 +net = mlp(nin, nhidden, nout, 'linear', alpha, beta); +net = mlpinit(net, 10); + +clc +disp('Next we take 100 samples from the posterior distribution. The first') +disp('200 samples at the start of the chain are omitted. As persistence') +disp('is not used, the momentum is randomised at each step. 100 iterations') +disp('are used at each step. The new state is accepted if the threshold') +disp('value is greater than a random number between 0 and 1.') +disp(' ') +disp('Negative step numbers indicate samples discarded from the start of the') +disp('chain.') +disp(' ') +disp('Press any key to continue.') +pause +% Set up vector of options for hybrid Monte Carlo. +nsamples = 100; % Number of retained samples. + +options = foptions; % Default options vector. +options(1) = 1; % Switch on diagnostics. +options(7) = 100; % Number of steps in trajectory. +options(14) = nsamples; % Number of Monte Carlo samples returned. +options(15) = 200; % Number of samples omitted at start of chain. +options(18) = 0.002; % Step size. + +w = mlppak(net); +% Initialise HMC +hmc('state', 42); +[samples, energies] = hmc('neterr', w, options, 'netgrad', net, x, t); + +clc +disp('The plot shows the underlying noise free function, the 100 samples') +disp('produced from the MLP, and their average as a Monte Carlo estimate') +disp('of the true posterior average.') +disp(' ') +disp('Press any key to continue.') +pause +nplot = 300; +plotvals = [0 : 1/(nplot - 1) : 1]'; +pred = zeros(size(plotvals)); +fh = figure; +for k = 1:nsamples + w2 = samples(k,:); + net2 = mlpunpak(net, w2); + y = mlpfwd(net2, plotvals); + % Average sample predictions as Monte Carlo estimate of true integral + pred = pred + y; + h4 = plot(plotvals, y, '-r', 'LineWidth', 1); + if k == 1 + hold on + end +end +pred = pred./nsamples; + +% Plot data +h1 = plot(x, t, 'ob', 'LineWidth', 2, 'MarkerFaceColor', 'blue'); +axis([0 1 -3 3]) + +% Plot function +[fx, fy] = fplot('sin(2*pi*x)', [0 1], '--g'); +h2 = plot(fx, fy, '--g', 'LineWidth', 2); +set(gca, 'box', 'on'); + +% Plot averaged prediction +h3 = plot(plotvals, pred, '-c', 'LineWidth', 2); +hold off + +lstrings = char('Data', 'Function', 'Prediction', 'Samples'); +legend([h1 h2 h3 h4], lstrings, 3); + +disp('Note how the predictions become much further from the true function') +disp('away from the region of high data density.') +disp(' ') +disp('Press any key to exit.') +pause +close(fh); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demhmc3.m b/sourcecodes/bnt-master/netlab3.3/demhmc3.m new file mode 100644 index 00000000..f2956ae5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demhmc3.m @@ -0,0 +1,135 @@ +%DEMHMC3 Demonstrate Bayesian regression with Hybrid Monte Carlo sampling. +% +% Description +% The problem consists of one input variable X and one target variable +% T with data generated by sampling X at equal intervals and then +% generating target data by computing SIN(2*PI*X) and adding Gaussian +% noise. The model is a 2-layer network with linear outputs, and the +% hybrid Monte Carlo algorithm (with persistence) is used to sample +% from the posterior distribution of the weights. The graph shows the +% underlying function, 300 samples from the function given by the +% posterior distribution of the weights, and the average prediction +% (weighted by the posterior probabilities). +% +% See also +% DEMHMC2, HMC, MLP, MLPERR, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the matrix of inputs x and targets t. +ndata = 20; % Number of data points. +noise = 0.1; % Standard deviation of noise distribution. +nin = 1; % Number of inputs. +nout = 1; % Number of outputs. + +seed = 42; % Seed for random number generators. +randn('state', seed); +rand('state', seed); + +x = 0.25 + 0.1*randn(ndata, nin); +t = sin(2*pi*x) + noise*randn(size(x)); + +clc +disp('This demonstration illustrates the use of the hybrid Monte Carlo') +disp('algorithm to sample from the posterior weight distribution of a') +disp('multi-layer perceptron.') +disp(' ') +disp('A regression problem is used, with the one-dimensional data drawn') +disp('from a noisy sine function. The x values are sampled from a normal') +disp('distribution with mean 0.25 and variance 0.01.') +disp(' ') +disp('First we initialise the network.') +disp(' ') +disp('Press any key to continue.') +pause + +% Set up network parameters. +nhidden = 5; % Number of hidden units. +alpha = 0.001; % Coefficient of weight-decay prior. +beta = 100.0; % Coefficient of data error. + +% Create and initialize network model. + +% Initialise weights reasonably close to 0 +net = mlp(nin, nhidden, nout, 'linear', alpha, beta); +net = mlpinit(net, 10); + +clc +disp('Next we take 100 samples from the posterior distribution. The first') +disp('300 samples at the start of the chain are omitted. As persistence') +disp('is used, the momentum has a small random component added at each step.') +disp('10 iterations are used at each step (compared with 100 in demhmc2).') +disp('The step size is 0.005 (compared with 0.002).') +disp('The new state is accepted if the threshold') +disp('value is greater than a random number between 0 and 1.') +disp(' ') +disp('Negative step numbers indicate samples discarded from the start of the') +disp('chain.') +disp(' ') +disp('Press any key to continue.') +pause + +% Set up vector of options for hybrid Monte Carlo. +nsamples = 100; % Number of retained samples. + +options = foptions; % Default options vector. +options(1) = 1; % Switch on diagnostics. +options(5) = 1; % Use persistence +options(7) = 10; % Number of steps in trajectory. +options(14) = nsamples; % Number of Monte Carlo samples returned. +options(15) = 300; % Number of samples omitted at start of chain. +options(17) = 0.95; % Alpha value in persistence +options(18) = 0.005; % Step size. + +w = mlppak(net); +% Initialise HMC +hmc('state', 42); +[samples, energies] = hmc('neterr', w, options, 'netgrad', net, x, t); + +clc +disp('The plot shows the underlying noise free function, the 100 samples') +disp('produced from the MLP, and their average as a Monte Carlo estimate') +disp('of the true posterior average.') +disp(' ') +disp('Press any key to continue.') +pause + +nplot = 300; +plotvals = [0 : 1/(nplot - 1) : 1]'; +pred = zeros(size(plotvals)); +fh1 = figure; +hold on +for k = 1:nsamples + w2 = samples(k,:); + net2 = mlpunpak(net, w2); + y = mlpfwd(net2, plotvals); + % Sum predictions + pred = pred + y; + h4 = plot(plotvals, y, '-r', 'LineWidth', 1); +end +pred = pred./nsamples; +% Plot data +h1 = plot(x, t, 'ob', 'LineWidth', 2, 'MarkerFaceColor', 'blue'); +axis([0 1 -3 3]) + +% Plot function +[fx, fy] = fplot('sin(2*pi*x)', [0 1], '--g'); +h2 = plot(fx, fy, '--g', 'LineWidth', 2); +set(gca, 'box', 'on'); + +% Plot averaged prediction +h3 = plot(plotvals, pred, '-c', 'LineWidth', 2); + +lstrings = char('Data', 'Function', 'Prediction', 'Samples'); +legend([h1 h2 h3 h4], lstrings, 3); +hold off + +disp('Note how the predictions become much further from the true function') +disp('away from the region of high data density.') +disp(' ') +disp('Press any key to exit.') +pause +close(fh1); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demkmn1.m b/sourcecodes/bnt-master/netlab3.3/demkmn1.m new file mode 100644 index 00000000..d9a7c6e2 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demkmn1.m @@ -0,0 +1,111 @@ +%DEMKMEAN Demonstrate simple clustering model trained with K-means. +% +% Description +% The problem consists of data in a two-dimensional space. The data is +% drawn from three spherical Gaussian distributions with priors 0.3, +% 0.5 and 0.2; centres (2, 3.5), (0, 0) and (0,2); and standard +% deviations 0.2, 0.5 and 1.0. The first figure contains a scatter plot +% of the data. The data is the same as in DEMGMM1. +% +% A cluster model with three components is trained using the batch K- +% means algorithm. The matrix of centres is printed after training. The +% second figure shows the data labelled with a colour derived from the +% corresponding cluster +% +% See also +% DEM2DDAT, DEMGMM1, KNN1, KMEANS +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Generate the data, fixing seeds for reproducible results +ndata = 250; +randn('state', 42); +rand('state', 42); +data = dem2ddat(ndata); + +% Randomise data order +data = data(randperm(ndata),:); + +clc +disp('This demonstration illustrates the use of a cluster model to') +disp('find centres that reflect the distribution of data points.') +disp('We begin by generating the data from a mixture of three Gaussians') +disp('in two-dimensional space and plotting it.') +disp(' ') +disp('Press any key to continue.') +pause + +fh1 = figure; +plot(data(:, 1), data(:, 2), 'o') +set(gca, 'Box', 'on') +title('Data') + +% Set up cluster model +ncentres = 3; +centres = zeros(ncentres, 2); + +% Set up vector of options for kmeans trainer +options = foptions; +options(1) = 1; % Prints out error values. +options(5) = 1; +options(14) = 10; % Number of iterations. + +clc +disp('The model is chosen to have three centres, which are initialised') +disp('at randomly selected data points. We now train the model using') +disp('the batch K-means algorithm with a maximum of 10 iterations and') +disp('stopping tolerance of 1e-4.') +disp(' ') +disp('Press any key to continue.') +pause + +% Train the centres from the data +[centres, options, post] = kmeans(centres, data, options); + +% Print out model +disp(' ') +disp('Note that training has terminated before 10 iterations as there') +disp('has been no change in the centres or error function.') +disp(' ') +disp('The trained model has centres:') +disp(centres); +disp('Press any key to continue.') +pause + +clc +disp('We now plot each data point coloured according to its classification') +disp('given by the nearest cluster centre. The cluster centres are denoted') +disp('by black crosses.') + +% Plot the result +fh2 = figure; + +hold on +colours = ['b.'; 'r.'; 'g.']; + +[tempi, tempj] = find(post); +hold on +for i = 1:3 + % Select data points closest to ith centre + thisX = data(tempi(tempj == i), 1); + thisY = data(tempi(tempj == i), 2); + hp(i) = plot(thisX, thisY, colours(i,:)); + set(hp(i), 'MarkerSize', 12); +end +set(gca, 'Box', 'on') +legend('Class 1', 'Class 2', 'Class 3', 2) +hold on +plot(centres(:, 1), centres(:,2), 'k+', 'LineWidth', 2, ... + 'MarkerSize', 8) +title('Centres and data labels') +hold off + +disp(' ') +disp('Press any key to end.') +pause + +close(fh1); +close(fh2); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demknn1.m b/sourcecodes/bnt-master/netlab3.3/demknn1.m new file mode 100644 index 00000000..6e3ec60a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demknn1.m @@ -0,0 +1,103 @@ +%DEMKNN1 Demonstrate nearest neighbour classifier. +% +% Description +% The problem consists of data in a two-dimensional space. The data is +% drawn from three spherical Gaussian distributions with priors 0.3, +% 0.5 and 0.2; centres (2, 3.5), (0, 0) and (0,2); and standard +% deviations 0.2, 0.5 and 1.0. The first figure contains a scatter plot +% of the data. The data is the same as in DEMGMM1. +% +% The second figure shows the data labelled with the corresponding +% class given by the classifier. +% +% See also +% DEM2DDAT, DEMGMM1, KNN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +clc +disp('This program demonstrates the use of the K nearest neighbour algorithm.') +disp(' ') +disp('Press any key to continue.') +pause +% Generate the test data +ndata = 250; +randn('state', 42); +rand('state', 42); + +[data, c] = dem2ddat(ndata); + +% Randomise data order +data = data(randperm(ndata),:); + +clc +disp('We generate the data in two-dimensional space from a mixture of') +disp('three spherical Gaussians. The centres are shown as black crosses') +disp('in the plot.') +disp(' ') +disp('Press any key to continue.') +pause +fh1 = figure; +plot(data(:, 1), data(:, 2), 'o') +set(gca, 'Box', 'on') +hold on +title('Data') +hp1 = plot(c(:, 1), c(:,2), 'k+') +% Increase size of crosses +set(hp1, 'MarkerSize', 8); +set(hp1, 'LineWidth', 2); +hold off + +clc +disp('We next use the centres as training examplars for the K nearest') +disp('neighbour algorithm.') +disp(' ') +disp('Press any key to continue.') +pause + +% Use centres as training data +train_labels = [1, 0, 0; 0, 1, 0; 0, 0, 1]; + +% Label the test data up to kmax neighbours +kmax = 1; +net = knn(2, 3, kmax, c, train_labels); +[y, l] = knnfwd(net, data); + +clc +disp('We now plot each data point coloured according to its classification.') +disp(' ') +disp('Press any key to continue.') +pause +% Plot the result +fh2 = figure; +colors = ['b.'; 'r.'; 'g.']; +for i = 1:3 + thisX = data(l == i,1); + thisY = data(l == i,2); + hp(i) = plot(thisX, thisY, colors(i,:)); + set(hp(i), 'MarkerSize', 12); + if i == 1 + hold on + end +end +set(gca, 'Box', 'on'); +legend('Class 1', 'Class 2', 'Class 3', 2) +hold on +labels = ['1', '2', '3']; +hp2 = plot(c(:, 1), c(:,2), 'k+'); +% Increase size of crosses +set(hp2, 'MarkerSize', 8); +set(hp2, 'LineWidth', 2); + +test_labels = labels(l(:,1)); + +title('Training data and data labels') +hold off + +disp('The demonstration is now complete: press any key to exit.') +pause +close(fh1); +close(fh2); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demmdn1.m b/sourcecodes/bnt-master/netlab3.3/demmdn1.m new file mode 100644 index 00000000..43c2d98b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demmdn1.m @@ -0,0 +1,211 @@ +%DEMMDN1 Demonstrate fitting a multi-valued function using a Mixture Density Network. +% +% Description +% The problem consists of one input variable X and one target variable +% T with data generated by sampling T at equal intervals and then +% generating target data by computing T + 0.3*SIN(2*PI*T) and adding +% Gaussian noise. A Mixture Density Network with 3 centres in the +% mixture model is trained by minimizing a negative log likelihood +% error function using the scaled conjugate gradient optimizer. +% +% The conditional means, mixing coefficients and variances are plotted +% as a function of X, and a contour plot of the full conditional +% density is also generated. +% +% See also +% MDN, MDNERR, MDNGRAD, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the matrix of inputs x and targets t. +seedn = 42; +seed = 42; +randn('state', seedn); +rand('state', seed); +ndata = 300; % Number of data points. +noise = 0.2; % Range of noise distribution. +t = [0:1/(ndata - 1):1]'; +x = t + 0.3*sin(2*pi*t) + noise*rand(ndata, 1) - noise/2; +axis_limits = [-0.2 1.2 -0.2 1.2]; + +clc +disp('This demonstration illustrates the use of a Mixture Density Network') +disp('to model multi-valued functions. The data is generated from the') +disp('mapping x = t + 0.3 sin(2 pi t) + e, where e is a noise term.') +disp('We begin by plotting the data.') +disp(' ') +disp('Press any key to continue') +pause +% Plot the data +fh1 = figure; +p1 = plot(x, t, 'ob'); +axis(axis_limits); +hold on +disp('Note that for x in the range 0.35 to 0.65, there are three possible') +disp('branches of the function.') +disp(' ') +disp('Press any key to continue') +pause + +% Set up network parameters. +nin = 1; % Number of inputs. +nhidden = 5; % Number of hidden units. +ncentres = 3; % Number of mixture components. +dim_target = 1; % Dimension of target space +mdntype = '0'; % Currently unused: reserved for future use +alpha = 100; % Inverse variance for weight initialisation + % Make variance small for good starting point + +% Create and initialize network weight vector. +net = mdn(nin, nhidden, ncentres, dim_target, mdntype); +init_options = zeros(1, 18); +init_options(1) = -1; % Suppress all messages +init_options(14) = 10; % 10 iterations of K means in gmminit +net = mdninit(net, alpha, t, init_options); + +% Set up vector of options for the optimiser. +options = foptions; +options(1) = 1; % This provides display of error values. +options(14) = 200; % Number of training cycles. + +clc +disp('We initialise the neural network model, which is an MLP with a') +disp('Gaussian mixture model with three components and spherical variance') +disp('as the error function. This enables us to model the complete') +disp('conditional density function.') +disp(' ') +disp('Next we train the model for 200 epochs using a scaled conjugate gradient') +disp('optimizer. The error function is the negative log likelihood of the') +disp('training data.') +disp(' ') +disp('Press any key to continue.') +pause + +% Train using scaled conjugate gradients. +[net, options] = netopt(net, options, x, t, 'scg'); + +disp(' ') +disp('Press any key to continue.') +pause + +clc +disp('We can also train a conventional MLP with sum of squares error function.') +disp('This will approximate the conditional mean, which is not always a') +disp('good representation of the data. Note that the error function is the') +disp('sum of squares error on the training data, which accounts for the') +disp('different values from training the MDN.') +disp(' ') +disp('We train the network with the quasi-Newton optimizer for 80 epochs.') +disp(' ') +disp('Press any key to continue.') +pause +mlp_nhidden = 8; +net2 = mlp(nin, mlp_nhidden, dim_target, 'linear'); +options(14) = 80; +[net2, options] = netopt(net2, options, x, t, 'quasinew'); +disp(' ') +disp('Press any key to continue.') +pause + +clc +disp('Now we plot the underlying function, the MDN prediction,') +disp('represented by the mode of the conditional distribution, and the') +disp('prediction of the conventional MLP.') +disp(' ') +disp('Press any key to continue.') +pause + +% Plot the original function, and the trained network function. +plotvals = [0:0.01:1]'; +mixes = mdn2gmm(mdnfwd(net, plotvals)); +axis(axis_limits); +yplot = t+0.3*sin(2*pi*t); +p2 = plot(yplot, t, '--y'); + +% Use the mode to represent the function +y = zeros(1, length(plotvals)); +priors = zeros(length(plotvals), ncentres); +c = zeros(length(plotvals), 3); +widths = zeros(length(plotvals), ncentres); +for i = 1:length(plotvals) + [m, j] = max(mixes(i).priors); + y(i) = mixes(i).centres(j,:); + c(i,:) = mixes(i).centres'; +end +p3 = plot(plotvals, y, '*r'); +p4 = plot(plotvals, mlpfwd(net2, plotvals), 'g'); +set(p4, 'LineWidth', 2); +legend([p1 p2 p3 p4], 'data', 'function', 'MDN mode', 'MLP mean', 4); +hold off + +clc +disp('We can also plot how the mixture model parameters depend on x.') +disp('First we plot the mixture centres, then the priors and finally') +disp('the variances.') +disp(' ') +disp('Press any key to continue.') +pause +fh2 = figure; +subplot(3, 1, 1) +plot(plotvals, c) +hold on +title('Mixture centres') +legend('centre 1', 'centre 2', 'centre 3') +hold off + +priors = reshape([mixes.priors], mixes(1).ncentres, size(mixes, 2))'; +%%fh3 = figure; +subplot(3, 1, 2) +plot(plotvals, priors) +hold on +title('Mixture priors') +legend('centre 1', 'centre 2', 'centre 3') +hold off + +variances = reshape([mixes.covars], mixes(1).ncentres, size(mixes, 2))'; +%%fh4 = figure; +subplot(3, 1, 3) +plot(plotvals, variances) +hold on +title('Mixture variances') +legend('centre 1', 'centre 2', 'centre 3') +hold off + +disp('The last figure is a contour plot of the conditional probability') +disp('density generated by the Mixture Density Network. Note how it') +disp('is well matched to the regions of high data density.') +disp(' ') +disp('Press any key to continue.') +pause +% Contour plot for MDN. +i = 0:0.01:1.0; +j = 0:0.01:1.0; + +[I, J] = meshgrid(i,j); +I = I(:); +J = J(:); +li = length(i); +lj = length(j); +Z = zeros(li, lj); +for k = 1:li; + Z(:,k) = gmmprob(mixes(k), j'); +end +fh5 = figure; +% Set up levels by hand to make a good figure +v = [2 2.5 3 3.5 5:3:18]; +contour(i, j, Z, v) +hold on +title('Contour plot of conditional density') +hold off + +disp(' ') +disp('Press any key to exit.') +pause +close(fh1); +close(fh2); +%%close(fh3); +%%close(fh4); +close(fh5); +%%clear all; diff --git a/sourcecodes/bnt-master/netlab3.3/demmet1.m b/sourcecodes/bnt-master/netlab3.3/demmet1.m new file mode 100644 index 00000000..804622a4 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demmet1.m @@ -0,0 +1,103 @@ +function samples=demmet1(plot_wait) +%DEMMET1 Demonstrate Markov Chain Monte Carlo sampling on a Gaussian. +% +% Description +% The problem consists of generating data from a Gaussian in two +% dimensions using a Markov Chain Monte Carlo algorithm. The points are +% plotted one after another to show the path taken by the chain. +% +% DEMMET1(PLOTWAIT) allows the user to set the time (in a whole number +% of seconds) between the plotting of points. This is passed to PAUSE +% +% See also +% DEMHMC1, METROP, GMM, DEMPOT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargin == 0 | plot_wait < 0 + plot_wait = 0; % No wait if not specified or incorrect +end +dim = 2; % Data dimension +ncentres = 1; % Number of centres in mixture model + +seed = 42; % Seed for random weight initialization. +randn('state', seed); +rand('state', seed); + +clc +disp('This demonstration illustrates the use of the Markov chain Monte Carlo') +disp('algorithm to sample from a Gaussian distribution.') +disp('The mean is at [0 0].') +disp(' ') +disp('First we set up the parameters of the mixture model we are sampling') +disp('from.') +disp(' ') +disp('Press any key to continue.') +pause + +% Set up mixture model to sample from +mix = gmm(dim, ncentres, 'spherical'); +mix.centres(1, :) = [0 0]; +x = [0 4]; % Start vector + +% Set up vector of options for hybrid Monte Carlo. + +nsamples = 150; % Number of retained samples. + +options = foptions; % Default options vector. +options(1) = 0; % Switch off diagnostics. +options(14) = nsamples; % Number of Monte Carlo samples returned. +options(18) = 0.1; + +clc +disp('Next we take 150 samples from the distribution.') +disp('Sampling starts at the point [0 4].') +disp('The new state is accepted if the threshold value is greater than') +disp('a random number between 0 and 1.') +disp(' ') +disp('Press any key to continue.') +pause + +[samples, energies] = metrop('dempot', x, options, '', mix); + +clc +disp('The plot shows the samples generated by the MCMC function in order') +disp('as an animation to show the path taken by the Markov chain.') +disp('The different colours are used to show that the first few samples') +disp('should be discarded as they lie too far from the mean.') +disp(' ') +disp('Press any key to continue.') +pause +probs = exp(-energies); +fh1 = figure; +g1end = floor(nsamples/4); + +for n = 1:nsamples + + if n < g1end + Marker = 'k.'; + p1 = plot(samples(n,1), samples(n,2), Marker, ... + 'EraseMode', 'none', 'MarkerSize', 12); + if n == 1 + axis([-3 5 -2 5]) + end + else + Marker = 'r.'; + p2 = plot(samples(n,1), samples(n,2), Marker, ... + 'EraseMode', 'none', 'MarkerSize', 12); + end + hold on + drawnow; % Force drawing immediately + pause(plot_wait); +end +lstrings = char(['Samples 1-' int2str(g1end)], ... + ['Samples ' int2str(g1end+1) '-' int2str(nsamples)]); +legend([p1 p2], lstrings, 1); + +disp(' ') +disp('Press any key to exit.') +pause +close(fh1); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demmlp1.m b/sourcecodes/bnt-master/netlab3.3/demmlp1.m new file mode 100644 index 00000000..4f102b73 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demmlp1.m @@ -0,0 +1,88 @@ +%DEMMLP1 Demonstrate simple regression using a multi-layer perceptron +% +% Description +% The problem consists of one input variable X and one target variable +% T with data generated by sampling X at equal intervals and then +% generating target data by computing SIN(2*PI*X) and adding Gaussian +% noise. A 2-layer network with linear outputs is trained by minimizing +% a sum-of-squares error function using the scaled conjugate gradient +% optimizer. +% +% See also +% MLP, MLPERR, MLPGRAD, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the matrix of inputs x and targets t. + +ndata = 20; % Number of data points. +noise = 0.2; % Standard deviation of noise distribution. +x = [0:1/(ndata - 1):1]'; +randn('state', 1); +t = sin(2*pi*x) + noise*randn(ndata, 1); + +clc +disp('This demonstration illustrates the use of a Multi-Layer Perceptron') +disp('network for regression problems. The data is generated from a noisy') +disp('sine function.') +disp(' ') +disp('Press any key to continue.') +pause + +% Set up network parameters. +nin = 1; % Number of inputs. +nhidden = 3; % Number of hidden units. +nout = 1; % Number of outputs. +alpha = 0.01; % Coefficient of weight-decay prior. + +% Create and initialize network weight vector. + +net = mlp(nin, nhidden, nout, 'linear', alpha); + +% Set up vector of options for the optimiser. + +options = zeros(1,18); +options(1) = 1; % This provides display of error values. +options(14) = 100; % Number of training cycles. + +clc +disp(['The network has ', num2str(nhidden), ' hidden units and a weight decay']) +disp(['coefficient of ', num2str(alpha), '.']) +disp(' ') +disp('After initializing the network, we train it use the scaled conjugate') +disp('gradients algorithm for 100 cycles.') +disp(' ') +disp('Press any key to continue') +pause + +% Train using scaled conjugate gradients. +[net, options] = netopt(net, options, x, t, 'scg'); + +disp(' ') +disp('Now we plot the data, underlying function, and network outputs') +disp('on a single graph to compare the results.') +disp(' ') +disp('Press any key to continue.') +pause + +% Plot the data, the original function, and the trained network function. +plotvals = [0:0.01:1]'; +y = mlpfwd(net, plotvals); +fh1 = figure; +plot(x, t, 'ob') +hold on +xlabel('Input') +ylabel('Target') +axis([0 1 -1.5 1.5]) +[fx, fy] = fplot('sin(2*pi*x)', [0 1]); +plot(fx, fy, '-r', 'LineWidth', 2) +plot(plotvals, y, '-k', 'LineWidth', 2) +legend('data', 'function', 'network'); + +disp(' ') +disp('Press any key to end.') +pause +close(fh1); +clear all; diff --git a/sourcecodes/bnt-master/netlab3.3/demmlp2.m b/sourcecodes/bnt-master/netlab3.3/demmlp2.m new file mode 100644 index 00000000..20a86cf4 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demmlp2.m @@ -0,0 +1,337 @@ +%DEMMLP2 Demonstrate simple classification using a multi-layer perceptron +% +% Description +% The problem consists of input data in two dimensions drawn from a +% mixture of three Gaussians: two of which are assigned to a single +% class. An MLP with logistic outputs trained with a quasi-Newton +% optimisation algorithm is compared with the optimal Bayesian decision +% rule. +% +% See also +% MLP, MLPFWD, NETERR, QUASINEW +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Set up some figure parameters +AxisShift = 0.05; +ClassSymbol1 = 'r.'; +ClassSymbol2 = 'y.'; +PointSize = 12; +titleSize = 10; + +% Fix the seeds +rand('state', 423); +randn('state', 423); + +clc +disp('This demonstration shows how an MLP with logistic outputs and') +disp('and cross entropy error function can be trained to model the') +disp('posterior class probabilities in a classification problem.') +disp('The results are compared with the optimal Bayes rule classifier,') +disp('which can be computed exactly as we know the form of the generating') +disp('distribution.') +disp(' ') +disp('Press any key to continue.') +pause + +fh1 = figure; +set(fh1, 'Name', 'True Data Distribution'); +whitebg(fh1, 'k'); + +% +% Generate the data +% +n=200; + +% Set up mixture model: 2d data with three centres +% Class 1 is first centre, class 2 from the other two +mix = gmm(2, 3, 'full'); +mix.priors = [0.5 0.25 0.25]; +mix.centres = [0 -0.1; 1 1; 1 -1]; +mix.covars(:,:,1) = [0.625 -0.2165; -0.2165 0.875]; +mix.covars(:,:,2) = [0.2241 -0.1368; -0.1368 0.9759]; +mix.covars(:,:,3) = [0.2375 0.1516; 0.1516 0.4125]; + +[data, label] = gmmsamp(mix, n); + +% +% Calculate some useful axis limits +% +x0 = min(data(:,1)); +x1 = max(data(:,1)); +y0 = min(data(:,2)); +y1 = max(data(:,2)); +dx = x1-x0; +dy = y1-y0; +expand = 5/100; % Add on 5 percent each way +x0 = x0 - dx*expand; +x1 = x1 + dx*expand; +y0 = y0 - dy*expand; +y1 = y1 + dy*expand; +resolution = 100; +step = dx/resolution; +xrange = [x0:step:x1]; +yrange = [y0:step:y1]; +% +% Generate the grid +% +[X Y]=meshgrid([x0:step:x1],[y0:step:y1]); +% +% Calculate the class conditional densities, the unconditional densities and +% the posterior probabilities +% +px_j = gmmactiv(mix, [X(:) Y(:)]); +px = reshape(px_j*(mix.priors)',size(X)); +post = gmmpost(mix, [X(:) Y(:)]); +p1_x = reshape(post(:, 1), size(X)); +p2_x = reshape(post(:, 2) + post(:, 3), size(X)); + +% +% Generate some pretty pictures !! +% +colormap(hot) +colorbar +subplot(1,2,1) +hold on +plot(data((label==1),1),data(label==1,2),ClassSymbol1, 'MarkerSize', PointSize) +plot(data((label>1),1),data(label>1,2),ClassSymbol2, 'MarkerSize', PointSize) +contour(xrange,yrange,p1_x,[0.5 0.5],'w-'); +axis([x0 x1 y0 y1]) +set(gca,'Box','On') +title('The Sampled Data'); +rect=get(gca,'Position'); +rect(1)=rect(1)-AxisShift; +rect(3)=rect(3)+AxisShift; +set(gca,'Position',rect) +hold off + +subplot(1,2,2) +imagesc(X(:),Y(:),px); +hold on +[cB, hB] = contour(xrange,yrange,p1_x,[0.5 0.5],'w:'); +set(hB,'LineWidth', 2); +axis([x0 x1 y0 y1]) +set(gca,'YDir','normal') +title('Probability Density p(x)') +hold off + +drawnow; +clc; +disp('The first figure shows the data sampled from a mixture of three') +disp('Gaussians, the first of which (whose centre is near the origin) is') +disp('labelled red and the other two are labelled yellow. The second plot') +disp('shows the unconditional density of the data with the optimal Bayesian') +disp('decision boundary superimposed.') +disp(' ') +disp('Press any key to continue.') +pause +fh2 = figure; +set(fh2, 'Name', 'Class-conditional Densities and Posterior Probabilities'); +whitebg(fh2, 'w'); + +subplot(2,2,1) +p1=reshape(px_j(:,1),size(X)); +imagesc(X(:),Y(:),p1); +colormap hot +colorbar +axis(axis) +set(gca,'YDir','normal') +hold on +plot(mix.centres(:,1),mix.centres(:,2),'b+','MarkerSize',8,'LineWidth',2) +title('Density p(x|red)') +hold off + +subplot(2,2,2) +p2=reshape((px_j(:,2)+px_j(:,3)),size(X)); +imagesc(X(:),Y(:),p2); +colorbar +set(gca,'YDir','normal') +hold on +plot(mix.centres(:,1),mix.centres(:,2),'b+','MarkerSize',8,'LineWidth',2) +title('Density p(x|yellow)') +hold off + +subplot(2,2,3) +imagesc(X(:),Y(:),p1_x); +set(gca,'YDir','normal') +colorbar +title('Posterior Probability p(red|x)') +hold on +plot(mix.centres(:,1),mix.centres(:,2),'b+','MarkerSize',8,'LineWidth',2) +hold off + +subplot(2,2,4) +imagesc(X(:),Y(:),p2_x); +set(gca,'YDir','normal') +colorbar +title('Posterior Probability p(yellow|x)') +hold on +plot(mix.centres(:,1),mix.centres(:,2),'b+','MarkerSize',8,'LineWidth',2) +hold off + +% Now set up and train the MLP +nhidden=6; +nout=1; +alpha = 0.2; % Weight decay +ncycles = 60; % Number of training cycles. +% Set up MLP network +net = mlp(2, nhidden, nout, 'logistic', alpha); +options = zeros(1,18); +options(1) = 1; % Print out error values +options(14) = ncycles; + +mlpstring = ['We now set up an MLP with ', num2str(nhidden), ... + ' hidden units, logistic output and cross']; +trainstring = ['entropy error function, and train it for ', ... + num2str(ncycles), ' cycles using the']; +wdstring = ['quasi-Newton optimisation algorithm with weight decay of ', ... + num2str(alpha), '.']; + +% Force out the figure before training the MLP +drawnow; +disp(' ') +disp('The second figure shows the class conditional densities and posterior') +disp('probabilities for each class. The blue crosses mark the centres of') +disp('the three Gaussians.') +disp(' ') +disp(mlpstring) +disp(trainstring) +disp(wdstring) +disp(' ') +disp('Press any key to continue.') +pause + +% Convert targets to 0-1 encoding +target=[label==1]; + +% Train using quasi-Newton. +[net] = netopt(net, options, data, target, 'quasinew'); +y = mlpfwd(net, data); +yg = mlpfwd(net, [X(:) Y(:)]); +yg = reshape(yg(:,1),size(X)); + +fh3 = figure; +set(fh3, 'Name', 'Network Output'); +whitebg(fh3, 'k') +subplot(1, 2, 1) +hold on +plot(data((label==1),1),data(label==1,2),'r.', 'MarkerSize', PointSize) +plot(data((label>1),1),data(label>1,2),'y.', 'MarkerSize', PointSize) +% Bayesian decision boundary +[cB, hB] = contour(xrange,yrange,p1_x,[0.5 0.5],'b-'); +[cN, hN] = contour(xrange,yrange,yg,[0.5 0.5],'r-'); +set(hB, 'LineWidth', 2); +set(hN, 'LineWidth', 2); +Chandles = [hB(1) hN(1)]; +legend(Chandles, 'Bayes', ... + 'Network', 3); + +axis([x0 x1 y0 y1]) +set(gca,'Box','on','XTick',[],'YTick',[]) + +title('Training Data','FontSize',titleSize); +hold off + +subplot(1, 2, 2) +imagesc(X(:),Y(:),yg); +colormap hot +colorbar +axis(axis) +set(gca,'YDir','normal','XTick',[],'YTick',[]) +title('Network Output','FontSize',titleSize) + +clc +disp('This figure shows the training data with the decision boundary') +disp('produced by the trained network and the network''s prediction of') +disp('the posterior probability of the red class.') +disp(' ') +disp('Press any key to continue.') +pause + +% +% Now generate and classify a test data set +% +[testdata testlabel] = gmmsamp(mix, n); +testlab=[testlabel==1 testlabel>1]; + +% This is the Bayesian classification +tpx_j = gmmpost(mix, testdata); +Bpost = [tpx_j(:,1), tpx_j(:,2)+tpx_j(:,3)]; +[Bcon Brate]=confmat(Bpost, [testlabel==1 testlabel>1]); + +% Compute network classification +yt = mlpfwd(net, testdata); +% Convert single output to posteriors for both classes +testpost = [yt 1-yt]; +[C trate]=confmat(testpost,[testlabel==1 testlabel>1]); + +fh4 = figure; +set(fh4, 'Name', 'Decision Boundaries'); +whitebg(fh4, 'k'); +hold on +plot(testdata((testlabel==1),1),testdata((testlabel==1),2),... + ClassSymbol1, 'MarkerSize', PointSize) +plot(testdata((testlabel>1),1),testdata((testlabel>1),2),... + ClassSymbol2, 'MarkerSize', PointSize) +% Bayesian decision boundary +[cB, hB] = contour(xrange,yrange,p1_x,[0.5 0.5],'b-'); +set(hB, 'LineWidth', 2); +% Network decision boundary +[cN, hN] = contour(xrange,yrange,yg,[0.5 0.5],'r-'); +set(hN, 'LineWidth', 2); +Chandles = [hB(1) hN(1)]; +legend(Chandles, 'Bayes decision boundary', ... + 'Network decision boundary', -1); +axis([x0 x1 y0 y1]) +title('Test Data') +set(gca,'Box','On','Xtick',[],'YTick',[]) + +clc +disp('This figure shows the test data with the decision boundary') +disp('produced by the trained network and the optimal Bayes rule.') +disp(' ') +disp('Press any key to continue.') +pause + +fh5 = figure; +set(fh5, 'Name', 'Test Set Performance'); +whitebg(fh5, 'w'); +% Bayes rule performance +subplot(1,2,1) +plotmat(Bcon,'b','k',12) +set(gca,'XTick',[0.5 1.5]) +set(gca,'YTick',[0.5 1.5]) +grid('off') +set(gca,'XTickLabel',['Red ' ; 'Yellow']) +set(gca,'YTickLabel',['Yellow' ; 'Red ']) +ylabel('True') +xlabel('Predicted') +title(['Bayes Confusion Matrix (' num2str(Brate(1)) '%)']) + +% Network performance +subplot(1,2, 2) +plotmat(C,'b','k',12) +set(gca,'XTick',[0.5 1.5]) +set(gca,'YTick',[0.5 1.5]) +grid('off') +set(gca,'XTickLabel',['Red ' ; 'Yellow']) +set(gca,'YTickLabel',['Yellow' ; 'Red ']) +ylabel('True') +xlabel('Predicted') +title(['Network Confusion Matrix (' num2str(trate(1)) '%)']) + +disp('The final figure shows the confusion matrices for the') +disp('two rules on the test set.') +disp(' ') +disp('Press any key to exit.') +pause +whitebg(fh1, 'w'); +whitebg(fh2, 'w'); +whitebg(fh3, 'w'); +whitebg(fh4, 'w'); +whitebg(fh5, 'w'); +close(fh1); close(fh2); close(fh3); +close(fh4); close(fh5); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demnlab.m b/sourcecodes/bnt-master/netlab3.3/demnlab.m new file mode 100644 index 00000000..11fa3e87 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demnlab.m @@ -0,0 +1,380 @@ +function demnlab(action); +%DEMNLAB A front-end Graphical User Interface to the demos +% +% Description +% This function will start a user interface allowing the user to select +% different demonstration functions to view. The demos are divided into +% 4 groups, with the demo being executed by selecting the desired +% option from a pop-up menu. +% +% See also +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% If run without parameters, initialise gui. +if nargin<1, + action='initialise'; +end; + +if strcmp(action,'initialise'), + + % Create figure + fig = figure( ... + 'Name', 'Netlab Demos', ... + 'NumberTitle', 'off', ... + 'Color', [0.7529 0.7529 0.7529], ... + 'Visible', 'on'); + + % Create GROUPS + % Bottom of demo buttons + group1_bot = 0.20; + group1_top = 0.75; + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'Position', [0.03 group1_bot 0.94 group1_top - group1_bot], ... + 'BackgroundColor', [0.5 0.5 0.5]); + + % Bottom of help and close buttons + group2_bot = 0.04; + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'Position', [0.03 group2_bot 0.94 0.12], ... + 'BackgroundColor', [0.5 0.5 0.5]); + + % Draw title + hLogoAxis = axes( ... + 'Units', 'normalized', ... + 'Position', [0.05 0.82 0.90 0.14], ... + 'Box', 'off', ... + 'XColor', [0 0 0], ... + 'YColor', [0 0 0], ... + 'Visible', 'on'); + + load netlogo; % load image and colour map + colormap(netcmap(1:3,:)); % change colour map: don't need many entries + image(nlogo); % draw logo + axis('image'); % ensures pixels on axis are square + axis off; % turn axes off + + % Create static text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.05 group1_top-0.1 0.90 0.08], ... + 'String', 'Select demo to run:'); + + % First row text offset + tRow1Offset = 0.14; + % Offset between text and button + TBoffset = 0.07; + % First row button offset + bRow1Offset = tRow1Offset+TBoffset; + % ONE text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.08 group1_top-tRow1Offset 0.36 0.05], ... + 'String', 'Regression'); + + popup1str(1) = {'Select Option'}; + popup1str(2) = {'Multi-Layer Perceptron'}; + popup1str(3) = {'Radial Basis Function'}; + popup1str(4) = {'Mixture Density Network'}; + % ONE popup + hPop1 = uicontrol(fig, ... + 'Style','popup', ... + 'Units','normalized', ... + 'String', popup1str, ... + 'Position', [0.08 group1_top-bRow1Offset 0.36 0.08], ... + 'BackgroundColor', [0.7 0.7 0.7], ... + 'Callback', 'demnlab popup1'); + + % TWO text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.56 group1_top-tRow1Offset 0.36 0.05], ... + 'String', 'Classification'); + + popup2str(1) = popup1str(1); + popup2str(2) = {'Generalised Linear Model (2 class)'}; + popup2str(3) = {'Generalised Linear Model (3 class)'}; + popup2str(4) = {'Multi-Layer Perceptron'}; + popup2str(5) = {'K nearest neighbour'}; + % TWO popup + hPop2 = uicontrol(fig, ... + 'Style','popup', ... + 'Units','normalized', ... + 'String', popup2str, ... + 'Position', [0.56 group1_top-bRow1Offset 0.36 0.08], ... + 'BackgroundColor', [0.7 0.7 0.7], ... + 'Callback', 'demnlab popup2'); + + tRow2Offset = 0.30; + bRow2Offset = tRow2Offset+TBoffset; + % THREE text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.08 group1_top - tRow2Offset 0.36 0.05], ... + 'String', 'Density Modelling and Clustering'); + + popup3str(1) = popup1str(1); + popup3str(2) = {'Gaussian Mixture (EM training)'}; + popup3str(3) = {'Gaussian Mixture (spherical)'}; + popup3str(4) = {'Gaussian Mixture (diagonal)'}; + popup3str(5) = {'Gaussian Mixture (full)'}; + popup3str(6) = {'Neuroscale'}; + popup3str(7) = {'GTM (EM training)'}; + popup3str(8) = {'GTM (visualisation)'}; + popup3str(9) = {'K-means clustering'}; + popup3str(10) = {'Self-Organising Map'}; + % TWO popup + % THREE popup + hPop3 = uicontrol(fig, ... + 'Style','popup', ... + 'Units','normalized', ... + 'String', popup3str, ... + 'Position', [0.08 group1_top - bRow2Offset 0.36 0.08], ... + 'BackgroundColor', [0.7 0.7 0.7], ... + 'Callback', 'demnlab popup3'); + + % FOUR text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.56 group1_top - tRow2Offset 0.36 0.05], ... + 'String', 'Bayesian Methods'); + + popup4str(1) = popup1str(1); + popup4str(2) = {'Sampling the MLP Prior'}; + popup4str(3) = {'Evidence Approximation for MLP'}; + popup4str(4) = {'Evidence Approximation for RBF'}; + popup4str(5) = {'Evidence Approximation in Classification'}; + popup4str(6) = {'ARD for MLP'}; + popup4str(7) = {'Sampling the GP Prior'}; + popup4str(8) = {'GPs for Regression'}; + popup4str(9) = {'ARD for GP'}; + % FOUR popup + hPop4 = uicontrol(fig, ... + 'Style','popup', ... + 'Units','normalized', ... + 'String', popup4str, ... + 'Position', [0.56 group1_top - bRow2Offset 0.36 0.08], ... + 'BackgroundColor', [0.7 0.7 0.7], ... + 'Callback', 'demnlab popup4'); + + + tRow3Offset = 0.45; + bRow3Offset = tRow3Offset+TBoffset; + % FIVE text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.08 group1_top - tRow3Offset 0.36 0.05], ... + 'String', 'Optimisation and Visualisation'); + + popup5str(1) = popup1str(1); + popup5str(2) = {'Algorithm Comparison'}; + popup5str(3) = {'On-line Gradient Descent'}; + popup5str(4) = {'Hinton Diagrams'}; + % FIVE popup + hPop5 = uicontrol(fig, ... + 'Style','popup', ... + 'Units','normalized', ... + 'String',popup5str, ... + 'Position', [0.08 group1_top - bRow3Offset 0.36 0.08], ... + 'BackgroundColor', [0.7 0.7 0.7], ... + 'Callback', 'demnlab popup5'); + + % SIX text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Position', [0.56 group1_top - tRow3Offset 0.36 0.05], ... + 'String', 'Sampling'); + + popup6str(1) = popup1str(1); + popup6str(2) = {'Sampling a Gaussian'}; + popup6str(3) = {'MCMC sampling (Metropolis)'}; + popup6str(4) = {'Hybrid MC (Gaussian mixture)'}; + popup6str(5) = {'Hybrid MC for MLP I'}; + popup6str(6) = {'Hybrid MC for MLP II'}; + % SIX popup + hPop6 = uicontrol(fig, ... + 'Style','popup', ... + 'Units','normalized', ... + 'String', popup6str, ... + 'Position', [0.56 group1_top - bRow3Offset 0.36 0.08], ... + 'BackgroundColor', [0.7 0.7 0.7], ... + 'Callback', 'demnlab popup6'); + + + % Create HELP button + uicontrol(fig, ... + 'Units', 'normalized', ... + 'Position' , [0.05 group2_bot+0.02 0.40 0.08], ... + 'String', 'Help', ... + 'Callback', 'demnlab help'); + + % Create CLOSE button + uicontrol(fig, ... + 'Units', 'normalized', ... + 'Position' , [0.55 group2_bot+0.02 0.40 0.08], ... + 'String', 'Close', ... + 'Callback', 'close(gcf)'); + + hndlList=[fig hPop1 hPop2 hPop3 hPop4 hPop5 hPop6]; + set(fig, 'UserData', hndlList); + set(fig, 'HandleVisibility', 'callback'); + +elseif strcmp(action, 'popup1'), + + hndlList=get(gcf,'UserData'); + hPop = hndlList(2); + + selected = get(hPop, 'Val'); + set(hPop, 'Val', [1]); + + switch selected + case 2 + demmlp1; + case 3 + demrbf1; + case 4 + demmdn1; + end; + +elseif strcmp(action,'popup2'), + + hndlList=get(gcf,'UserData'); + hPop = hndlList(3); + + selected = get(hPop, 'Val'); + set(hPop, 'Val', [1]); + + switch selected + case 2 + demglm1; + case 3 + demglm2; + case 4 + demmlp2; + case 5 + demknn1; + end + +elseif strcmp(action,'popup3'), + + hndlList=get(gcf,'UserData'); + hPop = hndlList(4); + + selected = get(hPop, 'Val'); + set(hPop, 'Val', [1]); + + switch selected + case 2 + demgmm1; + case 3 + demgmm2; + case 4 + demgmm3; + case 5 + demgmm4; + case 6 + demns1; + case 7 + demgtm1; + case 8 + demgtm2; + case 9 + demkmn1; + case 10 + demsom1; + end + +elseif strcmp(action,'popup4'), + + hndlList=get(gcf,'UserData'); + hPop = hndlList(5); + + selected = get(hPop, 'Val'); + set(hPop, 'Val', [1]); + + switch selected + case 2 + demprior; + case 3 + demev1; + case 4 + demev3; + case 5 + demev2; + case 6 + demard; + case 7 + demprgp; + case 8 + demgp; + case 9 + demgpard; + end + +elseif strcmp(action,'popup5'), + + hndlList=get(gcf,'UserData'); + hPop = hndlList(6); + + selected = get(hPop, 'Val'); + set(hPop, 'Val', [1]); + + switch selected + case 2 + demopt1; + case 3 + demolgd1; + case 4 + demhint; + end + + +elseif strcmp(action,'popup6'), + + hndlList=get(gcf,'UserData'); + hPop = hndlList(7); + + selected = get(hPop, 'Val'); + set(hPop, 'Val', [1]); + + switch selected + case 2 + demgauss; + case 3 + demmet1; + case 4 + demhmc1; + case 5 + demhmc2; + case 6 + demhmc3; + end + +elseif strcmp(action, 'help'), + + helpStr = {'To run a demo, press the appropriate button.'; ... + 'Instructions and information will appear in the Matlab';... + 'command window.'}; + + hHelpDlg = helpdlg(helpStr, 'Netlab Demo Help'); + +end; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demns1.m b/sourcecodes/bnt-master/netlab3.3/demns1.m new file mode 100644 index 00000000..029e1b48 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demns1.m @@ -0,0 +1,114 @@ +%DEMNS1 Demonstrate Neuroscale for visualisation. +% +% Description +% This script demonstrates the use of the Neuroscale algorithm for +% topographic projection and visualisation. A data sample is generated +% from a mixture of two Gaussians in 4d space, and an RBF is trained +% with the stress error function to project the data into 2d. The +% training data and a test sample are both plotted in this projection. +% +% See also +% RBF, RBFTRAIN, RBFPRIOR +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Generate the data +% Fix seeds for reproducible results +rand('state', 420); +randn('state', 420); + +input_dim = 4; +output_dim = 2; +mix = gmm(input_dim, 2, 'spherical'); +mix.centres = [1 1 1 1; 0 0 0 0]; +mix.priors = [0.5 0.5]; +mix.covars = [0.1 0.1]; + +ndata = 60; +[data, labels] = gmmsamp(mix, ndata); + +clc +disp('This demonstration illustrates the use of the Neuroscale model') +disp('to perform a topographic projection of data. We begin by generating') +disp('60 data points from a mixture of two Gaussians in 4 dimensional space.') +disp(' ') +disp('Press any key to continue') +pause + +ncentres = 10; +net = rbf(input_dim, ncentres, output_dim, 'tps', 'neuroscale'); +dstring = ['the Sammon mapping. The model has ', num2str(ncentres), ... + ' centres, two outputs, and uses']; +clc +disp('The Neuroscale model is an RBF with a Stress error measure as used in') +disp(dstring) +disp('thin plate spline basis functions.') +disp(' ') +disp('It is trained using the shadow targets algorithm for at most 60 iterations.') +disp(' ') +disp('Press any key to continue') +pause + +% First row controls shadow targets, second row controls rbfsetbf +options(1, :) = foptions; +options(2, :) = foptions; +options(1, 1) = 1; +options(1, 2) = 1e-2; +options(1, 3) = 1e-2; +options(1, 6) = 1; % Switch on PCA initialisation +options(1, 14) = 60; +options(2, 1) = -1; % Switch off all warnings +options(2, 5) = 1; +options(2, 14) = 10; +net2 = rbftrain(net, options, data); + +disp(' ') +disp('After training the model, we project the training data by a normal') +disp('forward propagation through the RBF network. Because there are two') +disp('outputs, the results can be plotted and visualised.') +disp(' ') +disp('Press any key to continue') +pause + +% Plot the result +y = rbffwd(net2, data); +ClassSymbol1 = 'r.'; +ClassSymbol2 = 'b.'; +PointSize = 12; +fh1 = figure; +hold on; +plot(y((labels==1),1),y(labels==1,2),ClassSymbol1, 'MarkerSize', PointSize) +plot(y((labels>1),1),y(labels>1,2),ClassSymbol2, 'MarkerSize', PointSize) + +disp(' ') +disp('In this plot, the red dots denote the first class and the blue') +disp('dots the second class.') +disp(' ') +disp('Press any key to continue.') +disp(' ') +pause + +disp('We now generate a further 100 data points from the original distribution') +disp('and plot their projection using star symbols. Note that a Sammon') +disp('mapping cannot be used to generalise to new data in this fashion.') + +[test_data, test_labels] = gmmsamp(mix, 100); +ytest = rbffwd(net2, test_data); +ClassSymbol1 = 'ro'; +ClassSymbol2 = 'bo'; +% Circles are rather large symbols +PointSize = 6; +hold on +plot(ytest((test_labels==1),1),ytest(test_labels==1,2), ... + ClassSymbol1, 'MarkerSize', PointSize) +plot(ytest((test_labels>1),1),ytest(test_labels>1,2),... + ClassSymbol2, 'MarkerSize', PointSize) +hold on +legend('Class 1', 'Class 2', 'Test Class 1', 'Test Class 2') +disp('Press any key to exit.') +pause + +close(fh1); +clear all; + diff --git a/sourcecodes/bnt-master/netlab3.3/demolgd1.m b/sourcecodes/bnt-master/netlab3.3/demolgd1.m new file mode 100644 index 00000000..fb18f760 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demolgd1.m @@ -0,0 +1,115 @@ +%DEMOLGD1 Demonstrate simple MLP optimisation with on-line gradient descent +% +% Description +% The problem consists of one input variable X and one target variable +% T with data generated by sampling X at equal intervals and then +% generating target data by computing SIN(2*PI*X) and adding Gaussian +% noise. A 2-layer network with linear outputs is trained by minimizing +% a sum-of-squares error function using on-line gradient descent. +% +% See also +% DEMMLP1, OLGD +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the matrix of inputs x and targets t. + +ndata = 20; % Number of data points. +noise = 0.2; % Standard deviation of noise distribution. +x = [0:1/(ndata - 1):1]'; +randn('state', 42); +rand('state', 42); +t = sin(2*pi*x) + noise*randn(ndata, 1); + +clc +disp('This demonstration illustrates the use of the on-line gradient') +disp('descent algorithm to train a Multi-Layer Perceptron network for') +disp('regression problems. It is intended to illustrate the drawbacks') +disp('of this algorithm compared to more powerful non-linear optimisation') +disp('algorithms, such as conjugate gradients.') +disp(' ') +disp('First we generate the data from a noisy sine function and construct') +disp('the network.') +disp(' ') +disp('Press any key to continue.') +pause +% Set up network parameters. +nin = 1; % Number of inputs. +nhidden = 3; % Number of hidden units. +nout = 1; % Number of outputs. +alpha = 0.01; % Coefficient of weight-decay prior. + +% Create and initialize network weight vector. +net = mlp(nin, nhidden, nout, 'linear'); +% Initialise weights reasonably close to 0 +net = mlpinit(net, 10); + +% Set up vector of options for the optimiser. +options = foptions; +options(1) = 1; % This provides display of error values. +options(14) = 20; % Number of training cycles. +options(18) = 0.1; % Learning rate +%options(17) = 0.4; % Momentum +options(17) = 0.4; % Momentum +options(5) = 1; % Do randomise pattern order +clc +disp('Then we set the options for the training algorithm.') +disp(['In the first phase of training, which lasts for ',... + num2str(options(14)), ' cycles,']) +disp(['the learning rate is ', num2str(options(18)), ... + ' and the momentum is ', num2str(options(17)), '.']) +disp('The error values are displayed at the end of each pass through the') +disp('entire pattern set.') +disp(' ') +disp('Press any key to continue.') +pause + +% Train using online gradient descent +[net, options] = olgd(net, options, x, t); + +% Now allow learning rate to decay and remove momentum +options(2) = 0; +options(3) = 0; +options(17) = 0.4; % Turn off momentum +options(5) = 1; % Randomise pattern order +options(6) = 1; % Set learning rate decay on +options(14) = 200; +options(18) = 0.1; % Initial learning rate + +disp(['In the second phase of training, which lasts for up to ',... + num2str(options(14)), ' cycles,']) +disp(['the learning rate starts at ', num2str(options(18)), ... + ', decaying at 1/t and the momentum is ', num2str(options(17)), '.']) +disp(' ') +disp('Press any key to continue.') +pause +[net, options] = olgd(net, options, x, t); + +clc +disp('Now we plot the data, underlying function, and network outputs') +disp('on a single graph to compare the results.') +disp(' ') +disp('Press any key to continue.') +pause + +% Plot the data, the original function, and the trained network function. +plotvals = [0:0.01:1]'; +y = mlpfwd(net, plotvals); +fh1 = figure; +plot(x, t, 'ob') +hold on +axis([0 1 -1.5 1.5]) +fplot('sin(2*pi*x)', [0 1], '--g') +plot(plotvals, y, '-r') +legend('data', 'function', 'network'); +hold off + +disp('Note the very poor fit to the data: this should be compared with') +disp('the results obtained in demmlp1.') +disp(' ') +disp('Press any key to exit.') +pause +close(fh1); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demopt1.m b/sourcecodes/bnt-master/netlab3.3/demopt1.m new file mode 100644 index 00000000..31b4a421 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demopt1.m @@ -0,0 +1,170 @@ +function demopt1(xinit) +%DEMOPT1 Demonstrate different optimisers on Rosenbrock's function. +% +% Description +% The four general optimisers (quasi-Newton, conjugate gradients, +% scaled conjugate gradients, and gradient descent) are applied to the +% minimisation of Rosenbrock's well known `banana' function. Each +% optimiser is run for at most 100 cycles, and a stopping criterion of +% 1.0e-4 is used for both position and function value. At the end, the +% trajectory of each algorithm is shown on a contour plot of the +% function. +% +% DEMOPT1(XINIT) allows the user to specify a row vector with two +% columns as the starting point. The default is the point [-1 1]. Note +% that the contour plot has an x range of [-1.5, 1.5] and a y range of +% [-0.5, 2.1], so it is best to choose a starting point in the same +% region. +% +% See also +% CONJGRAD, GRADDESC, QUASINEW, SCG, ROSEN, ROSEGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Initialise start point for search +if nargin < 1 | size(xinit) ~= [1 2] + xinit = [-1 1]; % Traditional start point +end + +% Find out if flops is available (i.e. pre-version 6 Matlab) +v = version; +if (str2num(strtok(v, '.')) >= 6) + flops_works = logical(0); +else + flops_works = logical(1); +end + +% Set up options +options = foptions; % Standard options +options(1) = -1; % Turn off printing completely +options(3) = 1e-8; % Tolerance in value of function +options(14) = 100; % Max. 100 iterations of algorithm + +clc +disp('This demonstration compares the performance of four generic') +disp('optimisation routines when finding the minimum of Rosenbrock''s') +disp('function y = 100*(x2-x1^2)^2 + (1-x1)^2.') +disp(' ') +disp('The global minimum of this function is at [1 1].') +disp(['Each algorithm starts at the point [' num2str(xinit(1))... + ' ' num2str(xinit(2)) '].']) +disp(' ') +disp('Press any key to continue.') +pause + +% Generate a contour plot of the function +a = -1.5:.02:1.5; +b = -0.5:.02:2.1; +[A, B] = meshgrid(a, b); +Z = rosen([A(:), B(:)]); +Z = reshape(Z, length(b), length(a)); +l = -1:6; +v = 2.^l; +fh1 = figure; +contour(a, b, Z, v) +title('Contour plot of Rosenbrock''s function') +hold on + +clc +disp('We now use quasi-Newton, conjugate gradient, scaled conjugate') +disp('gradient, and gradient descent with line search algorithms') +disp('to find a local minimum of this function. Each algorithm is stopped') +disp('when 100 cycles have elapsed, or if the change in function value') +disp('is less than 1.0e-8 or the change in the input vector is less than') +disp('1.0e-4 in magnitude.') +disp(' ') +disp('Press any key to continue.') +pause + +clc +x = xinit; +flops(0) +[x, options, errlog, pointlog] = quasinew('rosen', x, options, 'rosegrad'); +fprintf(1, 'For quasi-Newton method:\n') +fprintf(1, 'Final point is (%f, %f), value is %f\n', x(1), x(2), options(8)) +fprintf(1, 'Number of function evaluations is %d\n', options(10)) +fprintf(1, 'Number of gradient evaluations is %d\n', options(11)) +if flops_works + opt_flops = flops; + fprintf(1, 'Number of floating point operations is %d\n', opt_flops) +end +fprintf(1, 'Number of cycles is %d\n', size(pointlog, 1) - 1); +disp(' ') + +x = xinit; +flops(0) +[x, options, errlog2, pointlog2] = conjgrad('rosen', x, options, 'rosegrad'); +fprintf(1, 'For conjugate gradient method:\n') +fprintf(1, 'Final point is (%f, %f), value is %f\n', x(1), x(2), options(8)) +fprintf(1, 'Number of function evaluations is %d\n', options(10)) +fprintf(1, 'Number of gradient evaluations is %d\n', options(11)) +if flops_works + opt_flops = flops; + fprintf(1, 'Number of floating point operations is %d\n', ... + opt_flops) +end +fprintf(1, 'Number of cycles is %d\n', size(pointlog2, 1) - 1); +disp(' ') + +x = xinit; +flops(0) +[x, options, errlog3, pointlog3] = scg('rosen', x, options, 'rosegrad'); +fprintf(1, 'For scaled conjugate gradient method:\n') +fprintf(1, 'Final point is (%f, %f), value is %f\n', x(1), x(2), options(8)) +fprintf(1, 'Number of function evaluations is %d\n', options(10)) +fprintf(1, 'Number of gradient evaluations is %d\n', options(11)) +if flops_works + opt_flops = flops; + fprintf(1, 'Number of floating point operations is %d\n', opt_flops) +end +fprintf(1, 'Number of cycles is %d\n', size(pointlog3, 1) - 1); +disp(' ') + +x = xinit; +options(7) = 1; % Line minimisation used +flops(0) +[x, options, errlog4, pointlog4] = graddesc('rosen', x, options, 'rosegrad'); +fprintf(1, 'For gradient descent method:\n') +fprintf(1, 'Final point is (%f, %f), value is %f\n', x(1), x(2), options(8)) +fprintf(1, 'Number of function evaluations is %d\n', options(10)) +fprintf(1, 'Number of gradient evaluations is %d\n', options(11)) +if flops_works + opt_flops = flops; + fprintf(1, 'Number of floating point operations is %d\n', opt_flops) +end +fprintf(1, 'Number of cycles is %d\n', size(pointlog4, 1) - 1); +disp(' ') +disp('Note that gradient descent does not reach a local minimum in') +disp('100 cycles.') +disp(' ') +disp('On this problem, where the function is cheap to evaluate, the') +disp('computational effort is dominated by the algorithm overhead.') +disp('However on more complex optimisation problems (such as those') +disp('involving neural networks), computational effort is dominated by') +disp('the number of function and gradient evaluations. Counting these,') +disp('we can rank the algorithms: quasi-Newton (the best), conjugate') +disp('gradient, scaled conjugate gradient, gradient descent (the worst)') +disp(' ') +disp('Press any key to continue.') +pause +clc +disp('We now plot the trajectory of search points for each algorithm') +disp('superimposed on the contour plot.') +disp(' ') +disp('Press any key to continue.') +pause +plot(pointlog4(:,1), pointlog4(:,2), 'bd', 'MarkerSize', 6) +plot(pointlog3(:,1), pointlog3(:,2), 'mx', 'MarkerSize', 6, 'LineWidth', 2) +plot(pointlog(:,1), pointlog(:,2), 'k.', 'MarkerSize', 18) +plot(pointlog2(:,1), pointlog2(:,2), 'g+', 'MarkerSize', 6, 'LineWidth', 2) +lh = legend( 'Gradient Descent', 'Scaled Conjugate Gradients', ... + 'Quasi Newton', 'Conjugate Gradients'); + +hold off + +clc +disp('Press any key to end.') +pause +close(fh1); +clear all; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/dempot.m b/sourcecodes/bnt-master/netlab3.3/dempot.m new file mode 100644 index 00000000..87301bce --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/dempot.m @@ -0,0 +1,17 @@ +function e = dempot(x, mix) +%DEMPOT Computes the negative log likelihood for a mixture model. +% +% Description +% This function computes the negative log of the unconditional data +% density P(X) for a Gaussian mixture model. The data structure MIX +% defines the mixture model, while the matrix X contains the data +% vectors. +% +% See also +% DEMGPOT, DEMHMC1, DEMMET1 +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Computes the potential (negative log likelihood) +e = -log(gmmprob(mix, x)); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demprgp.m b/sourcecodes/bnt-master/netlab3.3/demprgp.m new file mode 100644 index 00000000..3cc7b8f5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demprgp.m @@ -0,0 +1,542 @@ +function demprgp(action); +%DEMPRGP Demonstrate sampling from a Gaussian Process prior. +% +% Description +% This function plots the functions represented by a Gaussian Process +% model. The hyperparameter values can be adjusted on a linear scale +% using the sliders (though the exponential of the parameters is used +% in the covariance function), or by typing values into the text boxes +% and pressing the return key. Both types of covariance function are +% supported. An extra function specific parameter is needed for the +% rational quadratic function. +% +% See also +% GP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargin<1, + action='initialize'; +end; + +if strcmp(action,'initialize') + + % Bounds on hyperparameter values + biasminval = -3.0; biasmaxval = 3.0; + noiseminval = -20; noisemaxval = -2; + fparminval = 0.0; fparmaxval = 2.0; + inwminval = 0; inwmaxval = 8; + % Initial hyperparameter values + bias = (biasminval+biasmaxval)/2; + noise = (noiseminval+noisemaxval)/2; + inweights = (inwminval+inwmaxval)/2; + fpar = (fparminval+fparmaxval)/2; + fpar2 = (fparminval+fparmaxval)/2; + + gptype = 'sqexp'; + + % Create FIGURE + fig=figure( ... + 'Name','Sampling from a Gaussian Process prior', ... + 'Position', [50 50 480 380], ... + 'NumberTitle','off', ... + 'Color', [0.8 0.8 0.8], ... + 'Visible','on'); + + % List box for covariance function type + nettype_box = uicontrol(fig, ... + 'Style', 'listbox', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'center', ... + 'Position', [0.52 0.77 0.40 0.12], ... + 'String', 'Squared Exponential|Rational Quadratic', ... + 'Max', 1, 'Min', 0, ... % Only allow one selection + 'Value', 1, ... % Initial value is squared exponential + 'BackgroundColor',[0.60 0.60 0.60],... + 'CallBack', 'demprgp GPtype'); + + % Title for list box + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'Position', [0.52 0.89 0.40 0.05], ... + 'String', 'Covariance Function Type', ... + 'BackgroundColor', get(fig, 'Color'), ... + 'HorizontalAlignment', 'center'); + + % Frames to enclose sliders + bottom_row = 0.04; + slider_frame_height = 0.15; + biasframe = uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'String', 'bias', ... + 'HorizontalAlignment', 'left', ... + 'Position', [0.05 bottom_row 0.35 slider_frame_height]); + + bpos = get(biasframe, 'Position'); + noise_frame_bottom = bpos(2) + bpos(4) + 0.02; + noiseframe = uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 noise_frame_bottom 0.35 slider_frame_height]); + + npos = get(noiseframe, 'Position'); + inw_frame_bottom = npos(2) + npos(4) + 0.02; + inwframe = uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 inw_frame_bottom 0.35 slider_frame_height]); + + inwpos = get(inwframe, 'Position'); + fpar_frame_bottom = inwpos(2) + inwpos(4) + 0.02; + % This frame sometimes has multiple parameters + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 fpar_frame_bottom 0.35 2*slider_frame_height]); + + % Frame text + slider_text_height = 0.05; + slider_text_voffset = 0.08; + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 bottom_row+slider_text_voffset ... + 0.06 slider_text_height], ... + 'String', 'bias'); + + % Frame text + noiseframe = uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 noise_frame_bottom+slider_text_voffset ... + 0.08 slider_text_height], ... + 'String', 'noise'); + + % Frame text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 inw_frame_bottom+slider_text_voffset ... + 0.14 slider_text_height], ... + 'String', 'inweights'); + + % Frame text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 fpar_frame_bottom+slider_frame_height+ ... + slider_text_voffset 0.06 slider_text_height], ... + 'String', 'fpar'); + + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 fpar_frame_bottom+slider_text_voffset ... + 0.06 slider_text_height], ... + 'String', 'fpar2', ... + 'Tag', 'fpar2text', ... + 'Enable', 'off'); + + % Slider + slider_left = 0.07; + slider_width = 0.31; + slider_frame_voffset = 0.02; + biasslide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', bias, ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [slider_left bottom_row+slider_frame_voffset ... + slider_width 0.05], ... + 'Min', biasminval, 'Max', biasmaxval, ... + 'Callback', 'demprgp update'); + + % Slider + noiseslide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', noise, ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [slider_left noise_frame_bottom+slider_frame_voffset ... + slider_width 0.05], ... + 'Min', noiseminval, 'Max', noisemaxval, ... + 'Callback', 'demprgp update'); + + % Slider + inweightsslide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', inweights, ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [slider_left inw_frame_bottom+slider_frame_voffset ... + slider_width 0.05], ... + 'Min', inwminval, 'Max', inwmaxval, ... + 'Callback', 'demprgp update'); + + % Slider + fparslide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', fpar, ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [slider_left fpar_frame_bottom+slider_frame_height+ ... + slider_frame_voffset slider_width 0.05], ... + 'Min', fparminval, 'Max', fparmaxval, ... + 'Callback', 'demprgp update'); + + fpar2slide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', fpar2, ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [slider_left fpar_frame_bottom+slider_frame_voffset ... + slider_width 0.05], ... + 'Min', fparminval, 'Max', fparmaxval, ... + 'Callback', 'demprgp update', ... + 'Tag', 'fpar2slider', ... + 'Enable', 'off'); + + % Text display of hyper-parameter values + + format = '%8f'; + + hp_left = 0.20; + hp_width = 0.17; + biasval = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [hp_left bottom_row+slider_text_voffset ... + hp_width slider_text_height], ... + 'String', sprintf(format, bias), ... + 'Callback', 'demprgp newval'); + + noiseval = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [hp_left noise_frame_bottom+slider_text_voffset ... + hp_width slider_text_height], ... + 'String', sprintf(format, noise), ... + 'Callback', 'demprgp newval'); + + inweightsval = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [hp_left inw_frame_bottom+slider_text_voffset ... + hp_width slider_text_height], ... + 'String', sprintf(format, inweights), ... + 'Callback', 'demprgp newval'); + + fparval = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [hp_left fpar_frame_bottom+slider_frame_height+ ... + slider_text_voffset hp_width slider_text_height], ... + 'String', sprintf(format, fpar), ... + 'Callback', 'demprgp newval'); + + fpar2val = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [hp_left fpar_frame_bottom+slider_text_voffset ... + hp_width slider_text_height], ... + 'String', sprintf(format, fpar), ... + 'Callback', 'demprgp newval', ... + 'Enable', 'off', ... + 'Tag', 'fpar2val'); + + + % The graph box + haxes = axes('Position', [0.5 0.28 0.45 0.45], ... + 'Units', 'normalized', ... + 'Visible', 'on'); + + % The SAMPLE button + uicontrol(fig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.5 bottom_row 0.13 0.1], ... + 'String','Sample', ... + 'Callback','demprgp replot'); + + % The CLOSE button + uicontrol(fig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.82 bottom_row 0.13 0.1], ... + 'String','Close', ... + 'Callback','close(gcf)'); + + % The HELP button + uicontrol(fig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.66 bottom_row 0.13 0.1], ... + 'String','Help', ... + 'Callback','demprgp help'); + + % Save handles to objects + + hndlList=[fig biasslide noiseslide inweightsslide fparslide ... + biasval noiseval inweightsval ... + fparval haxes nettype_box]; + set(fig, 'UserData', hndlList); + + demprgp('replot') + + +elseif strcmp(action, 'update'), + + % Update when a slider is moved. + + hndlList = get(gcf, 'UserData'); + biasslide = hndlList(2); + noiseslide = hndlList(3); + inweightsslide = hndlList(4); + fparslide = hndlList(5); + biasval = hndlList(6); + noiseval = hndlList(7); + inweightsval = hndlList(8); + fparval = hndlList(9); + haxes = hndlList(10); + nettype_box = hndlList(11); + + + bias = get(biasslide, 'Value'); + noise = get(noiseslide, 'Value'); + inweights = get(inweightsslide, 'Value'); + fpar = get(fparslide, 'Value'); + fpar2 = get(findobj('Tag', 'fpar2slider'), 'Value'); + + format = '%8f'; + set(biasval, 'String', sprintf(format, bias)); + set(noiseval, 'String', sprintf(format, noise)); + set(inweightsval, 'String', sprintf(format, inweights)); + set(fparval, 'String', sprintf(format, fpar)); + set(findobj('Tag', 'fpar2val'), 'String', ... + sprintf(format, fpar2)); + + demprgp('replot'); + +elseif strcmp(action, 'newval'), + + % Update when text is changed. + + hndlList = get(gcf, 'UserData'); + biasslide = hndlList(2); + noiseslide = hndlList(3); + inweightsslide = hndlList(4); + fparslide = hndlList(5); + biasval = hndlList(6); + noiseval = hndlList(7); + inweightsval = hndlList(8); + fparval = hndlList(9); + haxes = hndlList(10); + + bias = sscanf(get(biasval, 'String'), '%f'); + noise = sscanf(get(noiseval, 'String'), '%f'); + inweights = sscanf(get(inweightsval, 'String'), '%f'); + fpar = sscanf(get(fparval, 'String'), '%f'); + fpar2 = sscanf(get(findobj('Tag', 'fpar2val'), 'String'), '%f'); + + set(biasslide, 'Value', bias); + set(noiseslide, 'Value', noise); + set(inweightsslide, 'Value', inweights); + set(fparslide, 'Value', fpar); + set(findobj('Tag', 'fpar2slider'), 'Value', fpar2); + + demprgp('replot'); + +elseif strcmp(action, 'GPtype') + hndlList = get(gcf, 'UserData'); + nettype_box = hndlList(11); + gptval = get(nettype_box, 'Value'); + if gptval == 1 + % Squared exponential, so turn off fpar2 + set(findobj('Tag', 'fpar2text'), 'Enable', 'off'); + set(findobj('Tag', 'fpar2slider'), 'Enable', 'off'); + set(findobj('Tag', 'fpar2val'), 'Enable', 'off'); + else + % Rational quadratic, so turn on fpar2 + set(findobj('Tag', 'fpar2text'), 'Enable', 'on'); + set(findobj('Tag', 'fpar2slider'), 'Enable', 'on'); + set(findobj('Tag', 'fpar2val'), 'Enable', 'on'); + end + demprgp('replot'); + +elseif strcmp(action, 'replot'), + + % Re-sample from the prior and plot graphs. + + oldFigNumber=watchon; + + hndlList = get(gcf, 'UserData'); + biasslide = hndlList(2); + noiseslide = hndlList(3); + inweightsslide = hndlList(4); + fparslide = hndlList(5); + haxes = hndlList(10); + nettype_box = hndlList(11); + gptval = get(nettype_box, 'Value'); + if gptval == 1 + gptype = 'sqexp'; + else + gptype = 'ratquad'; + end + + bias = get(biasslide, 'Value'); + noise = get(noiseslide, 'Value'); + inweights = get(inweightsslide, 'Value'); + fpar = get(fparslide, 'Value'); + + + axes(haxes); + cla + set(gca, ... + 'Box', 'on', ... + 'Color', [0 0 0], ... + 'XColor', [0 0 0], ... + 'YColor', [0 0 0], ... + 'FontSize', 14); + ymin = -10; + ymax = 10; + axis([-1 1 ymin ymax]); + set(gca,'DefaultLineLineWidth', 2); + + xvals = (-1:0.01:1)'; + nsample = 10; % Number of samples from prior. + hold on + plot([-1 0; 1 0], [0 ymin; 0 ymax], 'b--'); + net = gp(1, gptype); + net.bias = bias; + net.noise = noise; + net.inweights = inweights; + if strcmp(gptype, 'sqexp') + net.fpar = fpar; + else + fpar2 = get(findobj('Tag', 'fpar2slider'), 'Value'); + net.fpar = [fpar fpar2]; + end + cn = gpcovar(net, xvals); + cninv = inv(cn); + cnchol = chol(cn); + set(gca, 'DefaultLineLineWidth', 1); + for n = 1:nsample + y = (cnchol') * randn(size(xvals)); + plot(xvals, y, 'y'); + end + + watchoff(oldFigNumber); + +elseif strcmp(action, 'help'), + + % Provide help to user. + + oldFigNumber=watchon; + + helpfig = figure('Position', [100 100 480 400], ... + 'Name', 'Help', ... + 'NumberTitle', 'off', ... + 'Color', [0.8 0.8 0.8], ... + 'Visible','on'); + + % The HELP TITLE BAR frame + uicontrol(helpfig, ... + 'Style','frame', ... + 'Units','normalized', ... + 'HorizontalAlignment', 'center', ... + 'Position', [0.05 0.82 0.9 0.1], ... + 'BackgroundColor',[0.60 0.60 0.60]); + + % The HELP TITLE BAR text + uicontrol(helpfig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.26 0.85 0.6 0.05], ... + 'HorizontalAlignment', 'left', ... + 'String', 'Help: Sampling from a Gaussian Process Prior'); + + helpstr1 = strcat(... + 'This demonstration shows the effects of sampling from a Gaussian', ... + ' process prior. The parameters bias, noise, inweights and fpar', ... + ' control the corresponding terms in the covariance function of the',... + ' Gaussian process. Their values can be adjusted on a linear scale',... + ' using the sliders, or by typing values into the text boxes and',... + ' pressing the return key. After setting these values, press the',... + ' ''Sample'' button to see a new sample from the prior.'); + + helpstr2 = strcat(... + 'Observe how inweights controls horizontal length-scale of the',... + ' variation in the functions, noise controls the roughness of the',... + ' functions, and the bias controls the size of the', ... + ' vertical offset of the signal.'); + helpstr3 = strcat(... + 'There are two types of covariance function supported by', ... + ' Netlab which can be selected using the ''Covariance Function', ... + ' Type'' menu.'); + helpstr4 = strcat(... + 'The squared exponential has a single fpar which', ... + ' controls the vertical scale of the process.'); + helpstr5 = strcat(... + 'The rational quadratic has two fpar values. The first is', ... + ' is a scale parameter inside the rational function like the',... + ' first fpar for the squared exponential covariance, while the', ... + ' second gives the exponent of the rational function (i.e. the',... + ' rate of decay of the covariance function.'); + % Set up cell array with help strings + hstr(1) = {helpstr1}; + hstr(2) = {''}; + hstr(3) = {helpstr2}; + hstr(4) = {''}; + hstr(5) = {helpstr3}; + hstr(6) = {''}; + hstr(7) = {helpstr4}; + hstr(8) = {''}; + hstr(9) = {helpstr5}; + + % The HELP text + helpui = uicontrol(helpfig, ... + 'Style', 'Text', ... + 'Units', 'normalized', ... + 'ForegroundColor', [0 0 0], ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [1 1 1], ... + 'Min', 0, ... + 'Max', 2, ... + 'Position', [0.05 0.2 0.9 0.57]); + [hstrw, newpos] = textwrap(helpui, hstr); + set(helpui, 'String', hstrw, 'Position', [0.05, 0.2, 0.9 newpos(4)]); + + % The CLOSE button + uicontrol(helpfig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.4 0.05 0.2 0.1], ... + 'String','Close', ... + 'Callback','close(gcf)'); + + watchoff(oldFigNumber); + +end; + diff --git a/sourcecodes/bnt-master/netlab3.3/demprior.m b/sourcecodes/bnt-master/netlab3.3/demprior.m new file mode 100644 index 00000000..4ad975a9 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demprior.m @@ -0,0 +1,406 @@ +function demprior(action); +%DEMPRIOR Demonstrate sampling from a multi-parameter Gaussian prior. +% +% Description +% This function plots the functions represented by a multi-layer +% perceptron network when the weights are set to values drawn from a +% Gaussian prior distribution. The parameters AW1, AB1 AW2 and AB2 +% control the inverse variances of the first-layer weights, the hidden +% unit biases, the second-layer weights and the output unit biases +% respectively. Their values can be adjusted on a logarithmic scale +% using the sliders, or by typing values into the text boxes and +% pressing the return key. +% +% See also +% MLP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargin<1, + action='initialize'; +end; + +if strcmp(action,'initialize') + + aw1 = 0.01; + ab1 = 0.1; + aw2 = 1.0; + ab2 = 1.0; + + % Create FIGURE + fig=figure( ... + 'Name','Sampling from a Gaussian prior', ... + 'Position', [50 50 480 380], ... + 'NumberTitle','off', ... + 'Color', [0.8 0.8 0.8], ... + 'Visible','on'); + + % The TITLE BAR frame + uicontrol(fig, ... + 'Style','frame', ... + 'Units','normalized', ... + 'HorizontalAlignment', 'center', ... + 'Position', [0.5 0.82 0.45 0.1], ... + 'BackgroundColor',[0.60 0.60 0.60]); + + % The TITLE BAR text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.54 0.85 0.40 0.05], ... + 'HorizontalAlignment', 'left', ... + 'String', 'Sampling from a Gaussian prior'); + + % Frames to enclose sliders + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 0.08 0.35 0.18]); + + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 0.3 0.35 0.18]); + + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 0.52 0.35 0.18]); + + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.05 0.74 0.35 0.18]); + + % Frame text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 0.17 0.06 0.07], ... + 'String', 'aw1'); + + % Frame text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 0.39 0.06 0.07], ... + 'String', 'ab1'); + + % Frame text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 0.61 0.06 0.07], ... + 'String', 'aw2'); + + % Frame text + uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.07 0.83 0.06 0.07], ... + 'String', 'ab2'); + + % Slider + minval = -5; maxval = 5; + aw1slide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', log10(aw1), ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [0.07 0.1 0.31 0.05], ... + 'Min', minval, 'Max', maxval, ... + 'Callback', 'demprior update'); + + % Slider + ab1slide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', log10(ab1), ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [0.07 0.32 0.31 0.05], ... + 'Min', minval, 'Max', maxval, ... + 'Callback', 'demprior update'); + + % Slider + aw2slide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', log10(aw2), ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [0.07 0.54 0.31 0.05], ... + 'Min', minval, 'Max', maxval, ... + 'Callback', 'demprior update'); + + % Slider + ab2slide = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Value', log10(ab2), ... + 'BackgroundColor', [0.8 0.8 0.8], ... + 'Position', [0.07 0.76 0.31 0.05], ... + 'Min', minval, 'Max', maxval, ... + 'Callback', 'demprior update'); + + % The graph box + haxes = axes('Position', [0.5 0.28 0.45 0.45], ... + 'Units', 'normalized', ... + 'Visible', 'on'); + + % Text display of hyper-parameter values + + format = '%8f'; + + aw1val = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [0.15 0.17 0.23 0.07], ... + 'String', sprintf(format, aw1), ... + 'Callback', 'demprior newval'); + + ab1val = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [0.15 0.39 0.23 0.07], ... + 'String', sprintf(format, ab1), ... + 'Callback', 'demprior newval'); + + aw2val = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [0.15 0.61 0.23 0.07], ... + 'String', sprintf(format, aw2), ... + 'Callback', 'demprior newval'); + + ab2val = uicontrol(fig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'Position', [0.15 0.83 0.23 0.07], ... + 'String', sprintf(format, ab2), ... + 'Callback', 'demprior newval'); + + % The SAMPLE button + uicontrol(fig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.5 0.08 0.13 0.1], ... + 'String','Sample', ... + 'Callback','demprior replot'); + + % The CLOSE button + uicontrol(fig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.82 0.08 0.13 0.1], ... + 'String','Close', ... + 'Callback','close(gcf)'); + + % The HELP button + uicontrol(fig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.66 0.08 0.13 0.1], ... + 'String','Help', ... + 'Callback','demprior help'); + + % Save handles to objects + + hndlList=[fig aw1slide ab1slide aw2slide ab2slide aw1val ab1val aw2val ... + ab2val haxes]; + set(fig, 'UserData', hndlList); + + demprior('replot') + + +elseif strcmp(action, 'update'), + + % Update when a slider is moved. + + hndlList = get(gcf, 'UserData'); + aw1slide = hndlList(2); + ab1slide = hndlList(3); + aw2slide = hndlList(4); + ab2slide = hndlList(5); + aw1val = hndlList(6); + ab1val = hndlList(7); + aw2val = hndlList(8); + ab2val = hndlList(9); + haxes = hndlList(10); + + aw1 = 10^get(aw1slide, 'Value'); + ab1 = 10^get(ab1slide, 'Value'); + aw2 = 10^get(aw2slide, 'Value'); + ab2 = 10^get(ab2slide, 'Value'); + + format = '%8f'; + set(aw1val, 'String', sprintf(format, aw1)); + set(ab1val, 'String', sprintf(format, ab1)); + set(aw2val, 'String', sprintf(format, aw2)); + set(ab2val, 'String', sprintf(format, ab2)); + + demprior('replot'); + +elseif strcmp(action, 'newval'), + + % Update when text is changed. + + hndlList = get(gcf, 'UserData'); + aw1slide = hndlList(2); + ab1slide = hndlList(3); + aw2slide = hndlList(4); + ab2slide = hndlList(5); + aw1val = hndlList(6); + ab1val = hndlList(7); + aw2val = hndlList(8); + ab2val = hndlList(9); + haxes = hndlList(10); + + aw1 = sscanf(get(aw1val, 'String'), '%f'); + ab1 = sscanf(get(ab1val, 'String'), '%f'); + aw2 = sscanf(get(aw2val, 'String'), '%f'); + ab2 = sscanf(get(ab2val, 'String'), '%f'); + + set(aw1slide, 'Value', log10(aw1)); + set(ab1slide, 'Value', log10(ab1)); + set(aw2slide, 'Value', log10(aw2)); + set(ab2slide, 'Value', log10(ab2)); + + demprior('replot'); + +elseif strcmp(action, 'replot'), + + % Re-sample from the prior and plot graphs. + + oldFigNumber=watchon; + + hndlList = get(gcf, 'UserData'); + aw1slide = hndlList(2); + ab1slide = hndlList(3); + aw2slide = hndlList(4); + ab2slide = hndlList(5); + haxes = hndlList(10); + + aw1 = 10^get(aw1slide, 'Value'); + ab1 = 10^get(ab1slide, 'Value'); + aw2 = 10^get(aw2slide, 'Value'); + ab2 = 10^get(ab2slide, 'Value'); + + axes(haxes); + cla + set(gca, ... + 'Box', 'on', ... + 'Color', [0 0 0], ... + 'XColor', [0 0 0], ... + 'YColor', [0 0 0], ... + 'FontSize', 14); + axis([-1 1 -10 10]); + set(gca,'DefaultLineLineWidth', 2); + + nhidden = 12; + prior = mlpprior(1, nhidden, 1, aw1, ab1, aw2, ab2); + xvals = -1:0.005:1; + nsample = 10; % Number of samples from prior. + hold on + plot([-1 0; 1 0], [0 -10; 0 10], 'b--'); + net = mlp(1, nhidden, 1, 'linear', prior); + for i = 1:nsample + net = mlpinit(net, prior); + yvals = mlpfwd(net, xvals'); + plot(xvals', yvals, 'y'); + end + + watchoff(oldFigNumber); + +elseif strcmp(action, 'help'), + + % Provide help to user. + + oldFigNumber=watchon; + + helpfig = figure('Position', [100 100 480 400], ... + 'Name', 'Help', ... + 'NumberTitle', 'off', ... + 'Color', [0.8 0.8 0.8], ... + 'Visible','on'); + + % The HELP TITLE BAR frame + uicontrol(helpfig, ... + 'Style','frame', ... + 'Units','normalized', ... + 'HorizontalAlignment', 'center', ... + 'Position', [0.05 0.82 0.9 0.1], ... + 'BackgroundColor',[0.60 0.60 0.60]); + + % The HELP TITLE BAR text + uicontrol(helpfig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position', [0.26 0.85 0.6 0.05], ... + 'HorizontalAlignment', 'left', ... + 'String', 'Help: Sampling from a Gaussian Prior'); + + helpstr1 = strcat( ... + 'This demonstration shows the effects of sampling from a Gaussian', ... + ' prior over weights for a two-layer feed-forward network. The', ... + ' parameters aw1, ab1, aw2 and ab2 control the inverse variances of', ... + ' the first-layer weights, the hidden unit biases, the second-layer', ... + ' weights and the output unit biases respectively. Their values can', ... + ' be adjusted on a logarithmic scale using the sliders, or by', ... + ' typing values into the text boxes and pressing the return key.', ... + ' After setting these values, press the ''Sample'' button to see a', ... + ' new sample from the prior. '); + helpstr2 = strcat( ... + 'Observe how aw1 controls the horizontal length-scale of the', ... + ' variation in the functions, ab1 controls the input range over', ... + ' such variations occur, aw2 sets the vertical scale of the output', ... + ' and ab2 sets the vertical off-set of the output. The network has', ... + ' 12 hidden units. '); + hstr(1) = {helpstr1}; + hstr(2) = {''}; + hstr(3) = {helpstr2}; + + % The HELP text + helpui = uicontrol(helpfig, ... + 'Style', 'edit', ... + 'Units', 'normalized', ... + 'ForegroundColor', [0 0 0], ... + 'HorizontalAlignment', 'left', ... + 'BackgroundColor', [1 1 1], ... + 'Min', 0, ... + 'Max', 2, ... + 'Position', [0.05 0.2 0.9 0.8]); + + [hstrw , newpos] = textwrap(helpui, hstr, 70); + set(helpui, 'String', hstrw, 'Position', [0.05, 0.2, 0.9, newpos(4)]); + + + % The CLOSE button + uicontrol(helpfig, ... + 'Style','push', ... + 'Units','normalized', ... + 'BackgroundColor', [0.6 0.6 0.6], ... + 'Position',[0.4 0.05 0.2 0.1], ... + 'String','Close', ... + 'Callback','close(gcf)'); + + watchoff(oldFigNumber); + +end; + diff --git a/sourcecodes/bnt-master/netlab3.3/demrbf1.m b/sourcecodes/bnt-master/netlab3.3/demrbf1.m new file mode 100644 index 00000000..ffd6fd59 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demrbf1.m @@ -0,0 +1,138 @@ +%DEMRBF1 Demonstrate simple regression using a radial basis function network. +% +% Description +% The problem consists of one input variable X and one target variable +% T with data generated by sampling X at equal intervals and then +% generating target data by computing SIN(2*PI*X) and adding Gaussian +% noise. This data is the same as that used in demmlp1. +% +% Three different RBF networks (with different activation functions) +% are trained in two stages. First, a Gaussian mixture model is trained +% using the EM algorithm, and the centres of this model are used to set +% the centres of the RBF. Second, the output weights (and biases) are +% determined using the pseudo-inverse of the design matrix. +% +% See also +% DEMMLP1, RBF, RBFFWD, GMM, GMMEM +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +% Generate the matrix of inputs x and targets t. +randn('state', 42); +rand('state', 42); +ndata = 20; % Number of data points. +noise = 0.2; % Standard deviation of noise distribution. +x = (linspace(0, 1, ndata))'; +t = sin(2*pi*x) + noise*randn(ndata, 1); +mu = mean(x); +sigma = std(x); +tr_in = (x - mu)./(sigma); + +clc +disp('This demonstration illustrates the use of a Radial Basis Function') +disp('network for regression problems. The data is generated from a noisy') +disp('sine function.') +disp(' ') +disp('Press any key to continue.') +pause +% Set up network parameters. +nin = 1; % Number of inputs. +nhidden = 7; % Number of hidden units. +nout = 1; % Number of outputs. + +clc +disp('We assess the effect of three different activation functions.') +disp('First we create a network with Gaussian activations.') +disp(' ') +disp('Press any key to continue.') +pause +% Create and initialize network weight and parameter vectors. +net = rbf(nin, nhidden, nout, 'gaussian'); + +disp('A two-stage training algorithm is used: it uses a small number of') +disp('iterations of EM to position the centres, and then the pseudo-inverse') +disp('of the design matrix to find the second layer weights.') +disp(' ') +disp('Press any key to continue.') +pause +disp('Error values from EM training.') +% Use fast training method +options = foptions; +options(1) = 1; % Display EM training +options(14) = 10; % number of iterations of EM +net = rbftrain(net, options, tr_in, t); + +disp(' ') +disp('Press any key to continue.') +pause +clc +disp('The second RBF network has thin plate spline activations.') +disp('The same centres are used again, so we just need to calculate') +disp('the second layer weights.') +disp(' ') +disp('Press any key to continue.') +pause +% Create a second RBF with thin plate spline functions +net2 = rbf(nin, nhidden, nout, 'tps'); + +% Re-use previous centres rather than calling rbftrain again +net2.c = net.c; +[y, act2] = rbffwd(net2, tr_in); + +% Solve for new output weights and biases from RBF activations +temp = pinv([act2 ones(ndata, 1)]) * t; +net2.w2 = temp(1:nhidden, :); +net2.b2 = temp(nhidden+1, :); + +disp('The third RBF network has r^4 log r activations.') +disp(' ') +disp('Press any key to continue.') +pause +% Create a third RBF with r^4 log r functions +net3 = rbf(nin, nhidden, nout, 'r4logr'); + +% Overwrite weight vector with parameters from first RBF +net3.c = net.c; +[y, act3] = rbffwd(net3, tr_in); +temp = pinv([act3 ones(ndata, 1)]) * t; +net3.w2 = temp(1:nhidden, :); +net3.b2 = temp(nhidden+1, :); + +disp('Now we plot the data, underlying function, and network outputs') +disp('on a single graph to compare the results.') +disp(' ') +disp('Press any key to continue.') +pause +% Plot the data, the original function, and the trained network functions. +plotvals = [x(1):0.01:x(end)]'; +inputvals = (plotvals-mu)./sigma; +y = rbffwd(net, inputvals); +y2 = rbffwd(net2, inputvals); +y3 = rbffwd(net3, inputvals); +fh1 = figure; + +plot(x, t, 'ob') +hold on +xlabel('Input') +ylabel('Target') +axis([x(1) x(end) -1.5 1.5]) +[fx, fy] = fplot('sin(2*pi*x)', [x(1) x(end)]); +plot(fx, fy, '-r', 'LineWidth', 2) +plot(plotvals, y, '--g', 'LineWidth', 2) +plot(plotvals, y2, 'k--', 'LineWidth', 2) +plot(plotvals, y3, '-.c', 'LineWidth', 2) +legend('data', 'function', 'Gaussian RBF', 'Thin plate spline RBF', ... + 'r^4 log r RBF'); +hold off + +disp('RBF training errors are'); +disp(['Gaussian ', num2str(rbferr(net, tr_in, t)), ' TPS ', ... +num2str(rbferr(net2, tr_in, t)), ' R4logr ', num2str(rbferr(net3, tr_in, t))]); + +disp(' ') +disp('Press any key to end.') +pause +close(fh1); +clear all; diff --git a/sourcecodes/bnt-master/netlab3.3/demsom1.m b/sourcecodes/bnt-master/netlab3.3/demsom1.m new file mode 100644 index 00000000..2f1fe93c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demsom1.m @@ -0,0 +1,113 @@ +%DEMSOM1 Demonstrate SOM for visualisation. +% +% Description +% This script demonstrates the use of a SOM with a two-dimensional +% grid to map onto data in two-dimensional space. Both on-line and +% batch training algorithms are shown. +% +% See also +% SOM, SOMPAK, SOMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +randn('state', 42); +rand('state', 42); +nin = 2; +ndata = 300; +% Give data an offset so that network has something to learn. +x = rand(ndata, nin) + ones(ndata, 1)*[1.5 1.5]; + +clc; +disp('This demonstration of the SOM, or Kohonen network, shows how the') +disp('network units after training lie in regions of high data density.') +disp('First we show the data, which is generated uniformly from a square.') +disp('Red crosses denote the data and black dots are the initial locations') +disp('of the SOM units.') +disp(' ') +disp('Press any key to continue.') +pause +net = som(nin, [8, 7]); +c1 = sompak(net); +h1 = figure; +plot(x(:, 1), x(:, 2), 'r+'); +hold on +plot(c1(:,1), c1(:, 2), 'k.'); +drawnow; % Force figure to be drawn before training starts +options = foptions; + +% Ordering phase +options(1) = 1; +options(14) = 50; +%options(14) = 5; % Just for testing +options(18) = 0.9; % Initial learning rate +options(16) = 0.05; % Final learning rate +options(17) = 8; % Initial neighbourhood size +options(15) = 1; % Final neighbourhood size + +disp('The SOM network is trained in two phases using an on-line algorithm.') +disp('Initially the neighbourhood is set to 8 and is then reduced') +disp('linearly to 1 over the first 50 iterations.') +disp('Each iteration consists of a pass through the complete') +disp('dataset, while the weights are adjusted after each pattern.') +disp('The learning rate is reduced linearly from 0.9 to 0.05.') +disp('This ordering phase puts the units in a rough grid shape.') +disp('Blue circles denote the units at the end of this phase.') +disp(' ') +disp('Press any key to continue.') +pause +net2 = somtrain(net, options, x); +c2 = sompak(net2); +plot(c2(:, 1), c2(:, 2), 'bo'); +drawnow; + +% Convergence phase +options(1) = 1; +options(14) = 400; +options(18) = 0.05; +options(16) = 0.01; +options(17) = 0; +options(15) = 0; + +disp('The second, convergence, phase of learning just updates the winning node.') +disp('The learning rate is reduced from 0.05 to 0.01 over 400 iterations.') +disp('Note how the error value does not decrease monotonically; it is') +disp('difficult to decide when training is complete in a principled way.') +disp('The units are plotted as green stars.') +disp(' ') +disp('Press any key to continue.') +pause +net3 = somtrain(net2, options, x); +c3 = sompak(net3); +plot(c3(:, 1), c3(:, 2), 'g*'); +drawnow; + +% Now try batch training +options(1) = 1; +options(6) = 1; +options(14) = 50; +options(17) = 3; +options(15) = 0; +disp('An alternative approach to the on-line algorithm is a batch update') +disp('rule. Each unit is updated to be the average weights') +disp('in a neighbourhood (which reduces from 3 to 0) over 50 iterations.'); +disp('Note how the error is even more unstable at first, though eventually') +disp('it does converge.') +disp('The final units are shown as black triangles.') +disp(' ') +disp('Press any key to continue.') +pause +net4 = somtrain(net, options, x); +c4 = sompak(net4); +plot(c4(:, 1), c4(:, 2), 'k^') +legend('Data', 'Initial weights', 'Weights after ordering', ... + 'Weights after convergence', 'Batch weights', 2); +drawnow; + +disp(' ') +disp('Press any key to end.') +disp(' ') +pause + +close(h1); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/demtrain.m b/sourcecodes/bnt-master/netlab3.3/demtrain.m new file mode 100644 index 00000000..61ebc16f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/demtrain.m @@ -0,0 +1,362 @@ +function demtrain(action); +%DEMTRAIN Demonstrate training of MLP network. +% +% Description +% DEMTRAIN brings up a simple GUI to show the training of an MLP +% network on classification and regression problems. The user should +% load in a dataset (which should be in Netlab format: see DATREAD), +% select the output activation function, the number of cycles and +% hidden units and then train the network. The scaled conjugate +% gradient algorithm is used. A graph shows the evolution of the error: +% the value is shown MAX(CEIL(ITERATIONS / 50), 5) cycles. +% +% Once the network is trained, it is saved to the file MLPTRAIN.NET. +% The results can then be viewed as a confusion matrix (for +% classification problems) or a plot of output versus target (for +% regression problems). +% +% See also +% CONFMAT, DATREAD, MLP, NETOPT, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% If run without parameters, initialise gui. +if nargin<1, + action='initialise'; +end; + +% Global variable to reference GUI figure +global DEMTRAIN_FIG +% Global array to reference sub-figures for results plots +global DEMTRAIN_RES_FIGS +global NUM_DEMTRAIN_RES_FIGS + +if strcmp(action,'initialise'), + + file = ''; + path = '.'; + + % Create FIGURE + fig = figure( ... + 'Name', 'Netlab Demo', ... + 'NumberTitle', 'off', ... + 'Menubar', 'none', ... + 'Color', [0.7529 0.7529 0.7529], ... + 'Visible', 'on'); + % Initialise the globals + DEMTRAIN_FIG = fig; + DEMTRAIN_RES_FIGS = 0; + NUM_DEMTRAIN_RES_FIGS = 0; + + % Create GROUP for buttons + uicontrol(fig, ... + 'Style', 'frame', ... + 'Units', 'normalized', ... + 'Position', [0.03 0.08 0.94 0.22], ... + 'BackgroundColor', [0.5 0.5 0.5]); + + % Create MAIN axis + hMain = axes( ... + 'Units', 'normalized', ... + 'Position', [0.10 0.5 0.80 0.40], ... + 'XColor', [0 0 0], ... + 'YColor', [0 0 0], ... + 'Visible', 'on'); + + % Create static text for FILENAME and PATH + hFilename = uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.7529 0.7529 0.7529], ... + 'Position', [0.05 0.32 0.90 0.05], ... + 'HorizontalAlignment', 'center', ... + 'String', 'Please load data file.', ... + 'Visible', 'on'); + hPath = uicontrol(fig, ... + 'Style', 'text', ... + 'Units', 'normalized', ... + 'BackgroundColor', [0.7529 0.7529 0.7529], ... + 'Position', [0.05 0.37 0.90 0.05], ... + 'HorizontalAlignment', 'center', ... + 'String', '', ... + 'Visible', 'on'); + + % Create NO OF HIDDEN UNITS slider and text + hSliderText = uicontrol(fig, ... + 'Style', 'text', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Units', 'normalized', ... + 'Position', [0.27 0.12 0.17 0.04], ... + 'HorizontalAlignment', 'right', ... + 'String', 'Hidden Units: 5'); + hSlider = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Position', [0.45 0.12 0.26 0.04], ... + 'String', 'Slider', ... + 'Min', 1, 'Max', 25, ... + 'Value', 5, ... + 'Callback', 'demtrain slider_moved'); + + % Create ITERATIONS slider and text + hIterationsText = uicontrol(fig, ... + 'Style', 'text', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Units', 'normalized', ... + 'Position', [0.27 0.21 0.17 0.04], ... + 'HorizontalAlignment', 'right', ... + 'String', 'Iterations: 50'); + hIterations = uicontrol(fig, ... + 'Style', 'slider', ... + 'Units', 'normalized', ... + 'Position', [0.45 0.21 0.26 0.04], ... + 'String', 'Slider', ... + 'Min', 10, 'Max', 500, ... + 'Value', 50, ... + 'Callback', 'demtrain iterations_moved'); + + % Create ACTIVATION FUNCTION popup and text + uicontrol(fig, ... + 'Style', 'text', ... + 'BackgroundColor', [0.5 0.5 0.5], ... + 'Units', 'normalized', ... + 'Position', [0.05 0.20 0.20 0.04], ... + 'HorizontalAlignment', 'center', ... + 'String', 'Activation Function:'); + hPopup = uicontrol(fig, ... + 'Style', 'popup', ... + 'Units', 'normalized', ... + 'Position' , [0.05 0.10 0.20 0.08], ... + 'String', 'Linear|Logistic|Softmax', ... + 'Callback', ''); + + % Create MENU + hMenu1 = uimenu('Label', 'Load Data file...', 'Callback', ''); + uimenu(hMenu1, 'Label', 'Select training data file', ... + 'Callback', 'demtrain get_ip_file'); + hMenu2 = uimenu('Label', 'Show Results...', 'Callback', ''); + uimenu(hMenu2, 'Label', 'Show classification results', ... + 'Callback', 'demtrain classify'); + uimenu(hMenu2, 'Label', 'Show regression results', ... + 'Callback', 'demtrain predict'); + + % Create START button + hStart = uicontrol(fig, ... + 'Units', 'normalized', ... + 'Position' , [0.75 0.2 0.20 0.08], ... + 'String', 'Start Training', ... + 'Enable', 'off',... + 'Callback', 'demtrain start'); + + % Create CLOSE button + uicontrol(fig, ... + 'Units', 'normalized', ... + 'Position' , [0.75 0.1 0.20 0.08], ... + 'String', 'Close', ... + 'Callback', 'demtrain close'); + + % Save handles of important UI objects + hndlList = [hSlider hSliderText hFilename hPath hPopup ... + hIterations hIterationsText hStart]; + set(fig, 'UserData', hndlList); + % Hide window from command line + set(fig, 'HandleVisibility', 'callback'); + + +elseif strcmp(action, 'slider_moved'), + + % Slider has been moved. + + hndlList = get(gcf, 'UserData'); + hSlider = hndlList(1); + hSliderText = hndlList(2); + + val = get(hSlider, 'Value'); + if rem(val, 1) < 0.5, % Force up and down arrows to work! + val = ceil(val); + else + val = floor(val); + end; + set(hSlider, 'Value', val); + set(hSliderText, 'String', ['Hidden Units: ' int2str(val)]); + + +elseif strcmp(action, 'iterations_moved'), + + % Slider has been moved. + + hndlList = get(gcf, 'UserData'); + hSlider = hndlList(6); + hSliderText = hndlList(7); + + val = get(hSlider, 'Value'); + set(hSliderText, 'String', ['Iterations: ' int2str(val)]); + +elseif strcmp(action, 'get_ip_file'), + + % Get data file button pressed. + + hndlList = get(gcf, 'UserData'); + + [file, path] = uigetfile('*.dat', 'Get Data File', 50, 50); + + if strcmp(file, '') | file == 0, + set(hndlList(3), 'String', 'No data file loaded.'); + set(hndlList(4), 'String', ''); + else + set(hndlList(3), 'String', file); + set(hndlList(4), 'String', path); + end; + + % Enable training button + set(hndlList(8), 'Enable', 'on'); + + set(gcf, 'UserData', hndlList); + +elseif strcmp(action, 'start'), + + % Start training + + % Get handles of and values from UI objects + hndlList = get(gcf, 'UserData'); + hSlider = hndlList(1); % No of hidden units + hIterations = hndlList(6); + iterations = get(hIterations, 'Value'); + + hFilename = hndlList(3); % Data file name + filename = get(hFilename, 'String'); + + hPath = hndlList(4); % Data file path + path = get(hPath, 'String'); + + hPopup = hndlList(5); % Activation function + if get(hPopup, 'Value') == 1, + act_fn = 'linear'; + elseif get(hPopup, 'Value') == 2, + act_fn = 'logistic'; + else + act_fn = 'softmax'; + end; + nhidden = get(hSlider, 'Value'); + + % Check data file exists + if fopen([path '/' filename]) == -1, + errordlg('Training data file has not been selected.', 'Error'); + else + % Load data file + [x,t,nin,nout,ndata] = datread([path filename]); + + % Call MLPTRAIN function repeatedly, while drawing training graph. + figure(DEMTRAIN_FIG); + hold on; + + title('Training - please wait.'); + + % Create net and find initial error + net = mlp(size(x, 2), nhidden, size(t, 2), act_fn); + % Initialise network with inverse variance of 10 + net = mlpinit(net, 10); + error = mlperr(net, x, t); + % Work out reporting step: should be sufficiently big to let training + % algorithm have a chance + step = max(ceil(iterations / 50), 5); + + % Refresh and rescale axis. + cla; + max = error; + min = max/10; + set(gca, 'YScale', 'log'); + ylabel('log Error'); + xlabel('No. iterations'); + axis([0 iterations min max+1]); + iold = 0; + errold = error; + % Plot circle to show error of last iteration + % Setting erase mode to none prevents screen flashing during + % training + plot(0, error, 'ro', 'EraseMode', 'none'); + hold on + drawnow; % Force redraw + for i = step-1:step:iterations, + [net, error] = mlptrain(net, x, t, step); + % Plot line from last point to new point. + line([iold i], [errold error], 'Color', 'r', 'EraseMode', 'none'); + iold = i; + errold = error; + + % If new point off scale, redraw axes. + if error > max, + max = error; + axis([0 iterations min max+1]); + end; + if error < min + min = error/10; + axis([0 iterations min max+1]); + end + % Plot circle to show error of last iteration + plot(i, error, 'ro', 'EraseMode', 'none'); + drawnow; % Force redraw + end; + save mlptrain.net net + zoom on; + + title(['Training complete. Final error=', num2str(error)]); + + end; + +elseif strcmp(action, 'close'), + + % Close all the figures we have created + close(DEMTRAIN_FIG); + for n = 1:NUM_DEMTRAIN_RES_FIGS + if ishandle(DEMTRAIN_RES_FIGS(n)) + close(DEMTRAIN_RES_FIGS(n)); + end + end + +elseif strcmp(action, 'classify'), + + if fopen('mlptrain.net') == -1, + errordlg('You have not yet trained the network.', 'Error'); + else + + hndlList = get(gcf, 'UserData'); + filename = get(hndlList(3), 'String'); + path = get(hndlList(4), 'String'); + [x,t,nin,nout,ndata] = datread([path filename]); + load mlptrain.net net -mat + y = mlpfwd(net, x); + + % Save results figure so that it can be closed later + NUM_DEMTRAIN_RES_FIGS = NUM_DEMTRAIN_RES_FIGS + 1; + DEMTRAIN_RES_FIGS(NUM_DEMTRAIN_RES_FIGS)=conffig(y,t); + + end; + +elseif strcmp(action, 'predict'), + + if fopen('mlptrain.net') == -1, + errordlg('You have not yet trained the network.', 'Error'); + else + + hndlList = get(gcf, 'UserData'); + filename = get(hndlList(3), 'String'); + path = get(hndlList(4), 'String'); + [x,t,nin,nout,ndata] = datread([path filename]); + load mlptrain.net net -mat + y = mlpfwd(net, x); + + for i = 1:size(y,2), + % Save results figure so that it can be closed later + NUM_DEMTRAIN_RES_FIGS = NUM_DEMTRAIN_RES_FIGS + 1; + DEMTRAIN_RES_FIGS(NUM_DEMTRAIN_RES_FIGS) = figure; + hold on; + title(['Output no ' num2str(i)]); + plot([0 1], [0 1], 'r:'); + plot(y(:,i),t(:,i), 'o'); + hold off; + end; + end; + +end; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/dist2.m b/sourcecodes/bnt-master/netlab3.3/dist2.m new file mode 100644 index 00000000..b5f1c646 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/dist2.m @@ -0,0 +1,31 @@ +function n2 = dist2(x, c) +%DIST2 Calculates squared distance between two sets of points. +% +% Description +% D = DIST2(X, C) takes two matrices of vectors and calculates the +% squared Euclidean distance between them. Both matrices must be of +% the same column dimension. If X has M rows and N columns, and C has +% L rows and N columns, then the result has M rows and L columns. The +% I, Jth entry is the squared distance from the Ith row of X to the +% Jth row of C. +% +% See also +% GMMACTIV, KMEANS, RBFFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[ndata, dimx] = size(x); +[ncentres, dimc] = size(c); +if dimx ~= dimc + error('Data dimension does not match dimension of centres') +end + +n2 = (ones(ncentres, 1) * sum((x.^2)', 1))' + ... + ones(ndata, 1) * sum((c.^2)',1) - ... + 2.*(x*(c')); + +% Rounding errors occasionally cause negative entries in n2 +if any(any(n2<0)) + n2(n2<0) = 0; +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/eigdec.m b/sourcecodes/bnt-master/netlab3.3/eigdec.m new file mode 100644 index 00000000..f5901d42 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/eigdec.m @@ -0,0 +1,55 @@ +function [evals, evec] = eigdec(x, N) +%EIGDEC Sorted eigendecomposition +% +% Description +% EVALS = EIGDEC(X, N computes the largest N eigenvalues of the +% matrix X in descending order. [EVALS, EVEC] = EIGDEC(X, N) also +% computes the corresponding eigenvectors. +% +% See also +% PCA, PPCA +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargout == 1 + evals_only = logical(1); +else + evals_only = logical(0); +end + +if N ~= round(N) | N < 1 | N > size(x, 2) + error('Number of PCs must be integer, >0, < dim'); +end + +% Find the eigenvalues of the data covariance matrix +if evals_only + % Use eig function as always more efficient than eigs here + temp_evals = eig(x); +else + % Use eig function unless fraction of eigenvalues required is tiny + if (N/size(x, 2)) > 0.04 + [temp_evec, temp_evals] = eig(x); + else + options.disp = 0; + [temp_evec, temp_evals] = eigs(x, N, 'LM', options); + end + temp_evals = diag(temp_evals); +end + +% Eigenvalues nearly always returned in descending order, but just +% to make sure..... +[evals perm] = sort(-temp_evals); +evals = -evals(1:N); +if ~evals_only + if evals == temp_evals(1:N) + % Originals were in order + evec = temp_evec(:, 1:N); + return + else + % Need to reorder the eigenvectors + for i=1:N + evec(:,i) = temp_evec(:,perm(i)); + end + end +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/errbayes.m b/sourcecodes/bnt-master/netlab3.3/errbayes.m new file mode 100644 index 00000000..7c2a330b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/errbayes.m @@ -0,0 +1,49 @@ +function [e, edata, eprior] = errbayes(net, edata) +%ERRBAYES Evaluate Bayesian error function for network. +% +% Description +% E = ERRBAYES(NET, EDATA) takes a network data structure NET together +% the data contribution to the error for a set of inputs and targets. +% It returns the regularised error using any zero mean Gaussian priors +% on the weights defined in NET. +% +% [E, EDATA, EPRIOR] = ERRBAYES(NET, X, T) additionally returns the +% data and prior components of the error. +% +% See also +% GLMERR, MLPERR, RBFERR +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Evaluate the data contribution to the error. +if isfield(net, 'beta') + e1 = net.beta*edata; +else + e1 = edata; +end + +% Evaluate the prior contribution to the error. +if isfield(net, 'alpha') + w = netpak(net); + if size(net.alpha) == [1 1] + eprior = 0.5*(w*w'); + e2 = eprior*net.alpha; + else + if (isfield(net, 'mask')) + nindx_cols = size(net.index, 2); + nmask_rows = size(find(net.mask), 1); + index = reshape(net.index(logical(repmat(net.mask, ... + 1, nindx_cols))), nmask_rows, nindx_cols); + else + index = net.index; + end + eprior = 0.5*(w.^2)*index; + e2 = eprior*net.alpha; + end +else + eprior = 0; + e2 = 0; +end + +e = e1 + e2; diff --git a/sourcecodes/bnt-master/netlab3.3/evidence.m b/sourcecodes/bnt-master/netlab3.3/evidence.m new file mode 100644 index 00000000..ab023cb8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/evidence.m @@ -0,0 +1,114 @@ +function [net, gamma, logev] = evidence(net, x, t, num) +%EVIDENCE Re-estimate hyperparameters using evidence approximation. +% +% Description +% [NET] = EVIDENCE(NET, X, T) re-estimates the hyperparameters ALPHA +% and BETA by applying Bayesian re-estimation formulae for NUM +% iterations. The hyperparameter ALPHA can be a simple scalar +% associated with an isotropic prior on the weights, or can be a vector +% in which each component is associated with a group of weights as +% defined by the INDEX matrix in the NET data structure. These more +% complex priors can be set up for an MLP using MLPPRIOR. Initial +% values for the iterative re-estimation are taken from the network +% data structure NET passed as an input argument, while the return +% argument NET contains the re-estimated values. +% +% [NET, GAMMA, LOGEV] = EVIDENCE(NET, X, T, NUM) allows the re- +% estimation formula to be applied for NUM cycles in which the re- +% estimated values for the hyperparameters from each cycle are used to +% re-evaluate the Hessian matrix for the next cycle. The return value +% GAMMA is the number of well-determined parameters and LOGEV is the +% log of the evidence. +% +% See also +% MLPPRIOR, NETGRAD, NETHESS, DEMEV1, DEMARD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, '', x, t); +if ~isempty(errstring) + error(errstring); +end + +ndata = size(x, 1); +if nargin == 3 + num = 1; +end + +% Extract weights from network +w = netpak(net); + +% Evaluate data-dependent contribution to the Hessian matrix. +[h, dh] = nethess(w, net, x, t); +clear h; % To save memory when Hessian is large +if (~isfield(net, 'beta')) + local_beta = 1; +end + +[evec, evl] = eig(dh); +% Now set the negative eigenvalues to zero. +evl = evl.*(evl > 0); +% safe_evl is used to avoid taking log of zero +safe_evl = evl + eps.*(evl <= 0); + +[e, edata, eprior] = neterr(w, net, x, t); + +if size(net.alpha) == [1 1] + % Form vector of eigenvalues + evl = diag(evl); + safe_evl = diag(safe_evl); +else + ngroups = size(net.alpha, 1); + gams = zeros(1, ngroups); + logas = zeros(1, ngroups); + % Reconstruct data hessian with negative eigenvalues set to zero. + dh = evec*evl*evec'; +end + +% Do the re-estimation. +for k = 1 : num + % Re-estimate alpha. + if size(net.alpha) == [1 1] + % Evaluate number of well-determined parameters. + L = evl; + if isfield(net, 'beta') + L = net.beta*L; + end + gamma = sum(L./(L + net.alpha)); + net.alpha = 0.5*gamma/eprior; + % Partially evaluate log evidence: only include unmasked weights + logev = 0.5*length(w)*log(net.alpha); + else + hinv = inv(hbayes(net, dh)); + for m = 1 : ngroups + group_nweights = sum(net.index(:, m)); + gams(m) = group_nweights - ... + net.alpha(m)*sum(diag(hinv).*net.index(:,m)); + net.alpha(m) = real(gams(m)/(2*eprior(m))); + % Weight alphas by number of weights in group + logas(m) = 0.5*group_nweights*log(net.alpha(m)); + end + gamma = sum(gams, 2); + logev = sum(logas); + end + % Re-estimate beta. + if isfield(net, 'beta') + net.beta = 0.5*(net.nout*ndata - gamma)/edata; + logev = logev + 0.5*ndata*log(net.beta) - 0.5*ndata*log(2*pi); + local_beta = net.beta; + end + + % Evaluate new log evidence + e = errbayes(net, edata); + if size(net.alpha) == [1 1] + logev = logev - e - 0.5*sum(log(local_beta*safe_evl+net.alpha)); + else + for m = 1:ngroups + logev = logev - e - ... + 0.5*sum(log(local_beta*(safe_evl*net.index(:, m))+... + net.alpha(m))); + end + end +end + diff --git a/sourcecodes/bnt-master/netlab3.3/fevbayes.m b/sourcecodes/bnt-master/netlab3.3/fevbayes.m new file mode 100644 index 00000000..d5681596 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/fevbayes.m @@ -0,0 +1,59 @@ +function [extra, invhess] = fevbayes(net, y, a, x, t, x_test, invhess) +%FEVBAYES Evaluate Bayesian regularisation for network forward propagation. +% +% Description +% EXTRA = FEVBAYES(NET, Y, A, X, T, X_TEST) takes a network data +% structure NET together with a set of hidden unit activations A from +% test inputs X_TEST, training data inputs X and T and outputs a matrix +% of extra information EXTRA that consists of error bars (variance) for +% a regression problem or moderated outputs for a classification +% problem. The optional argument (and return value) INVHESS is the +% inverse of the network Hessian computed on the training data inputs +% and targets. Passing it in avoids recomputing it, which can be a +% significant saving for large training sets. +% +% This is called by network-specific functions such as MLPEVFWD which +% are needed since the return values (predictions and hidden unit +% activations) for different network types are in different orders (for +% good reasons). +% +% See also +% MLPEVFWD, RBFEVFWD, GLMEVFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +w = netpak(net); +g = netderiv(w, net, x_test); +if nargin < 7 + % Need to compute inverse hessian + hess = nethess(w, net, x, t); + invhess = inv(hess); +end + +ntest = size(x_test, 1); +var = zeros(ntest, 1); +for idx = 1:1:net.nout, + for n = 1:1:ntest, + grad = squeeze(g(n,:,idx)); + var(n,idx) = grad*invhess*grad'; + end +end + +switch net.outfn + case 'linear' + % extra is variance + extra = ones(size(var))./net.beta + var; + case 'logistic' + % extra is moderated output + kappa = 1./(sqrt(ones(size(var)) + (pi.*var)./8)); + extra = 1./(1 + exp(-kappa.*a)); + case 'softmax' + % Use extended Mackay formula; beware that this may not + % be very accurate + kappa = 1./(sqrt(ones(size(var)) + (pi.*var)./8)); + temp = exp(kappa.*a); + extra = temp./(sum(temp, 2)*ones(1, net.nout)); + otherwise + error(['Unknown activation function ', net.outfn]); +end diff --git a/sourcecodes/bnt-master/netlab3.3/gauss.m b/sourcecodes/bnt-master/netlab3.3/gauss.m new file mode 100644 index 00000000..33e23136 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gauss.m @@ -0,0 +1,33 @@ +function y = gauss(mu, covar, x) +%GAUSS Evaluate a Gaussian distribution. +% +% Description +% +% Y = GAUSS(MU, COVAR, X) evaluates a multi-variate Gaussian density +% in D-dimensions at a set of points given by the rows of the matrix X. +% The Gaussian density has mean vector MU and covariance matrix COVAR. +% +% See also +% GSAMP, DEMGAUSS +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[n, d] = size(x); + +[j, k] = size(covar); + +% Check that the covariance matrix is the correct dimension +if ((j ~= d) | (k ~=d)) + error('Dimension of the covariance matrix and data should match'); +end + +invcov = inv(covar); +mu = reshape(mu, 1, d); % Ensure that mu is a row vector + +x = x - ones(n, 1)*mu; +fact = sum(((x*invcov).*x), 2); + +y = exp(-0.5*fact); + +y = y./sqrt((2*pi)^d*det(covar)); diff --git a/sourcecodes/bnt-master/netlab3.3/gbayes.m b/sourcecodes/bnt-master/netlab3.3/gbayes.m new file mode 100644 index 00000000..7b814e97 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gbayes.m @@ -0,0 +1,56 @@ +function [g, gdata, gprior] = gbayes(net, gdata) +%GBAYES Evaluate gradient of Bayesian error function for network. +% +% Description +% G = GBAYES(NET, GDATA) takes a network data structure NET together +% the data contribution to the error gradient for a set of inputs and +% targets. It returns the regularised error gradient using any zero +% mean Gaussian priors on the weights defined in NET. In addition, if +% a MASK is defined in NET, then the entries in G that correspond to +% weights with a 0 in the mask are removed. +% +% [G, GDATA, GPRIOR] = GBAYES(NET, GDATA) additionally returns the data +% and prior components of the error. +% +% See also +% ERRBAYES, GLMGRAD, MLPGRAD, RBFGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Evaluate the data contribution to the gradient. +if (isfield(net, 'mask')) + gdata = gdata(logical(net.mask)); +end +if isfield(net, 'beta') + g1 = gdata*net.beta; +else + g1 = gdata; +end + +% Evaluate the prior contribution to the gradient. +if isfield(net, 'alpha') + w = netpak(net); + if size(net.alpha) == [1 1] + gprior = w; + g2 = net.alpha*gprior; + else + if (isfield(net, 'mask')) + nindx_cols = size(net.index, 2); + nmask_rows = size(find(net.mask), 1); + index = reshape(net.index(logical(repmat(net.mask, ... + 1, nindx_cols))), nmask_rows, nindx_cols); + else + index = net.index; + end + + ngroups = size(net.alpha, 1); + gprior = index'.*(ones(ngroups, 1)*w); + g2 = net.alpha'*gprior; + end +else + gprior = 0; + g2 = 0; +end + +g = g1 + g2; diff --git a/sourcecodes/bnt-master/netlab3.3/glm.m b/sourcecodes/bnt-master/netlab3.3/glm.m new file mode 100644 index 00000000..7741171a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glm.m @@ -0,0 +1,82 @@ +function net = glm(nin, nout, outfunc, prior, beta) +%GLM Create a generalized linear model. +% +% Description +% +% NET = GLM(NIN, NOUT, FUNC) takes the number of inputs and outputs for +% a generalized linear model, together with a string FUNC which +% specifies the output unit activation function, and returns a data +% structure NET. The weights are drawn from a zero mean, isotropic +% Gaussian, with variance scaled by the fan-in of the output units. +% This makes use of the Matlab function RANDN and so the seed for the +% random weight initialization can be set using RANDN('STATE', S) +% where S is the seed value. The optional argument ALPHA sets the +% inverse variance for the weight initialization. +% +% The fields in NET are +% type = 'glm' +% nin = number of inputs +% nout = number of outputs +% nwts = total number of weights and biases +% actfn = string describing the output unit activation function: +% 'linear' +% 'logistic' +% 'softmax' +% w1 = first-layer weight matrix +% b1 = first-layer bias vector +% +% NET = GLM(NIN, NOUT, FUNC, PRIOR), in which PRIOR is a scalar, allows +% the field NET.ALPHA in the data structure NET to be set, +% corresponding to a zero-mean isotropic Gaussian prior with inverse +% variance with value PRIOR. Alternatively, PRIOR can consist of a data +% structure with fields ALPHA and INDEX, allowing individual Gaussian +% priors to be set over groups of weights in the network. Here ALPHA is +% a column vector in which each element corresponds to a separate +% group of weights, which need not be mutually exclusive. The +% membership of the groups is defined by the matrix INDEX in which the +% columns correspond to the elements of ALPHA. Each column has one +% element for each weight in the matrix, in the order defined by the +% function GLMPAK, and each element is 1 or 0 according to whether the +% weight is a member of the corresponding group or not. +% +% NET = GLM(NIN, NOUT, FUNC, PRIOR, BETA) also sets the additional +% field NET.BETA in the data structure NET, where beta corresponds to +% the inverse noise variance. +% +% See also +% GLMPAK, GLMUNPAK, GLMFWD, GLMERR, GLMGRAD, GLMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.type = 'glm'; +net.nin = nin; +net.nout = nout; +net.nwts = (nin + 1)*nout; + +outtfns = {'linear', 'logistic', 'softmax'}; + +if sum(strcmp(outfunc, outtfns)) == 0 + error('Undefined activation function. Exiting.'); +else + net.outfn = outfunc; +end + +if nargin > 3 + if isstruct(prior) + net.alpha = prior.alpha; + net.index = prior.index; + elseif size(prior) == [1 1] + net.alpha = prior; + else + error('prior must be a scalar or structure'); + end +end + +net.w1 = randn(nin, nout)/sqrt(nin + 1); +net.b1 = randn(1, nout)/sqrt(nin + 1); + +if nargin == 5 + net.beta = beta; +end + diff --git a/sourcecodes/bnt-master/netlab3.3/glmderiv.m b/sourcecodes/bnt-master/netlab3.3/glmderiv.m new file mode 100644 index 00000000..bc8de671 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmderiv.m @@ -0,0 +1,40 @@ +function g = glmderiv(net, x) +%GLMDERIV Evaluate derivatives of GLM outputs with respect to weights. +% +% Description +% G = GLMDERIV(NET, X) takes a network data structure NET and a matrix +% of input vectors X and returns a three-index matrix mat{g} whose I, +% J, K element contains the derivative of network output K with respect +% to weight or bias parameter J for input pattern I. The ordering of +% the weight and bias parameters is defined by GLMUNPAK. +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm', x); +if ~isempty(errstring) + error(errstring); +end + +ndata = size(x, 1); +if isfield(net, 'mask') + nwts = size(find(net.mask), 1); + mask_array = logical(net.mask)*ones(1, net.nout); +else + nwts = net.nwts; +end +g = zeros(ndata, nwts, net.nout); + +temp = zeros(net.nwts, net.nout); +for n = 1:ndata + % Weight matrix w1 + temp(1:(net.nin*net.nout), :) = kron(eye(net.nout), (x(n, :))'); + % Bias term b1 + temp(net.nin*net.nout+1:end, :) = eye(net.nout); + if isfield(net, 'mask') + g(n, :, :) = reshape(temp(find(mask_array)), nwts, net.nout); + else + g(n, :, :) = temp; + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/glmerr.m b/sourcecodes/bnt-master/netlab3.3/glmerr.m new file mode 100644 index 00000000..bab849c3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmerr.m @@ -0,0 +1,49 @@ +function [e, edata, eprior, y, a] = glmerr(net, x, t) +%GLMERR Evaluate error function for generalized linear model. +% +% Description +% E = GLMERR(NET, X, T) takes a generalized linear model data +% structure NET together with a matrix X of input vectors and a matrix +% T of target vectors, and evaluates the error function E. The choice +% of error function corresponds to the output unit activation function. +% Each row of X corresponds to one input vector and each row of T +% corresponds to one target vector. +% +% [E, EDATA, EPRIOR, Y, A] = GLMERR(NET, X, T) also returns the data +% and prior components of the total error. +% +% [E, EDATA, EPRIOR, Y, A] = GLMERR(NET, X) also returns a matrix Y +% giving the outputs of the models and a matrix A giving the summed +% inputs to each output unit, where each row corresponds to one +% pattern. +% +% See also +% GLM, GLMPAK, GLMUNPAK, GLMFWD, GLMGRAD, GLMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm', x, t); +if ~isempty(errstring); + error(errstring); +end + +[y, a] = glmfwd(net, x); + +switch net.outfn + + case 'linear' % Linear outputs + edata = 0.5*sum(sum((y - t).^2)); + + case 'logistic' % Logistic outputs + edata = - sum(sum(t.*log(y) + (1 - t).*log(1 - y))); + + case 'softmax' % Softmax outputs + edata = - sum(sum(t.*log(y))); + + otherwise + error(['Unknown activation function ', net.outfn]); +end + +[e, edata, eprior] = errbayes(net, edata); diff --git a/sourcecodes/bnt-master/netlab3.3/glmevfwd.m b/sourcecodes/bnt-master/netlab3.3/glmevfwd.m new file mode 100644 index 00000000..d445a367 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmevfwd.m @@ -0,0 +1,28 @@ +function [y, extra, invhess] = glmevfwd(net, x, t, x_test, invhess) +%GLMEVFWD Forward propagation with evidence for GLM +% +% Description +% Y = GLMEVFWD(NET, X, T, X_TEST) takes a network data structure NET +% together with the input X and target T training data and input test +% data X_TEST. It returns the normal forward propagation through the +% network Y together with a matrix EXTRA which consists of error bars +% (variance) for a regression problem or moderated outputs for a +% classification problem. +% +% The optional argument (and return value) INVHESS is the inverse of +% the network Hessian computed on the training data inputs and targets. +% Passing it in avoids recomputing it, which can be a significant +% saving for large training sets. +% +% See also +% FEVBAYES +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[y, a] = glmfwd(net, x_test); +if nargin == 4 + [extra, invhess] = fevbayes(net, y, a, x, t, x_test); +else + [extra, invhess] = fevbayes(net, y, a, x, t, x_test, invhess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/glmfwd.m b/sourcecodes/bnt-master/netlab3.3/glmfwd.m new file mode 100644 index 00000000..5a9519cd --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmfwd.m @@ -0,0 +1,62 @@ +function [y, a] = glmfwd(net, x) +%GLMFWD Forward propagation through generalized linear model. +% +% Description +% Y = GLMFWD(NET, X) takes a generalized linear model data structure +% NET together with a matrix X of input vectors, and forward propagates +% the inputs through the network to generate a matrix Y of output +% vectors. Each row of X corresponds to one input vector and each row +% of Y corresponds to one output vector. +% +% [Y, A] = GLMFWD(NET, X) also returns a matrix A giving the summed +% inputs to each output unit, where each row corresponds to one +% pattern. +% +% See also +% GLM, GLMPAK, GLMUNPAK, GLMERR, GLMGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm', x); +if ~isempty(errstring); + error(errstring); +end + +ndata = size(x, 1); + +a = x*net.w1 + ones(ndata, 1)*net.b1; + +switch net.outfn + + case 'linear' % Linear outputs + y = a; + + case 'logistic' % Logistic outputs + % Prevent overflow and underflow: use same bounds as glmerr + % Ensure that log(1-y) is computable: need exp(a) > eps + maxcut = -log(eps); + % Ensure that log(y) is computable + mincut = -log(1/realmin - 1); + a = min(a, maxcut); + a = max(a, mincut); + y = 1./(1 + exp(-a)); + + case 'softmax' % Softmax outputs + nout = size(a,2); + % Prevent overflow and underflow: use same bounds as glmerr + % Ensure that sum(exp(a), 2) does not overflow + maxcut = log(realmax) - log(nout); + % Ensure that exp(a) > 0 + mincut = log(realmin); + a = min(a, maxcut); + a = max(a, mincut); + temp = exp(a); + y = temp./(sum(temp, 2)*ones(1,nout)); + % Ensure that log(y) is computable + y(y<realmin) = realmin; + + otherwise + error(['Unknown activation function ', net.outfn]); +end diff --git a/sourcecodes/bnt-master/netlab3.3/glmgrad.m b/sourcecodes/bnt-master/netlab3.3/glmgrad.m new file mode 100644 index 00000000..d967804e --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmgrad.m @@ -0,0 +1,36 @@ +function [g, gdata, gprior] = glmgrad(net, x, t) +%GLMGRAD Evaluate gradient of error function for generalized linear model. +% +% Description +% G = GLMGRAD(NET, X, T) takes a generalized linear model data +% structure NET together with a matrix X of input vectors and a matrix +% T of target vectors, and evaluates the gradient G of the error +% function with respect to the network weights. The error function +% corresponds to the choice of output unit activation function. Each +% row of X corresponds to one input vector and each row of T +% corresponds to one target vector. +% +% [G, GDATA, GPRIOR] = GLMGRAD(NET, X, T) also returns separately the +% data and prior contributions to the gradient. +% +% See also +% GLM, GLMPAK, GLMUNPAK, GLMFWD, GLMERR, GLMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm', x, t); +if ~isempty(errstring); + error(errstring); +end + +y = glmfwd(net, x); +delout = y - t; + +gw1 = x'*delout; +gb1 = sum(delout, 1); + +gdata = [gw1(:)', gb1]; + +[g, gdata, gprior] = gbayes(net, gdata); diff --git a/sourcecodes/bnt-master/netlab3.3/glmhess.m b/sourcecodes/bnt-master/netlab3.3/glmhess.m new file mode 100644 index 00000000..21fc8ad2 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmhess.m @@ -0,0 +1,106 @@ +function [h, hdata] = glmhess(net, x, t, hdata) +%GLMHESS Evaluate the Hessian matrix for a generalised linear model. +% +% Description +% H = GLMHESS(NET, X, T) takes a GLM network data structure NET, a +% matrix X of input values, and a matrix T of target values and returns +% the full Hessian matrix H corresponding to the second derivatives of +% the negative log posterior distribution, evaluated for the current +% weight and bias values as defined by NET. Note that the target data +% is not required in the calculation, but is included to make the +% interface uniform with NETHESS. For linear and logistic outputs, the +% computation is very simple and is done (in effect) in one line in +% GLMTRAIN. +% +% [H, HDATA] = GLMHESS(NET, X, T) returns both the Hessian matrix H and +% the contribution HDATA arising from the data dependent term in the +% Hessian. +% +% H = GLMHESS(NET, X, T, HDATA) takes a network data structure NET, a +% matrix X of input values, and a matrix T of target values, together +% with the contribution HDATA arising from the data dependent term in +% the Hessian, and returns the full Hessian matrix H corresponding to +% the second derivatives of the negative log posterior distribution. +% This version saves computation time if HDATA has already been +% evaluated for the current weight and bias values. +% +% See also +% GLM, GLMTRAIN, HESSCHEK, NETHESS +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm', x, t); +if ~isempty(errstring); + error(errstring); +end + +ndata = size(x, 1); +nparams = net.nwts; +nout = net.nout; +p = glmfwd(net, x); +inputs = [x ones(ndata, 1)]; + +if nargin == 3 + hdata = zeros(nparams); % Full Hessian matrix + % Calculate data component of Hessian + switch net.outfn + + case 'linear' + % No weighting function here + out_hess = [x ones(ndata, 1)]'*[x ones(ndata, 1)]; + for j = 1:nout + hdata = rearrange_hess(net, j, out_hess, hdata); + end + case 'logistic' + % Each output is independent + e = ones(1, net.nin+1); + link_deriv = p.*(1-p); + out_hess = zeros(net.nin+1); + for j = 1:nout + inputs = [x ones(ndata, 1)].*(sqrt(link_deriv(:,j))*e); + out_hess = inputs'*inputs; % Hessian for this output + hdata = rearrange_hess(net, j, out_hess, hdata); + end + + case 'softmax' + bb_start = nparams - nout + 1; % Start of bias weights block + ex_hess = zeros(nparams); % Contribution to Hessian from single example + for m = 1:ndata + X = x(m,:)'*x(m,:); + a = diag(p(m,:))-((p(m,:)')*p(m,:)); + ex_hess(1:nparams-nout,1:nparams-nout) = kron(a, X); + ex_hess(bb_start:nparams, bb_start:nparams) = a.*ones(net.nout, net.nout); + temp = kron(a, x(m,:)); + ex_hess(bb_start:nparams, 1:nparams-nout) = temp; + ex_hess(1:nparams-nout, bb_start:nparams) = temp'; + hdata = hdata + ex_hess; + end + otherwise + error(['Unknown activation function ', net.outfn]); + end +end + +[h, hdata] = hbayes(net, hdata); + +function hdata = rearrange_hess(net, j, out_hess, hdata) + +% Because all the biases come after all the input weights, +% we have to rearrange the blocks that make up the network Hessian. +% This function assumes that we are on the jth output and that all outputs +% are independent. + +bb_start = net.nwts - net.nout + 1; % Start of bias weights block +ob_start = 1+(j-1)*net.nin; % Start of weight block for jth output +ob_end = j*net.nin; % End of weight block for jth output +b_index = bb_start+(j-1); % Index of bias weight +% Put input weight block in right place +hdata(ob_start:ob_end, ob_start:ob_end) = out_hess(1:net.nin, 1:net.nin); +% Put second derivative of bias weight in right place +hdata(b_index, b_index) = out_hess(net.nin+1, net.nin+1); +% Put cross terms (input weight v bias weight) in right place +hdata(b_index, ob_start:ob_end) = out_hess(net.nin+1,1:net.nin); +hdata(ob_start:ob_end, b_index) = out_hess(1:net.nin, net.nin+1); + +return \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/glminit.m b/sourcecodes/bnt-master/netlab3.3/glminit.m new file mode 100644 index 00000000..3dc47bc7 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glminit.m @@ -0,0 +1,36 @@ +function net = glminit(net, prior) +%GLMINIT Initialise the weights in a generalized linear model. +% +% Description +% +% NET = GLMINIT(NET, PRIOR) takes a generalized linear model NET and +% sets the weights and biases by sampling from a Gaussian distribution. +% If PRIOR is a scalar, then all of the parameters (weights and biases) +% are sampled from a single isotropic Gaussian with inverse variance +% equal to PRIOR. If PRIOR is a data structure similar to that in +% MLPPRIOR but for a single layer of weights, then the parameters are +% sampled from multiple Gaussians according to their groupings (defined +% by the INDEX field) with corresponding variances (defined by the +% ALPHA field). +% +% See also +% GLM, GLMPAK, GLMUNPAK, MLPINIT, MLPPRIOR +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'glm'); +if ~isempty(errstring); + error(errstring); +end +if isstruct(prior) + sig = 1./sqrt(prior.index*prior.alpha); + w = sig'.*randn(1, net.nwts); +elseif size(prior) == [1 1] + w = randn(1, net.nwts).*sqrt(1/prior); +else + error('prior must be a scalar or a structure'); +end + +net = glmunpak(net, w); + diff --git a/sourcecodes/bnt-master/netlab3.3/glmpak.m b/sourcecodes/bnt-master/netlab3.3/glmpak.m new file mode 100644 index 00000000..300960fc --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmpak.m @@ -0,0 +1,20 @@ +function w = glmpak(net) +%GLMPAK Combines weights and biases into one weights vector. +% +% Description +% W = GLMPAK(NET) takes a network data structure NET and combines them +% into a single row vector W. +% +% See also +% GLM, GLMUNPAK, GLMFWD, GLMERR, GLMGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'glm'); +if ~errstring + error(errstring); +end + +w = [net.w1(:)', net.b1]; + diff --git a/sourcecodes/bnt-master/netlab3.3/glmtrain.m b/sourcecodes/bnt-master/netlab3.3/glmtrain.m new file mode 100644 index 00000000..8dd0e3db --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmtrain.m @@ -0,0 +1,206 @@ +function [net, options] = glmtrain(net, options, x, t) +%GLMTRAIN Specialised training of generalized linear model +% +% Description +% NET = GLMTRAIN(NET, OPTIONS, X, T) uses the iterative reweighted +% least squares (IRLS) algorithm to set the weights in the generalized +% linear model structure NET. This is a more efficient alternative to +% using GLMERR and GLMGRAD and a non-linear optimisation routine +% through NETOPT. Note that for linear outputs, a single pass through +% the algorithm is all that is required, since the error function is +% quadratic in the weights. The algorithm also handles scalar ALPHA +% and BETA terms. If you want to use more complicated priors, you +% should use general-purpose non-linear optimisation algorithms. +% +% For logistic and softmax outputs, general priors can be handled, +% although this requires the pseudo-inverse of the Hessian, giving up +% the better conditioning and some of the speed advantage of the normal +% form equations. +% +% The error function value at the final set of weights is returned in +% OPTIONS(8). Each row of X corresponds to one input vector and each +% row of T corresponds to one target vector. +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values during training. If +% OPTIONS(1) is set to 0, then only warning messages are displayed. If +% OPTIONS(1) is -1, then nothing is displayed. +% +% OPTIONS(2) is a measure of the precision required for the value of +% the weights W at the solution. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. Both this and the previous condition must +% be satisfied for termination. +% +% OPTIONS(5) is set to 1 if an approximation to the Hessian (which +% assumes that all outputs are independent) is used for softmax +% outputs. With the default value of 0 the exact Hessian (which is more +% expensive to compute) is used. +% +% OPTIONS(14) is the maximum number of iterations for the IRLS +% algorithm; default 100. +% +% See also +% GLM, GLMERR, GLMGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm', x, t); +if ~errstring + error(errstring); +end + +if(~options(14)) + options(14) = 100; +end + +display = options(1); +% Do we need to test for termination? +test = (options(2) | options(3)); + +ndata = size(x, 1); +% Add a column of ones for the bias +inputs = [x ones(ndata, 1)]; + +% Linear outputs are a special case as they can be found in one step +if strcmp(net.outfn, 'linear') + if ~isfield(net, 'alpha') + % Solve for the weights and biases using left matrix divide + temp = inputs\t; + elseif size(net.alpha == [1 1]) + if isfield(net, 'beta') + beta = net.beta; + else + beta = 1.0; + end + % Use normal form equation + hessian = beta*(inputs'*inputs) + net.alpha*eye(net.nin+1); + temp = pinv(hessian)*(beta*(inputs'*t)); + else + error('Only scalar alpha allowed'); + end + net.w1 = temp(1:net.nin, :); + net.b1 = temp(net.nin+1, :); + % Store error value in options vector + options(8) = glmerr(net, x, t); + return; +end + +% Otherwise need to use iterative reweighted least squares +e = ones(1, net.nin+1); +for n = 1:options(14) + + switch net.outfn + case 'logistic' + if n == 1 + % Initialise model + p = (t+0.5)/2; + act = log(p./(1-p)); + wold = glmpak(net); + end + link_deriv = p.*(1-p); + weights = sqrt(link_deriv); % sqrt of weights + if (min(min(weights)) < eps) + warning('ill-conditioned weights in glmtrain') + return + end + z = act + (t-p)./link_deriv; + if ~isfield(net, 'alpha') + % Treat each output independently with relevant set of weights + for j = 1:net.nout + indep = inputs.*(weights(:,j)*e); + dep = z(:,j).*weights(:,j); + temp = indep\dep; + net.w1(:,j) = temp(1:net.nin); + net.b1(j) = temp(net.nin+1); + end + else + gradient = glmgrad(net, x, t); + Hessian = glmhess(net, x, t); + deltaw = -gradient*pinv(Hessian); + w = wold + deltaw; + net = glmunpak(net, w); + end + [err, edata, eprior, p, act] = glmerr(net, x, t); + if n == 1 + errold = err; + wold = netpak(net); + else + w = netpak(net); + end + case 'softmax' + if n == 1 + % Initialise model: ensure that row sum of p is one no matter + % how many classes there are + p = (t + (1/size(t, 2)))/2; + act = log(p./(1-p)); + end + if options(5) == 1 | n == 1 + link_deriv = p.*(1-p); + weights = sqrt(link_deriv); % sqrt of weights + if (min(min(weights)) < eps) + warning('ill-conditioned weights in glmtrain') + return + end + z = act + (t-p)./link_deriv; + % Treat each output independently with relevant set of weights + for j = 1:net.nout + indep = inputs.*(weights(:,j)*e); + dep = z(:,j).*weights(:,j); + temp = indep\dep; + net.w1(:,j) = temp(1:net.nin); + net.b1(j) = temp(net.nin+1); + end + [err, edata, eprior, p, act] = glmerr(net, x, t); + if n == 1 + errold = err; + wold = netpak(net); + else + w = netpak(net); + end + else + % Exact method of calculation after w first initialised + % Start by working out Hessian + Hessian = glmhess(net, x, t); + gradient = glmgrad(net, x, t); + % Now compute modification to weights + deltaw = -gradient*pinv(Hessian); + w = wold + deltaw; + net = glmunpak(net, w); + [err, edata, eprior, p] = glmerr(net, x, t); + end + + otherwise + error(['Unknown activation function ', net.outfn]); + end + if options(1) + fprintf(1, 'Cycle %4d Error %11.6f\n', n, err) + end + % Test for termination + % Terminate if error increases + if err > errold + errold = err; + w = wold; + options(8) = err; + fprintf(1, 'Error has increased: terminating\n') + return; + end + if test && n > 1 + if (max(abs(w - wold)) < options(2) && abs(err-errold) < options(3)) + options(8) = err; + return; + else + errold = err; + wold = w; + end + end +end + +options(8) = err; +if (options(1) >= 0) + disp(maxitmess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/glmunpak.m b/sourcecodes/bnt-master/netlab3.3/glmunpak.m new file mode 100644 index 00000000..478ef657 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/glmunpak.m @@ -0,0 +1,30 @@ +function net = glmunpak(net, w) +%GLMUNPAK Separates weights vector into weight and bias matrices. +% +% Description +% NET = GLMUNPAK(NET, W) takes a glm network data structure NET and a +% weight vector W, and returns a network data structure identical to +% the input network, except that the first-layer weight matrix W1 and +% the first-layer bias vector B1 have been set to the corresponding +% elements of W. +% +% See also +% GLM, GLMPAK, GLMFWD, GLMERR, GLMGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'glm'); +if ~errstring + error(errstring); +end + +if net.nwts ~= length(w) + error('Invalid weight vector length') +end + +nin = net.nin; +nout = net.nout; +net.w1 = reshape(w(1:nin*nout), nin, nout); +net.b1 = reshape(w(nin*nout + 1: (nin + 1)*nout), 1, nout); diff --git a/sourcecodes/bnt-master/netlab3.3/gmm.m b/sourcecodes/bnt-master/netlab3.3/gmm.m new file mode 100644 index 00000000..248de8f5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmm.m @@ -0,0 +1,113 @@ +function mix = gmm(dim, ncentres, covar_type, ppca_dim) +%GMM Creates a Gaussian mixture model with specified architecture. +% +% Description +% MIX = GMM(DIM, NCENTRES, COVARTYPE) takes the dimension of the space +% DIM, the number of centres in the mixture model and the type of the +% mixture model, and returns a data structure MIX. The mixture model +% type defines the covariance structure of each component Gaussian: +% 'spherical' = single variance parameter for each component: stored as a vector +% 'diag' = diagonal matrix for each component: stored as rows of a matrix +% 'full' = full matrix for each component: stored as 3d array +% 'ppca' = probabilistic PCA: stored as principal components (in a 3d array +% and associated variances and off-subspace noise +% MIX = GMM(DIM, NCENTRES, COVARTYPE, PPCA_DIM) also sets the +% dimension of the PPCA sub-spaces: the default value is one. +% +% The priors are initialised to equal values summing to one, and the +% covariances are all the identity matrix (or equivalent). The centres +% are initialised randomly from a zero mean unit variance Gaussian. +% This makes use of the MATLAB function RANDN and so the seed for the +% random weight initialisation can be set using RANDN('STATE', S) where +% S is the state value. +% +% The fields in MIX are +% +% type = 'gmm' +% nin = the dimension of the space +% ncentres = number of mixture components +% covartype = string for type of variance model +% priors = mixing coefficients +% centres = means of Gaussians: stored as rows of a matrix +% covars = covariances of Gaussians +% The additional fields for mixtures of PPCA are +% U = principal component subspaces +% lambda = in-space covariances: stored as rows of a matrix +% The off-subspace noise is stored in COVARS. +% +% See also +% GMMPAK, GMMUNPAK, GMMSAMP, GMMINIT, GMMEM, GMMACTIV, GMMPOST, +% GMMPROB +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if ncentres < 1 + error('Number of centres must be greater than zero') +end + +mix.type = 'gmm'; +mix.nin = dim; +mix.ncentres = ncentres; + +vartypes = {'spherical', 'diag', 'full', 'ppca'}; + +if sum(strcmp(covar_type, vartypes)) == 0 + error('Undefined covariance type') +else + mix.covar_type = covar_type; +end + +% Make default dimension of PPCA subspaces one. +if strcmp(covar_type, 'ppca') + if nargin < 4 + ppca_dim = 1; + end + if ppca_dim > dim + error('Dimension of PPCA subspaces must be less than data.') + end + mix.ppca_dim = ppca_dim; +end + +% Initialise priors to be equal and summing to one +mix.priors = ones(1,mix.ncentres) ./ mix.ncentres; + +% Initialise centres +mix.centres = randn(mix.ncentres, mix.nin); + +% Initialise all the variances to unity +switch mix.covar_type + +case 'spherical' + mix.covars = ones(1, mix.ncentres); + mix.nwts = mix.ncentres + mix.ncentres*mix.nin + mix.ncentres; +case 'diag' + % Store diagonals of covariance matrices as rows in a matrix + mix.covars = ones(mix.ncentres, mix.nin); + mix.nwts = mix.ncentres + mix.ncentres*mix.nin + ... + mix.ncentres*mix.nin; +case 'full' + % Store covariance matrices in a row vector of matrices + mix.covars = repmat(eye(mix.nin), [1 1 mix.ncentres]); + mix.nwts = mix.ncentres + mix.ncentres*mix.nin + ... + mix.ncentres*mix.nin*mix.nin; +case 'ppca' + % This is the off-subspace noise: make it smaller than + % lambdas + mix.covars = 0.1*ones(1, mix.ncentres); + % Also set aside storage for principal components and + % associated variances + init_space = eye(mix.nin); + init_space = init_space(:, 1:mix.ppca_dim); + init_space(mix.ppca_dim+1:mix.nin, :) = ... + ones(mix.nin - mix.ppca_dim, mix.ppca_dim); + mix.U = repmat(init_space , [1 1 mix.ncentres]); + mix.lambda = ones(mix.ncentres, mix.ppca_dim); + % Take account of additional parameters + mix.nwts = mix.ncentres + mix.ncentres*mix.nin + ... + mix.ncentres + mix.ncentres*mix.ppca_dim + ... + mix.ncentres*mix.nin*mix.ppca_dim; +otherwise + error(['Unknown covariance type ', mix.covar_type]); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/gmmactiv.m b/sourcecodes/bnt-master/netlab3.3/gmmactiv.m new file mode 100644 index 00000000..117e1cdc --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmactiv.m @@ -0,0 +1,77 @@ +function a = gmmactiv(mix, x) +%GMMACTIV Computes the activations of a Gaussian mixture model. +% +% Description +% This function computes the activations A (i.e. the probability +% P(X|J) of the data conditioned on each component density) for a +% Gaussian mixture model. For the PPCA model, each activation is the +% conditional probability of X given that it is generated by the +% component subspace. The data structure MIX defines the mixture model, +% while the matrix X contains the data vectors. Each row of X +% represents a single vector. +% +% See also +% GMM, GMMPOST, GMMPROB +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check that inputs are consistent +errstring = consist(mix, 'gmm', x); +if ~isempty(errstring) + error(errstring); +end + +ndata = size(x, 1); +a = zeros(ndata, mix.ncentres); % Preallocate matrix + +switch mix.covar_type + +case 'spherical' + % Calculate squared norm matrix, of dimension (ndata, ncentres) + n2 = dist2(x, mix.centres); + + % Calculate width factors + wi2 = ones(ndata, 1) * (2 .* mix.covars); + normal = (pi .* wi2) .^ (mix.nin/2); + + % Now compute the activations + a = exp(-(n2./wi2))./ normal; + +case 'diag' + normal = (2*pi)^(mix.nin/2); + s = prod(sqrt(mix.covars), 2); + for j = 1:mix.ncentres + diffs = x - (ones(ndata, 1) * mix.centres(j, :)); + a(:, j) = exp(-0.5*sum((diffs.*diffs)./(ones(ndata, 1) * ... + mix.covars(j, :)), 2)) ./ (normal*s(j)); + end + +case 'full' + normal = (2*pi)^(mix.nin/2); + for j = 1:mix.ncentres + diffs = x - (ones(ndata, 1) * mix.centres(j, :)); + % Use Cholesky decomposition of covariance matrix to speed computation + c = chol(mix.covars(:, :, j)); + temp = diffs/c; + a(:, j) = exp(-0.5*sum(temp.*temp, 2))./(normal*prod(diag(c))); + end +case 'ppca' + log_normal = mix.nin*log(2*pi); + d2 = zeros(ndata, mix.ncentres); + logZ = zeros(1, mix.ncentres); + for i = 1:mix.ncentres + k = 1 - mix.covars(i)./mix.lambda(i, :); + logZ(i) = log_normal + mix.nin*log(mix.covars(i)) - ... + sum(log(1 - k)); + diffs = x - ones(ndata, 1)*mix.centres(i, :); + proj = diffs*mix.U(:, :, i); + d2(:,i) = (sum(diffs.*diffs, 2) - ... + sum((proj.*(ones(ndata, 1)*k)).*proj, 2)) / ... + mix.covars(i); + end + a = exp(-0.5*(d2 + ones(ndata, 1)*logZ)); +otherwise + error(['Unknown covariance type ', mix.covar_type]); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/gmmem.m b/sourcecodes/bnt-master/netlab3.3/gmmem.m new file mode 100644 index 00000000..42066538 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmem.m @@ -0,0 +1,181 @@ +function [mix, options, errlog] = gmmem(mix, x, options) +%GMMEM EM algorithm for Gaussian mixture model. +% +% Description +% [MIX, OPTIONS, ERRLOG] = GMMEM(MIX, X, OPTIONS) uses the Expectation +% Maximization algorithm of Dempster et al. to estimate the parameters +% of a Gaussian mixture model defined by a data structure MIX. The +% matrix X represents the data whose expectation is maximized, with +% each row corresponding to a vector. The optional parameters have +% the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG. If OPTIONS(1) is set to 0, then +% only warning messages are displayed. If OPTIONS(1) is -1, then +% nothing is displayed. +% +% OPTIONS(3) is a measure of the absolute precision required of the +% error function at the solution. If the change in log likelihood +% between two steps of the EM algorithm is less than this value, then +% the function terminates. +% +% OPTIONS(5) is set to 1 if a covariance matrix is reset to its +% original value when any of its singular values are too small (less +% than MIN_COVAR which has the value eps). With the default value of +% 0 no action is taken. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% The optional return value OPTIONS contains the final error value +% (i.e. data log likelihood) in OPTIONS(8). +% +% See also +% GMM, GMMINIT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check that inputs are consistent +errstring = consist(mix, 'gmm', x); +if ~isempty(errstring) + error(errstring); +end + +[ndata, xdim] = size(x); + +% Sort out the options +if (options(14)) + niters = options(14); +else + niters = 100; +end + +display = options(1); +store = 0; +if (nargout > 2) + store = 1; % Store the error values to return them + errlog = zeros(1, niters); +end +test = 0; +if options(3) > 0.0 + test = 1; % Test log likelihood for termination +end + +check_covars = 0; +if options(5) >= 1 + if display >= 0 + disp('check_covars is on'); + end + check_covars = 1; % Ensure that covariances don't collapse + MIN_COVAR = eps; % Minimum singular value of covariance matrix + init_covars = mix.covars; +end + +% Main loop of algorithm +for n = 1:niters + + % Calculate posteriors based on old parameters + [post, act] = gmmpost(mix, x); + + % Calculate error value if needed + if (display || store || test) + prob = act*(mix.priors)'; + % Error value is negative log likelihood of data + e = - sum(log(prob)); + if store + errlog(n) = e; + end + if display > 0 + fprintf(1, 'Cycle %4d Error %11.6f\n', n, e); + end + if test + if (n > 1 && abs(e - eold) < options(3)) + options(8) = e; + return; + else + eold = e; + end + end + end + + % Adjust the new estimates for the parameters + new_pr = sum(post, 1); + new_c = post' * x; + + % Now move new estimates to old parameter vectors + mix.priors = new_pr ./ ndata; + + mix.centres = new_c ./ (new_pr' * ones(1, mix.nin)); + + switch mix.covar_type + case 'spherical' + n2 = dist2(x, mix.centres); + for j = 1:mix.ncentres + v(j) = (post(:,j)'*n2(:,j)); + end + mix.covars = ((v./new_pr))./mix.nin; + if check_covars + % Ensure that no covariance is too small + for j = 1:mix.ncentres + if mix.covars(j) < MIN_COVAR + mix.covars(j) = init_covars(j); + end + end + end + case 'diag' + for j = 1:mix.ncentres + diffs = x - (ones(ndata, 1) * mix.centres(j,:)); + mix.covars(j,:) = sum((diffs.*diffs).*(post(:,j)*ones(1, ... + mix.nin)), 1)./new_pr(j); + end + if check_covars + % Ensure that no covariance is too small + for j = 1:mix.ncentres + if min(mix.covars(j,:)) < MIN_COVAR + mix.covars(j,:) = init_covars(j,:); + end + end + end + case 'full' + for j = 1:mix.ncentres + diffs = x - (ones(ndata, 1) * mix.centres(j,:)); + diffs = diffs.*(sqrt(post(:,j))*ones(1, mix.nin)); + mix.covars(:,:,j) = (diffs'*diffs)/new_pr(j); + end + if check_covars + % Ensure that no covariance is too small + for j = 1:mix.ncentres + if min(svd(mix.covars(:,:,j))) < MIN_COVAR + mix.covars(:,:,j) = init_covars(:,:,j); + end + end + end + case 'ppca' + for j = 1:mix.ncentres + diffs = x - (ones(ndata, 1) * mix.centres(j,:)); + diffs = diffs.*(sqrt(post(:,j))*ones(1, mix.nin)); + [tempcovars, tempU, templambda] = ... + ppca((diffs'*diffs)/new_pr(j), mix.ppca_dim); + if length(templambda) ~= mix.ppca_dim + error('Unable to extract enough components'); + else + mix.covars(j) = tempcovars; + mix.U(:, :, j) = tempU; + mix.lambda(j, :) = templambda; + end + end + if check_covars + if mix.covars(j) < MIN_COVAR + mix.covars(j) = init_covars(j); + end + end + otherwise + error(['Unknown covariance type ', mix.covar_type]); + end +end + +options(8) = -sum(log(gmmprob(mix, x))); +if (display >= 0) + disp(maxitmess); +end + \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gmminit.m b/sourcecodes/bnt-master/netlab3.3/gmminit.m new file mode 100644 index 00000000..30dc974d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmminit.m @@ -0,0 +1,94 @@ +function mix = gmminit(mix, x, options) +%GMMINIT Initialises Gaussian mixture model from data +% +% Description +% MIX = GMMINIT(MIX, X, OPTIONS) uses a dataset X to initialise the +% parameters of a Gaussian mixture model defined by the data structure +% MIX. The k-means algorithm is used to determine the centres. The +% priors are computed from the proportion of examples belonging to each +% cluster. The covariance matrices are calculated as the sample +% covariance of the points associated with (i.e. closest to) the +% corresponding centres. For a mixture of PPCA model, the PPCA +% decomposition is calculated for the points closest to a given centre. +% This initialisation can be used as the starting point for training +% the model using the EM algorithm. +% +% See also +% GMM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[ndata, xdim] = size(x); + +% Check that inputs are consistent +errstring = consist(mix, 'gmm', x); +if ~isempty(errstring) + error(errstring); +end + +% Arbitrary width used if variance collapses to zero: make it 'large' so +% that centre is responsible for a reasonable number of points. +GMM_WIDTH = 1.0; + +% Use kmeans algorithm to set centres +options(5) = 1; +[mix.centres, options, post] = kmeansNetlab(mix.centres, x, options); + +% Set priors depending on number of points in each cluster +cluster_sizes = max(sum(post, 1), 1); % Make sure that no prior is zero +mix.priors = cluster_sizes/sum(cluster_sizes); % Normalise priors + +switch mix.covar_type +case 'spherical' + if mix.ncentres > 1 + % Determine widths as distance to nearest centre + % (or a constant if this is zero) + cdist = dist2(mix.centres, mix.centres); + cdist = cdist + diag(ones(mix.ncentres, 1)*realmax); + mix.covars = min(cdist); + mix.covars = mix.covars + GMM_WIDTH*(mix.covars < eps); + else + % Just use variance of all data points averaged over all + % dimensions + mix.covars = mean(diag(cov(x))); + end + case 'diag' + for j = 1:mix.ncentres + % Pick out data points belonging to this centre + c = x(find(post(:, j)),:); + diffs = c - (ones(size(c, 1), 1) * mix.centres(j, :)); + mix.covars(j, :) = sum((diffs.*diffs), 1)/size(c, 1); + % Replace small entries by GMM_WIDTH value + mix.covars(j, :) = mix.covars(j, :) + GMM_WIDTH.*(mix.covars(j, :)<eps); + end + case 'full' + for j = 1:mix.ncentres + % Pick out data points belonging to this centre + c = x(find(post(:, j)),:); + diffs = c - (ones(size(c, 1), 1) * mix.centres(j, :)); + mix.covars(:,:,j) = (diffs'*diffs)/(size(c, 1)); + % Add GMM_WIDTH*Identity to rank-deficient covariance matrices + if rank(mix.covars(:,:,j)) < mix.nin + mix.covars(:,:,j) = mix.covars(:,:,j) + GMM_WIDTH.*eye(mix.nin); + end + end + case 'ppca' + for j = 1:mix.ncentres + % Pick out data points belonging to this centre + c = x(find(post(:,j)),:); + diffs = c - (ones(size(c, 1), 1) * mix.centres(j, :)); + [tempcovars, tempU, templambda] = ... + ppca((diffs'*diffs)/size(c, 1), mix.ppca_dim); + if length(templambda) ~= mix.ppca_dim + error('Unable to extract enough components'); + else + mix.covars(j) = tempcovars; + mix.U(:, :, j) = tempU; + mix.lambda(j, :) = templambda; + end + end + otherwise + error(['Unknown covariance type ', mix.covar_type]); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/gmmpak.m b/sourcecodes/bnt-master/netlab3.3/gmmpak.m new file mode 100644 index 00000000..8e307fab --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmpak.m @@ -0,0 +1,22 @@ +function p = gmmpak(mix) +%GMMPAK Combines all the parameters in a Gaussian mixture model into one vector. +% +% Description +% P = GMMPAK(NET) takes a mixture data structure MIX and combines the +% component parameter matrices into a single row vector P. +% +% See also +% GMM, GMMUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(mix, 'gmm'); +if ~errstring + error(errstring); +end + +p = [mix.priors, mix.centres(:)', mix.covars(:)']; +if strcmp(mix.covar_type, 'ppca') + p = [p, mix.lambda(:)', mix.U(:)']; +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gmmpost.m b/sourcecodes/bnt-master/netlab3.3/gmmpost.m new file mode 100644 index 00000000..f2b7e224 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmpost.m @@ -0,0 +1,36 @@ +function [post, a] = gmmpost(mix, x) +%GMMPOST Computes the class posterior probabilities of a Gaussian mixture model. +% +% Description +% This function computes the posteriors POST (i.e. the probability of +% each component conditioned on the data P(J|X)) for a Gaussian mixture +% model. The data structure MIX defines the mixture model, while the +% matrix X contains the data vectors. Each row of X represents a +% single vector. +% +% See also +% GMM, GMMACTIV, GMMPROB +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check that inputs are consistent +errstring = consist(mix, 'gmm', x); +if ~isempty(errstring) + error(errstring); +end + +ndata = size(x, 1); + +a = gmmactiv(mix, x); + +post = (ones(ndata, 1)*mix.priors).*a; +s = sum(post, 2); +if any(s==0) + warning('Some zero posterior probabilities') + % Set any zeros to one before dividing + zero_rows = find(s==0); + s = s + (s==0); + post(zero_rows, :) = 1/mix.ncentres; +end +post = post./(s*ones(1, mix.ncentres)); diff --git a/sourcecodes/bnt-master/netlab3.3/gmmprob.m b/sourcecodes/bnt-master/netlab3.3/gmmprob.m new file mode 100644 index 00000000..8f438b66 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmprob.m @@ -0,0 +1,26 @@ +function prob = gmmprob(mix, x) +%GMMPROB Computes the data probability for a Gaussian mixture model. +% +% Description +% This function computes the unconditional data density P(X) for a +% Gaussian mixture model. The data structure MIX defines the mixture +% model, while the matrix X contains the data vectors. Each row of X +% represents a single vector. +% +% See also +% GMM, GMMPOST, GMMACTIV +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check that inputs are consistent +errstring = consist(mix, 'gmm', x); +if ~isempty(errstring) + error(errstring); +end + +% Compute activations +a = gmmactiv(mix, x); + +% Form dot product with priors +prob = a * (mix.priors)'; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gmmsamp.m b/sourcecodes/bnt-master/netlab3.3/gmmsamp.m new file mode 100644 index 00000000..8550ddf9 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmsamp.m @@ -0,0 +1,66 @@ +function [data, label] = gmmsamp(mix, n) +%GMMSAMP Sample from a Gaussian mixture distribution. +% +% Description +% +% DATA = GSAMP(MIX, N) generates a sample of size N from a Gaussian +% mixture distribution defined by the MIX data structure. The matrix X +% has N rows in which each row represents a MIX.NIN-dimensional sample +% vector. +% +% [DATA, LABEL] = GMMSAMP(MIX, N) also returns a column vector of +% classes (as an index 1..N) LABEL. +% +% See also +% GSAMP, GMM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check input arguments +errstring = consist(mix, 'gmm'); +if ~isempty(errstring) + error(errstring); +end +if n < 1 + error('Number of data points must be positive') +end + +% Determine number to sample from each component +priors = rand(1, n); + +% Pre-allocate data array +data = zeros(n, mix.nin); +if nargout > 1 + label = zeros(n, 1); +end +cum_prior = 0; % Cumulative sum of priors +total_samples = 0; % Cumulative sum of number of sampled points +for j = 1:mix.ncentres + num_samples = sum(priors >= cum_prior & ... + priors < cum_prior + mix.priors(j)); + % Form a full covariance matrix + switch mix.covar_type + case 'spherical' + covar = mix.covars(j) * eye(mix.nin); + case 'diag' + covar = diag(mix.covars(j, :)); + case 'full' + covar = mix.covars(:, :, j); + case 'ppca' + covar = mix.covars(j) * eye(mix.nin) + ... + mix.U(:, :, j)* ... + (diag(mix.lambda(j, :))-(mix.covars(j)*eye(mix.ppca_dim)))* ... + (mix.U(:, :, j)'); + otherwise + error(['Unknown covariance type ', mix.covar_type]); + end + data(total_samples+1:total_samples+num_samples, :) = ... + gsamp(mix.centres(j, :), covar, num_samples); + if nargout > 1 + label(total_samples+1:total_samples+num_samples) = j; + end + cum_prior = cum_prior + mix.priors(j); + total_samples = total_samples + num_samples; +end + diff --git a/sourcecodes/bnt-master/netlab3.3/gmmunpak.m b/sourcecodes/bnt-master/netlab3.3/gmmunpak.m new file mode 100644 index 00000000..9a503816 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gmmunpak.m @@ -0,0 +1,54 @@ +function mix = gmmunpak(mix, p) +%GMMUNPAK Separates a vector of Gaussian mixture model parameters into its components. +% +% Description +% MIX = GMMUNPAK(MIX, P) takes a GMM data structure MIX and a single +% row vector of parameters P and returns a mixture data structure +% identical to the input MIX, except that the mixing coefficients +% PRIORS, centres CENTRES and covariances COVARS (and, for PPCA, the +% lambdas and U (PCA sub-spaces)) are all set to the corresponding +% elements of P. +% +% See also +% GMM, GMMPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(mix, 'gmm'); +if ~errstring + error(errstring); +end +if mix.nwts ~= length(p) + error('Invalid weight vector length') +end + +mark1 = mix.ncentres; +mark2 = mark1 + mix.ncentres*mix.nin; + +mix.priors = reshape(p(1:mark1), 1, mix.ncentres); +mix.centres = reshape(p(mark1 + 1:mark2), mix.ncentres, mix.nin); +switch mix.covar_type + case 'spherical' + mark3 = mix.ncentres*(2 + mix.nin); + mix.covars = reshape(p(mark2 + 1:mark3), 1, mix.ncentres); + case 'diag' + mark3 = mix.ncentres*(1 + mix.nin + mix.nin); + mix.covars = reshape(p(mark2 + 1:mark3), mix.ncentres, mix.nin); + case 'full' + mark3 = mix.ncentres*(1 + mix.nin + mix.nin*mix.nin); + mix.covars = reshape(p(mark2 + 1:mark3), mix.nin, mix.nin, ... + mix.ncentres); + case 'ppca' + mark3 = mix.ncentres*(2 + mix.nin); + mix.covars = reshape(p(mark2 + 1:mark3), 1, mix.ncentres); + % Now also extract k and eigenspaces + mark4 = mark3 + mix.ncentres*mix.ppca_dim; + mix.lambda = reshape(p(mark3 + 1:mark4), mix.ncentres, ... + mix.ppca_dim); + mix.U = reshape(p(mark4 + 1:end), mix.nin, mix.ppca_dim, ... + mix.ncentres); + otherwise + error(['Unknown covariance type ', mix.covar_type]); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/gp.m b/sourcecodes/bnt-master/netlab3.3/gp.m new file mode 100644 index 00000000..54643faa --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gp.m @@ -0,0 +1,93 @@ +function net = gp(nin, covar_fn, prior) +%GP Create a Gaussian Process. +% +% Description +% +% NET = GP(NIN, COVARFN) takes the number of inputs NIN for a Gaussian +% Process model with a single output, together with a string COVARFN +% which specifies the type of the covariance function, and returns a +% data structure NET. The parameters are set to zero. +% +% The fields in NET are +% type = 'gp' +% nin = number of inputs +% nout = number of outputs: always 1 +% nwts = total number of weights and covariance function parameters +% bias = logarithm of constant offset in covariance function +% noise = logarithm of output noise variance +% inweights = logarithm of inverse length scale for each input +% covarfn = string describing the covariance function: +% 'sqexp' +% 'ratquad' +% fpar = covariance function specific parameters (1 for squared exponential, +% 2 for rational quadratic) +% trin = training input data (initially empty) +% trtargets = training target data (initially empty) +% +% NET = GP(NIN, COVARFN, PRIOR) sets a Gaussian prior on the parameters +% of the model. PRIOR must contain the fields PR_MEAN and PR_VARIANCE. +% If PR_MEAN is a scalar, then the Gaussian is assumed to be isotropic +% and the additional fields NET.PR_MEAN and PR_VARIANCE are set. +% Otherwise, the Gaussian prior has a mean defined by a column vector +% of parameters PRIOR.PR_MEAN and covariance defined by a column vector +% of parameters PRIOR.PR_VARIANCE. Each element of PRMEAN corresponds +% to a separate group of parameters, which need not be mutually +% exclusive. The membership of the groups is defined by the matrix +% PRIOR.INDEX in which the columns correspond to the elements of +% PRMEAN. Each column has one element for each weight in the matrix, in +% the order defined by the function GPPAK, and each element is 1 or 0 +% according to whether the parameter is a member of the corresponding +% group or not. The additional field NET.INDEX is set in this case. +% +% See also +% GPPAK, GPUNPAK, GPFWD, GPERR, GPCOVAR, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.type = 'gp'; +net.nin = nin; +net.nout = 1; % Only do single output GP + +% Store log parameters +net.bias = 0; +net.min_noise = sqrt(eps); % Prevent output noise collapsing completely +net.noise = 0; +net.inweights = zeros(1,nin); % Weights on inputs in covariance function + +covarfns = {'sqexp', 'ratquad'}; + +if sum(strcmp(covar_fn, covarfns)) == 0 + error('Undefined activation function. Exiting.'); +else + net.covar_fn = covar_fn; +end + +switch covar_fn + + case 'sqexp' % Squared exponential + net.fpar = zeros(1,1); % One function specific parameter + + case 'ratquad' % Rational quadratic + net.fpar = zeros(1, 2); % Two function specific parameters + + otherwise + error(['Unknown covariance function ', covar_fn]); +end + +net.nwts = 2 + nin + length(net.fpar); + +if nargin >= 3 + if size(prior.pr_mean) == [1 1] + net.pr_mean = prior.pr_mean; + net.pr_var = prior.pr_var; + else + net.pr_mean = prior.pr_mean; + net.pr_var = prior.pr_var; + net.index = prior.index; + end +end + +% Store training data as needed for gpfwd +net.tr_in = []; +net.tr_targets = []; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gpcovar.m b/sourcecodes/bnt-master/netlab3.3/gpcovar.m new file mode 100644 index 00000000..f71fa4db --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpcovar.m @@ -0,0 +1,38 @@ +function [cov, covf] = gpcovar(net, x) +%GPCOVAR Calculate the covariance for a Gaussian Process. +% +% Description +% +% COV = GPCOVAR(NET, X) takes a Gaussian Process data structure NET +% together with a matrix X of input vectors, and computes the +% covariance matrix COV. The inverse of this matrix is used when +% calculating the mean and variance of the predictions made by NET. +% +% [COV, COVF] = GPCOVAR(NET, X) also generates the covariance matrix +% due to the covariance function specified by NET.COVARFN as calculated +% by GPCOVARF. +% +% See also +% GP, GPPAK, GPUNPAK, GPCOVARP, GPCOVARF, GPFWD, GPERR, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'gp', x); +if ~isempty(errstring); + error(errstring); +end + +ndata = size(x, 1); + +% Compute prior covariance +if nargout >= 2 + [covp, covf] = gpcovarp(net, x, x); +else + covp = gpcovarp(net, x, x); +end + +% Add output noise variance +cov = covp + (net.min_noise + exp(net.noise))*eye(ndata); + diff --git a/sourcecodes/bnt-master/netlab3.3/gpcovarf.m b/sourcecodes/bnt-master/netlab3.3/gpcovarf.m new file mode 100644 index 00000000..ed66e60e --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpcovarf.m @@ -0,0 +1,44 @@ +function covf = gpcovarf(net, x1, x2) +%GPCOVARF Calculate the covariance function for a Gaussian Process. +% +% Description +% +% COVF = GPCOVARF(NET, X1, X2) takes a Gaussian Process data structure +% NET together with two matrices X1 and X2 of input vectors, and +% computes the matrix of the covariance function values COVF. +% +% See also +% GP, GPCOVAR, GPCOVARP, GPERR, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gp', x1); +if ~isempty(errstring); + error(errstring); +end + +if size(x1, 2) ~= size(x2, 2) + error('Number of variables in x1 and x2 must be the same'); +end + +n1 = size(x1, 1); +n2 = size(x2, 1); +beta = diag(exp(net.inweights)); + +% Compute the weighted squared distances between x1 and x2 +z = (x1.*x1)*beta*ones(net.nin, n2) - 2*x1*beta*x2' ... + + ones(n1, net.nin)*beta*(x2.*x2)'; + +switch net.covar_fn + + case 'sqexp' % Squared exponential + covf = exp(net.fpar(1) - 0.5*z); + + case 'ratquad' % Rational quadratic + nu = exp(net.fpar(2)); + covf = exp(net.fpar(1))*((ones(size(z)) + z).^(-nu)); + + otherwise + error(['Unknown covariance function ', net.covar_fn]); +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gpcovarp.m b/sourcecodes/bnt-master/netlab3.3/gpcovarp.m new file mode 100644 index 00000000..203c405c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpcovarp.m @@ -0,0 +1,30 @@ +function [covp, covf] = gpcovarp(net, x1, x2) +%GPCOVARP Calculate the prior covariance for a Gaussian Process. +% +% Description +% +% COVP = GPCOVARP(NET, X1, X2) takes a Gaussian Process data structure +% NET together with two matrices X1 and X2 of input vectors, and +% computes the matrix of the prior covariance. This is the function +% component of the covariance plus the exponential of the bias term. +% +% [COVP, COVF] = GPCOVARP(NET, X1, X2) also returns the function +% component of the covariance. +% +% See also +% GP, GPCOVAR, GPCOVARF, GPERR, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gp', x1); +if ~isempty(errstring); + error(errstring); +end + +if size(x1, 2) ~= size(x2, 2) + error('Number of variables in x1 and x2 must be the same'); +end + +covf = gpcovarf(net, x1, x2); +covp = covf + exp(net.bias); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gperr.m b/sourcecodes/bnt-master/netlab3.3/gperr.m new file mode 100644 index 00000000..402d63bd --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gperr.m @@ -0,0 +1,51 @@ +function [e, edata, eprior] = gperr(net, x, t) +%GPERR Evaluate error function for Gaussian Process. +% +% Description +% E = GPERR(NET, X, T) takes a Gaussian Process data structure NET +% together with a matrix X of input vectors and a matrix T of target +% vectors, and evaluates the error function E. Each row of X +% corresponds to one input vector and each row of T corresponds to one +% target vector. +% +% [E, EDATA, EPRIOR] = GPERR(NET, X, T) additionally returns the data +% and hyperprior components of the error, assuming a Gaussian prior on +% the weights with mean and variance parameters PRMEAN and PRVARIANCE +% taken from the network data structure NET. +% +% See also +% GP, GPCOVAR, GPFWD, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gp', x, t); +if ~isempty(errstring); + error(errstring); +end + +cn = gpcovar(net, x); + +edata = 0.5*(sum(log(eig(cn, 'nobalance'))) + t'*inv(cn)*t); + +% Evaluate the hyperprior contribution to the error. +% The hyperprior is Gaussian with mean pr_mean and variance +% pr_variance +if isfield(net, 'pr_mean') + w = gppak(net); + m = repmat(net.pr_mean, size(w)); + if size(net.pr_mean) == [1 1] + eprior = 0.5*((w-m)*(w-m)'); + e2 = eprior/net.pr_var; + else + wpr = repmat(w, size(net.pr_mean, 1), 1)'; + eprior = 0.5*(((wpr - m').^2).*net.index); + e2 = (sum(eprior, 1))*(1./net.pr_var); + end +else + e2 = 0; + eprior = 0; +end + +e = edata + e2; + diff --git a/sourcecodes/bnt-master/netlab3.3/gpfwd.m b/sourcecodes/bnt-master/netlab3.3/gpfwd.m new file mode 100644 index 00000000..b2d390e5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpfwd.m @@ -0,0 +1,51 @@ +function [y, sigsq] = gpfwd(net, x, cninv) +%GPFWD Forward propagation through Gaussian Process. +% +% Description +% Y = GPFWD(NET, X) takes a Gaussian Process data structure NET +% together with a matrix X of input vectors, and forward propagates +% the inputs through the model to generate a matrix Y of output +% vectors. Each row of X corresponds to one input vector and each row +% of Y corresponds to one output vector. This assumes that the +% training data (both inputs and targets) has been stored in NET by a +% call to GPINIT; these are needed to compute the training data +% covariance matrix. +% +% [Y, SIGSQ] = GPFWD(NET, X) also generates a column vector SIGSQ of +% conditional variances (or squared error bars) where each value +% corresponds to a pattern. +% +% [Y, SIGSQ] = GPFWD(NET, X, CNINV) uses the pre-computed inverse +% covariance matrix CNINV in the forward propagation. This increases +% efficiency if several calls to GPFWD are made. +% +% See also +% GP, DEMGP, GPINIT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gp', x); +if ~isempty(errstring); + error(errstring); +end + +if ~(isfield(net, 'tr_in') & isfield(net, 'tr_targets')) + error('Require training inputs and targets'); +end + +if nargin == 2 + % Inverse covariance matrix not supplied. + cninv = inv(gpcovar(net, net.tr_in)); +end +ktest = gpcovarp(net, x, net.tr_in); + +% Predict mean +y = ktest*cninv*net.tr_targets; + +if nargout >= 2 + % Predict error bar + ndata = size(x, 1); + sigsq = (ones(ndata, 1) * gpcovarp(net, x(1,:), x(1,:))) ... + - sum((ktest*cninv).*ktest, 2); +end diff --git a/sourcecodes/bnt-master/netlab3.3/gpgrad.m b/sourcecodes/bnt-master/netlab3.3/gpgrad.m new file mode 100644 index 00000000..7ea531bc --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpgrad.m @@ -0,0 +1,100 @@ +function g = gpgrad(net, x, t) +%GPGRAD Evaluate error gradient for Gaussian Process. +% +% Description +% G = GPGRAD(NET, X, T) takes a Gaussian Process data structure NET +% together with a matrix X of input vectors and a matrix T of target +% vectors, and evaluates the error gradient G. Each row of X +% corresponds to one input vector and each row of T corresponds to one +% target vector. +% +% See also +% GP, GPCOVAR, GPFWD, GPERR +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gp', x, t); +if ~isempty(errstring); + error(errstring); +end + +% Evaluate derivatives with respect to each hyperparameter in turn. +ndata = size(x, 1); +[cov, covf] = gpcovar(net, x); +cninv = inv(cov); +trcninv = trace(cninv); +cninvt = cninv*t; + +% Function parameters +switch net.covar_fn + + case 'sqexp' % Squared exponential + gfpar = trace(cninv*covf) - cninvt'*covf*cninvt; + + case 'ratquad' % Rational quadratic + beta = diag(exp(net.inweights)); + gfpar(1) = trace(cninv*covf) - cninvt'*covf*cninvt; + D2 = (x.*x)*beta*ones(net.nin, ndata) - 2*x*beta*x' ... + + ones(ndata, net.nin)*beta*(x.*x)'; + E = ones(size(D2)); + L = - exp(net.fpar(2)) * covf .* log(E + D2); % d(cn)/d(nu) + gfpar(2) = trace(cninv*L) - cninvt'*L*cninvt; + + otherwise + error(['Unknown covariance function ', net.covar_fn]); +end + +% Bias derivative +ndata = size(x, 1); +fac = exp(net.bias)*ones(ndata); +gbias = trace(cninv*fac) - cninvt'*fac*cninvt; + +% Noise derivative +gnoise = exp(net.noise)*(trcninv - cninvt'*cninvt); + +% Input weight derivatives +if strcmp(net.covar_fn, 'ratquad') + F = (exp(net.fpar(2))*E)./(E + D2); +end + +nparams = length(net.inweights); +for l = 1 : nparams + vect = x(:, l); + matx = (vect.*vect)*ones(1, ndata) ... + - 2.0*vect*vect' ... + + ones(ndata, 1)*(vect.*vect)'; + switch net.covar_fn + case 'sqexp' % Squared exponential + dmat = -0.5*exp(net.inweights(l))*covf.*matx; + + case 'ratquad' % Rational quadratic + dmat = - exp(net.inweights(l))*covf.*matx.*F; + otherwise + error(['Unknown covariance function ', net.covar_fn]); + end + + gw1(l) = trace(cninv*dmat) - cninvt'*dmat*cninvt; +end + +g1 = [gbias, gnoise, gw1, gfpar]; +g1 = 0.5*g1; + +% Evaluate the prior contribution to the gradient. +if isfield(net, 'pr_mean') + w = gppak(net); + m = repmat(net.pr_mean, size(w)); + if size(net.pr_mean) == [1 1] + gprior = w - m; + g2 = gprior/net.pr_var; + else + ngroups = size(net.pr_mean, 1); + gprior = net.index'.*(ones(ngroups, 1)*w - m); + g2 = (1./net.pr_var)'*gprior; + end +else + gprior = 0; + g2 = 0; +end + +g = g1 + g2; diff --git a/sourcecodes/bnt-master/netlab3.3/gpinit.m b/sourcecodes/bnt-master/netlab3.3/gpinit.m new file mode 100644 index 00000000..c355c0d5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpinit.m @@ -0,0 +1,43 @@ +function net = gpinit(net, tr_in, tr_targets, prior) +%GPINIT Initialise Gaussian Process model. +% +% Description +% NET = GPINIT(NET, TRIN, TRTARGETS) takes a Gaussian Process data +% structure NET together with a matrix TRIN of training input vectors +% and a matrix TRTARGETS of training target vectors, and stores them +% in NET. These datasets are required if the corresponding inverse +% covariance matrix is not supplied to GPFWD. This is important if the +% data structure is saved and then reloaded before calling GPFWD. Each +% row of TRIN corresponds to one input vector and each row of TRTARGETS +% corresponds to one target vector. +% +% NET = GPINIT(NET, TRIN, TRTARGETS, PRIOR) additionally initialises +% the parameters in NET from the PRIOR data structure which contains +% the mean and variance of the Gaussian distribution which is sampled +% from. +% +% See also +% GP, GPFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gp', tr_in, tr_targets); +if ~isempty(errstring); + error(errstring); +end + +if nargin >= 4 + % Initialise weights at random + if size(prior.pr_mean) == [1 1] + w = randn(1, net.nwts).*sqrt(prior.pr_var) + ... + repmat(prior.pr_mean, 1, net.nwts); + else + sig = sqrt(prior.index*prior.pr_var); + w = sig'.*randn(1, net.nwts) + (prior.index*prior.pr_mean)'; + end + net = gpunpak(net, w); +end + +net.tr_in = tr_in; +net.tr_targets = tr_targets; diff --git a/sourcecodes/bnt-master/netlab3.3/gppak.m b/sourcecodes/bnt-master/netlab3.3/gppak.m new file mode 100644 index 00000000..bb1c52a8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gppak.m @@ -0,0 +1,19 @@ +function hp = gppak(net) +%GPPAK Combines GP hyperparameters into one vector. +% +% Description +% HP = GPPAK(NET) takes a Gaussian Process data structure NET and +% combines the hyperparameters into a single row vector HP. +% +% See also +% GP, GPUNPAK, GPFWD, GPERR, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'gp'); +if ~isempty(errstring); + error(errstring); +end +hp = [net.bias, net.noise, net.inweights, net.fpar]; diff --git a/sourcecodes/bnt-master/netlab3.3/gpunpak.m b/sourcecodes/bnt-master/netlab3.3/gpunpak.m new file mode 100644 index 00000000..8e1058e4 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gpunpak.m @@ -0,0 +1,36 @@ +function net = gpunpak(net, hp) +%GPUNPAK Separates hyperparameter vector into components. +% +% Description +% NET = GPUNPAK(NET, HP) takes an Gaussian Process data structure NET +% and a hyperparameter vector HP, and returns a Gaussian Process data +% structure identical to the input model, except that the covariance +% bias BIAS, output noise NOISE, the input weight vector INWEIGHTS and +% the vector of covariance function specific parameters FPAR have all +% been set to the corresponding elements of HP. +% +% See also +% GP, GPPAK, GPFWD, GPERR, GPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'gp'); +if ~isempty(errstring); + error(errstring); +end +if net.nwts ~= length(hp) + error('Invalid weight vector length'); +end + +net.bias = hp(1); +net.noise = hp(2); + +% Unpack input weights +mark1 = 2 + net.nin; +net.inweights = hp(3:mark1); + +% Unpack function specific parameters +net.fpar = hp(mark1 + 1:size(hp, 2)); + diff --git a/sourcecodes/bnt-master/netlab3.3/gradchek.m b/sourcecodes/bnt-master/netlab3.3/gradchek.m new file mode 100644 index 00000000..3f9298fb --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gradchek.m @@ -0,0 +1,49 @@ +function [gradient, delta] = gradchek(w, func, grad, varargin) +%GRADCHEK Checks a user-defined gradient function using finite differences. +% +% Description +% This function is intended as a utility for other netlab functions +% (particularly optimisation functions) to use. It enables the user to +% check whether a gradient calculation has been correctly implmented +% for a given function. GRADCHEK(W, FUNC, GRAD) checks how accurate the +% gradient GRAD of a function FUNC is at a parameter vector X. A +% central difference formula with step size 1.0e-6 is used, and the +% results for both gradient function and finite difference +% approximation are printed. The optional return value GRADIENT is the +% gradient calculated using the function GRAD and the return value +% DELTA is the difference between the functional and finite difference +% methods of calculating the graident. +% +% GRADCHEK(X, FUNC, GRAD, P1, P2, ...) allows additional arguments to +% be passed to FUNC and GRAD. +% +% See also +% CONJGRAD, GRADDESC, HMC, OLGD, QUASINEW, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Reasonable value for step size +epsilon = 1.0e-6; + +func = fcnchk(func, length(varargin)); +grad = fcnchk(grad, length(varargin)); + +% Treat +nparams = length(w); +deltaf = zeros(1, nparams); +step = zeros(1, nparams); +for i = 1:nparams + % Move a small way in the ith coordinate of w + step(i) = 1.0; + fplus = feval('linef', epsilon, func, w, step, varargin{:}); + fminus = feval('linef', -epsilon, func, w, step, varargin{:}); + % Use central difference formula for approximation + deltaf(i) = 0.5*(fplus - fminus)/epsilon; + step(i) = 0.0; +end +gradient = feval(grad, w, varargin{:}); +fprintf(1, 'Checking gradient ...\n\n'); +delta = gradient - deltaf; +fprintf(1, ' analytic diffs delta\n\n'); +disp([gradient', deltaf', delta']) diff --git a/sourcecodes/bnt-master/netlab3.3/graddesc.m b/sourcecodes/bnt-master/netlab3.3/graddesc.m new file mode 100644 index 00000000..3cc64141 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/graddesc.m @@ -0,0 +1,180 @@ +function [x, options, flog, pointlog] = graddesc(f, x, options, gradf, ... + varargin) +%GRADDESC Gradient descent optimization. +% +% Description +% [X, OPTIONS, FLOG, POINTLOG] = GRADDESC(F, X, OPTIONS, GRADF) uses +% batch gradient descent to find a local minimum of the function F(X) +% whose gradient is given by GRADF(X). A log of the function values +% after each cycle is (optionally) returned in ERRLOG, and a log of the +% points visited is (optionally) returned in POINTLOG. +% +% Note that X is a row vector and F returns a scalar value. The point +% at which F has a local minimum is returned as X. The function value +% at that point is returned in OPTIONS(8). +% +% GRADDESC(F, X, OPTIONS, GRADF, P1, P2, ...) allows additional +% arguments to be passed to F() and GRADF(). +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG, and the points visited in the +% return argument POINTSLOG. If OPTIONS(1) is set to 0, then only +% warning messages are displayed. If OPTIONS(1) is -1, then nothing is +% displayed. +% +% OPTIONS(2) is the absolute precision required for the value of X at +% the solution. If the absolute difference between the values of X +% between two successive steps is less than OPTIONS(2), then this +% condition is satisfied. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. If the absolute difference between the +% objective function values between two successive steps is less than +% OPTIONS(3), then this condition is satisfied. Both this and the +% previous condition must be satisfied for termination. +% +% OPTIONS(7) determines the line minimisation method used. If it is +% set to 1 then a line minimiser is used (in the direction of the +% negative gradient). If it is 0 (the default), then each parameter +% update is a fixed multiple (the learning rate) of the negative +% gradient added to a fixed multiple (the momentum) of the previous +% parameter update. +% +% OPTIONS(9) should be set to 1 to check the user defined gradient +% function GRADF with GRADCHEK. This is carried out at the initial +% parameter vector X. +% +% OPTIONS(10) returns the total number of function evaluations +% (including those in any line searches). +% +% OPTIONS(11) returns the total number of gradient evaluations. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% OPTIONS(15) is the precision in parameter space of the line search; +% default FOPTIONS(2). +% +% OPTIONS(17) is the momentum; default 0.5. It should be scaled by the +% inverse of the number of data points. +% +% OPTIONS(18) is the learning rate; default 0.01. It should be scaled +% by the inverse of the number of data points. +% +% See also +% CONJGRAD, LINEMIN, OLGD, MINBRACK, QUASINEW, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set up the options. +if length(options) < 18 + error('Options vector too short') +end + +if (options(14)) + niters = options(14); +else + niters = 100; +end + +line_min_flag = 0; % Flag for line minimisation option +if (round(options(7)) == 1) + % Use line minimisation + line_min_flag = 1; + % Set options for line minimiser + line_options = foptions; + if options(15) > 0 + line_options(2) = options(15); + end +else + % Learning rate: must be positive + if (options(18) > 0) + eta = options(18); + else + eta = 0.01; + end + % Momentum term: allow zero momentum + if (options(17) >= 0) + mu = options(17); + else + mu = 0.5; + end +end + +% Check function string +f = fcnchk(f, length(varargin)); +gradf = fcnchk(gradf, length(varargin)); + +% Display information if options(1) > 0 +display = options(1) > 0; + +% Work out if we need to compute f at each iteration. +% Needed if using line search or if display results or if termination +% criterion requires it. +fcneval = (options(7) | display | options(3)); + +% Check gradients +if (options(9) > 0) + feval('gradchek', x, f, gradf, varargin{:}); +end + +dxold = zeros(1, size(x, 2)); +xold = x; +fold = 0; % Must be initialised so that termination test can be performed +if fcneval + fnew = feval(f, x, varargin{:}); + options(10) = options(10) + 1; + fold = fnew; +end + +% Main optimization loop. +for j = 1:niters + xold = x; + grad = feval(gradf, x, varargin{:}); + options(11) = options(11) + 1; % Increment gradient evaluation counter + if (line_min_flag ~= 1) + dx = mu*dxold - eta*grad; + x = x + dx; + dxold = dx; + if fcneval + fold = fnew; + fnew = feval(f, x, varargin{:}); + options(10) = options(10) + 1; + end + else + sd = - grad./norm(grad); % New search direction. + fold = fnew; + % Do a line search: normalise search direction to have length 1 + [lmin, line_options] = feval('linemin', f, x, sd, fold, ... + line_options, varargin{:}); + options(10) = options(10) + line_options(10); + x = xold + lmin*sd; + fnew = line_options(8); + end + if nargout >= 3 + flog(j) = fnew; + if nargout >= 4 + pointlog(j, :) = x; + end + end + if display + fprintf(1, 'Cycle %5d Function %11.8f\n', j, fnew); + end + if (max(abs(x - xold)) < options(2) & abs(fnew - fold) < options(3)) + % Termination criteria are met + options(8) = fnew; + return; + end +end + +if fcneval + options(8) = fnew; +else + options(8) = feval(f, x, varargin{:}); + options(10) = options(10) + 1; +end +if (options(1) >= 0) + disp(maxitmess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/gsamp.m b/sourcecodes/bnt-master/netlab3.3/gsamp.m new file mode 100644 index 00000000..58b1ba1d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gsamp.m @@ -0,0 +1,32 @@ +function x = gsamp(mu, covar, nsamp) +%GSAMP Sample from a Gaussian distribution. +% +% Description +% +% X = GSAMP(MU, COVAR, NSAMP) generates a sample of size NSAMP from a +% D-dimensional Gaussian distribution. The Gaussian density has mean +% vector MU and covariance matrix COVAR, and the matrix X has NSAMP +% rows in which each row represents a D-dimensional sample vector. +% +% See also +% GAUSS, DEMGAUSS +% + +% Copyright (c) Ian T Nabney (1996-2001) + +d = size(covar, 1); + +mu = reshape(mu, 1, d); % Ensure that mu is a row vector + +[evec, eval] = eig(covar); + +deig=diag(eval); + +if (~isreal(deig)) | any(deig<0), + warning('Covariance Matrix is not OK, redefined to be positive definite'); + eval=abs(eval); +end + +coeffs = randn(nsamp, d)*sqrt(eval); + +x = ones(nsamp, 1)*mu + coeffs*evec'; diff --git a/sourcecodes/bnt-master/netlab3.3/gtm.m b/sourcecodes/bnt-master/netlab3.3/gtm.m new file mode 100644 index 00000000..9d1b3ab3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtm.m @@ -0,0 +1,61 @@ +function net = gtm(dim_latent, nlatent, dim_data, ncentres, rbfunc, ... + prior) +%GTM Create a Generative Topographic Map. +% +% Description +% +% NET = GTM(DIMLATENT, NLATENT, DIMDATA, NCENTRES, RBFUNC), takes the +% dimension of the latent space DIMLATENT, the number of data points +% sampled in the latent space NLATENT, the dimension of the data space +% DIMDATA, the number of centres in the RBF model NCENTRES, the +% activation function for the RBF RBFUNC and returns a data structure +% NET. The parameters in the RBF and GMM sub-models are set by calls to +% the corresponding creation routines RBF and GMM. +% +% The fields in NET are +% type = 'gtm' +% nin = dimension of data space +% dimlatent = dimension of latent space +% rbfnet = RBF network data structure +% gmmnet = GMM data structure +% X = sample of latent points +% +% NET = GTM(DIMLATENT, NLATENT, DIMDATA, NCENTRES, RBFUNC, PRIOR), +% sets a Gaussian zero mean prior on the parameters of the RBF model. +% PRIOR must be a scalar and represents the inverse variance of the +% prior distribution. This gives rise to a weight decay term in the +% error function. +% +% See also +% GTMFWD, GTMPOST, RBF, GMM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.type = 'gtm'; +% Input to functions is data +net.nin = dim_data; +net.dim_latent = dim_latent; + +% Default is no regularisation +if nargin == 5 + prior = 0.0; +end + +% Only allow scalar prior +if isstruct(prior) | size(prior) ~= [1 1] + error('Prior must be a scalar'); +end + +% Create RBF network +net.rbfnet = rbf(dim_latent, ncentres, dim_data, rbfunc, ... + 'linear', prior); + +% Mask all but output weights +net.rbfnet.mask = rbfprior(rbfunc, dim_latent, ncentres, dim_data); + +% Create field for GMM output model +net.gmmnet = gmm(dim_data, nlatent, 'spherical'); + +% Create empty latent data sample +net.X = []; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/gtmem.m b/sourcecodes/bnt-master/netlab3.3/gtmem.m new file mode 100644 index 00000000..47dd84a5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmem.m @@ -0,0 +1,135 @@ +function [net, options, errlog] = gtmem(net, t, options) +%GTMEM EM algorithm for Generative Topographic Mapping. +% +% Description +% [NET, OPTIONS, ERRLOG] = GTMEM(NET, T, OPTIONS) uses the Expectation +% Maximization algorithm to estimate the parameters of a GTM defined by +% a data structure NET. The matrix T represents the data whose +% expectation is maximized, with each row corresponding to a vector. +% It is assumed that the latent data NET.X has been set following a +% call to GTMINIT, for example. The optional parameters have the +% following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG. If OPTIONS(1) is set to 0, then +% only warning messages are displayed. If OPTIONS(1) is -1, then +% nothing is displayed. +% +% OPTIONS(3) is a measure of the absolute precision required of the +% error function at the solution. If the change in log likelihood +% between two steps of the EM algorithm is less than this value, then +% the function terminates. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% The optional return value OPTIONS contains the final error value +% (i.e. data log likelihood) in OPTIONS(8). +% +% See also +% GTM, GTMINIT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check that inputs are consistent +errstring = consist(net, 'gtm', t); +if ~isempty(errstring) + error(errstring); +end + +% Sort out the options +if (options(14)) + niters = options(14); +else + niters = 100; +end + +display = options(1); +store = 0; +if (nargout > 2) + store = 1; % Store the error values to return them + errlog = zeros(1, niters); +end +test = 0; +if options(3) > 0.0 + test = 1; % Test log likelihood for termination +end + +% Calculate various quantities that remain constant during training +[ndata, tdim] = size(t); +ND = ndata*tdim; +[net.gmmnet.centres, Phi] = rbffwd(net.rbfnet, net.X); +Phi = [Phi ones(size(net.X, 1), 1)]; +PhiT = Phi'; +[K, Mplus1] = size(Phi); + +A = zeros(Mplus1, Mplus1); +cholDcmp = zeros(Mplus1, Mplus1); +% Use a sparse representation for the weight regularizing matrix. +if (net.rbfnet.alpha > 0) + Alpha = net.rbfnet.alpha*speye(Mplus1); + Alpha(Mplus1, Mplus1) = 0; +end + +for n = 1:niters + % Calculate responsibilities + [R, act] = gtmpost(net, t); + % Calculate error value if needed + if (display | store | test) + prob = act*(net.gmmnet.priors)'; + % Error value is negative log likelihood of data + e = - sum(log(max(prob,eps))); + if store + errlog(n) = e; + end + if display > 0 + fprintf(1, 'Cycle %4d Error %11.6f\n', n, e); + end + if test + if (n > 1 & abs(e - eold) < options(3)) + options(8) = e; + return; + else + eold = e; + end + end + end + + % Calculate matrix be inverted (Phi'*G*Phi + alpha*I in the papers). + % Sparse representation of G normally executes faster and saves + % memory + if (net.rbfnet.alpha > 0) + A = full(PhiT*spdiags(sum(R)', 0, K, K)*Phi + ... + (Alpha.*net.gmmnet.covars(1))); + else + A = full(PhiT*spdiags(sum(R)', 0, K, K)*Phi); + end + % A is a symmetric matrix likely to be positive definite, so try + % fast Cholesky decomposition to calculate W, otherwise use SVD. + % (PhiT*(R*t)) is computed right-to-left, as R + % and t are normally (much) larger than PhiT. + [cholDcmp singular] = chol(A); + if (singular) + if (display) + fprintf(1, ... + 'gtmem: Warning -- M-Step matrix singular, using pinv.\n'); + end + W = pinv(A)*(PhiT*(R'*t)); + else + W = cholDcmp \ (cholDcmp' \ (PhiT*(R'*t))); + end + % Put new weights into network to calculate responsibilities + % net.rbfnet = netunpak(net.rbfnet, W); + net.rbfnet.w2 = W(1:net.rbfnet.nhidden, :); + net.rbfnet.b2 = W(net.rbfnet.nhidden+1, :); + % Calculate new distances + d = dist2(t, Phi*W); + + % Calculate new value for beta + net.gmmnet.covars = ones(1, net.gmmnet.ncentres)*(sum(sum(d.*R))/ND); +end + +options(8) = -sum(log(gtmprob(net, t))); +if (display >= 0) + disp(maxitmess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/gtmfwd.m b/sourcecodes/bnt-master/netlab3.3/gtmfwd.m new file mode 100644 index 00000000..bb84d344 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmfwd.m @@ -0,0 +1,17 @@ +function mix = gtmfwd(net) +%GTMFWD Forward propagation through GTM. +% +% Description +% MIX = GTMFWD(NET) takes a GTM structure NET, and forward propagates +% the latent data sample NET.X through the GTM to generate the +% structure MIX which represents the Gaussian mixture model in data +% space. +% +% See also +% GTM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.gmmnet.centres = rbffwd(net.rbfnet, net.X); +mix = net.gmmnet; diff --git a/sourcecodes/bnt-master/netlab3.3/gtminit.m b/sourcecodes/bnt-master/netlab3.3/gtminit.m new file mode 100644 index 00000000..ae646a09 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtminit.m @@ -0,0 +1,153 @@ +function net = gtminit(net, options, data, samp_type, varargin) +%GTMINIT Initialise the weights and latent sample in a GTM. +% +% Description +% NET = GTMINIT(NET, OPTIONS, DATA, SAMPTYPE) takes a GTM NET and +% generates a sample of latent data points and sets the centres (and +% widths if appropriate) of NET.RBFNET. +% +% If the SAMPTYPE is 'REGULAR', then regular grids of latent data +% points and RBF centres are created. The dimension of the latent data +% space must be 1 or 2. For one-dimensional latent space, the +% LSAMPSIZE parameter gives the number of latent points and the +% RBFSAMPSIZE parameter gives the number of RBF centres. For a two- +% dimensional latent space, these parameters must be vectors of length +% 2 with the number of points in each of the x and y directions to +% create a rectangular grid. The widths of the RBF basis functions are +% set by a call to RBFSETFW passing OPTIONS(7) as the scaling +% parameter. +% +% If the SAMPTYPE is 'UNIFORM' or 'GAUSSIAN' then the latent data is +% found by sampling from a uniform or Gaussian distribution +% correspondingly. The RBF basis function parameters are set by a call +% to RBFSETBF with the DATA parameter as dataset and the OPTIONS +% vector. +% +% Finally, the output layer weights of the RBF are initialised by +% mapping the mean of the latent variable to the mean of the target +% variable, and the L-dimensional latent variale variance to the +% variance of the targets along the first L principal components. +% +% See also +% GTM, GTMEM, PCA, RBFSETBF, RBFSETFW +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check for consistency +errstring = consist(net, 'gtm', data); +if ~isempty(errstring) + error(errstring); +end + +% Check type of sample +stypes = {'regular', 'uniform', 'gaussian'}; +if (strcmp(samp_type, stypes)) == 0 + error('Undefined sample type.') +end + +if net.dim_latent > size(data, 2) + error('Latent space dimension must not be greater than data dimension') +end +nlatent = net.gmmnet.ncentres; +nhidden = net.rbfnet.nhidden; + +% Create latent data sample and set RBF centres + +switch samp_type +case 'regular' + if nargin ~= 6 + error('Regular type must specify latent and RBF shapes'); + end + l_samp_size = varargin{1}; + rbf_samp_size = varargin{2}; + if round(l_samp_size) ~= l_samp_size + error('Latent sample specification must contain integers') + end + % Check existence and size of rbf specification + if any(size(rbf_samp_size) ~= [1 net.dim_latent]) | ... + prod(rbf_samp_size) ~= nhidden + error('Incorrect specification of RBF centres') + end + % Check dimension and type of latent data specification + if any(size(l_samp_size) ~= [1 net.dim_latent]) | ... + prod(l_samp_size) ~= nlatent + error('Incorrect dimension of latent sample spec.') + end + if net.dim_latent == 1 + net.X = [-1:2/(l_samp_size-1):1]'; + net.rbfnet.c = [-1:2/(rbf_samp_size-1):1]'; + net.rbfnet = rbfsetfw(net.rbfnet, options(7)); + elseif net.dim_latent == 2 + net.X = gtm_rctg(l_samp_size); + net.rbfnet.c = gtm_rctg(rbf_samp_size); + net.rbfnet = rbfsetfw(net.rbfnet, options(7)); + else + error('For regular sample, input dimension must be 1 or 2.') + end + + +case {'uniform', 'gaussian'} + if strcmp(samp_type, 'uniform') + net.X = 2 * (rand(nlatent, net.dim_latent) - 0.5); + else + % Sample from N(0, 0.25) distribution to ensure most latent + % data is inside square + net.X = randn(nlatent, net.dim_latent)/2; + end + net.rbfnet = rbfsetbf(net.rbfnet, options, net.X); +otherwise + % Shouldn't get here + error('Invalid sample type'); + +end + +% Latent data sample and basis function parameters chosen. +% Now set output weights +[PCcoeff, PCvec] = pca(data); + +% Scale PCs by eigenvalues +A = PCvec(:, 1:net.dim_latent)*diag(sqrt(PCcoeff(1:net.dim_latent))); + +[temp, Phi] = rbffwd(net.rbfnet, net.X); +% Normalise X to ensure 1:1 mapping of variances and calculate weights +% as solution of Phi*W = normX*A' +normX = (net.X - ones(size(net.X))*diag(mean(net.X)))*diag(1./std(net.X)); +net.rbfnet.w2 = Phi \ (normX*A'); +% Bias is mean of target data +net.rbfnet.b2 = mean(data); + +% Must also set initial value of variance +% Find average distance between nearest centres +% Ensure that distance of centre to itself is excluded by setting diagonal +% entries to realmax +net.gmmnet.centres = rbffwd(net.rbfnet, net.X); +d = dist2(net.gmmnet.centres, net.gmmnet.centres) + ... + diag(ones(net.gmmnet.ncentres, 1)*realmax); +sigma = mean(min(d))/2; + +% Now set covariance to minimum of this and next largest eigenvalue +if net.dim_latent < size(data, 2) + sigma = min(sigma, PCcoeff(net.dim_latent+1)); +end +net.gmmnet.covars = sigma*ones(1, net.gmmnet.ncentres); + +% Sub-function to create the sample data in 2d +function sample = gtm_rctg(samp_size) + +xDim = samp_size(1); +yDim = samp_size(2); +% Produce a grid with the right number of rows and columns +[X, Y] = meshgrid([0:1:(xDim-1)], [(yDim-1):-1:0]); + +% Change grid representation +sample = [X(:), Y(:)]; + +% Shift grid to correct position and scale it +maxXY= max(sample); +sample(:,1) = 2*(sample(:,1) - maxXY(1)/2)./maxXY(1); +sample(:,2) = 2*(sample(:,2) - maxXY(2)/2)./maxXY(2); +return; + + + diff --git a/sourcecodes/bnt-master/netlab3.3/gtmlmean.m b/sourcecodes/bnt-master/netlab3.3/gtmlmean.m new file mode 100644 index 00000000..8aad8103 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmlmean.m @@ -0,0 +1,22 @@ +function means = gtmlmean(net, data) +%GTMLMEAN Mean responsibility for data in a GTM. +% +% Description +% MEANS = GTMLMEAN(NET, DATA) takes a GTM structure NET, and computes +% the means of the responsibility distributions for each data point in +% DATA. +% +% See also +% GTM, GTMPOST, GTMLMODE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check for consistency +errstring = consist(net, 'gtm', data); +if ~isempty(errstring) + error(errstring); +end + +R = gtmpost(net, data); +means = R*net.X; diff --git a/sourcecodes/bnt-master/netlab3.3/gtmlmode.m b/sourcecodes/bnt-master/netlab3.3/gtmlmode.m new file mode 100644 index 00000000..6fee04a8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmlmode.m @@ -0,0 +1,25 @@ +function modes = gtmlmode(net, data) +%GTMLMODE Mode responsibility for data in a GTM. +% +% Description +% MODES = GTMLMODE(NET, DATA) takes a GTM structure NET, and computes +% the modes of the responsibility distributions for each data point in +% DATA. These will always lie at one of the latent space sample points +% NET.X. +% +% See also +% GTM, GTMPOST, GTMLMEAN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check for consistency +errstring = consist(net, 'gtm', data); +if ~isempty(errstring) + error(errstring); +end + +R = gtmpost(net, data); +% Mode is maximum responsibility +[max_resp, max_index] = max(R, [], 2); +modes = net.X(max_index, :); diff --git a/sourcecodes/bnt-master/netlab3.3/gtmmag.m b/sourcecodes/bnt-master/netlab3.3/gtmmag.m new file mode 100644 index 00000000..f0014971 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmmag.m @@ -0,0 +1,27 @@ +function mags = gtmmag(net, latent_data) +%GTMMAG Magnification factors for a GTM +% +% Description +% MAGS = GTMMAG(NET, LATENTDATA) takes a GTM structure NET, and +% computes the magnification factors for each point the latent space +% contained in LATENTDATA. +% +% See also +% GTM, GTMPOST, GTMLMEAN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'gtm'); +if ~isempty(errstring) + error(errstring); +end + +Jacs = rbfjacob(net.rbfnet, latent_data); +nlatent = size(latent_data, 1); +mags = zeros(nlatent, 1); +temp = zeros(net.rbfnet.nin, net.rbfnet.nout); +for m = 1:nlatent + temp = squeeze(Jacs(m, :, :)); % Turn into a 2d matrix + mags(m) = sqrt(det(temp*temp')); +end diff --git a/sourcecodes/bnt-master/netlab3.3/gtmpost.m b/sourcecodes/bnt-master/netlab3.3/gtmpost.m new file mode 100644 index 00000000..d9182af0 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmpost.m @@ -0,0 +1,26 @@ +function [post, a] = gtmpost(net, data) +%GTMPOST Latent space responsibility for data in a GTM. +% +% Description +% POST = GTMPOST(NET, DATA) takes a GTM structure NET, and computes +% the responsibility at each latent space sample point NET.X for each +% data point in DATA. +% +% [POST, A] = GTMPOST(NET, DATA) also returns the activations A of the +% GMM NET.GMMNET as computed by GMMPOST. +% +% See also +% GTM, GTMEM, GTMLMEAN, GMLMODE, GMMPROB +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check for consistency +errstring = consist(net, 'gtm', data); +if ~isempty(errstring) + error(errstring); +end + +net.gmmnet.centres = rbffwd(net.rbfnet, net.X); + +[post, a] = gmmpost(net.gmmnet, data); diff --git a/sourcecodes/bnt-master/netlab3.3/gtmprob.m b/sourcecodes/bnt-master/netlab3.3/gtmprob.m new file mode 100644 index 00000000..203216a2 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/gtmprob.m @@ -0,0 +1,22 @@ +function prob = gtmprob(net, data) +%GTMPROB Probability for data under a GTM. +% +% Description +% PROB = GTMPROB(NET, DATA) takes a GTM structure NET, and computes +% the probability of each point in the dataset DATA. +% +% See also +% GTM, GTMEM, GTMLMEAN, GTMLMODE, GTMPOST +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check for consistency +errstring = consist(net, 'gtm', data); +if ~isempty(errstring) + error(errstring); +end + +net.gmmnet.centres = rbffwd(net.rbfnet, net.X); + +prob = gmmprob(net.gmmnet, data); diff --git a/sourcecodes/bnt-master/netlab3.3/hbayes.m b/sourcecodes/bnt-master/netlab3.3/hbayes.m new file mode 100644 index 00000000..65864783 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/hbayes.m @@ -0,0 +1,49 @@ +function [h, hdata] = hbayes(net, hdata) +%HBAYES Evaluate Hessian of Bayesian error function for network. +% +% Description +% H = HBAYES(NET, HDATA) takes a network data structure NET together +% the data contribution to the Hessian for a set of inputs and targets. +% It returns the regularised Hessian using any zero mean Gaussian +% priors on the weights defined in NET. In addition, if a MASK is +% defined in NET, then the entries in H that correspond to weights with +% a 0 in the mask are removed. +% +% [H, HDATA] = HBAYES(NET, HDATA) additionally returns the data +% component of the Hessian. +% +% See also +% GBAYES, GLMHESS, MLPHESS, RBFHESS +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if (isfield(net, 'mask')) + % Extract relevant entries in Hessian + nmask_rows = size(find(net.mask), 1); + hdata = reshape(hdata(logical(net.mask*(net.mask'))), ... + nmask_rows, nmask_rows); + nwts = nmask_rows; +else + nwts = net.nwts; +end +if isfield(net, 'beta') + h = net.beta*hdata; +else + h = hdata; +end + +if isfield(net, 'alpha') + if size(net.alpha) == [1 1] + h = h + net.alpha*eye(nwts); + else + if isfield(net, 'mask') + nindx_cols = size(net.index, 2); + index = reshape(net.index(logical(repmat(net.mask, ... + 1, nindx_cols))), nmask_rows, nindx_cols); + else + index = net.index; + end + h = h + diag(index*net.alpha); + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/hesschek.m b/sourcecodes/bnt-master/netlab3.3/hesschek.m new file mode 100644 index 00000000..ffd7253d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/hesschek.m @@ -0,0 +1,60 @@ +function h = hesschek(net, x, t) +%HESSCHEK Use central differences to confirm correct evaluation of Hessian matrix. +% +% Description +% +% HESSCHEK(NET, X, T) takes a network data structure NET, together with +% input and target data matrices X and T, and compares the evaluation +% of the Hessian matrix using the function NETHESS and using central +% differences with the function NETERR. +% +% The optional return value H is the Hessian computed using NETHESS. +% +% See also +% NETHESS, NETERR +% + +% Copyright (c) Ian T Nabney (1996-2001) + +w0 = netpak(net); +nwts = length(w0); +h = nethess(w0, net, x, t); + +w = w0; +hcent = zeros(nwts, nwts); +h1 = 0.0; h2 = 0.0; h3 = 0.0; h4 = 0.0; +epsilon = 1.0e-4; +fprintf(1, 'Checking Hessian ...\n\n'); +for k = 1:nwts; + for l = 1:nwts; + if(l == k) + w(k) = w0(k) + 2.0*epsilon; + h1 = neterr(w, net, x, t); + w(k) = w0(k) - 2.0*epsilon; + h2 = neterr(w, net, x, t); + w(k) = w0(k); + h3 = neterr(w, net, x, t); + hcent(k, k) = (h1 + h2 - 2.0*h3)/(4.0*epsilon^2); + else + w(k) = w0(k) + epsilon; + w(l) = w0(l) + epsilon; + h1 = neterr(w, net, x, t); + w(k) = w0(k) - epsilon; + w(l) = w0(l) - epsilon; + h2 = neterr(w, net, x, t); + w(k) = w0(k) + epsilon; + w(l) = w0(l) - epsilon; + h3 = neterr(w, net, x, t); + w(k) = w0(k) - epsilon; + w(l) = w0(l) + epsilon; + h4 = neterr(w, net, x, t); + hcent(k, l) = (h1 + h2 - h3 - h4)/(4.0*epsilon^2); + w(k) = w0(k); + w(l) = w0(l); + end + end +end + +fprintf(1, ' analytical numerical delta\n\n'); +temp = [h(:), hcent(:), (h(:) - hcent(:))]; +fprintf(1, '%12.6f %12.6f %12.6f\n', temp'); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/hintmat.m b/sourcecodes/bnt-master/netlab3.3/hintmat.m new file mode 100644 index 00000000..24617921 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/hintmat.m @@ -0,0 +1,41 @@ +function [xvals, yvals, color] = hintmat(w); +%HINTMAT Evaluates the coordinates of the patches for a Hinton diagram. +% +% Description +% [xvals, yvals, color] = hintmat(w) +% takes a matrix W and returns coordinates XVALS, YVALS for the +% patches comrising the Hinton diagram, together with a vector COLOR +% labelling the color (black or white) of the corresponding elements +% according to their sign. +% +% See also +% HINTON +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set scale to be up to 0.9 of maximum absolute weight value, where scale +% defined so that area of box proportional to weight value. + +w = flipud(w); +[nrows, ncols] = size(w); + +scale = 0.45*sqrt(abs(w)/max(max(abs(w)))); +scale = scale(:); +color = 0.5*(sign(w(:)) + 3); + +delx = 1; +dely = 1; +[X, Y] = meshgrid(0.5*delx:delx:(ncols-0.5*delx), 0.5*dely:dely:(nrows-0.5*dely)); + +% Now convert from matrix format to column vector format, and then duplicate +% columns with appropriate offsets determined by normalized weight magnitudes. + +xtemp = X(:); +ytemp = Y(:); + +xvals = [xtemp-delx*scale, xtemp+delx*scale, ... + xtemp+delx*scale, xtemp-delx*scale]; +yvals = [ytemp-dely*scale, ytemp-dely*scale, ... + ytemp+dely*scale, ytemp+dely*scale]; + diff --git a/sourcecodes/bnt-master/netlab3.3/hinton.m b/sourcecodes/bnt-master/netlab3.3/hinton.m new file mode 100644 index 00000000..c2c4b2be --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/hinton.m @@ -0,0 +1,57 @@ +function h = hinton(w); +%HINTON Plot Hinton diagram for a weight matrix. +% +% Description +% +% HINTON(W) takes a matrix W and plots the Hinton diagram. +% +% H = HINTON(NET) also returns the figure handle H which can be used, +% for instance, to delete the figure when it is no longer needed. +% +% To print the figure correctly in black and white, you should call +% SET(H, 'INVERTHARDCOPY', 'OFF') before printing. +% +% See also +% DEMHINT, HINTMAT, MLPHINT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set scale to be up to 0.9 of maximum absolute weight value, where scale +% defined so that area of box proportional to weight value. + +% Use no more than 640x480 pixels +xmax = 640; ymax = 480; + +% Offset bottom left hand corner +x01 = 40; y01 = 40; +x02 = 80; y02 = 80; + +% Need to allow 5 pixels border for window frame: but 30 at top +border = 5; +top_border = 30; + +ymax = ymax - top_border; +xmax = xmax - border; + +% First layer + +[xvals, yvals, color] = hintmat(w); +% Try to preserve aspect ratio approximately +if (8*size(w, 1) < 6*size(w, 2)) + delx = xmax; dely = xmax*size(w, 1)/(size(w, 2)); +else + delx = ymax*size(w, 2)/size(w, 1); dely = ymax; +end + +h = figure('Color', [0.5 0.5 0.5], ... + 'Name', 'Hinton diagram', ... + 'NumberTitle', 'off', ... + 'Colormap', [0 0 0; 1 1 1], ... + 'Units', 'pixels', ... + 'Position', [x01 y01 delx dely]); +set(gca, 'Visible', 'off', 'Position', [0 0 1 1]); +hold on +patch(xvals', yvals', color', 'Edgecolor', 'none'); +axis equal; + diff --git a/sourcecodes/bnt-master/netlab3.3/histp.m b/sourcecodes/bnt-master/netlab3.3/histp.m new file mode 100644 index 00000000..7b7fb31b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/histp.m @@ -0,0 +1,29 @@ +function h = histp(x, xmin, xmax, nbins) +%HISTP Histogram estimate of 1-dimensional probability distribution. +% +% Description +% +% HISTP(X, XMIN, XMAX, NBINS) takes a column vector X of data values +% and generates a normalized histogram plot of the distribution. The +% histogram has NBINS bins lying in the range XMIN to XMAX. +% +% H = HISTP(...) returns a vector of patch handles. +% +% See also +% DEMGAUSS +% + +% Copyright (c) Ian T Nabney (1996-2001) + +ndata = length(x); + +bins = linspace(xmin, xmax, nbins); + +binwidth = (xmax - xmin)/nbins; + +num = hist(x, bins); + +num = num/(ndata*binwidth); + +h = bar(bins, num, 0.6); + diff --git a/sourcecodes/bnt-master/netlab3.3/hmc.m b/sourcecodes/bnt-master/netlab3.3/hmc.m new file mode 100644 index 00000000..dab42be8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/hmc.m @@ -0,0 +1,281 @@ +function [samples, energies, diagn] = hmc(f, x, options, gradf, varargin) +%HMC Hybrid Monte Carlo sampling. +% +% Description +% SAMPLES = HMC(F, X, OPTIONS, GRADF) uses a hybrid Monte Carlo +% algorithm to sample from the distribution P ~ EXP(-F), where F is the +% first argument to HMC. The Markov chain starts at the point X, and +% the function GRADF is the gradient of the `energy' function F. +% +% HMC(F, X, OPTIONS, GRADF, P1, P2, ...) allows additional arguments to +% be passed to F() and GRADF(). +% +% [SAMPLES, ENERGIES, DIAGN] = HMC(F, X, OPTIONS, GRADF) also returns a +% log of the energy values (i.e. negative log probabilities) for the +% samples in ENERGIES and DIAGN, a structure containing diagnostic +% information (position, momentum and acceptance threshold) for each +% step of the chain in DIAGN.POS, DIAGN.MOM and DIAGN.ACC respectively. +% All candidate states (including rejected ones) are stored in +% DIAGN.POS. +% +% [SAMPLES, ENERGIES, DIAGN] = HMC(F, X, OPTIONS, GRADF) also returns +% the ENERGIES (i.e. negative log probabilities) corresponding to the +% samples. The DIAGN structure contains three fields: +% +% POS the position vectors of the dynamic process. +% +% MOM the momentum vectors of the dynamic process. +% +% ACC the acceptance thresholds. +% +% S = HMC('STATE') returns a state structure that contains the state of +% the two random number generators RAND and RANDN and the momentum of +% the dynamic process. These are contained in fields randstate, +% randnstate and mom respectively. The momentum state is only used for +% a persistent momentum update. +% +% HMC('STATE', S) resets the state to S. If S is an integer, then it +% is passed to RAND and RANDN and the momentum variable is randomised. +% If S is a structure returned by HMC('STATE') then it resets the +% generator to exactly the same state. +% +% The optional parameters in the OPTIONS vector have the following +% interpretations. +% +% OPTIONS(1) is set to 1 to display the energy values and rejection +% threshold at each step of the Markov chain. If the value is 2, then +% the position vectors at each step are also displayed. +% +% OPTIONS(5) is set to 1 if momentum persistence is used; default 0, +% for complete replacement of momentum variables. +% +% OPTIONS(7) defines the trajectory length (i.e. the number of leap- +% frog steps at each iteration). Minimum value 1. +% +% OPTIONS(9) is set to 1 to check the user defined gradient function. +% +% OPTIONS(14) is the number of samples retained from the Markov chain; +% default 100. +% +% OPTIONS(15) is the number of samples omitted from the start of the +% chain; default 0. +% +% OPTIONS(17) defines the momentum used when a persistent update of +% (leap-frog) momentum is used. This is bounded to the interval [0, +% 1). +% +% OPTIONS(18) is the step size used in leap-frogs; default 1/trajectory +% length. +% +% See also +% METROP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Global variable to store state of momentum variables: set by set_state +% Used to initialise variable if set +global HMC_MOM +if nargin <= 2 + if ~strcmp(f, 'state') + error('Unknown argument to hmc'); + end + switch nargin + case 1 + samples = get_state(f); + return; + case 2 + set_state(f, x); + return; + end +end + +display = options(1); +if (round(options(5) == 1)) + persistence = 1; + % Set alpha to lie in [0, 1) + alpha = max(0, options(17)); + alpha = min(1, alpha); + salpha = sqrt(1-alpha*alpha); +else + persistence = 0; +end +L = max(1, options(7)); % At least one step in leap-frogging +if options(14) > 0 + nsamples = options(14); +else + nsamples = 100; % Default +end +if options(15) >= 0 + nomit = options(15); +else + nomit = 0; +end +if options(18) > 0 + step_size = options(18); % Step size. +else + step_size = 1/L; % Default +end +x = x(:)'; % Force x to be a row vector +nparams = length(x); + +% Set up strings for evaluating potential function and its gradient. +f = fcnchk(f, length(varargin)); +gradf = fcnchk(gradf, length(varargin)); + +% Check the gradient evaluation. +if (options(9)) + % Check gradients + feval('gradchek', x, f, gradf, varargin{:}); +end + +samples = zeros(nsamples, nparams); % Matrix of returned samples. +if nargout >= 2 + en_save = 1; + energies = zeros(nsamples, 1); +else + en_save = 0; +end +if nargout >= 3 + diagnostics = 1; + diagn_pos = zeros(nsamples, nparams); + diagn_mom = zeros(nsamples, nparams); + diagn_acc = zeros(nsamples, 1); +else + diagnostics = 0; +end + +n = - nomit + 1; +Eold = feval(f, x, varargin{:}); % Evaluate starting energy. +nreject = 0; +if (~persistence | isempty(HMC_MOM)) + p = randn(1, nparams); % Initialise momenta at random +else + p = HMC_MOM; % Initialise momenta from stored state +end +lambda = 1; + +% Main loop. +while n <= nsamples + + xold = x; % Store starting position. + pold = p; % Store starting momenta + Hold = Eold + 0.5*(p*p'); % Recalculate Hamiltonian as momenta have changed + + if ~persistence + % Choose a direction at random + if (rand < 0.5) + lambda = -1; + else + lambda = 1; + end + end + % Perturb step length. + epsilon = lambda*step_size*(1.0 + 0.1*randn(1)); + + % First half-step of leapfrog. + p = p - 0.5*epsilon*feval(gradf, x, varargin{:}); + x = x + epsilon*p; + + % Full leapfrog steps. + for m = 1 : L - 1 + p = p - epsilon*feval(gradf, x, varargin{:}); + x = x + epsilon*p; + end + + % Final half-step of leapfrog. + p = p - 0.5*epsilon*feval(gradf, x, varargin{:}); + + % Now apply Metropolis algorithm. + Enew = feval(f, x, varargin{:}); % Evaluate new energy. + p = -p; % Negate momentum + Hnew = Enew + 0.5*p*p'; % Evaluate new Hamiltonian. + a = exp(Hold - Hnew); % Acceptance threshold. + if (diagnostics & n > 0) + diagn_pos(n,:) = x; + diagn_mom(n,:) = p; + diagn_acc(n,:) = a; + end + if (display > 1) + fprintf(1, 'New position is\n'); + disp(x); + end + + if a > rand(1) % Accept the new state. + Eold = Enew; % Update energy + if (display > 0) + fprintf(1, 'Finished step %4d Threshold: %g\n', n, a); + end + else % Reject the new state. + if n > 0 + nreject = nreject + 1; + end + x = xold; % Reset position + p = pold; % Reset momenta + if (display > 0) + fprintf(1, ' Sample rejected %4d. Threshold: %g\n', n, a); + end + end + if n > 0 + samples(n,:) = x; % Store sample. + if en_save + energies(n) = Eold; % Store energy. + end + end + + % Set momenta for next iteration + if persistence + p = -p; + % Adjust momenta by a small random amount. + p = alpha.*p + salpha.*randn(1, nparams); + else + p = randn(1, nparams); % Replace all momenta. + end + + n = n + 1; +end + +if (display > 0) + fprintf(1, '\nFraction of samples rejected: %g\n', ... + nreject/(nsamples)); +end +if diagnostics + diagn.pos = diagn_pos; + diagn.mom = diagn_mom; + diagn.acc = diagn_acc; +end +% Store final momentum value in global so that it can be retrieved later +HMC_MOM = p; +return + +% Return complete state of sampler (including momentum) +function state = get_state(f) + +global HMC_MOM +state.randstate = rand('state'); +state.randnstate = randn('state'); +state.mom = HMC_MOM; +return + +% Set complete state of sampler (including momentum) or just set randn +% and rand with integer argument. +function set_state(f, x) + +global HMC_MOM +if isnumeric(x) + rand('state', x); + randn('state', x); + HMC_MOM = []; +else + if ~isstruct(x) + error('Second argument to hmc must be number or state structure'); + end + if (~isfield(x, 'randstate') | ~isfield(x, 'randnstate') ... + | ~isfield(x, 'mom')) + error('Second argument to hmc must contain correct fields') + end + rand('state', x.randstate); + randn('state', x.randnstate); + HMC_MOM = x.mom; +end +return diff --git a/sourcecodes/bnt-master/netlab3.3/kmeansNetlab.m b/sourcecodes/bnt-master/netlab3.3/kmeansNetlab.m new file mode 100644 index 00000000..09935f7b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/kmeansNetlab.m @@ -0,0 +1,124 @@ +function [centres, options, post, errlog] = kmeans(centres, data, options) +%KMEANS Trains a k means cluster model. +% +% Description +% CENTRES = KMEANS(CENTRES, DATA, OPTIONS) uses the batch K-means +% algorithm to set the centres of a cluster model. The matrix DATA +% represents the data which is being clustered, with each row +% corresponding to a vector. The sum of squares error function is used. +% The point at which a local minimum is achieved is returned as +% CENTRES. The error value at that point is returned in OPTIONS(8). +% +% [CENTRES, OPTIONS, POST, ERRLOG] = KMEANS(CENTRES, DATA, OPTIONS) +% also returns the cluster number (in a one-of-N encoding) for each +% data point in POST and a log of the error values after each cycle in +% ERRLOG. The optional parameters have the following +% interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG. If OPTIONS(1) is set to 0, then +% only warning messages are displayed. If OPTIONS(1) is -1, then +% nothing is displayed. +% +% OPTIONS(2) is a measure of the absolute precision required for the +% value of CENTRES at the solution. If the absolute difference between +% the values of CENTRES between two successive steps is less than +% OPTIONS(2), then this condition is satisfied. +% +% OPTIONS(3) is a measure of the precision required of the error +% function at the solution. If the absolute difference between the +% error functions between two successive steps is less than OPTIONS(3), +% then this condition is satisfied. Both this and the previous +% condition must be satisfied for termination. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% See also +% GMMINIT, GMMEM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[ndata, data_dim] = size(data); +[ncentres, dim] = size(centres); + +if dim ~= data_dim + error('Data dimension does not match dimension of centres') +end + +if (ncentres > ndata) + error('More centres than data') +end + +% Sort out the options +if (options(14)) + niters = options(14); +else + niters = 100; +end + +store = 0; +if (nargout > 3) + store = 1; + errlog = zeros(1, niters); +end + +% Check if centres and posteriors need to be initialised from data +if (options(5) == 1) + % Do the initialisation + perm = randperm(ndata); + perm = perm(1:ncentres); + + % Assign first ncentres (permuted) data points as centres + centres = data(perm, :); +end +% Matrix to make unit vectors easy to construct +id = eye(ncentres); + +% Main loop of algorithm +for n = 1:niters + + % Save old centres to check for termination + old_centres = centres; + + % Calculate posteriors based on existing centres + d2 = dist2(data, centres); + % Assign each point to nearest centre + [minvals, index] = min(d2', [], 1); + post = id(index,:); + + num_points = sum(post, 1); + % Adjust the centres based on new posteriors + for j = 1:ncentres + if (num_points(j) > 0) + centres(j,:) = sum(data(find(post(:,j)),:), 1)/num_points(j); + end + end + + % Error value is total squared distance from cluster centres + e = sum(minvals); + if store + errlog(n) = e; + end + if options(1) > 0 + fprintf(1, 'Cycle %4d Error %11.6f\n', n, e); + end + + if n > 1 + % Test for termination + if max(max(abs(centres - old_centres))) < options(2) & ... + abs(old_e - e) < options(3) + options(8) = e; + return; + end + end + old_e = e; +end + +% If we get here, then we haven't terminated in the given number of +% iterations. +options(8) = e; +if (options(1) >= 0) + disp(maxitmess); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/knn.m b/sourcecodes/bnt-master/netlab3.3/knn.m new file mode 100644 index 00000000..2a171804 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/knn.m @@ -0,0 +1,34 @@ +function net = knn(nin, nout, k, tr_in, tr_targets) +%KNN Creates a K-nearest-neighbour classifier. +% +% Description +% NET = KNN(NIN, NOUT, K, TR_IN, TR_TARGETS) creates a KNN model NET +% with input dimension NIN, output dimension NOUT and K neighbours. +% The training data is also stored in the data structure and the +% targets are assumed to be using a 1-of-N coding. +% +% The fields in NET are +% type = 'knn' +% nin = number of inputs +% nout = number of outputs +% tr_in = training input data +% tr_targets = training target data +% +% See also +% KMEANS, KNNFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +net.type = 'knn'; +net.nin = nin; +net.nout = nout; +net.k = k; +errstring = consist(net, 'knn', tr_in, tr_targets); +if ~isempty(errstring) + error(errstring); +end +net.tr_in = tr_in; +net.tr_targets = tr_targets; + diff --git a/sourcecodes/bnt-master/netlab3.3/knnfwd.m b/sourcecodes/bnt-master/netlab3.3/knnfwd.m new file mode 100644 index 00000000..db450bae --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/knnfwd.m @@ -0,0 +1,53 @@ +function [y, l] = knnfwd(net, x) +%KNNFWD Forward propagation through a K-nearest-neighbour classifier. +% +% Description +% [Y, L] = KNNFWD(NET, X) takes a matrix X of input vectors (one vector +% per row) and uses the K-nearest-neighbour rule on the training data +% contained in NET to produce a matrix Y of outputs and a matrix L of +% classification labels. The nearest neighbours are determined using +% Euclidean distance. The IJth entry of Y counts the number of +% occurrences that an example from class J is among the K closest +% training examples to example I from X. The matrix L contains the +% predicted class labels as an index 1..N, not as 1-of-N coding. +% +% See also +% KMEANS, KNN +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +errstring = consist(net, 'knn', x); +if ~isempty(errstring) + error(errstring); +end + +ntest = size(x, 1); % Number of input vectors. +nclass = size(net.tr_targets, 2); % Number of classes. + +% Compute matrix of squared distances between input vectors from the training +% and test sets. The matrix distsq has dimensions (ntrain, ntest). + +distsq = dist2(net.tr_in, x); + +% Now sort the distances. This generates a matrix kind of the same +% dimensions as distsq, in which each column gives the indices of the +% elements in the corresponding column of distsq in ascending order. + +[vals, kind] = sort(distsq); +y = zeros(ntest, nclass); + +for k=1:net.k + % We now look at the predictions made by the Kth nearest neighbours alone, + % and represent this as a 1-of-N coded matrix, and then accumulate the + % predictions so far. + + y = y + net.tr_targets(kind(k,:),:); + +end + +if nargout == 2 + % Convert this set of outputs to labels, randomly breaking ties + [temp, l] = max((y + 0.1*rand(size(y))), [], 2); +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/linef.m b/sourcecodes/bnt-master/netlab3.3/linef.m new file mode 100644 index 00000000..4f0ddd32 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/linef.m @@ -0,0 +1,21 @@ +function y = linef(lambda, fn, x, d, varargin) +%LINEF Calculate function value along a line. +% +% Description +% LINEF(LAMBDA, FN, X, D) calculates the value of the function FN at +% the point X+LAMBDA*D. Here X is a row vector and LAMBDA is a scalar. +% +% LINEF(LAMBDA, FN, X, D, P1, P2, ...) allows additional arguments to +% be passed to FN(). This function is used for convenience in some of +% the optimisation routines. +% +% See also +% GRADCHEK, LINEMIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check function string +fn = fcnchk(fn, length(varargin)); + +y = feval(fn, x+lambda.*d, varargin{:}); diff --git a/sourcecodes/bnt-master/netlab3.3/linemin.m b/sourcecodes/bnt-master/netlab3.3/linemin.m new file mode 100644 index 00000000..8e390a43 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/linemin.m @@ -0,0 +1,152 @@ +function [x, options] = linemin(f, pt, dir, fpt, options, ... + varargin) +%LINEMIN One dimensional minimization. +% +% Description +% [X, OPTIONS] = LINEMIN(F, PT, DIR, FPT, OPTIONS) uses Brent's +% algorithm to find the minimum of the function F(X) along the line DIR +% through the point PT. The function value at the starting point is +% FPT. The point at which F has a local minimum is returned as X. The +% function value at that point is returned in OPTIONS(8). +% +% LINEMIN(F, PT, DIR, FPT, OPTIONS, P1, P2, ...) allows additional +% arguments to be passed to F(). +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values. +% +% OPTIONS(2) is a measure of the absolute precision required for the +% value of X at the solution. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. Both this and the previous condition must +% be satisfied for termination. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% See also +% CONJGRAD, MINBRACK, QUASINEW +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set up the options. +if(options(14)) + niters = options(14); +else + niters = 100; +end +options(10) = 0; % Initialise count of function evaluations + +display = options(1); + +% Check function string +f = fcnchk(f, length(varargin)); + +% Value of golden section (1 + sqrt(5))/2.0 +phi = 1.6180339887499; +cphi = 1 - 1/phi; +TOL = sqrt(eps); % Maximal fractional precision +TINY = 1.0e-10; % Can't use fractional precision when minimum is at 0 + +% Bracket the minimum +[br_min, br_mid, br_max, num_evals] = feval('minbrack', 'linef', ... + 0.0, 1.0, fpt, f, pt, dir, varargin{:}); +options(10) = options(10) + num_evals; % Increment number of fn. evals + % No gradient evals in minbrack + +% Use Brent's algorithm to find minimum +% Initialise the points and function values +w = br_mid; % Where second from minimum is +v = br_mid; % Previous value of w +x = v; % Where current minimum is +e = 0.0; % Distance moved on step before last +fx = feval('linef', x, f, pt, dir, varargin{:}); +options(10) = options(10) + 1; +fv = fx; fw = fx; + +for n = 1:niters + xm = 0.5.*(br_min+br_max); % Middle of bracket + % Make sure that tolerance is big enough + tol1 = TOL * (max(abs(x))) + TINY; + % Decide termination on absolute precision required by options(2) + if (max(abs(x - xm)) <= options(2) & br_max-br_min < 4*options(2)) + options(8) = fx; + return; + end + % Check if step before last was big enough to try a parabolic step. + % Note that this will fail on first iteration, which must be a golden + % section step. + if (max(abs(e)) > tol1) + % Construct a trial parabolic fit through x, v and w + r = (fx - fv) .* (x - w); + q = (fx - fw) .* (x - v); + p = (x - v).*q - (x - w).*r; + q = 2.0 .* (q - r); + if (q > 0.0) p = -p; end + q = abs(q); + % Test if the parabolic fit is OK + if (abs(p) >= abs(0.5*q*e) | p <= q*(br_min-x) | p >= q*(br_max-x)) + % No it isn't, so take a golden section step + if (x >= xm) + e = br_min-x; + else + e = br_max-x; + end + d = cphi*e; + else + % Yes it is, so take the parabolic step + e = d; + d = p/q; + u = x+d; + if (u-br_min < 2*tol1 | br_max-u < 2*tol1) + d = sign(xm-x)*tol1; + end + end + else + % Step before last not big enough, so take a golden section step + if (x >= xm) + e = br_min - x; + else + e = br_max - x; + end + d = cphi*e; + end + % Make sure that step is big enough + if (abs(d) >= tol1) + u = x+d; + else + u = x + sign(d)*tol1; + end + % Evaluate function at u + fu = feval('linef', u, f, pt, dir, varargin{:}); + options(10) = options(10) + 1; + % Reorganise bracket + if (fu <= fx) + if (u >= x) + br_min = x; + else + br_max = x; + end + v = w; w = x; x = u; + fv = fw; fw = fx; fx = fu; + else + if (u < x) + br_min = u; + else + br_max = u; + end + if (fu <= fw | w == x) + v = w; w = u; + fv = fw; fw = fu; + elseif (fu <= fv | v == x | v == w) + v = u; + fv = fu; + end + end + if (display == 1) + fprintf(1, 'Cycle %4d Error %11.6f\n', n, fx); + end +end +options(8) = fx; diff --git a/sourcecodes/bnt-master/netlab3.3/maxitmess.m b/sourcecodes/bnt-master/netlab3.3/maxitmess.m new file mode 100644 index 00000000..338be6d7 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/maxitmess.m @@ -0,0 +1,16 @@ +function s = maxitmess() +%MAXITMESS Create a standard error message when training reaches max. iterations. +% +% Description +% S = MAXITMESS returns a standard string that it used by training +% algorithms when the maximum number of iterations (as specified in +% OPTIONS(14) is reached. +% +% See also +% CONJGRAD, GLMTRAIN, GMMEM, GRADDESC, GTMEM, KMEANS, OLGD, QUASINEW, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +s = 'Maximum number of iterations has been exceeded'; + diff --git a/sourcecodes/bnt-master/netlab3.3/mdn.m b/sourcecodes/bnt-master/netlab3.3/mdn.m new file mode 100644 index 00000000..964baa1d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdn.m @@ -0,0 +1,77 @@ +function net = mdn(nin, nhidden, ncentres, dim_target, mix_type, ... + prior, beta) +%MDN Creates a Mixture Density Network with specified architecture. +% +% Description +% NET = MDN(NIN, NHIDDEN, NCENTRES, DIMTARGET) takes the number of +% inputs, hidden units for a 2-layer feed-forward network and the +% number of centres and target dimension for the mixture model whose +% parameters are set from the outputs of the neural network. The fifth +% argument MIXTYPE is used to define the type of mixture model. +% (Currently there is only one type supported: a mixture of Gaussians +% with a single covariance parameter for each component.) For this +% model, the mixture coefficients are computed from a group of softmax +% outputs, the centres are equal to a group of linear outputs, and the +% variances are obtained by applying the exponential function to a +% third group of outputs. +% +% The network is initialised by a call to MLP, and the arguments PRIOR, +% and BETA have the same role as for that function. Weight +% initialisation uses the Matlab function RANDN and so the seed for +% the random weight initialization can be set using RANDN('STATE', S) +% where S is the seed value. A specialised data structure (rather than +% GMM) is used for the mixture model outputs to improve the efficiency +% of error and gradient calculations in network training. The fields +% are described in MDNFWD where they are set up. +% +% The fields in NET are +% +% type = 'mdn' +% nin = number of input variables +% nout = dimension of target space (not number of network outputs) +% nwts = total number of weights and biases +% mdnmixes = data structure for mixture model output +% mlp = data structure for MLP network +% +% See also +% MDNFWD, MDNERR, MDN2GMM, MDNGRAD, MDNPAK, MDNUNPAK, MLP +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Currently ignore type argument: reserved for future use +net.type = 'mdn'; + +% Set up the mixture model part of the structure +% For efficiency we use a specialised data structure in place of GMM +mdnmixes.type = 'mdnmixes'; +mdnmixes.ncentres = ncentres; +mdnmixes.dim_target = dim_target; + +% This calculation depends on spherical variances +mdnmixes.nparams = ncentres + ncentres*dim_target + ncentres; + +% Make the weights in the mdnmixes structure null +mdnmixes.mixcoeffs = []; +mdnmixes.centres = []; +mdnmixes.covars = []; + +% Number of output nodes = number of parameters in mixture model +nout = mdnmixes.nparams; + +% Set up the MLP part of the network +if (nargin == 5) + mlpnet = mlp(nin, nhidden, nout, 'linear'); +elseif (nargin == 6) + mlpnet = mlp(nin, nhidden, nout, 'linear', prior); +elseif (nargin == 7) + mlpnet = mlp(nin, nhidden, nout, 'linear', prior, beta); +end + +% Create descriptor +net.mdnmixes = mdnmixes; +net.mlp = mlpnet; +net.nin = nin; +net.nout = dim_target; +net.nwts = mlpnet.nwts; diff --git a/sourcecodes/bnt-master/netlab3.3/mdn2gmm.m b/sourcecodes/bnt-master/netlab3.3/mdn2gmm.m new file mode 100644 index 00000000..bad5ed9f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdn2gmm.m @@ -0,0 +1,41 @@ +function gmmmixes = mdn2gmm(mdnmixes) +%MDN2GMM Converts an MDN mixture data structure to array of GMMs. +% +% Description +% GMMMIXES = MDN2GMM(MDNMIXES) takes an MDN mixture data structure +% MDNMIXES containing three matrices (for priors, centres and +% variances) where each row represents the corresponding parameter +% values for a different mixture model and creates an array of GMMs. +% These can then be used with the standard Netlab Gaussian mixture +% model functions. +% +% See also +% GMM, MDN, MDNFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Check argument for consistency +errstring = consist(mdnmixes, 'mdnmixes'); +if ~isempty(errstring) + error(errstring); +end + +nmixes = size(mdnmixes.centres, 1); +% Construct ndata structures containing the mixture model information. +% First allocate the memory. +tempmix = gmm(mdnmixes.dim_target, mdnmixes.ncentres, 'spherical'); +f = fieldnames(tempmix); +gmmmixes = cell(size(f, 1), 1, nmixes); +gmmmixes = cell2struct(gmmmixes, f,1); + +% Then fill each structure in turn using gmmunpak. Assume that spherical +% covariance structure is used. +for i = 1:nmixes + centres = reshape(mdnmixes.centres(i, :), mdnmixes.dim_target, ... + mdnmixes.ncentres)'; + gmmmixes(i) = gmmunpak(tempmix, [mdnmixes.mixcoeffs(i,:), ... + centres(:)', mdnmixes.covars(i,:)]); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/mdndist2.m b/sourcecodes/bnt-master/netlab3.3/mdndist2.m new file mode 100644 index 00000000..d7ed93d3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdndist2.m @@ -0,0 +1,51 @@ +function n2 = mdndist2(mixparams, t) +%MDNDIST2 Calculates squared distance between centres of Gaussian kernels and data +% +% Description +% N2 = MDNDIST2(MIXPARAMS, T) takes takes the centres of the Gaussian +% contained in MIXPARAMS and the target data matrix, T, and computes +% the squared Euclidean distance between them. If T has M rows and N +% columns, then the CENTRES field in the MIXPARAMS structure should +% have M rows and N*MIXPARAMS.NCENTRES columns: the centres in each row +% relate to the corresponding row in T. The result has M rows and +% MIXPARAMS.NCENTRES columns. The I, Jth entry is the squared distance +% from the Ith row of X to the Jth centre in the Ith row of +% MIXPARAMS.CENTRES. +% +% See also +% MDNFWD, MDNPROB +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Check arguments for consistency +errstring = consist(mixparams, 'mdnmixes'); +if ~isempty(errstring) + error(errstring); +end + +ncentres = mixparams.ncentres; +dim_target = mixparams.dim_target; +ntarget = size(t, 1); +if ntarget ~= size(mixparams.centres, 1) + error('Number of targets does not match number of mixtures') +end +if size(t, 2) ~= mixparams.dim_target + error('Target dimension does not match mixture dimension') +end + +% Build t that suits parameters, that is repeat t for each centre +t = kron(ones(1, ncentres), t); + +% Do subtraction and square +diff2 = (t - mixparams.centres).^2; + +% Reshape and sum each component +diff2 = reshape(diff2', dim_target, (ntarget*ncentres))'; +n2 = sum(diff2, 2); + +% Calculate the sum of distance, and reshape +% so that we have a distance for each centre per target +n2 = reshape(n2, ncentres, ntarget)'; + diff --git a/sourcecodes/bnt-master/netlab3.3/mdnerr.m b/sourcecodes/bnt-master/netlab3.3/mdnerr.m new file mode 100644 index 00000000..a0b686e0 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnerr.m @@ -0,0 +1,33 @@ +function e = mdnerr(net, x, t) +%MDNERR Evaluate error function for Mixture Density Network. +% +% Description +% E = MDNERR(NET, X, T) takes a mixture density network data structure +% NET, a matrix X of input vectors and a matrix T of target vectors, +% and evaluates the error function E. The error function is the +% negative log likelihood of the target data under the conditional +% density given by the mixture model parameterised by the MLP. Each +% row of X corresponds to one input vector and each row of T +% corresponds to one target vector. +% +% See also +% MDN, MDNFWD, MDNGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Check arguments for consistency +errstring = consist(net, 'mdn', x, t); +if ~isempty(errstring) + error(errstring); +end + +% Get the output mixture models +mixparams = mdnfwd(net, x); + +% Compute the probabilities of mixtures +probs = mdnprob(mixparams, t); +% Compute the error +e = sum( -log(max(eps, sum(probs, 2)))); + diff --git a/sourcecodes/bnt-master/netlab3.3/mdnfwd.m b/sourcecodes/bnt-master/netlab3.3/mdnfwd.m new file mode 100644 index 00000000..962fbe08 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnfwd.m @@ -0,0 +1,93 @@ +function [mixparams, y, z, a] = mdnfwd(net, x) +%MDNFWD Forward propagation through Mixture Density Network. +% +% Description +% MIXPARAMS = MDNFWD(NET, X) takes a mixture density network data +% structure NET and a matrix X of input vectors, and forward propagates +% the inputs through the network to generate a structure MIXPARAMS +% which contains the parameters of several mixture models. Each row +% of X represents one input vector and the corresponding row of the +% matrices in MIXPARAMS represents the parameters of a mixture model +% for the conditional probability of target vectors given the input +% vector. This is not represented as an array of GMM structures to +% improve the efficiency of MDN training. +% +% The fields in MIXPARAMS are +% type = 'mdnmixes' +% ncentres = number of mixture components +% dimtarget = dimension of target space +% mixcoeffs = mixing coefficients +% centres = means of Gaussians: stored as one row per pattern +% covars = covariances of Gaussians +% nparams = number of parameters +% +% [MIXPARAMS, Y, Z] = MDNFWD(NET, X) also generates a matrix Y of the +% outputs of the MLP and a matrix Z of the hidden unit activations +% where each row corresponds to one pattern. +% +% [MIXPARAMS, Y, Z, A] = MLPFWD(NET, X) also returns a matrix A giving +% the summed inputs to each output unit, where each row corresponds to +% one pattern. +% +% See also +% MDN, MDN2GMM, MDNERR, MDNGRAD, MLPFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Check arguments for consistency +errstring = consist(net, 'mdn', x); +if ~isempty(errstring) + error(errstring); +end + +% Extract mlp and mixture model descriptors +mlpnet = net.mlp; +mixes = net.mdnmixes; + +ncentres = mixes.ncentres; % Number of components in mixture model +dim_target = mixes.dim_target; % Dimension of targets +nparams = mixes.nparams; % Number of parameters in mixture model + +% Propagate forwards through MLP +[y, z, a] = mlpfwd(mlpnet, x); + +% Compute the postion for each parameter in the whole +% matrix. Used to define the mixparams structure +mixcoeff = [1:1:ncentres]; +centres = [ncentres+1:1:(ncentres*(1+dim_target))]; +variances = [(ncentres*(1+dim_target)+1):1:nparams]; + +% Convert output values into mixture model parameters + +% Use softmax to calculate priors +% Prevent overflow and underflow: use same bounds as glmfwd +% Ensure that sum(exp(y), 2) does not overflow +maxcut = log(realmax) - log(ncentres); +% Ensure that exp(y) > 0 +mincut = log(realmin); +temp = min(y(:,1:ncentres), maxcut); +temp = max(temp, mincut); +temp = exp(temp); +mixpriors = temp./(sum(temp, 2)*ones(1,ncentres)); + +% Centres are just copies of network outputs +mixcentres = y(:,(ncentres+1):ncentres*(1+dim_target)); + +% Variances are exp of network outputs +mixwidths = exp(y(:,(ncentres*(1+dim_target)+1):nparams)); + +% Now build up all the mixture model weight vectors +ndata = size(x, 1); + +% Return parameters +mixparams.type = mixes.type; +mixparams.ncentres = mixes.ncentres; +mixparams.dim_target = mixes.dim_target; +mixparams.nparams = mixes.nparams; + +mixparams.mixcoeffs = mixpriors; +mixparams.centres = mixcentres; +mixparams.covars = mixwidths; + diff --git a/sourcecodes/bnt-master/netlab3.3/mdngrad.m b/sourcecodes/bnt-master/netlab3.3/mdngrad.m new file mode 100644 index 00000000..6a0b6ea8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdngrad.m @@ -0,0 +1,66 @@ +function g = mdngrad(net, x, t) +%MDNGRAD Evaluate gradient of error function for Mixture Density Network. +% +% Description +% G = MDNGRAD(NET, X, T) takes a mixture density network data +% structure NET, a matrix X of input vectors and a matrix T of target +% vectors, and evaluates the gradient G of the error function with +% respect to the network weights. The error function is negative log +% likelihood of the target data. Each row of X corresponds to one +% input vector and each row of T corresponds to one target vector. +% +% See also +% MDN, MDNFWD, MDNERR, MDNPROB, MLPBKP +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Check arguments for consistency +errstring = consist(net, 'mdn', x, t); +if ~isempty(errstring) + error(errstring); +end + +[mixparams, y, z] = mdnfwd(net, x); + +% Compute gradients at MLP outputs: put the answer in deltas +ncentres = net.mdnmixes.ncentres; +dim_target = net.mdnmixes.dim_target; +nmixparams = net.mdnmixes.nparams; +ntarget = size(t, 1); +deltas = zeros(ntarget, net.mlp.nout); +e = ones(ncentres, 1); +f = ones(1, dim_target); + +post = mdnpost(mixparams, t); + +% Calculate prior derivatives +deltas(:,1:ncentres) = mixparams.mixcoeffs - post; + +% Calculate centre derivatives +long_t = kron(ones(1, ncentres), t); +centre_err = mixparams.centres - long_t; + +% Get the post to match each u_jk: +% this array will be (ntarget, (ncentres*dim_target)) +long_post = kron(ones(dim_target, 1), post); +long_post = reshape(long_post, ntarget, (ncentres*dim_target)); + +% Get the variance to match each u_jk: +var = mixparams.covars; +var = kron(ones(dim_target, 1), var); +var = reshape(var, ntarget, (ncentres*dim_target)); + +% Compute centre deltas +deltas(:, (ncentres+1):(ncentres*(1+dim_target))) = ... + (centre_err.*long_post)./var; + +% Compute variance deltas +dist2 = mdndist2(mixparams, t); +c = dim_target*ones(ntarget, ncentres); +deltas(:, (ncentres*(1+dim_target)+1):nmixparams) = ... + post.*((dist2./mixparams.covars)-c)./(-2); + +% Now back-propagate deltas through MLP +g = mlpbkp(net.mlp, x, z, deltas); diff --git a/sourcecodes/bnt-master/netlab3.3/mdninit.m b/sourcecodes/bnt-master/netlab3.3/mdninit.m new file mode 100644 index 00000000..20f23d0e --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdninit.m @@ -0,0 +1,53 @@ +function net = mdninit(net, prior, t, options) +%MDNINIT Initialise the weights in a Mixture Density Network. +% +% Description +% +% NET = MDNINIT(NET, PRIOR) takes a Mixture Density Network NET and +% sets the weights and biases by sampling from a Gaussian distribution. +% It calls MLPINIT for the MLP component of NET. +% +% NET = MDNINIT(NET, PRIOR, T, OPTIONS) uses the target data T to +% initialise the biases for the output units after initialising the +% other weights as above. It calls GMMINIT, with T and OPTIONS as +% arguments, to obtain a model of the unconditional density of T. The +% biases are then set so that NET will output the values in the +% Gaussian mixture model. +% +% See also +% MDN, MLP, MLPINIT, GMMINIT +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Initialise network weights from prior: this gives noise around values +% determined later +net.mlp = mlpinit(net.mlp, prior); + +if nargin > 2 + % Initialise priors, centres and variances from target data + temp_mix = gmm(net.mdnmixes.dim_target, net.mdnmixes.ncentres, 'spherical'); + temp_mix = gmminit(temp_mix, t, options); + + ncentres = net.mdnmixes.ncentres; + dim_target = net.mdnmixes.dim_target; + + % Now set parameters in MLP to yield the right values. + % This involves setting the biases correctly. + + % Priors + net.mlp.b2(1:ncentres) = temp_mix.priors; + + % Centres are arranged in mlp such that we have + % u11, u12, u13, ..., u1c, ... , uj1, uj2, uj3, ..., ujc, ..., um1, uM2, + % ..., uMc + % This is achieved by transposing temp_mix.centres before reshaping + end_centres = ncentres*(dim_target+1); + net.mlp.b2(ncentres+1:end_centres) = ... + reshape(temp_mix.centres', 1, ncentres*dim_target); + + % Variances + net.mlp.b2((end_centres+1):net.mlp.nout) = ... + log(temp_mix.covars); +end diff --git a/sourcecodes/bnt-master/netlab3.3/mdnnet.mat b/sourcecodes/bnt-master/netlab3.3/mdnnet.mat new file mode 100644 index 00000000..724f8d43 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnnet.mat Binary files differdiff --git a/sourcecodes/bnt-master/netlab3.3/mdnpak.m b/sourcecodes/bnt-master/netlab3.3/mdnpak.m new file mode 100644 index 00000000..47da11a6 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnpak.m @@ -0,0 +1,19 @@ +function w = mdnpak(net) +%MDNPAK Combines weights and biases into one weights vector. +% +% Description +% W = MDNPAK(NET) takes a mixture density network data structure NET +% and combines the network weights into a single row vector W. +% +% See also +% MDN, MDNUNPAK, MDNFWD, MDNERR, MDNGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +errstring = consist(net, 'mdn'); +if ~errstring + error(errstring); +end +w = mlppak(net.mlp); diff --git a/sourcecodes/bnt-master/netlab3.3/mdnpost.m b/sourcecodes/bnt-master/netlab3.3/mdnpost.m new file mode 100644 index 00000000..2a27849e --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnpost.m @@ -0,0 +1,26 @@ +function [post, a] = mdnpost(mixparams, t) +%MDNPOST Computes the posterior probability for each MDN mixture component. +% +% Description +% POST = MDNPOST(MIXPARAMS, T) computes the posterior probability +% P(J|T) of each data vector in T under the Gaussian mixture model +% represented by the corresponding entries in MIXPARAMS. Each row of T +% represents a single vector. +% +% [POST, A] = MDNPOST(MIXPARAMS, T) also computes the activations A +% (i.e. the probability P(T|J) of the data conditioned on each +% component density) for a Gaussian mixture model. +% +% See also +% MDNGRAD, MDNPROB +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +[prob a] = mdnprob(mixparams, t); + +s = sum(prob, 2); +% Set any zeros to one before dividing +s = s + (s==0); +post = prob./(s*ones(1, mixparams.ncentres)); diff --git a/sourcecodes/bnt-master/netlab3.3/mdnprob.m b/sourcecodes/bnt-master/netlab3.3/mdnprob.m new file mode 100644 index 00000000..3c828eac --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnprob.m @@ -0,0 +1,52 @@ +function [prob,a] = mdnprob(mixparams, t) +%MDNPROB Computes the data probability likelihood for an MDN mixture structure. +% +% Description +% PROB = MDNPROB(MIXPARAMS, T) computes the probability P(T) of each +% data vector in T under the Gaussian mixture model represented by the +% corresponding entries in MIXPARAMS. Each row of T represents a single +% vector. +% +% [PROB, A] = MDNPROB(MIXPARAMS, T) also computes the activations A +% (i.e. the probability P(T|J) of the data conditioned on each +% component density) for a Gaussian mixture model. +% +% See also +% MDNERR, MDNPOST +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +% Check arguments for consistency +errstring = consist(mixparams, 'mdnmixes'); +if ~isempty(errstring) + error(errstring); +end + +ntarget = size(t, 1); +if ntarget ~= size(mixparams.centres, 1) + error('Number of targets does not match number of mixtures') +end +if size(t, 2) ~= mixparams.dim_target + error('Target dimension does not match mixture dimension') +end + +dim_target = mixparams.dim_target; +ntarget = size(t, 1); + +% Calculate squared norm matrix, of dimension (ndata, ncentres) +% vector (ntarget * ncentres) +dist2 = mdndist2(mixparams, t); + +% Calculate variance factors +variance = 2.*mixparams.covars; + +% Compute the normalisation term +normal = ((2.*pi).*mixparams.covars).^(dim_target./2); + +% Now compute the activations +a = exp(-(dist2./variance))./normal; + +% Accumulate negative log likelihood of targets +prob = mixparams.mixcoeffs.*a; diff --git a/sourcecodes/bnt-master/netlab3.3/mdnunpak.m b/sourcecodes/bnt-master/netlab3.3/mdnunpak.m new file mode 100644 index 00000000..0b88cb7a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mdnunpak.m @@ -0,0 +1,25 @@ +function net = mdnunpak(net, w) +%MDNUNPAK Separates weights vector into weight and bias matrices. +% +% Description +% NET = MDNUNPAK(NET, W) takes an mdn network data structure NET and a +% weight vector W, and returns a network data structure identical to +% the input network, except that the weights in the MLP sub-structure +% are set to the corresponding elements of W. +% +% See also +% MDN, MDNPAK, MDNFWD, MDNERR, MDNGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) +% David J Evans (1998) + +errstring = consist(net, 'mdn'); +if ~errstring + error(errstring); +end +if net.nwts ~= length(w) + error('Invalid weight vector length') +end + +net.mlp = mlpunpak(net.mlp, w); diff --git a/sourcecodes/bnt-master/netlab3.3/metrop.m b/sourcecodes/bnt-master/netlab3.3/metrop.m new file mode 100644 index 00000000..730afc6b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/metrop.m @@ -0,0 +1,193 @@ +function [samples, energies, diagn] = metrop(f, x, options, gradf, varargin) +%METROP Markov Chain Monte Carlo sampling with Metropolis algorithm. +% +% Description +% SAMPLES = METROP(F, X, OPTIONS) uses the Metropolis algorithm to +% sample from the distribution P ~ EXP(-F), where F is the first +% argument to METROP. The Markov chain starts at the point X and each +% candidate state is picked from a Gaussian proposal distribution and +% accepted or rejected according to the Metropolis criterion. +% +% SAMPLES = METROP(F, X, OPTIONS, [], P1, P2, ...) allows additional +% arguments to be passed to F(). The fourth argument is ignored, but +% is included for compatibility with HMC and the optimisers. +% +% [SAMPLES, ENERGIES, DIAGN] = METROP(F, X, OPTIONS) also returns a log +% of the energy values (i.e. negative log probabilities) for the +% samples in ENERGIES and DIAGN, a structure containing diagnostic +% information (position and acceptance threshold) for each step of the +% chain in DIAGN.POS and DIAGN.ACC respectively. All candidate states +% (including rejected ones) are stored in DIAGN.POS. +% +% S = METROP('STATE') returns a state structure that contains the state +% of the two random number generators RAND and RANDN. These are +% contained in fields randstate, randnstate. +% +% METROP('STATE', S) resets the state to S. If S is an integer, then +% it is passed to RAND and RANDN. If S is a structure returned by +% METROP('STATE') then it resets the generator to exactly the same +% state. +% +% The optional parameters in the OPTIONS vector have the following +% interpretations. +% +% OPTIONS(1) is set to 1 to display the energy values and rejection +% threshold at each step of the Markov chain. If the value is 2, then +% the position vectors at each step are also displayed. +% +% OPTIONS(14) is the number of samples retained from the Markov chain; +% default 100. +% +% OPTIONS(15) is the number of samples omitted from the start of the +% chain; default 0. +% +% OPTIONS(18) is the variance of the proposal distribution; default 1. +% +% See also +% HMC +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargin <= 2 + if ~strcmp(f, 'state') + error('Unknown argument to metrop'); + end + switch nargin + case 1 + % Return state of sampler + samples = get_state(f); % Function defined in this module + return; + case 2 + % Set the state of the sampler + set_state(f, x); % Function defined in this module + return; + end +end + +if 0 +seed = 42; +randn('state', seed); +rand('state', seed) +end + +display = options(1); +if options(14) > 0 + nsamples = options(14); +else + nsamples = 100; +end +if options(15) >= 0 + nomit = options(15); +else + nomit = 0; +end +if options(18) > 0.0 + std_dev = sqrt(options(18)); +else + std_dev = 1.0; % default +end +nparams = length(x); + +% Set up string for evaluating potential function. +f = fcnchk(f, length(varargin)); + +samples = zeros(nsamples, nparams); % Matrix of returned samples. +if nargout >= 2 + en_save = 1; + energies = zeros(nsamples, 1); +else + en_save = 0; +end +if nargout >= 3 + diagnostics = 1; + diagn_pos = zeros(nsamples, nparams); + diagn_acc = zeros(nsamples, 1); +else + diagnostics = 0; +end + +% Main loop. +n = - nomit + 1; +Eold = feval(f, x, varargin{:}); % Evaluate starting energy. +nreject = 0; % Initialise count of rejected states. +while n <= nsamples + + xold = x; + % Sample a new point from the proposal distribution + x = xold + randn(1, nparams)*std_dev; + %fprintf('netlab propose: xold = %5.3f,%5.3f, xnew = %5.3f,%5.3f\n',... + % xold(1), xold(2), x(1), x(2)); + + % Now apply Metropolis algorithm. + Enew = feval(f, x, varargin{:}); % Evaluate new energy. + a = exp(Eold - Enew); % Acceptance threshold. + if (diagnostics & n > 0) + diagn_pos(n,:) = x; + diagn_acc(n,:) = a; + end + if (display > 1) + fprintf(1, 'New position is\n'); + disp(x); + end + + r = rand(1); + %fprintf('netlab: n=%d, a=%f/%f=%5.3f (%5.3f), r=%5.3f\n',... + % n, exp(-Enew), exp(-Eold), a, exp(-Enew)/exp(-Eold), r); + if a > r % Accept the new state. + Eold = Enew; + if (display > 0) + fprintf(1, 'Finished step %4d Threshold: %g\n', n, a); + end + else % Reject the new state + if n > 0 + nreject = nreject + 1; + end + x = xold; % Reset position + if (display > 0) + fprintf(1, ' Sample rejected %4d. Threshold: %g\n', n, a); + end + end + if n > 0 + samples(n,:) = x; % Store sample. + if en_save + energies(n) = Eold; % Store energy. + end + end + n = n + 1; +end + +if (display > 0) + fprintf(1, '\nFraction of samples rejected: %g\n', ... + nreject/(nsamples)); +end + +if diagnostics + diagn.pos = diagn_pos; + diagn.acc = diagn_acc; +end + +% Return complete state of the sampler. +function state = get_state(f) + +state.randstate = rand('state'); +state.randnstate = randn('state'); +return + +% Set state of sampler, either from full state, or with an integer +function set_state(f, x) + +if isnumeric(x) + rand('state', x); + randn('state', x); +else + if ~isstruct(x) + error('Second argument to metrop must be number or state structure'); + end + if (~isfield(x, 'randstate') | ~isfield(x, 'randnstate')) + error('Second argument to metrop must contain correct fields') + end + rand('state', x.randstate); + randn('state', x.randnstate); +end +return diff --git a/sourcecodes/bnt-master/netlab3.3/minbrack.m b/sourcecodes/bnt-master/netlab3.3/minbrack.m new file mode 100644 index 00000000..31b93c42 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/minbrack.m @@ -0,0 +1,127 @@ +function [br_min, br_mid, br_max, num_evals] = minbrack(f, a, b, fa, ... + varargin) +%MINBRACK Bracket a minimum of a function of one variable. +% +% Description +% BRMIN, BRMID, BRMAX, NUMEVALS] = MINBRACK(F, A, B, FA) finds a +% bracket of three points around a local minimum of F. The function F +% must have a one dimensional domain. A < B is an initial guess at the +% minimum and maximum points of a bracket, but MINBRACK will search +% outside this interval if necessary. The bracket consists of three +% points (in increasing order) such that F(BRMID) < F(BRMIN) and +% F(BRMID) < F(BRMAX). FA is the value of the function at A: it is +% included to avoid unnecessary function evaluations in the +% optimization routines. The return value NUMEVALS is the number of +% function evaluations in MINBRACK. +% +% MINBRACK(F, A, B, FA, P1, P2, ...) allows additional arguments to be +% passed to F +% +% See also +% LINEMIN, LINEF +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check function string +f = fcnchk(f, length(varargin)); + +% Value of golden section (1 + sqrt(5))/2.0 +phi = 1.6180339887499; + +% Initialise count of number of function evaluations +num_evals = 0; + +% A small non-zero number to avoid dividing by zero in quadratic interpolation +TINY = 1.e-10; + +% Maximal proportional step to take: don't want to make this too big +% as then spend a lot of time finding the minimum inside the bracket +max_step = 10.0; + +fb = feval(f, b, varargin{:}); +num_evals = num_evals + 1; + +% Assume that we know going from a to b is downhill initially +% (usually because gradf(a) < 0). +if (fb > fa) + % Minimum must lie between a and b: do golden section until we find point + % low enough to be middle of bracket + c = b; + b = a + (c-a)/phi; + fb = feval(f, b, varargin{:}); + num_evals = num_evals + 1; + while (fb > fa) + c = b; + b = a + (c-a)/phi; + fb = feval(f, b, varargin{:}); + num_evals = num_evals + 1; + end +else + % There is a valid bracket upper bound greater than b + c = b + phi*(b-a); + fc = feval(f, c, varargin{:}); + num_evals = num_evals + 1; + bracket_found = 0; + + while (fb > fc) + % Do a quadratic interpolation (i.e. to minimum of quadratic) + r = (b-a).*(fb-fc); + q = (b-c).*(fb-fa); + u = b - ((b-c)*q - (b-a)*r)/(2.0*(sign(q-r)*max([abs(q-r), TINY]))); + ulimit = b + max_step*(c-b); + + if ((b-u)'*(u-c) > 0.0) + % Interpolant lies between b and c + fu = feval(f, u, varargin{:}); + num_evals = num_evals + 1; + if (fu < fc) + % Have a minimum between b and c + br_min = b; + br_mid = u; + br_max = c; + return; + elseif (fu > fb) + % Have a minimum between a and u + br_min = a; + br_mid = c; + br_max = u; + return; + end + % Quadratic interpolation didn't give a bracket, so take a golden step + u = c + phi*(c-b); + elseif ((c-u)'*(u-ulimit) > 0.0) + % Interpolant lies between c and limit + fu = feval(f, u, varargin{:}); + num_evals = num_evals + 1; + if (fu < fc) + % Move bracket along, and then take a golden section step + b = c; + c = u; + u = c + phi*(c-b); + else + bracket_found = 1; + end + elseif ((u-ulimit)'*(ulimit-c) >= 0.0) + % Limit parabolic u to maximum value + u = ulimit; + else + % Reject parabolic u and use golden section step + u = c + phi*(c-b); + end + if ~bracket_found + fu = feval(f, u, varargin{:}); + num_evals = num_evals + 1; + end + a = b; b = c; c = u; + fa = fb; fb = fc; fc = fu; + end % while loop +end % bracket found +br_mid = b; +if (a < c) + br_min = a; + br_max = c; +else + br_min = c; + br_max = a; +end diff --git a/sourcecodes/bnt-master/netlab3.3/mlp.m b/sourcecodes/bnt-master/netlab3.3/mlp.m new file mode 100644 index 00000000..d1dc1cf1 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlp.m @@ -0,0 +1,92 @@ +function net = mlp(nin, nhidden, nout, outfunc, prior, beta) +%MLP Create a 2-layer feedforward network. +% +% Description +% NET = MLP(NIN, NHIDDEN, NOUT, FUNC) takes the number of inputs, +% hidden units and output units for a 2-layer feed-forward network, +% together with a string FUNC which specifies the output unit +% activation function, and returns a data structure NET. The weights +% are drawn from a zero mean, unit variance isotropic Gaussian, with +% varianced scaled by the fan-in of the hidden or output units as +% appropriate. This makes use of the Matlab function RANDN and so the +% seed for the random weight initialization can be set using +% RANDN('STATE', S) where S is the seed value. The hidden units use +% the TANH activation function. +% +% The fields in NET are +% type = 'mlp' +% nin = number of inputs +% nhidden = number of hidden units +% nout = number of outputs +% nwts = total number of weights and biases +% actfn = string describing the output unit activation function: +% 'linear' +% 'logistic +% 'softmax' +% w1 = first-layer weight matrix +% b1 = first-layer bias vector +% w2 = second-layer weight matrix +% b2 = second-layer bias vector +% Here W1 has dimensions NIN times NHIDDEN, B1 has dimensions 1 times +% NHIDDEN, W2 has dimensions NHIDDEN times NOUT, and B2 has dimensions +% 1 times NOUT. +% +% NET = MLP(NIN, NHIDDEN, NOUT, FUNC, PRIOR), in which PRIOR is a +% scalar, allows the field NET.ALPHA in the data structure NET to be +% set, corresponding to a zero-mean isotropic Gaussian prior with +% inverse variance with value PRIOR. Alternatively, PRIOR can consist +% of a data structure with fields ALPHA and INDEX, allowing individual +% Gaussian priors to be set over groups of weights in the network. Here +% ALPHA is a column vector in which each element corresponds to a +% separate group of weights, which need not be mutually exclusive. The +% membership of the groups is defined by the matrix INDX in which the +% columns correspond to the elements of ALPHA. Each column has one +% element for each weight in the matrix, in the order defined by the +% function MLPPAK, and each element is 1 or 0 according to whether the +% weight is a member of the corresponding group or not. A utility +% function MLPPRIOR is provided to help in setting up the PRIOR data +% structure. +% +% NET = MLP(NIN, NHIDDEN, NOUT, FUNC, PRIOR, BETA) also sets the +% additional field NET.BETA in the data structure NET, where beta +% corresponds to the inverse noise variance. +% +% See also +% MLPPRIOR, MLPPAK, MLPUNPAK, MLPFWD, MLPERR, MLPBKP, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.type = 'mlp'; +net.nin = nin; +net.nhidden = nhidden; +net.nout = nout; +net.nwts = (nin + 1)*nhidden + (nhidden + 1)*nout; + +outfns = {'linear', 'logistic', 'softmax'}; + +if sum(strcmp(outfunc, outfns)) == 0 + error('Undefined output function. Exiting.'); +else + net.outfn = outfunc; +end + +if nargin > 4 + if isstruct(prior) + net.alpha = prior.alpha; + net.index = prior.index; + elseif size(prior) == [1 1] + net.alpha = prior; + else + error('prior must be a scalar or a structure'); + end +end + +net.w1 = randn(nin, nhidden)/sqrt(nin + 1); +net.b1 = randn(1, nhidden)/sqrt(nin + 1); +net.w2 = randn(nhidden, nout)/sqrt(nhidden + 1); +net.b2 = randn(1, nout)/sqrt(nhidden + 1); + +if nargin == 6 + net.beta = beta; +end diff --git a/sourcecodes/bnt-master/netlab3.3/mlpbkp.m b/sourcecodes/bnt-master/netlab3.3/mlpbkp.m new file mode 100644 index 00000000..5e45c8b8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpbkp.m @@ -0,0 +1,37 @@ +function g = mlpbkp(net, x, z, deltas) +%MLPBKP Backpropagate gradient of error function for 2-layer network. +% +% Description +% G = MLPBKP(NET, X, Z, DELTAS) takes a network data structure NET +% together with a matrix X of input vectors, a matrix Z of hidden unit +% activations, and a matrix DELTAS of the gradient of the error +% function with respect to the values of the output units (i.e. the +% summed inputs to the output units, before the activation function is +% applied). The return value is the gradient G of the error function +% with respect to the network weights. Each row of X corresponds to one +% input vector. +% +% This function is provided so that the common backpropagation +% algorithm can be used by multi-layer perceptron network models to +% compute gradients for mixture density networks as well as standard +% error functions. +% +% See also +% MLP, MLPGRAD, MLPDERIV, MDNGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Evaluate second-layer gradients. +gw2 = z'*deltas; +gb2 = sum(deltas, 1); + +% Now do the backpropagation. +delhid = deltas*net.w2'; +delhid = delhid.*(1.0 - z.*z); + +% Finally, evaluate the first-layer gradients. +gw1 = x'*delhid; +gb1 = sum(delhid, 1); + +g = [gw1(:)', gb1, gw2(:)', gb2]; diff --git a/sourcecodes/bnt-master/netlab3.3/mlpderiv.m b/sourcecodes/bnt-master/netlab3.3/mlpderiv.m new file mode 100644 index 00000000..9dd96c03 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpderiv.m @@ -0,0 +1,47 @@ +function g = mlpderiv(net, x) +%MLPDERIV Evaluate derivatives of network outputs with respect to weights. +% +% Description +% G = MLPDERIV(NET, X) takes a network data structure NET and a matrix +% of input vectors X and returns a three-index matrix G whose I, J, K +% element contains the derivative of network output K with respect to +% weight or bias parameter J for input pattern I. The ordering of the +% weight and bias parameters is defined by MLPUNPAK. +% +% See also +% MLP, MLPPAK, MLPGRAD, MLPBKP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp', x); +if ~isempty(errstring); + error(errstring); +end + +[y, z] = mlpfwd(net, x); + +ndata = size(x, 1); + +if isfield(net, 'mask') + nwts = size(find(net.mask), 1); + temp = zeros(1, net.nwts); +else + nwts = net.nwts; +end + +g = zeros(ndata, nwts, net.nout); +for k = 1 : net.nout + delta = zeros(1, net.nout); + delta(1, k) = 1; + for n = 1 : ndata + if isfield(net, 'mask') + temp = mlpbkp(net, x(n, :), z(n, :), delta); + g(n, :, k) = temp(logical(net.mask)); + else + g(n, :, k) = mlpbkp(net, x(n, :), z(n, :),... + delta); + end + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/mlperr.m b/sourcecodes/bnt-master/netlab3.3/mlperr.m new file mode 100644 index 00000000..7e3efe47 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlperr.m @@ -0,0 +1,62 @@ +function [e, edata, eprior] = mlperr(net, x, t) +%MLPERR Evaluate error function for 2-layer network. +% +% Description +% E = MLPERR(NET, X, T) takes a network data structure NET together +% with a matrix X of input vectors and a matrix T of target vectors, +% and evaluates the error function E. The choice of error function +% corresponds to the output unit activation function. Each row of X +% corresponds to one input vector and each row of T corresponds to one +% target vector. +% +% [E, EDATA, EPRIOR] = MLPERR(NET, X, T) additionally returns the data +% and prior components of the error, assuming a zero mean Gaussian +% prior on the weights with inverse variance parameters ALPHA and BETA +% taken from the network data structure NET. +% +% See also +% MLP, MLPPAK, MLPUNPAK, MLPFWD, MLPBKP, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp', x, t); +if ~isempty(errstring); + error(errstring); +end +[y, z, a] = mlpfwd(net, x); + +switch net.outfn + + case 'linear' % Linear outputs + edata = 0.5*sum(sum((y - t).^2)); + + case 'logistic' % Logistic outputs + % Ensure that log(1-y) is computable: need exp(a) > eps + maxcut = -log(eps); + % Ensure that log(y) is computable + mincut = -log(1/realmin - 1); + a = min(a, maxcut); + a = max(a, mincut); + y = 1./(1 + exp(-a)); + edata = - sum(sum(t.*log(y) + (1 - t).*log(1 - y))); + + case 'softmax' % Softmax outputs + nout = size(a,2); + % Ensure that sum(exp(a), 2) does not overflow + maxcut = log(realmax) - log(nout); + % Ensure that exp(a) > 0 + mincut = log(realmin); + a = min(a, maxcut); + a = max(a, mincut); + temp = exp(a); + y = temp./(sum(temp, 2)*ones(1,nout)); + % Ensure that log(y) is computable + y(y<realmin) = realmin; + edata = - sum(sum(t.*log(y))); + + otherwise + error(['Unknown activation function ', net.outfn]); +end +[e, edata, eprior] = errbayes(net, edata); diff --git a/sourcecodes/bnt-master/netlab3.3/mlpevfwd.m b/sourcecodes/bnt-master/netlab3.3/mlpevfwd.m new file mode 100644 index 00000000..e6d6898e --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpevfwd.m @@ -0,0 +1,26 @@ +function [y, extra, invhess] = mlpevfwd(net, x, t, x_test, invhess) +%MLPEVFWD Forward propagation with evidence for MLP +% +% Description +% Y = MLPEVFWD(NET, X, T, X_TEST) takes a network data structure NET +% together with the input X and target T training data and input test +% data X_TEST. It returns the normal forward propagation through the +% network Y together with a matrix EXTRA which consists of error bars +% (variance) for a regression problem or moderated outputs for a +% classification problem. The optional argument (and return value) +% INVHESS is the inverse of the network Hessian computed on the +% training data inputs and targets. Passing it in avoids recomputing +% it, which can be a significant saving for large training sets. +% +% See also +% FEVBAYES +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[y, z, a] = mlpfwd(net, x_test); +if nargin == 4 + [extra, invhess] = fevbayes(net, y, a, x, t, x_test); +else + [extra, invhess] = fevbayes(net, y, a, x, t, x_test, invhess); +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/mlpfwd.m b/sourcecodes/bnt-master/netlab3.3/mlpfwd.m new file mode 100644 index 00000000..d470ac96 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpfwd.m @@ -0,0 +1,65 @@ +function [y, z, a] = mlpfwd(net, x) +%MLPFWD Forward propagation through 2-layer network. +% +% Description +% Y = MLPFWD(NET, X) takes a network data structure NET together with a +% matrix X of input vectors, and forward propagates the inputs through +% the network to generate a matrix Y of output vectors. Each row of X +% corresponds to one input vector and each row of Y corresponds to one +% output vector. +% +% [Y, Z] = MLPFWD(NET, X) also generates a matrix Z of the hidden unit +% activations where each row corresponds to one pattern. +% +% [Y, Z, A] = MLPFWD(NET, X) also returns a matrix A giving the summed +% inputs to each output unit, where each row corresponds to one +% pattern. +% +% See also +% MLP, MLPPAK, MLPUNPAK, MLPERR, MLPBKP, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp', x); +if ~isempty(errstring); + error(errstring); +end + +ndata = size(x, 1); + +z = tanh(x*net.w1 + ones(ndata, 1)*net.b1); +a = z*net.w2 + ones(ndata, 1)*net.b2; + +switch net.outfn + + case 'linear' % Linear outputs + + y = a; + + case 'logistic' % Logistic outputs + % Prevent overflow and underflow: use same bounds as mlperr + % Ensure that log(1-y) is computable: need exp(a) > eps + maxcut = -log(eps); + % Ensure that log(y) is computable + mincut = -log(1/realmin - 1); + a = min(a, maxcut); + a = max(a, mincut); + y = 1./(1 + exp(-a)); + + case 'softmax' % Softmax outputs + + % Prevent overflow and underflow: use same bounds as glmerr + % Ensure that sum(exp(a), 2) does not overflow + maxcut = log(realmax) - log(net.nout); + % Ensure that exp(a) > 0 + mincut = log(realmin); + a = min(a, maxcut); + a = max(a, mincut); + temp = exp(a); + y = temp./(sum(temp, 2)*ones(1, net.nout)); + + otherwise + error(['Unknown activation function ', net.outfn]); +end diff --git a/sourcecodes/bnt-master/netlab3.3/mlpgrad.m b/sourcecodes/bnt-master/netlab3.3/mlpgrad.m new file mode 100644 index 00000000..c5a2a349 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpgrad.m @@ -0,0 +1,33 @@ +function [g, gdata, gprior] = mlpgrad(net, x, t) +%MLPGRAD Evaluate gradient of error function for 2-layer network. +% +% Description +% G = MLPGRAD(NET, X, T) takes a network data structure NET together +% with a matrix X of input vectors and a matrix T of target vectors, +% and evaluates the gradient G of the error function with respect to +% the network weights. The error funcion corresponds to the choice of +% output unit activation function. Each row of X corresponds to one +% input vector and each row of T corresponds to one target vector. +% +% [G, GDATA, GPRIOR] = MLPGRAD(NET, X, T) also returns separately the +% data and prior contributions to the gradient. In the case of multiple +% groups in the prior, GPRIOR is a matrix with a row for each group and +% a column for each weight parameter. +% +% See also +% MLP, MLPPAK, MLPUNPAK, MLPFWD, MLPERR, MLPBKP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp', x, t); +if ~isempty(errstring); + error(errstring); +end +[y, z] = mlpfwd(net, x); +delout = y - t; + +gdata = mlpbkp(net, x, z, delout); + +[g, gdata, gprior] = gbayes(net, gdata); diff --git a/sourcecodes/bnt-master/netlab3.3/mlphdotv.m b/sourcecodes/bnt-master/netlab3.3/mlphdotv.m new file mode 100644 index 00000000..63b54622 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlphdotv.m @@ -0,0 +1,79 @@ +function hdv = mlphdotv(net, x, t, v) +%MLPHDOTV Evaluate the product of the data Hessian with a vector. +% +% Description +% +% HDV = MLPHDOTV(NET, X, T, V) takes an MLP network data structure NET, +% together with the matrix X of input vectors, the matrix T of target +% vectors and an arbitrary row vector V whose length equals the number +% of parameters in the network, and returns the product of the data- +% dependent contribution to the Hessian matrix with V. The +% implementation is based on the R-propagation algorithm of +% Pearlmutter. +% +% See also +% MLP, MLPHESS, HESSCHEK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp', x, t); +if ~isempty(errstring); + error(errstring); +end + +ndata = size(x, 1); + +[y, z] = mlpfwd(net, x); % Standard forward propagation. +zprime = (1 - z.*z); % Hidden unit first derivatives. +zpprime = -2.0*z.*zprime; % Hidden unit second derivatives. + +vnet = mlpunpak(net, v); % Unpack the v vector. + +% Do the R-forward propagation. + +ra1 = x*vnet.w1 + ones(ndata, 1)*vnet.b1; +rz = zprime.*ra1; +ra2 = rz*net.w2 + z*vnet.w2 + ones(ndata, 1)*vnet.b2; + +switch net.outfn + + case 'linear' % Linear outputs + + ry = ra2; + + case 'logistic' % Logistic outputs + + ry = y.*(1 - y).*ra2; + + case 'softmax' % Softmax outputs + + nout = size(t, 2); + ry = y.*ra2 - y.*(sum(y.*ra2, 2)*ones(1, nout)); + + otherwise + error(['Unknown activation function ', net.outfn]); +end + +% Evaluate delta for the output units. + +delout = y - t; + +% Do the standard backpropagation. + +delhid = zprime.*(delout*net.w2'); + +% Now do the R-backpropagation. + +rdelhid = zpprime.*ra1.*(delout*net.w2') + zprime.*(delout*vnet.w2') + ... + zprime.*(ry*net.w2'); + +% Finally, evaluate the components of hdv and then merge into long vector. + +hw1 = x'*rdelhid; +hb1 = sum(rdelhid, 1); +hw2 = z'*ry + rz'*delout; +hb2 = sum(ry, 1); + +hdv = [hw1(:)', hb1, hw2(:)', hb2]; diff --git a/sourcecodes/bnt-master/netlab3.3/mlphess.m b/sourcecodes/bnt-master/netlab3.3/mlphess.m new file mode 100644 index 00000000..bd7c44e9 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlphess.m @@ -0,0 +1,51 @@ +function [h, hdata] = mlphess(net, x, t, hdata) +%MLPHESS Evaluate the Hessian matrix for a multi-layer perceptron network. +% +% Description +% H = MLPHESS(NET, X, T) takes an MLP network data structure NET, a +% matrix X of input values, and a matrix T of target values and returns +% the full Hessian matrix H corresponding to the second derivatives of +% the negative log posterior distribution, evaluated for the current +% weight and bias values as defined by NET. +% +% [H, HDATA] = MLPHESS(NET, X, T) returns both the Hessian matrix H and +% the contribution HDATA arising from the data dependent term in the +% Hessian. +% +% H = MLPHESS(NET, X, T, HDATA) takes a network data structure NET, a +% matrix X of input values, and a matrix T of target values, together +% with the contribution HDATA arising from the data dependent term in +% the Hessian, and returns the full Hessian matrix H corresponding to +% the second derivatives of the negative log posterior distribution. +% This version saves computation time if HDATA has already been +% evaluated for the current weight and bias values. +% +% See also +% MLP, HESSCHEK, MLPHDOTV, EVIDENCE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp', x, t); +if ~isempty(errstring); + error(errstring); +end + +if nargin == 3 + % Data term in Hessian needs to be computed + hdata = datahess(net, x, t); +end + +[h, hdata] = hbayes(net, hdata); + +% Sub-function to compute data part of Hessian +function hdata = datahess(net, x, t) + +hdata = zeros(net.nwts, net.nwts); + +for v = eye(net.nwts); + hdata(find(v),:) = mlphdotv(net, x, t, v); +end + +return diff --git a/sourcecodes/bnt-master/netlab3.3/mlphint.m b/sourcecodes/bnt-master/netlab3.3/mlphint.m new file mode 100644 index 00000000..3546dfd3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlphint.m @@ -0,0 +1,92 @@ +function [h1, h2] = mlphint(net); +%MLPHINT Plot Hinton diagram for 2-layer feed-forward network. +% +% Description +% +% MLPHINT(NET) takes a network structure NET and plots the Hinton +% diagram comprised of two figure windows, one displaying the first- +% layer weights and biases, and one displaying the second-layer weights +% and biases. +% +% [H1, H2] = MLPHINT(NET) also returns handles H1 and H2 to the +% figures which can be used, for instance, to delete the figures when +% they are no longer needed. +% +% To print the figure correctly, you should call SET(H, +% 'INVERTHARDCOPY', 'ON') before printing. +% +% See also +% DEMHINT, HINTMAT, MLP, MLPPAK, MLPUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set scale to be up to 0.9 of maximum absolute weight value, where scale +% defined so that area of box proportional to weight value. + +% Use no more than 640x480 pixels +xmax = 640; ymax = 480; + +% Offset bottom left hand corner +x01 = 40; y01 = 40; +x02 = 80; y02 = 80; + +% Need to allow 5 pixels border for window frame: but 30 at top +border = 5; +top_border = 30; + +ymax = ymax - top_border; +xmax = xmax - border; + +% First layer + +wb1 = [net.w1; net.b1]; +[xvals, yvals, color] = hintmat(wb1'); +% Try to preserve aspect ratio approximately +if (8*net.nhidden < 6*(net.nin + 1)) + delx = xmax; dely = xmax*net.nhidden/(net.nin + 1); +else + delx = ymax*(net.nin + 1)/net.nhidden; dely = ymax; +end + +h1 = figure('Color', [0.5 0.5 0.5], ... + 'Name', 'Hinton diagram: first-layer weights and biases', ... + 'NumberTitle', 'off', ... + 'Colormap', [0 0 0; 1 1 1], ... + 'Units', 'pixels', ... + 'Position', [x01 y01 delx dely]); +set(gca, 'Visible', 'off', 'Position', [0 0 1 1]); +hold on + +cmap = [0 0 0; 1 1 1]; +colors(1, :, :) = cmap(color, :); +patch(xvals', yvals', colors, 'Edgecolor', 'none'); +axis equal; +xpos = net.nin; +line([xpos xpos], [0 net.nhidden], 'color', 'red', 'linewidth', 3); + +% Second layer + +wb2 = [net.w2; net.b2]; +[xvals, yvals, color] = hintmat(wb2'); +if (8*net.nout < 6*(net.nhidden + 1)) + delx = xmax; dely = xmax*net.nout/(net.nhidden + 1); +else + delx = ymax*(net.nhidden + 1)/net.nout; dely = ymax; +end + +h2 = figure('Color', [0.5 0.5 0.5], ... + 'Name', 'Hinton diagram: second-layer weights and biases', ... + 'NumberTitle', 'off', ... + 'Colormap', [0 0 0; 1 1 1], ... + 'Units', 'pixels', ... + 'Position', [x02 y02 delx dely]); +set(gca, 'Visible', 'off', 'Position', [0 0 1 1]); + +hold on +colors2(1, :, :) = cmap(color, :); +patch(xvals', yvals', colors2, 'Edgecolor', 'none'); +axis equal; +xpos = net.nhidden; +line([xpos xpos], [0 net.nout], 'color', 'red', 'linewidth', 3); + diff --git a/sourcecodes/bnt-master/netlab3.3/mlpinit.m b/sourcecodes/bnt-master/netlab3.3/mlpinit.m new file mode 100644 index 00000000..ae4e80b5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpinit.m @@ -0,0 +1,31 @@ +function net = mlpinit(net, prior) +%MLPINIT Initialise the weights in a 2-layer feedforward network. +% +% Description +% +% NET = MLPINIT(NET, PRIOR) takes a 2-layer feedforward network NET and +% sets the weights and biases by sampling from a Gaussian distribution. +% If PRIOR is a scalar, then all of the parameters (weights and biases) +% are sampled from a single isotropic Gaussian with inverse variance +% equal to PRIOR. If PRIOR is a data structure of the kind generated by +% MLPPRIOR, then the parameters are sampled from multiple Gaussians +% according to their groupings (defined by the INDEX field) with +% corresponding variances (defined by the ALPHA field). +% +% See also +% MLP, MLPPRIOR, MLPPAK, MLPUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if isstruct(prior) + sig = 1./sqrt(prior.index*prior.alpha); + w = sig'.*randn(1, net.nwts); +elseif size(prior) == [1 1] + w = randn(1, net.nwts).*sqrt(1/prior); +else + error('prior must be a scalar or a structure'); +end + +net = mlpunpak(net, w); + diff --git a/sourcecodes/bnt-master/netlab3.3/mlppak.m b/sourcecodes/bnt-master/netlab3.3/mlppak.m new file mode 100644 index 00000000..30ec375d --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlppak.m @@ -0,0 +1,31 @@ +function w = mlppak(net) +%MLPPAK Combines weights and biases into one weights vector. +% +% Description +% W = MLPPAK(NET) takes a network data structure NET and combines the +% component weight matrices bias vectors into a single row vector W. +% The facility to switch between these two representations for the +% network parameters is useful, for example, in training a network by +% error function minimization, since a single vector of parameters can +% be handled by general-purpose optimization routines. +% +% The ordering of the paramters in W is defined by +% w = [net.w1(:)', net.b1, net.w2(:)', net.b2]; +% where W1 is the first-layer weight matrix, B1 is the first-layer +% bias vector, W2 is the second-layer weight matrix, and B2 is the +% second-layer bias vector. +% +% See also +% MLP, MLPUNPAK, MLPFWD, MLPERR, MLPBKP, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp'); +if ~isempty(errstring); + error(errstring); +end + +w = [net.w1(:)', net.b1, net.w2(:)', net.b2]; + diff --git a/sourcecodes/bnt-master/netlab3.3/mlpprior.m b/sourcecodes/bnt-master/netlab3.3/mlpprior.m new file mode 100644 index 00000000..8f7415c5 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpprior.m @@ -0,0 +1,62 @@ +function prior = mlpprior(nin, nhidden, nout, aw1, ab1, aw2, ab2) +%MLPPRIOR Create Gaussian prior for mlp. +% +% Description +% PRIOR = MLPPRIOR(NIN, NHIDDEN, NOUT, AW1, AB1, AW2, AB2) generates a +% data structure PRIOR, with fields PRIOR.ALPHA and PRIOR.INDEX, which +% specifies a Gaussian prior distribution for the network weights in a +% two-layer feedforward network. Two different cases are possible. In +% the first case, AW1, AB1, AW2 and AB2 are all scalars and represent +% the regularization coefficients for four groups of parameters in the +% network corresponding to first-layer weights, first-layer biases, +% second-layer weights, and second-layer biases respectively. Then +% PRIOR.ALPHA represents a column vector of length 4 containing the +% parameters, and PRIOR.INDEX is a matrix specifying which weights +% belong in each group. Each column has one element for each weight in +% the matrix, using the standard ordering as defined in MLPPAK, and +% each element is 1 or 0 according to whether the weight is a member of +% the corresponding group or not. In the second case the parameter AW1 +% is a vector of length equal to the number of inputs in the network, +% and the corresponding matrix PRIOR.INDEX now partitions the first- +% layer weights into groups corresponding to the weights fanning out of +% each input unit. This prior is appropriate for the technique of +% automatic relevance determination. +% +% See also +% MLP, MLPERR, MLPGRAD, EVIDENCE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +nextra = nhidden + (nhidden + 1)*nout; +nwts = nin*nhidden + nextra; + +if size(aw1) == [1,1] + + indx = [ones(1, nin*nhidden), zeros(1, nextra)]'; + +elseif size(aw1) == [1, nin] + + indx = kron(ones(nhidden, 1), eye(nin)); + indx = [indx; zeros(nextra, nin)]; + +else + + error('Parameter aw1 of invalid dimensions'); + +end + +extra = zeros(nwts, 3); + +mark1 = nin*nhidden; +mark2 = mark1 + nhidden; +extra(mark1 + 1:mark2, 1) = ones(nhidden,1); +mark3 = mark2 + nhidden*nout; +extra(mark2 + 1:mark3, 2) = ones(nhidden*nout,1); +mark4 = mark3 + nout; +extra(mark3 + 1:mark4, 3) = ones(nout,1); + +indx = [indx, extra]; + +prior.index = indx; +prior.alpha = [aw1, ab1, aw2, ab2]'; diff --git a/sourcecodes/bnt-master/netlab3.3/mlptrain.m b/sourcecodes/bnt-master/netlab3.3/mlptrain.m new file mode 100644 index 00000000..395689e9 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlptrain.m @@ -0,0 +1,24 @@ +function [net, error] = mlptrain(net, x, t, its); +%MLPTRAIN Utility to train an MLP network for demtrain +% +% Description +% +% [NET, ERROR] = MLPTRAIN(NET, X, T, ITS) trains a network data +% structure NET using the scaled conjugate gradient algorithm for ITS +% cycles with input data X, target data T. +% +% See also +% DEMTRAIN, SCG, NETOPT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +options = zeros(1,18); +options(1) = -1; % To prevent any messages at all +options(9) = 0; +options(14) = its; + +[net, options] = netopt(net, options, x, t, 'scg'); + +error = options(8); + diff --git a/sourcecodes/bnt-master/netlab3.3/mlpunpak.m b/sourcecodes/bnt-master/netlab3.3/mlpunpak.m new file mode 100644 index 00000000..c2b16b5f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/mlpunpak.m @@ -0,0 +1,39 @@ +function net = mlpunpak(net, w) +%MLPUNPAK Separates weights vector into weight and bias matrices. +% +% Description +% NET = MLPUNPAK(NET, W) takes an mlp network data structure NET and a +% weight vector W, and returns a network data structure identical to +% the input network, except that the first-layer weight matrix W1, the +% first-layer bias vector B1, the second-layer weight matrix W2 and the +% second-layer bias vector B2 have all been set to the corresponding +% elements of W. +% +% See also +% MLP, MLPPAK, MLPFWD, MLPERR, MLPBKP, MLPGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'mlp'); +if ~isempty(errstring); + error(errstring); +end + +if net.nwts ~= length(w) + error('Invalid weight vector length') +end + +nin = net.nin; +nhidden = net.nhidden; +nout = net.nout; + +mark1 = nin*nhidden; +net.w1 = reshape(w(1:mark1), nin, nhidden); +mark2 = mark1 + nhidden; +net.b1 = reshape(w(mark1 + 1: mark2), 1, nhidden); +mark3 = mark2 + nhidden*nout; +net.w2 = reshape(w(mark2 + 1: mark3), nhidden, nout); +mark4 = mark3 + nout; +net.b2 = reshape(w(mark3 + 1: mark4), 1, nout); diff --git a/sourcecodes/bnt-master/netlab3.3/netderiv.m b/sourcecodes/bnt-master/netlab3.3/netderiv.m new file mode 100644 index 00000000..2c6b379c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netderiv.m @@ -0,0 +1,19 @@ +function g = netderiv(w, net, x) +%NETDERIV Evaluate derivatives of network outputs by weights generically. +% +% Description +% +% G = NETDERIV(W, NET, X) takes a weight vector W and a network data +% structure NET, together with the matrix X of input vectors, and +% returns the gradient of the outputs with respect to the weights +% evaluated at W. +% +% See also +% NETEVFWD, NETOPT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +fstr = [net.type, 'deriv']; +net = netunpak(net, w); +g = feval(fstr, net, x); diff --git a/sourcecodes/bnt-master/netlab3.3/neterr.m b/sourcecodes/bnt-master/netlab3.3/neterr.m new file mode 100644 index 00000000..6679630c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/neterr.m @@ -0,0 +1,29 @@ +function [e, varargout] = neterr(w, net, x, t) +%NETERR Evaluate network error function for generic optimizers +% +% Description +% +% E = NETERR(W, NET, X, T) takes a weight vector W and a network data +% structure NET, together with the matrix X of input vectors and the +% matrix T of target vectors, and returns the value of the error +% function evaluated at W. +% +% [E, VARARGOUT] = NETERR(W, NET, X, T) also returns any additional +% return values from the error function. +% +% See also +% NETGRAD, NETHESS, NETOPT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstr = [net.type, 'err']; +net = netunpak(net, w); + +[s{1:nargout}] = feval(errstr, net, x, t); +e = s{1}; +if nargout > 1 + for i = 2:nargout + varargout{i-1} = s{i}; + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/netevfwd.m b/sourcecodes/bnt-master/netlab3.3/netevfwd.m new file mode 100644 index 00000000..26bb25aa --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netevfwd.m @@ -0,0 +1,29 @@ +function [y, extra, invhess] = netevfwd(w, net, x, t, x_test, invhess) +%NETEVFWD Generic forward propagation with evidence for network +% +% Description +% [Y, EXTRA] = NETEVFWD(W, NET, X, T, X_TEST) takes a network data +% structure NET together with the input X and target T training data +% and input test data X_TEST. It returns the normal forward propagation +% through the network Y together with a matrix EXTRA which consists of +% error bars (variance) for a regression problem or moderated outputs +% for a classification problem. +% +% The optional argument (and return value) INVHESS is the inverse of +% the network Hessian computed on the training data inputs and targets. +% Passing it in avoids recomputing it, which can be a significant +% saving for large training sets. +% +% See also +% MLPEVFWD, RBFEVFWD, GLMEVFWD, FEVBAYES +% + +% Copyright (c) Ian T Nabney (1996-2001) + +func = [net.type, 'evfwd']; +net = netunpak(net, w); +if nargin == 5 + [y, extra, invhess] = feval(func, net, x, t, x_test); +else + [y, extra, invhess] = feval(func, net, x, t, x_test, invhess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/netgrad.m b/sourcecodes/bnt-master/netlab3.3/netgrad.m new file mode 100644 index 00000000..afab88e0 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netgrad.m @@ -0,0 +1,21 @@ +function g = netgrad(w, net, x, t) +%NETGRAD Evaluate network error gradient for generic optimizers +% +% Description +% +% G = NETGRAD(W, NET, X, T) takes a weight vector W and a network data +% structure NET, together with the matrix X of input vectors and the +% matrix T of target vectors, and returns the gradient of the error +% function evaluated at W. +% +% See also +% MLP, NETERR, NETOPT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +gradstr = [net.type, 'grad']; + +net = netunpak(net, w); + +g = feval(gradstr, net, x, t); diff --git a/sourcecodes/bnt-master/netlab3.3/nethess.m b/sourcecodes/bnt-master/netlab3.3/nethess.m new file mode 100644 index 00000000..119fe502 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/nethess.m @@ -0,0 +1,29 @@ +function [h, varargout] = nethess(w, net, x, t, varargin) +%NETHESS Evaluate network Hessian +% +% Description +% +% H = NETHESS(W, NET, X, T) takes a weight vector W and a network data +% structure NET, together with the matrix X of input vectors and the +% matrix T of target vectors, and returns the value of the Hessian +% evaluated at W. +% +% [E, VARARGOUT] = NETHESS(W, NET, X, T, VARARGIN) also returns any +% additional return values from the network Hessian function, and +% passes additional arguments to that function. +% +% See also +% NETERR, NETGRAD, NETOPT +% + +% Copyright (c) Ian T Nabney (1996-2001) + +hess_str = [net.type, 'hess']; + +net = netunpak(net, w); + +[s{1:nargout}] = feval(hess_str, net, x, t, varargin{:}); +h = s{1}; +for i = 2:nargout + varargout{i-1} = s{i}; +end diff --git a/sourcecodes/bnt-master/netlab3.3/netinit.m b/sourcecodes/bnt-master/netlab3.3/netinit.m new file mode 100644 index 00000000..f94e30b3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netinit.m @@ -0,0 +1,45 @@ +function net = netinit(net, prior) +%NETINIT Initialise the weights in a network. +% +% Description +% +% NET = NETINIT(NET, PRIOR) takes a network data structure NET and sets +% the weights and biases by sampling from a Gaussian distribution. If +% PRIOR is a scalar, then all of the parameters (weights and biases) +% are sampled from a single isotropic Gaussian with inverse variance +% equal to PRIOR. If PRIOR is a data structure of the kind generated by +% MLPPRIOR, then the parameters are sampled from multiple Gaussians +% according to their groupings (defined by the INDEX field) with +% corresponding variances (defined by the ALPHA field). +% +% See also +% MLPPRIOR, NETUNPAK, RBFPRIOR +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if isstruct(prior) + if (isfield(net, 'mask')) + if find(sum(prior.index, 2)) ~= find(net.mask) + error('Index does not match mask'); + end + sig = sqrt(prior.index*prior.alpha); + % Weights corresponding to zeros in mask will not be used anyway + % Set their priors to one to avoid division by zero + sig = sig + (sig == 0); + sig = 1./sqrt(sig); + else + sig = 1./sqrt(prior.index*prior.alpha); + end + w = sig'.*randn(1, net.nwts); +elseif size(prior) == [1 1] + w = randn(1, net.nwts).*sqrt(1/prior); +else + error('prior must be a scalar or a structure'); +end + +if (isfield(net, 'mask')) + w = w(logical(net.mask)); +end +net = netunpak(net, w); + diff --git a/sourcecodes/bnt-master/netlab3.3/netlab3.3.zip b/sourcecodes/bnt-master/netlab3.3/netlab3.3.zip new file mode 100644 index 00000000..96bcfaa3 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netlab3.3.zip Binary files differdiff --git a/sourcecodes/bnt-master/netlab3.3/netlogo.mat b/sourcecodes/bnt-master/netlab3.3/netlogo.mat new file mode 100644 index 00000000..743716f8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netlogo.mat Binary files differdiff --git a/sourcecodes/bnt-master/netlab3.3/netopt.m b/sourcecodes/bnt-master/netlab3.3/netopt.m new file mode 100644 index 00000000..76807903 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netopt.m @@ -0,0 +1,52 @@ +function [net, options, varargout] = netopt(net, options, x, t, alg); +%NETOPT Optimize the weights in a network model. +% +% Description +% +% NETOPT is a helper function which facilitates the training of +% networks using the general purpose optimizers as well as sampling +% from the posterior distribution of parameters using general purpose +% Markov chain Monte Carlo sampling algorithms. It can be used with any +% function that searches in parameter space using error and gradient +% functions. +% +% [NET, OPTIONS] = NETOPT(NET, OPTIONS, X, T, ALG) takes a network +% data structure NET, together with a vector OPTIONS of parameters +% governing the behaviour of the optimization algorithm, a matrix X of +% input vectors and a matrix T of target vectors, and returns the +% trained network as well as an updated OPTIONS vector. The string ALG +% determines which optimization algorithm (CONJGRAD, QUASINEW, SCG, +% etc.) or Monte Carlo algorithm (such as HMC) will be used. +% +% [NET, OPTIONS, VARARGOUT] = NETOPT(NET, OPTIONS, X, T, ALG) also +% returns any additional return values from the optimisation algorithm. +% +% See also +% NETGRAD, BFGS, CONJGRAD, GRADDESC, HMC, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +optstring = [alg, '(''neterr'', w, options, ''netgrad'', net, x, t)']; + +% Extract weights from network as single vector +w = netpak(net); + +% Carry out optimisation +[s{1:nargout}] = eval(optstring); +w = s{1}; + +if nargout > 1 + options = s{2}; + + % If there are additional arguments, extract them + nextra = nargout - 2; + if nextra > 0 + for i = 1:nextra + varargout{i} = s{i+2}; + end + end +end + +% Pack the weights back into the network +net = netunpak(net, w); diff --git a/sourcecodes/bnt-master/netlab3.3/netpak.m b/sourcecodes/bnt-master/netlab3.3/netpak.m new file mode 100644 index 00000000..03d9bd58 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netpak.m @@ -0,0 +1,25 @@ +function w = netpak(net) +%NETPAK Combines weights and biases into one weights vector. +% +% Description +% W = NETPAK(NET) takes a network data structure NET and combines the +% component weight matrices into a single row vector W. The facility +% to switch between these two representations for the network +% parameters is useful, for example, in training a network by error +% function minimization, since a single vector of parameters can be +% handled by general-purpose optimization routines. This function also +% takes into account a MASK defined as a field in NET by removing any +% weights that correspond to entries of 0 in the mask. +% +% See also +% NET, NETUNPAK, NETFWD, NETERR, NETGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +pakstr = [net.type, 'pak']; +w = feval(pakstr, net); +% Return masked subset of weights +if (isfield(net, 'mask')) + w = w(logical(net.mask)); +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/netunpak.m b/sourcecodes/bnt-master/netlab3.3/netunpak.m new file mode 100644 index 00000000..7f253d2f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/netunpak.m @@ -0,0 +1,34 @@ +function net = netunpak(net, w) +%NETUNPAK Separates weights vector into weight and bias matrices. +% +% Description +% NET = NETUNPAK(NET, W) takes an net network data structure NET and a +% weight vector W, and returns a network data structure identical to +% the input network, except that the componenet weight matrices have +% all been set to the corresponding elements of W. If there is a MASK +% field in the NET data structure, then the weights in W are placed in +% locations corresponding to non-zero entries in the mask (so W should +% have the same length as the number of non-zero entries in the MASK). +% +% See also +% NETPAK, NETFWD, NETERR, NETGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +unpakstr = [net.type, 'unpak']; + +% Check if we are being passed a masked set of weights +if (isfield(net, 'mask')) + if length(w) ~= size(find(net.mask), 1) + error('Weight vector length does not match mask length') + end + % Do a full pack of all current network weights + pakstr = [net.type, 'pak']; + fullw = feval(pakstr, net); + % Replace current weights with new ones + fullw(logical(net.mask)) = w; + w = fullw; +end + +net = feval(unpakstr, net, w); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/oilTrn.dat b/sourcecodes/bnt-master/netlab3.3/oilTrn.dat new file mode 100644 index 00000000..ba12723b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/oilTrn.dat @@ -0,0 +1,503 @@ + nin 12 + nout 2 + ndata 500 + 3.315000e-01 2.156000e-01 6.802000e-01 1.434000e-01 6.825000e-01 2.720000e-01 6.223000e-01 2.092000e-01 7.961000e-01 1.530000e-01 5.856000e-01 2.573000e-01 3.440000e-01 1.401000e-01 + 9.390000e-02 1.008900e+00 3.650000e-02 6.944000e-01 9.080000e-02 4.961000e-01 7.220000e-02 6.521000e-01 -1.300000e-02 6.085000e-01 6.310000e-02 6.597000e-01 5.140000e-02 4.459000e-01 + 5.184000e-01 2.283000e-01 5.300000e-01 6.884000e-01 7.456000e-01 6.171000e-01 6.136000e-01 5.928000e-01 7.678000e-01 6.130000e-01 6.705000e-01 5.202000e-01 3.710000e-01 3.214000e-01 + 4.208000e-01 6.740000e-01 1.651000e-01 7.592000e-01 1.810000e-01 5.448000e-01 1.707000e-01 7.554000e-01 1.635000e-01 5.492000e-01 2.598000e-01 6.455000e-01 1.667000e-01 4.177000e-01 + 3.130000e-01 6.465000e-01 5.908000e-01 6.924000e-01 7.664000e-01 6.262000e-01 1.717700e+00 1.500000e-02 8.510000e-02 1.904600e+00 -1.650000e-02 2.210000e-02 3.378000e-01 4.184000e-01 + 1.145800e+00 -4.670000e-02 4.056000e-01 5.662000e-01 3.123000e-01 4.580000e-01 3.636000e-01 6.134000e-01 3.305000e-01 4.132000e-01 4.167000e-01 5.514000e-01 3.249000e-01 2.790000e-01 +-1.900000e-03 1.732000e-01 5.700000e-03 4.882000e-01 2.076000e-01 3.910000e-01 8.600000e-03 1.719800e+00 2.150000e-02 -2.580000e-02 6.730000e-02 -8.290000e-02 5.100000e-02 2.123000e-01 + 7.800000e-03 4.615000e-01 1.181000e-01 6.590000e-01 2.587000e-01 6.352000e-01 -1.910000e-02 1.749100e+00 1.098000e-01 -1.315000e-01 4.070000e-02 -4.850000e-02 6.210000e-02 3.856000e-01 + 7.305000e-01 1.189000e-01 1.062600e+00 1.013000e-01 1.138500e+00 1.486000e-01 1.763100e+00 -4.160000e-02 2.073100e+00 -8.720000e-02 -4.390000e-02 5.300000e-02 6.070000e-01 8.130000e-02 + 6.199000e-01 4.779000e-01 9.528000e-01 7.463000e-01 9.741000e-01 8.633000e-01 1.677100e+00 6.200000e-02 2.027100e+00 -3.010000e-02 1.510000e-02 1.714700e+00 5.065000e-01 4.589000e-01 + 1.130800e+00 -2.860000e-02 1.000500e+00 7.387000e-01 8.390000e-01 1.874000e-01 1.084800e+00 6.374000e-01 8.534000e-01 1.373000e-01 1.042200e+00 6.836000e-01 6.215000e-01 1.323000e-01 + 5.093000e-01 6.038000e-01 2.874000e-01 8.162000e-01 2.267000e-01 6.181000e-01 2.646000e-01 8.490000e-01 3.365000e-01 4.648000e-01 2.815000e-01 8.254000e-01 2.280000e-01 4.243000e-01 + 7.600000e-02 5.010000e-01 1.870000e-01 7.011000e-01 1.728000e-01 8.475000e-01 2.300000e-01 6.536000e-01 1.616000e-01 8.732000e-01 1.603000e-01 7.331000e-01 9.550000e-02 4.203000e-01 + 4.662000e-01 1.209000e-01 7.657000e-01 1.390000e-01 8.886000e-01 1.304000e-01 1.694000e+00 4.530000e-02 -1.170000e-02 2.011100e+00 -1.940000e-02 2.170000e-02 4.422000e-01 7.900000e-02 + 1.125400e+00 -2.520000e-02 6.776000e-01 1.793000e-01 4.287000e-01 2.804000e-01 6.003000e-01 2.737000e-01 4.541000e-01 2.370000e-01 6.775000e-01 1.850000e-01 4.369000e-01 1.284000e-01 + 3.334000e-01 5.468000e-01 6.072000e-01 7.549000e-01 6.294000e-01 9.322000e-01 5.850000e-01 7.859000e-01 6.324000e-01 9.568000e-01 6.026000e-01 7.616000e-01 3.277000e-01 4.629000e-01 + 3.760000e-02 7.113000e-01 4.992000e-01 5.377000e-01 6.049000e-01 5.530000e-01 1.663500e+00 8.210000e-02 -1.890000e-02 2.024100e+00 1.400000e-03 -3.000000e-03 2.534000e-01 3.568000e-01 + 1.703000e-01 2.234000e-01 2.677000e-01 3.477000e-01 2.734000e-01 4.324000e-01 1.222000e-01 5.213000e-01 2.333000e-01 4.926000e-01 2.044000e-01 4.215000e-01 1.407000e-01 2.175000e-01 + 1.328000e-01 2.144000e-01 2.466000e-01 2.912000e-01 3.244000e-01 2.815000e-01 1.586000e-01 3.982000e-01 3.165000e-01 3.005000e-01 3.038000e-01 2.239000e-01 1.300000e-01 1.831000e-01 + 1.106900e+00 -1.500000e-03 1.787100e+00 -7.230000e-02 1.038900e+00 7.086000e-01 1.672800e+00 7.010000e-02 1.008300e+00 5.901000e-01 1.740500e+00 -1.880000e-02 7.498000e-01 2.096000e-01 + 1.083700e+00 2.940000e-02 6.675000e-01 5.421000e-01 5.040000e-01 3.997000e-01 7.271000e-01 4.673000e-01 6.061000e-01 2.526000e-01 7.957000e-01 3.861000e-01 4.794000e-01 1.997000e-01 + 6.379000e-01 3.709000e-01 9.139000e-01 6.785000e-01 1.117100e+00 6.825000e-01 9.074000e-01 6.917000e-01 1.152700e+00 6.739000e-01 9.866000e-01 5.957000e-01 5.590000e-01 3.562000e-01 + 6.520000e-02 4.575000e-01 5.170000e-01 2.900000e-01 6.257000e-01 3.041000e-01 1.700700e+00 3.410000e-02 -1.700000e-03 2.400000e-03 -4.430000e-02 5.270000e-02 2.800000e-01 1.838000e-01 + 5.460000e-02 4.317000e-01 2.260000e-02 8.026000e-01 2.228000e-01 7.077000e-01 1.370000e-02 1.717800e+00 -5.640000e-02 6.540000e-02 -9.100000e-03 1.070000e-02 7.470000e-02 3.874000e-01 + 6.250000e-02 1.036100e+00 -3.160000e-02 9.835000e-01 1.020000e-02 7.331000e-01 -5.260000e-02 1.005500e+00 -5.360000e-02 7.915000e-01 -2.270000e-02 9.754000e-01 3.200000e-03 5.922000e-01 + 8.813000e-01 2.321000e-01 4.637000e-01 1.117000e-01 2.749000e-01 2.270000e-01 4.328000e-01 1.497000e-01 3.223000e-01 1.589000e-01 4.639000e-01 1.134000e-01 2.964000e-01 1.282000e-01 + 2.830000e-02 1.087400e+00 1.069000e-01 6.261000e-01 1.089000e-01 4.879000e-01 -3.670000e-02 7.978000e-01 -6.910000e-02 6.871000e-01 4.170000e-02 7.048000e-01 3.750000e-02 4.698000e-01 + 1.116000e+00 -1.490000e-02 1.075900e+00 6.281000e-01 6.518000e-01 9.179000e-01 9.002000e-01 8.356000e-01 6.325000e-01 8.442000e-01 8.748000e-01 8.595000e-01 5.796000e-01 3.492000e-01 + 3.371000e-01 7.714000e-01 7.103000e-01 8.678000e-01 8.329000e-01 8.652000e-01 1.701600e+00 3.270000e-02 -1.300000e-01 2.157200e+00 -8.280000e-02 1.834700e+00 3.773000e-01 5.367000e-01 + 7.327000e-01 3.195000e-01 9.908000e-01 6.799000e-01 1.154500e+00 7.437000e-01 1.039000e+00 6.200000e-01 1.251800e+00 6.617000e-01 9.425000e-01 7.395000e-01 5.773000e-01 3.877000e-01 + 3.568000e-01 4.784000e-01 7.597000e-01 3.709000e-01 7.185000e-01 5.614000e-01 1.689700e+00 4.820000e-02 -5.500000e-02 2.069800e+00 1.110000e-02 -1.070000e-02 3.974000e-01 2.757000e-01 + 8.451000e-01 2.537000e-01 1.175900e+00 3.158000e-01 1.157700e+00 4.833000e-01 1.690800e+00 5.060000e-02 1.869400e+00 1.534000e-01 -8.700000e-02 1.834000e+00 6.629000e-01 2.139000e-01 + 1.053400e+00 6.240000e-02 9.920000e-01 7.462000e-01 6.545000e-01 9.606000e-01 9.666000e-01 7.742000e-01 7.771000e-01 7.086000e-01 9.877000e-01 7.512000e-01 6.243000e-01 3.112000e-01 + 7.150000e-02 1.045000e+00 1.133000e-01 8.252000e-01 4.200000e-02 7.057000e-01 1.016000e-01 8.422000e-01 7.360000e-02 6.540000e-01 7.840000e-02 8.680000e-01 7.220000e-02 5.227000e-01 + 1.002600e+00 1.198000e-01 6.942000e-01 1.048700e+00 5.669000e-01 5.787000e-01 7.905000e-01 9.402000e-01 6.875000e-01 3.929000e-01 7.361000e-01 1.000800e+00 5.116000e-01 2.856000e-01 + 4.843000e-01 6.102000e-01 2.827000e-01 1.441700e+00 2.176000e-01 8.856000e-01 3.216000e-01 1.399100e+00 8.600000e-02 1.002300e+00 2.466000e-01 1.485100e+00 1.914000e-01 5.923000e-01 + 3.972000e-01 1.078000e-01 5.178000e-01 2.948000e-01 5.923000e-01 3.344000e-01 5.037000e-01 3.121000e-01 6.118000e-01 3.270000e-01 5.425000e-01 2.633000e-01 3.099000e-01 1.574000e-01 + 3.522000e-01 3.868000e-01 5.201000e-01 6.377000e-01 5.522000e-01 7.776000e-01 5.076000e-01 6.553000e-01 6.258000e-01 7.105000e-01 4.515000e-01 7.196000e-01 3.029000e-01 3.665000e-01 + 2.230000e-01 8.879000e-01 1.593000e-01 7.129000e-01 1.089000e-01 5.914000e-01 1.294000e-01 7.501000e-01 1.590000e-01 5.180000e-01 1.132000e-01 7.677000e-01 1.232000e-01 4.428000e-01 + 4.409000e-01 -1.820000e-02 6.190000e-01 5.930000e-02 5.689000e-01 2.286000e-01 4.717000e-01 2.311000e-01 5.974000e-01 2.056000e-01 4.849000e-01 2.192000e-01 3.067000e-01 9.330000e-02 + 7.310000e-02 4.657000e-01 1.022000e-01 7.503000e-01 5.510000e-02 9.277000e-01 7.300000e-02 7.852000e-01 1.180000e-01 8.651000e-01 1.219000e-01 7.260000e-01 5.260000e-02 4.397000e-01 + 6.670000e-02 5.461000e-01 4.370000e-01 4.789000e-01 5.523000e-01 4.830000e-01 1.668600e+00 7.700000e-02 1.280000e-02 1.978700e+00 3.340000e-02 -4.030000e-02 2.290000e-01 3.043000e-01 + 8.427000e-01 2.270000e-01 1.094300e+00 3.023000e-01 1.294900e+00 2.022000e-01 1.886300e+00 -1.803000e-01 2.044700e+00 -4.780000e-02 2.900000e-03 1.732800e+00 6.900000e-01 1.222000e-01 + 1.126600e+00 -2.560000e-02 5.215000e-01 1.210700e+00 4.669000e-01 6.685000e-01 5.571000e-01 1.170500e+00 4.676000e-01 6.308000e-01 4.948000e-01 1.244900e+00 3.906000e-01 4.083000e-01 + 5.600000e-01 3.640000e-01 8.255000e-01 6.307000e-01 9.831000e-01 6.679000e-01 8.842000e-01 5.626000e-01 9.357000e-01 7.488000e-01 8.266000e-01 6.312000e-01 4.738000e-01 3.681000e-01 + 4.810000e-02 1.007400e+00 4.296000e-01 9.304000e-01 4.478000e-01 1.047200e+00 1.756000e+00 -3.380000e-02 -7.480000e-02 2.089300e+00 -4.250000e-02 5.140000e-02 2.391000e-01 5.596000e-01 + 1.095200e+00 1.350000e-02 8.862000e-01 8.475000e-01 7.144000e-01 3.892000e-01 9.740000e-01 7.529000e-01 7.114000e-01 3.571000e-01 9.248000e-01 8.077000e-01 5.859000e-01 1.979000e-01 + 2.006000e-01 3.454000e-01 3.986000e-01 4.469000e-01 3.417000e-01 6.387000e-01 2.808000e-01 5.793000e-01 4.392000e-01 5.374000e-01 3.424000e-01 5.096000e-01 2.043000e-01 2.865000e-01 +-2.480000e-02 8.079000e-01 3.514000e-01 7.237000e-01 4.603000e-01 7.386000e-01 8.570000e-02 1.632300e+00 6.240000e-02 1.922700e+00 -2.600000e-02 3.120000e-02 1.639000e-01 4.770000e-01 + 4.297000e-01 1.825000e-01 5.939000e-01 3.794000e-01 7.233000e-01 3.816000e-01 6.731000e-01 2.893000e-01 7.694000e-01 3.435000e-01 7.308000e-01 2.192000e-01 3.835000e-01 1.722000e-01 + 6.357000e-01 4.547000e-01 2.637000e-01 7.217000e-01 2.175000e-01 5.563000e-01 2.751000e-01 7.127000e-01 1.261000e-01 6.451000e-01 2.460000e-01 7.426000e-01 2.248000e-01 3.850000e-01 + 1.124800e+00 -2.590000e-02 4.916000e-01 1.229300e+00 3.932000e-01 9.462000e-01 5.271000e-01 1.185500e+00 4.211000e-01 8.591000e-01 5.142000e-01 1.206400e+00 3.246000e-01 5.502000e-01 + 1.131100e+00 -2.980000e-02 1.013700e+00 7.144000e-01 8.036000e-01 2.317000e-01 1.076000e+00 6.498000e-01 7.927000e-01 2.099000e-01 9.650000e-01 7.713000e-01 6.183000e-01 1.358000e-01 + 3.026000e-01 4.889000e-01 6.114000e-01 4.941000e-01 6.889000e-01 5.464000e-01 1.615600e+00 1.361000e-01 5.000000e-04 2.004300e+00 3.290000e-02 -3.970000e-02 3.152000e-01 3.408000e-01 +-1.580000e-02 4.553000e-01 1.158000e-01 6.442000e-01 3.486000e-01 5.106000e-01 -1.019000e-01 1.854600e+00 2.450000e-02 -3.050000e-02 6.260000e-02 -7.460000e-02 1.096000e-01 3.147000e-01 +-8.610000e-02 1.207600e+00 3.407000e-01 1.090100e+00 5.373000e-01 9.981000e-01 -3.530000e-02 1.776400e+00 -1.200000e-03 2.001700e+00 2.530000e-02 1.701800e+00 1.707000e-01 6.720000e-01 + 3.425000e-01 1.699000e-01 6.039000e-01 1.853000e-01 5.608000e-01 3.606000e-01 5.748000e-01 2.185000e-01 6.062000e-01 3.205000e-01 4.801000e-01 3.313000e-01 3.026000e-01 1.613000e-01 + 8.624000e-01 2.389000e-01 1.129600e+00 5.381000e-01 1.292500e+00 4.896000e-01 1.810500e+00 -9.300000e-02 2.041200e+00 -4.980000e-02 4.800000e-03 1.730400e+00 6.792000e-01 2.692000e-01 +-2.360000e-02 6.987000e-01 1.493000e-01 8.415000e-01 2.933000e-01 8.121000e-01 -3.410000e-02 1.772800e+00 -7.390000e-02 2.085800e+00 -5.190000e-02 6.100000e-02 1.021000e-01 4.729000e-01 + 1.140600e+00 -3.760000e-02 5.592000e-01 1.172900e+00 3.989000e-01 8.188000e-01 6.176000e-01 1.108100e+00 4.037000e-01 7.717000e-01 5.580000e-01 1.172800e+00 4.104000e-01 4.151000e-01 + 4.442000e-01 2.989000e-01 6.370000e-01 5.436000e-01 8.215000e-01 5.003000e-01 7.841000e-01 3.647000e-01 7.794000e-01 5.730000e-01 6.997000e-01 4.653000e-01 3.936000e-01 2.806000e-01 + 7.410000e-02 3.967000e-01 4.133000e-01 3.645000e-01 4.413000e-01 4.767000e-01 -4.360000e-02 1.781800e+00 4.830000e-02 -5.760000e-02 1.240000e-02 -1.260000e-02 1.716000e-01 2.843000e-01 + 3.583000e-01 7.460000e-01 1.489000e-01 6.681000e-01 1.327000e-01 5.254000e-01 1.705000e-01 6.434000e-01 1.733000e-01 4.643000e-01 1.570000e-01 6.552000e-01 1.646000e-01 3.708000e-01 + 5.144000e-01 7.100000e-03 7.310000e-01 1.166000e-01 8.438000e-01 1.292000e-01 1.745400e+00 -2.040000e-02 5.240000e-02 -6.220000e-02 2.000000e-04 -1.200000e-03 4.207000e-01 6.780000e-02 + 6.726000e-01 1.323000e-01 1.144800e+00 9.590000e-02 1.362000e+00 3.870000e-02 1.169700e+00 6.840000e-02 1.369600e+00 5.110000e-02 1.182100e+00 5.370000e-02 6.784000e-01 3.530000e-02 + 1.674000e-01 3.897000e-01 4.111000e-01 4.741000e-01 6.153000e-01 3.688000e-01 1.697900e+00 3.140000e-02 -5.520000e-02 2.065200e+00 -4.320000e-02 5.020000e-02 2.435000e-01 2.632000e-01 + 3.785000e-01 7.282000e-01 1.824000e-01 7.266000e-01 1.605000e-01 5.580000e-01 1.435000e-01 7.714000e-01 1.922000e-01 5.056000e-01 3.317000e-01 5.522000e-01 1.777000e-01 3.997000e-01 + 3.145000e-01 7.818000e-01 1.241000e-01 5.976000e-01 1.230000e-01 4.677000e-01 1.220000e-01 6.060000e-01 1.649000e-01 4.058000e-01 7.690000e-02 6.546000e-01 1.322000e-01 3.602000e-01 + 2.918000e-01 2.719000e-01 6.459000e-01 2.241000e-01 7.378000e-01 2.574000e-01 1.683400e+00 6.250000e-02 2.860000e-02 1.964400e+00 -5.600000e-03 6.500000e-03 3.631000e-01 1.400000e-01 + 1.151300e+00 -5.120000e-02 6.183000e-01 3.049000e-01 6.627000e-01 3.790000e-02 7.937000e-01 9.420000e-02 5.698000e-01 1.317000e-01 8.385000e-01 4.390000e-02 4.815000e-01 9.850000e-02 + 1.269000e-01 9.731000e-01 4.120000e-01 1.036700e+00 5.733000e-01 9.856000e-01 1.699400e+00 3.680000e-02 -2.470000e-02 2.024300e+00 -1.100000e-02 1.743800e+00 2.299000e-01 6.160000e-01 +-9.200000e-03 6.616000e-01 5.080000e-02 9.117000e-01 1.767000e-01 9.075000e-01 2.000000e-02 1.709800e+00 -6.340000e-02 2.076700e+00 -2.020000e-02 2.450000e-02 4.400000e-02 5.188000e-01 + 3.096000e-01 3.670000e-01 4.894000e-01 5.677000e-01 5.287000e-01 6.841000e-01 4.190000e-01 6.544000e-01 5.844000e-01 6.375000e-01 5.973000e-01 4.401000e-01 2.783000e-01 3.331000e-01 + 5.330000e-02 8.748000e-01 1.818000e-01 1.072900e+00 2.553000e-01 1.123700e+00 -2.450000e-02 1.760300e+00 -3.000000e-03 2.009000e+00 4.320000e-02 -5.350000e-02 1.072000e-01 6.315000e-01 + 1.054700e+00 5.700000e-02 9.906000e-01 9.390000e-02 8.846000e-01 -8.220000e-02 1.024500e+00 4.920000e-02 7.418000e-01 6.930000e-02 1.035600e+00 3.830000e-02 6.174000e-01 2.600000e-02 + 4.399000e-01 3.581000e-01 8.365000e-01 3.885000e-01 9.079000e-01 4.923000e-01 7.725000e-01 4.573000e-01 7.929000e-01 6.507000e-01 7.882000e-01 4.434000e-01 4.279000e-01 2.874000e-01 + 4.167000e-01 5.007000e-01 6.382000e-01 7.986000e-01 8.046000e-01 8.237000e-01 7.404000e-01 6.831000e-01 7.403000e-01 9.233000e-01 6.321000e-01 8.118000e-01 3.938000e-01 4.348000e-01 + 1.641000e-01 5.116000e-01 5.404000e-01 4.425000e-01 5.521000e-01 5.661000e-01 1.699300e+00 3.610000e-02 2.810000e-02 1.964300e+00 6.130000e-02 -7.280000e-02 2.508000e-01 3.306000e-01 + 6.288000e-01 4.748000e-01 3.036000e-01 1.429400e+00 1.956000e-01 1.249500e+00 3.526000e-01 1.368400e+00 2.819000e-01 1.080300e+00 2.704000e-01 1.468300e+00 2.507000e-01 6.484000e-01 + 2.369000e-01 6.611000e-01 5.288000e-01 6.834000e-01 6.557000e-01 6.726000e-01 1.709300e+00 2.570000e-02 -3.230000e-02 2.038300e+00 -7.010000e-02 8.680000e-02 2.784000e-01 4.395000e-01 + 8.934000e-01 2.195000e-01 4.395000e-01 9.475000e-01 3.611000e-01 6.075000e-01 3.924000e-01 1.003200e+00 3.487000e-01 5.886000e-01 4.447000e-01 9.433000e-01 2.960000e-01 4.255000e-01 + 3.127000e-01 7.936000e-01 2.430000e-01 1.480400e+00 1.331000e-01 1.843300e+00 1.919000e-01 1.538600e+00 8.090000e-02 1.620200e+00 2.307000e-01 1.493000e+00 1.468000e-01 8.283000e-01 + 4.778000e-01 5.678000e-01 8.714000e-01 7.462000e-01 9.055000e-01 9.499000e-01 8.423000e-01 7.811000e-01 9.141000e-01 9.747000e-01 7.781000e-01 8.622000e-01 4.561000e-01 4.868000e-01 + 8.896000e-01 2.207000e-01 2.578000e-01 1.349300e+00 2.169000e-01 8.216000e-01 4.123000e-01 1.165400e+00 3.797000e-01 5.948000e-01 4.365000e-01 1.129600e+00 2.961000e-01 4.484000e-01 + 1.164000e-01 5.170000e-01 9.500000e-03 1.017200e+00 1.090000e-02 1.156900e+00 -2.770000e-02 1.060900e+00 -2.000000e-03 1.186900e+00 7.900000e-02 9.352000e-01 2.830000e-02 5.596000e-01 + 4.084000e-01 6.935000e-01 7.813000e-01 7.456000e-01 8.580000e-01 7.981000e-01 1.763400e+00 -3.380000e-02 3.120000e-02 1.962900e+00 4.640000e-02 1.680600e+00 4.056000e-01 4.892000e-01 + 4.115000e-01 3.150000e-01 6.251000e-01 5.168000e-01 6.864000e-01 6.216000e-01 6.157000e-01 5.283000e-01 5.501000e-01 8.035000e-01 5.848000e-01 5.660000e-01 3.420000e-01 3.209000e-01 + 5.793000e-01 5.256000e-01 2.275000e-01 6.283000e-01 2.350000e-01 4.414000e-01 3.038000e-01 5.316000e-01 2.404000e-01 4.183000e-01 2.991000e-01 5.356000e-01 2.314000e-01 3.192000e-01 + 1.988000e-01 4.499000e-01 2.277000e-01 8.006000e-01 2.805000e-01 8.852000e-01 2.238000e-01 8.069000e-01 2.915000e-01 8.895000e-01 2.605000e-01 7.604000e-01 1.524000e-01 4.373000e-01 + 5.380000e-02 4.115000e-01 1.528000e-01 5.627000e-01 5.870000e-02 7.814000e-01 1.204000e-01 6.025000e-01 2.900000e-03 8.575000e-01 8.210000e-02 6.511000e-01 4.320000e-02 3.794000e-01 + 5.143000e-01 4.670000e-01 6.954000e-01 6.313000e-01 8.395000e-01 5.981000e-01 1.833600e+00 -1.195000e-01 6.730000e-02 1.927100e+00 4.320000e-02 -5.020000e-02 4.050000e-01 3.727000e-01 + 1.093200e+00 1.700000e-02 9.920000e-01 2.131000e-01 8.428000e-01 4.310000e-02 1.037400e+00 1.593000e-01 8.353000e-01 2.400000e-02 1.127100e+00 5.930000e-02 6.564000e-01 1.860000e-02 + 1.529000e-01 7.166000e-01 4.760000e-01 7.066000e-01 6.900000e-01 5.941000e-01 1.790100e+00 -6.710000e-02 8.600000e-03 1.987800e+00 -4.000000e-03 4.600000e-03 3.005000e-01 3.884000e-01 + 6.414000e-01 1.886000e-01 9.415000e-01 2.036000e-01 9.966000e-01 2.832000e-01 1.774400e+00 -5.000000e-02 2.061900e+00 -7.050000e-02 2.750000e-02 -3.310000e-02 5.474000e-01 1.283000e-01 + 1.100400e+00 1.080000e-02 1.541000e+00 1.791000e-01 9.895000e-01 8.366000e-01 1.446100e+00 2.887000e-01 9.364000e-01 7.076000e-01 1.512700e+00 2.114000e-01 7.341000e-01 2.329000e-01 + 1.063300e+00 4.970000e-02 6.328000e-01 1.103200e+00 4.612000e-01 1.008900e+00 7.123000e-01 1.013200e+00 5.231000e-01 8.612000e-01 7.706000e-01 9.377000e-01 4.556000e-01 4.502000e-01 + 9.390000e-02 2.852000e-01 1.502000e-01 5.744000e-01 3.306000e-01 4.972000e-01 2.500000e-02 1.699300e+00 2.940000e-02 -3.570000e-02 -5.610000e-02 6.750000e-02 1.023000e-01 3.008000e-01 + 6.954000e-01 2.670000e-01 1.011800e+00 2.593000e-01 1.137400e+00 2.561000e-01 1.706800e+00 3.340000e-02 2.083900e+00 -9.270000e-02 1.300000e-02 -1.780000e-02 6.024000e-01 1.478000e-01 + 1.190800e+00 -9.960000e-02 3.961000e-01 1.285800e+00 3.905000e-01 6.359000e-01 4.426000e-01 1.233500e+00 3.446000e-01 6.614000e-01 4.912000e-01 1.180000e+00 3.361000e-01 4.130000e-01 + 5.374000e-01 3.805000e-01 8.830000e-01 5.444000e-01 1.049100e+00 5.771000e-01 9.036000e-01 5.234000e-01 1.049500e+00 5.964000e-01 9.324000e-01 4.881000e-01 5.132000e-01 3.125000e-01 + 1.039100e+00 7.860000e-02 1.266200e+00 4.742000e-01 9.161000e-01 1.626000e-01 1.197900e+00 5.470000e-01 9.137000e-01 1.269000e-01 1.213700e+00 5.333000e-01 6.993000e-01 7.170000e-02 + 6.458000e-01 4.649000e-01 2.589000e-01 9.217000e-01 3.329000e-01 5.323000e-01 2.311000e-01 9.548000e-01 1.899000e-01 6.802000e-01 3.269000e-01 8.420000e-01 2.573000e-01 4.138000e-01 + 1.052100e+00 6.650000e-02 7.528000e-01 9.807000e-01 6.024000e-01 8.703000e-01 7.803000e-01 9.460000e-01 7.222000e-01 6.507000e-01 7.261000e-01 1.012000e+00 5.141000e-01 3.938000e-01 + 5.840000e-02 5.279000e-01 8.240000e-02 8.389000e-01 8.580000e-02 9.652000e-01 7.990000e-02 8.364000e-01 1.640000e-02 1.055800e+00 1.320000e-01 7.804000e-01 6.160000e-02 4.665000e-01 + 4.050000e-02 9.263000e-01 3.432000e-01 9.408000e-01 4.826000e-01 9.161000e-01 -1.620000e-02 1.747900e+00 -1.530000e-02 2.022200e+00 6.260000e-02 -7.430000e-02 1.872000e-01 5.699000e-01 + 5.268000e-01 3.336000e-01 6.997000e-01 4.976000e-01 9.724000e-01 3.158000e-01 1.717100e+00 1.850000e-02 1.060000e-02 1.983800e+00 -2.130000e-02 2.460000e-02 4.486000e-01 2.498000e-01 + 1.870000e-02 6.936000e-01 1.526000e-01 8.764000e-01 2.685000e-01 8.772000e-01 -2.400000e-02 1.761100e+00 5.660000e-02 1.933300e+00 3.520000e-02 -4.000000e-02 7.850000e-02 5.252000e-01 + 1.103300e+00 3.900000e-03 8.285000e-01 3.832000e-01 6.649000e-01 2.246000e-01 6.963000e-01 5.402000e-01 5.935000e-01 2.886000e-01 7.667000e-01 4.494000e-01 5.473000e-01 1.339000e-01 + 8.410000e-02 1.032800e+00 4.392000e-01 1.048400e+00 6.082000e-01 9.935000e-01 1.738900e+00 -3.800000e-03 -2.100000e-02 2.026900e+00 -1.067000e-01 1.858800e+00 2.398000e-01 6.295000e-01 + 4.689000e-01 4.092000e-01 6.791000e-01 7.029000e-01 8.472000e-01 7.169000e-01 6.373000e-01 7.540000e-01 7.336000e-01 8.768000e-01 7.611000e-01 6.114000e-01 3.997000e-01 3.986000e-01 + 5.368000e-01 2.121000e-01 7.638000e-01 3.146000e-01 8.883000e-01 3.072000e-01 1.674800e+00 6.920000e-02 9.300000e-03 1.989600e+00 6.700000e-03 -9.100000e-03 4.426000e-01 1.885000e-01 + 4.240000e-01 4.900000e-01 7.232000e-01 5.014000e-01 8.591000e-01 4.838000e-01 1.772600e+00 -4.870000e-02 4.240000e-02 1.954800e+00 -1.780000e-02 2.100000e-02 4.348000e-01 2.854000e-01 + 1.050000e-01 1.003300e+00 2.230000e-02 5.880000e-01 3.430000e-02 4.697000e-01 1.249000e-01 4.665000e-01 6.710000e-02 4.225000e-01 1.030000e-01 4.903000e-01 6.130000e-02 3.688000e-01 + 3.111000e-01 3.751000e-01 4.651000e-01 6.121000e-01 6.362000e-01 5.681000e-01 5.020000e-01 5.630000e-01 6.580000e-01 5.583000e-01 5.059000e-01 5.646000e-01 3.052000e-01 3.081000e-01 + 4.478000e-01 -1.400000e-03 6.501000e-01 6.110000e-02 7.771000e-01 2.590000e-02 6.717000e-01 3.060000e-02 7.627000e-01 5.250000e-02 6.493000e-01 5.720000e-02 3.869000e-01 2.020000e-02 + 7.303000e-01 3.720000e-01 3.490000e-01 6.100000e-01 2.788000e-01 4.756000e-01 2.771000e-01 6.963000e-01 3.090000e-01 4.241000e-01 2.837000e-01 6.863000e-01 2.655000e-01 3.350000e-01 + 5.140000e-02 1.062500e+00 3.940000e-02 4.118000e-01 -9.000000e-03 4.001000e-01 1.314000e-01 3.037000e-01 7.130000e-02 2.978000e-01 7.100000e-02 3.778000e-01 4.670000e-02 2.925000e-01 + 4.980000e-01 5.100000e-01 8.111000e-01 5.115000e-01 8.700000e-01 5.826000e-01 1.682700e+00 5.570000e-02 7.700000e-03 1.987000e+00 3.500000e-02 -4.180000e-02 4.798000e-01 2.986000e-01 + 2.034000e-01 6.543000e-01 2.535000e-01 1.109500e+00 3.152000e-01 1.231200e+00 2.384000e-01 1.129500e+00 3.808000e-01 1.179800e+00 2.254000e-01 1.140700e+00 1.744000e-01 6.073000e-01 + 3.153000e-01 4.310000e-01 6.277000e-01 5.140000e-01 6.222000e-01 6.974000e-01 6.016000e-01 5.469000e-01 7.290000e-01 5.904000e-01 5.691000e-01 5.835000e-01 3.592000e-01 3.007000e-01 + 8.961000e-01 2.310000e-02 1.048800e+00 2.182000e-01 1.305600e+00 6.030000e-02 1.701700e+00 3.300000e-02 2.162900e+00 -1.954000e-01 -5.500000e-03 6.700000e-03 6.851000e-01 5.080000e-02 + 1.099300e+00 6.800000e-03 4.745000e-01 1.250200e+00 3.631000e-01 8.284000e-01 4.291000e-01 1.304800e+00 3.799000e-01 7.691000e-01 5.049000e-01 1.217200e+00 3.288000e-01 4.910000e-01 + 1.124000e-01 4.549000e-01 1.585000e-01 7.363000e-01 7.680000e-02 9.628000e-01 1.012000e-01 8.069000e-01 9.650000e-02 9.527000e-01 1.245000e-01 7.763000e-01 5.500000e-02 4.688000e-01 + 2.914000e-01 8.203000e-01 1.384000e-01 9.877000e-01 1.531000e-01 6.968000e-01 2.093000e-01 9.102000e-01 2.100000e-01 6.084000e-01 1.684000e-01 9.538000e-01 1.537000e-01 5.024000e-01 + 2.503000e-01 5.377000e-01 4.508000e-01 7.689000e-01 6.041000e-01 7.665000e-01 4.468000e-01 7.721000e-01 6.211000e-01 7.645000e-01 4.442000e-01 7.789000e-01 2.925000e-01 4.055000e-01 + 5.522000e-01 9.600000e-02 8.623000e-01 9.830000e-02 9.783000e-01 1.028000e-01 1.732200e+00 1.400000e-03 6.040000e-02 1.928700e+00 -1.110000e-02 1.420000e-02 4.638000e-01 1.047000e-01 + 6.990000e-01 1.695000e-01 1.112800e+00 2.379000e-01 1.287200e+00 2.463000e-01 1.054400e+00 3.047000e-01 1.330800e+00 2.267000e-01 1.103400e+00 2.486000e-01 6.255000e-01 1.588000e-01 + 1.206000e-01 5.177000e-01 2.957000e-01 6.815000e-01 2.742000e-01 8.485000e-01 2.064000e-01 7.865000e-01 3.213000e-01 8.104000e-01 2.374000e-01 7.500000e-01 1.375000e-01 4.325000e-01 + 7.560000e-02 4.799000e-01 1.069000e-01 7.654000e-01 2.251000e-01 7.443000e-01 2.162000e-01 6.340000e-01 1.732000e-01 8.273000e-01 1.715000e-01 6.874000e-01 9.100000e-02 4.062000e-01 + 2.584000e-01 8.386000e-01 7.620000e-02 7.278000e-01 1.070000e-01 5.378000e-01 7.110000e-02 7.384000e-01 7.870000e-02 5.573000e-01 4.850000e-02 7.629000e-01 1.041000e-01 4.274000e-01 + 7.270000e-01 3.828000e-01 1.070200e+00 4.521000e-01 1.323100e+00 3.022000e-01 1.663400e+00 8.400000e-02 2.080600e+00 -9.090000e-02 7.430000e-02 1.642100e+00 6.176000e-01 2.725000e-01 + 2.311000e-01 8.728000e-01 1.756000e-01 5.309000e-01 1.360000e-01 4.464000e-01 1.481000e-01 5.650000e-01 6.090000e-02 5.215000e-01 1.231000e-01 5.944000e-01 1.064000e-01 3.859000e-01 + 6.257000e-01 4.252000e-01 8.745000e-01 5.017000e-01 8.145000e-01 7.101000e-01 1.767800e+00 -4.060000e-02 3.790000e-02 1.954800e+00 -3.610000e-02 1.777500e+00 4.962000e-01 3.155000e-01 + 3.711000e-01 3.448000e-01 6.690000e-01 3.564000e-01 7.493000e-01 4.080000e-01 1.665500e+00 8.190000e-02 4.510000e-02 1.949600e+00 1.120000e-02 -1.470000e-02 3.586000e-01 2.472000e-01 + 1.091400e+00 1.500000e-02 7.516000e-01 9.932000e-01 6.739000e-01 4.224000e-01 9.437000e-01 7.683000e-01 6.294000e-01 4.399000e-01 8.063000e-01 9.298000e-01 5.482000e-01 2.326000e-01 + 4.256000e-01 2.780000e-02 6.816000e-01 2.220000e-02 7.522000e-01 5.630000e-02 6.717000e-01 3.390000e-02 8.997000e-01 -1.075000e-01 6.933000e-01 9.500000e-03 3.776000e-01 3.240000e-02 + 1.028000e+00 9.510000e-02 5.619000e-01 4.529000e-01 5.648000e-01 2.058000e-01 6.508000e-01 3.510000e-01 4.077000e-01 3.744000e-01 5.686000e-01 4.476000e-01 4.042000e-01 2.186000e-01 +-4.200000e-02 7.610000e-01 1.212000e-01 9.064000e-01 1.753000e-01 9.878000e-01 5.340000e-02 1.670300e+00 -1.660000e-02 2.013600e+00 6.680000e-02 -7.970000e-02 7.740000e-02 5.239000e-01 + 3.907000e-01 5.587000e-01 6.918000e-01 5.787000e-01 7.508000e-01 6.533000e-01 1.765900e+00 -3.900000e-02 -4.400000e-03 2.007300e+00 1.150000e-02 -1.440000e-02 3.759000e-01 3.750000e-01 + 5.883000e-01 4.444000e-01 9.442000e-01 6.742000e-01 1.223800e+00 5.907000e-01 1.045900e+00 5.541000e-01 1.129100e+00 7.246000e-01 1.068700e+00 5.271000e-01 5.945000e-01 3.288000e-01 + 1.261000e-01 4.338000e-01 2.383000e-01 6.331000e-01 1.393000e-01 8.796000e-01 7.800000e-02 8.233000e-01 1.462000e-01 8.875000e-01 1.771000e-01 7.083000e-01 8.940000e-02 4.235000e-01 + 2.951000e-01 7.971000e-01 2.091000e-01 7.775000e-01 8.550000e-02 6.993000e-01 1.987000e-01 7.923000e-01 5.940000e-02 7.144000e-01 1.031000e-01 9.043000e-01 1.108000e-01 5.057000e-01 + 1.100700e+00 5.800000e-03 5.735000e-01 5.508000e-01 4.843000e-01 3.643000e-01 5.151000e-01 6.179000e-01 4.244000e-01 4.098000e-01 6.194000e-01 4.928000e-01 4.094000e-01 2.485000e-01 + 8.864000e-01 1.600000e-03 1.375200e+00 2.020000e-02 1.571500e+00 1.050000e-02 1.415500e+00 -3.190000e-02 1.655200e+00 -6.190000e-02 1.409500e+00 -2.720000e-02 7.862000e-01 1.790000e-02 + 1.152100e+00 -5.580000e-02 7.192000e-01 1.030300e+00 7.694000e-01 4.801000e-01 8.318000e-01 8.979000e-01 6.653000e-01 5.582000e-01 9.178000e-01 7.947000e-01 5.415000e-01 3.093000e-01 + 5.044000e-01 1.540000e-02 8.126000e-01 -2.700000e-03 8.906000e-01 4.030000e-02 7.963000e-01 1.780000e-02 9.854000e-01 -5.490000e-02 7.248000e-01 1.000000e-01 4.532000e-01 1.820000e-02 +-2.180000e-02 8.299000e-01 1.599000e-01 9.649000e-01 3.780000e-01 8.483000e-01 2.180000e-02 1.707200e+00 -4.570000e-02 2.054800e+00 -1.020000e-02 1.240000e-02 1.121000e-01 5.458000e-01 + 9.160000e-02 1.036900e+00 1.755000e-01 6.179000e-01 8.330000e-02 5.723000e-01 1.340000e-01 6.709000e-01 5.480000e-02 5.914000e-01 6.290000e-02 7.533000e-01 1.001000e-01 4.355000e-01 + 3.507000e-01 4.682000e-01 6.535000e-01 4.860000e-01 7.487000e-01 5.119000e-01 1.678900e+00 5.970000e-02 -5.820000e-02 2.068200e+00 -2.410000e-02 2.870000e-02 3.420000e-01 3.332000e-01 + 1.034700e+00 8.190000e-02 5.016000e-01 1.232800e+00 4.562000e-01 1.218400e+00 4.686000e-01 1.269500e+00 4.205000e-01 1.132200e+00 5.104000e-01 1.223000e+00 3.742000e-01 5.762000e-01 + 6.282000e-01 1.157000e-01 8.571000e-01 3.314000e-01 1.049100e+00 2.918000e-01 8.451000e-01 3.437000e-01 1.130400e+00 2.179000e-01 9.457000e-01 2.238000e-01 5.257000e-01 1.544000e-01 + 1.087900e+00 1.780000e-02 5.849000e-01 9.080000e-02 4.400000e-01 1.295000e-01 5.795000e-01 1.014000e-01 4.978000e-01 4.730000e-02 6.493000e-01 2.220000e-02 4.243000e-01 4.870000e-02 + 1.043200e+00 7.570000e-02 3.362000e-01 8.387000e-01 3.922000e-01 4.720000e-01 4.663000e-01 6.843000e-01 3.137000e-01 5.435000e-01 4.333000e-01 7.240000e-01 3.211000e-01 3.463000e-01 + 1.763000e-01 5.283000e-01 4.547000e-01 5.701000e-01 6.420000e-01 4.928000e-01 1.767600e+00 -4.320000e-02 3.570000e-02 1.956900e+00 -3.270000e-02 3.990000e-02 2.514000e-01 3.485000e-01 + 4.510000e-01 6.430000e-02 6.271000e-01 2.025000e-01 7.699000e-01 1.555000e-01 7.216000e-01 8.540000e-02 8.444000e-01 8.560000e-02 8.332000e-01 -4.750000e-02 4.062000e-01 6.300000e-02 + 7.900000e-02 5.267000e-01 8.690000e-02 8.697000e-01 1.671000e-01 9.115000e-01 5.880000e-02 9.035000e-01 6.870000e-02 1.042700e+00 8.080000e-02 8.789000e-01 6.730000e-02 4.822000e-01 + 2.188000e-01 3.369000e-01 3.669000e-01 5.010000e-01 3.112000e-01 6.960000e-01 2.375000e-01 6.537000e-01 3.345000e-01 6.777000e-01 2.797000e-01 6.031000e-01 1.660000e-01 3.429000e-01 +-9.450000e-02 1.214000e+00 -7.660000e-02 1.790700e+00 1.769000e-01 1.628300e+00 -1.730000e-02 1.752100e+00 2.320000e-02 1.976100e+00 -8.350000e-02 1.829900e+00 2.030000e-02 9.413000e-01 + 1.486000e-01 6.692000e-01 1.849000e-01 1.112100e+00 2.417000e-01 1.229800e+00 3.131000e-01 9.606000e-01 3.608000e-01 1.105200e+00 1.707000e-01 1.123700e+00 1.341000e-01 6.086000e-01 + 1.151900e+00 -5.420000e-02 6.338000e-01 3.940000e-01 4.542000e-01 3.527000e-01 5.905000e-01 4.522000e-01 5.814000e-01 1.835000e-01 6.035000e-01 4.320000e-01 4.249000e-01 2.042000e-01 + 1.479000e-01 2.251000e-01 1.986000e-01 3.961000e-01 2.674000e-01 3.986000e-01 1.340000e-01 4.690000e-01 2.304000e-01 4.534000e-01 1.999000e-01 3.911000e-01 1.029000e-01 2.407000e-01 + 5.890000e-02 1.864000e-01 3.178000e-01 2.535000e-01 4.522000e-01 2.345000e-01 9.190000e-02 1.622500e+00 -7.700000e-03 9.000000e-03 -7.670000e-02 9.070000e-02 1.755000e-01 1.388000e-01 + 9.400000e-02 6.238000e-01 3.969000e-01 6.334000e-01 4.998000e-01 6.599000e-01 1.790100e+00 -6.620000e-02 -7.930000e-02 2.093800e+00 2.700000e-03 -4.400000e-03 2.079000e-01 3.997000e-01 + 2.319000e-01 5.927000e-01 3.022000e-01 1.001700e+00 3.991000e-01 1.072700e+00 4.055000e-01 8.795000e-01 4.504000e-01 1.036000e+00 3.112000e-01 9.839000e-01 1.925000e-01 5.553000e-01 + 3.815000e-01 3.332000e-01 6.462000e-01 3.913000e-01 7.058000e-01 4.557000e-01 1.709500e+00 2.520000e-02 -6.140000e-02 2.076500e+00 2.690000e-02 -3.110000e-02 3.664000e-01 2.403000e-01 + 1.078300e+00 3.410000e-02 8.201000e-01 9.103000e-01 6.938000e-01 1.278200e+00 7.104000e-01 1.039200e+00 5.932000e-01 1.278000e+00 7.361000e-01 1.003500e+00 5.368000e-01 4.586000e-01 + 9.140000e-02 5.724000e-01 3.504000e-01 6.380000e-01 4.148000e-01 7.028000e-01 1.752300e+00 -2.260000e-02 -2.290000e-02 2.024300e+00 -1.770000e-02 1.910000e-02 2.003000e-01 3.734000e-01 + 4.475000e-01 6.681000e-01 3.219000e-01 5.937000e-01 2.144000e-01 5.219000e-01 2.775000e-01 6.470000e-01 1.795000e-01 5.440000e-01 2.650000e-01 6.645000e-01 2.055000e-01 3.837000e-01 + 1.052400e+00 6.480000e-02 5.225000e-01 8.387000e-01 4.788000e-01 4.712000e-01 5.038000e-01 8.560000e-01 3.832000e-01 5.550000e-01 5.543000e-01 7.987000e-01 3.811000e-01 3.338000e-01 + 9.849000e-01 1.433000e-01 5.062000e-01 2.203000e-01 5.441000e-01 2.260000e-02 5.696000e-01 1.404000e-01 4.719000e-01 9.920000e-02 4.997000e-01 2.260000e-01 3.843000e-01 1.097000e-01 + 5.748000e-01 1.579000e-01 9.100000e-01 2.326000e-01 9.980000e-01 3.141000e-01 9.007000e-01 2.419000e-01 9.729000e-01 3.628000e-01 8.942000e-01 2.515000e-01 5.003000e-01 1.643000e-01 + 4.395000e-01 4.679000e-01 8.962000e-01 2.943000e-01 9.701000e-01 3.516000e-01 1.817900e+00 -9.650000e-02 2.730000e-02 1.960900e+00 2.100000e-03 -1.700000e-03 4.865000e-01 2.228000e-01 + 4.840000e-01 2.094000e-01 7.491000e-01 3.479000e-01 9.198000e-01 3.134000e-01 7.565000e-01 3.362000e-01 7.934000e-01 4.768000e-01 6.991000e-01 4.020000e-01 4.212000e-01 2.110000e-01 + 4.958000e-01 1.443000e-01 8.970000e-01 8.420000e-02 8.679000e-01 2.843000e-01 7.258000e-01 2.898000e-01 8.606000e-01 3.076000e-01 7.934000e-01 2.101000e-01 4.459000e-01 1.359000e-01 + 1.079100e+00 3.290000e-02 5.166000e-01 7.448000e-01 4.626000e-01 4.487000e-01 5.806000e-01 6.720000e-01 4.977000e-01 3.901000e-01 5.378000e-01 7.246000e-01 4.012000e-01 2.923000e-01 + 5.550000e-02 7.274000e-01 3.617000e-01 7.326000e-01 5.602000e-01 6.382000e-01 1.931200e+00 -2.333000e-01 7.000000e-03 1.992700e+00 -7.110000e-02 8.280000e-02 2.292000e-01 4.078000e-01 + 1.089100e+00 2.140000e-02 4.152000e-01 8.053000e-01 3.788000e-01 5.185000e-01 4.597000e-01 7.495000e-01 3.572000e-01 5.192000e-01 4.332000e-01 7.823000e-01 3.524000e-01 3.287000e-01 + 5.581000e-01 5.780000e-02 7.462000e-01 2.461000e-01 8.369000e-01 2.983000e-01 8.010000e-01 1.832000e-01 9.550000e-01 1.747000e-01 8.798000e-01 8.960000e-02 4.791000e-01 8.590000e-02 + 1.096500e+00 3.900000e-03 4.117000e-01 4.921000e-01 3.664000e-01 3.479000e-01 4.433000e-01 4.572000e-01 3.868000e-01 3.088000e-01 4.363000e-01 4.653000e-01 3.154000e-01 2.638000e-01 + 1.103600e+00 -9.000000e-04 5.596000e-01 1.163800e+00 4.281000e-01 9.927000e-01 5.554000e-01 1.170500e+00 4.449000e-01 9.107000e-01 5.766000e-01 1.139700e+00 3.750000e-01 5.223000e-01 + 4.741000e-01 2.965000e-01 7.502000e-01 4.499000e-01 7.687000e-01 6.150000e-01 6.862000e-01 5.282000e-01 8.357000e-01 5.573000e-01 6.292000e-01 5.912000e-01 4.165000e-01 2.794000e-01 + 3.807000e-01 4.936000e-01 5.540000e-01 8.272000e-01 8.500000e-01 6.754000e-01 6.042000e-01 7.640000e-01 7.195000e-01 8.542000e-01 6.354000e-01 7.275000e-01 3.726000e-01 4.133000e-01 + 8.364000e-01 3.270000e-02 1.186400e+00 1.968000e-01 1.436000e+00 1.226000e-01 1.236400e+00 1.442000e-01 1.383800e+00 2.110000e-01 1.171300e+00 2.177000e-01 7.280000e-01 6.490000e-02 +-1.560000e-02 4.196000e-01 4.650000e-02 5.764000e-01 6.000000e-02 6.460000e-01 3.050000e-02 5.935000e-01 4.570000e-02 6.741000e-01 -3.280000e-02 6.699000e-01 1.860000e-02 3.420000e-01 + 5.736000e-01 5.383000e-01 2.750000e-01 1.314900e+00 2.674000e-01 7.521000e-01 3.316000e-01 1.253500e+00 2.474000e-01 7.465000e-01 2.832000e-01 1.306000e+00 2.464000e-01 4.982000e-01 + 7.007000e-01 3.733000e-01 9.031000e-01 7.899000e-01 1.038900e+00 7.671000e-01 1.784000e+00 -5.930000e-02 2.500000e-02 1.970800e+00 2.790000e-02 1.699700e+00 4.946000e-01 4.653000e-01 + 1.101300e+00 3.700000e-03 4.778000e-01 5.740000e-02 3.817000e-01 6.810000e-02 3.378000e-01 2.228000e-01 3.602000e-01 8.800000e-02 3.097000e-01 2.582000e-01 3.379000e-01 5.670000e-02 +-5.860000e-02 7.000000e-02 2.469000e-01 5.640000e-02 2.655000e-01 1.748000e-01 -4.000000e-03 5.900000e-03 2.730000e-02 -3.150000e-02 5.900000e-03 -7.100000e-03 1.159000e-01 4.810000e-02 + 1.120600e+00 -1.640000e-02 5.178000e-01 1.212100e+00 3.926000e-01 1.017600e+00 5.052000e-01 1.224700e+00 3.007000e-01 1.063800e+00 4.734000e-01 1.256600e+00 3.547000e-01 5.385000e-01 + 2.983000e-01 8.036000e-01 2.176000e-01 9.156000e-01 1.722000e-01 6.839000e-01 1.845000e-01 9.522000e-01 1.204000e-01 7.224000e-01 2.503000e-01 8.755000e-01 1.339000e-01 5.293000e-01 + 1.063100e+00 5.430000e-02 8.058000e-01 9.270000e-01 5.826000e-01 9.213000e-01 7.334000e-01 1.003300e+00 6.819000e-01 7.321000e-01 7.287000e-01 1.006000e+00 5.168000e-01 4.008000e-01 + 4.254000e-01 3.643000e-01 7.034000e-01 5.253000e-01 6.943000e-01 7.253000e-01 8.134000e-01 4.001000e-01 7.932000e-01 6.344000e-01 7.578000e-01 4.625000e-01 4.057000e-01 3.053000e-01 + 4.290000e-02 7.170000e-01 2.490000e-01 8.308000e-01 4.208000e-01 7.661000e-01 -2.320000e-02 1.762900e+00 1.860000e-02 1.978800e+00 -6.870000e-02 8.130000e-02 1.143000e-01 5.234000e-01 +-3.980000e-02 6.514000e-01 1.588000e-01 7.603000e-01 2.672000e-01 7.724000e-01 -1.780000e-02 1.750400e+00 1.890000e-02 1.978200e+00 -5.620000e-02 6.840000e-02 9.140000e-02 4.407000e-01 + 2.062000e-01 4.098000e-01 2.551000e-01 7.148000e-01 4.466000e-01 6.359000e-01 2.715000e-01 6.969000e-01 2.490000e-01 8.829000e-01 2.201000e-01 7.591000e-01 1.590000e-01 4.010000e-01 + 3.410000e-01 4.307000e-01 7.206000e-01 3.524000e-01 7.949000e-01 4.069000e-01 1.602900e+00 1.539000e-01 -1.459000e-01 2.171600e+00 1.000000e-02 -1.250000e-02 3.856000e-01 2.496000e-01 +-2.410000e-02 6.119000e-01 2.913000e-01 5.930000e-01 2.959000e-01 7.341000e-01 6.990000e-02 1.644200e+00 -4.070000e-02 2.050400e+00 9.500000e-02 -1.145000e-01 1.324000e-01 3.856000e-01 + 8.864000e-01 1.457000e-01 1.449500e+00 1.612000e-01 1.558100e+00 2.872000e-01 1.467800e+00 1.383000e-01 1.645400e+00 2.126000e-01 1.511700e+00 8.060000e-02 8.334000e-01 9.520000e-02 + 4.516000e-01 3.031000e-01 6.539000e-01 4.383000e-01 8.696000e-01 3.244000e-01 1.766400e+00 -3.990000e-02 -4.020000e-02 2.044000e+00 5.800000e-02 -6.990000e-02 3.998000e-01 2.355000e-01 + 1.500000e-03 1.039600e+00 -6.270000e-02 1.707800e+00 -1.400000e-02 1.880800e+00 6.470000e-02 1.564000e+00 1.572000e-01 1.698000e+00 1.540000e-02 1.620800e+00 1.990000e-02 9.212000e-01 + 4.896000e-01 2.424000e-01 8.585000e-01 1.687000e-01 1.021900e+00 1.227000e-01 1.728400e+00 -1.600000e-03 3.230000e-02 1.961700e+00 -3.790000e-02 4.570000e-02 4.665000e-01 1.439000e-01 + 1.028500e+00 2.200000e-03 1.519100e+00 1.153000e-01 1.891400e+00 -6.680000e-02 1.574600e+00 4.710000e-02 1.882900e+00 -2.460000e-02 1.660200e+00 -5.800000e-02 9.215000e-01 1.180000e-02 + 5.700000e-01 1.949000e-01 6.662000e-01 5.775000e-01 7.901000e-01 6.218000e-01 7.056000e-01 5.306000e-01 7.961000e-01 6.351000e-01 7.592000e-01 4.700000e-01 4.177000e-01 2.949000e-01 + 2.964000e-01 2.839000e-01 3.532000e-01 5.773000e-01 4.329000e-01 6.187000e-01 3.322000e-01 6.025000e-01 4.204000e-01 6.566000e-01 3.175000e-01 6.181000e-01 2.053000e-01 3.326000e-01 + 1.110100e+00 -6.900000e-03 5.430000e-01 7.135000e-01 3.813000e-01 5.501000e-01 4.775000e-01 7.901000e-01 4.455000e-01 4.477000e-01 5.519000e-01 7.022000e-01 3.929000e-01 3.006000e-01 + 4.494000e-01 3.016000e-01 6.661000e-01 5.118000e-01 8.272000e-01 5.039000e-01 7.380000e-01 4.324000e-01 8.066000e-01 5.531000e-01 7.674000e-01 3.959000e-01 4.210000e-01 2.538000e-01 + 1.201700e+00 -1.156000e-01 3.379000e-01 6.891000e-01 3.519000e-01 4.317000e-01 4.171000e-01 5.947000e-01 2.714000e-01 5.115000e-01 3.865000e-01 6.303000e-01 3.125000e-01 3.083000e-01 + 4.175000e-01 6.920000e-01 7.238000e-01 8.324000e-01 8.213000e-01 8.528000e-01 1.684100e+00 5.680000e-02 9.100000e-03 1.987300e+00 -1.150000e-02 1.749900e+00 4.226000e-01 4.766000e-01 + 4.360000e-01 3.713000e-01 6.214000e-01 6.574000e-01 7.203000e-01 7.235000e-01 5.825000e-01 6.956000e-01 7.206000e-01 7.458000e-01 6.731000e-01 5.917000e-01 3.731000e-01 3.595000e-01 + 3.848000e-01 2.801000e-01 5.886000e-01 4.593000e-01 7.417000e-01 4.387000e-01 5.876000e-01 4.592000e-01 7.156000e-01 4.903000e-01 6.483000e-01 3.882000e-01 3.588000e-01 2.423000e-01 + 1.096400e+00 1.260000e-02 8.966000e-01 3.421000e-01 6.822000e-01 2.301000e-01 8.664000e-01 3.788000e-01 6.956000e-01 1.860000e-01 8.982000e-01 3.447000e-01 5.707000e-01 1.193000e-01 + 4.805000e-01 5.688000e-01 5.959000e-01 1.079200e+00 6.938000e-01 1.213100e+00 6.770000e-01 9.835000e-01 7.859000e-01 1.129700e+00 7.537000e-01 8.900000e-01 3.702000e-01 5.925000e-01 + 1.090700e+00 1.510000e-02 4.532000e-01 8.947000e-01 3.829000e-01 5.780000e-01 4.569000e-01 8.886000e-01 3.970000e-01 5.348000e-01 5.703000e-01 7.581000e-01 3.893000e-01 3.213000e-01 + 4.580000e-02 5.139000e-01 -1.840000e-02 3.196000e-01 -3.710000e-02 3.004000e-01 9.080000e-02 1.905000e-01 -1.390000e-02 2.675000e-01 1.530000e-02 2.785000e-01 9.600000e-03 2.236000e-01 + 1.131200e+00 -3.020000e-02 8.929000e-01 7.040000e-02 7.703000e-01 -2.560000e-02 8.951000e-01 6.580000e-02 6.570000e-01 8.810000e-02 8.834000e-01 8.170000e-02 5.877000e-01 1.170000e-02 +-5.820000e-02 8.150000e-01 3.495000e-01 7.002000e-01 4.372000e-01 7.411000e-01 -8.800000e-02 1.829700e+00 6.840000e-02 1.920900e+00 -3.980000e-02 4.890000e-02 1.860000e-01 4.308000e-01 + 3.479000e-01 6.417000e-01 6.093000e-01 7.009000e-01 7.565000e-01 6.621000e-01 1.628200e+00 1.224000e-01 1.492000e-01 1.828500e+00 -1.960000e-02 2.300000e-02 3.743000e-01 3.924000e-01 + 4.160000e-01 1.654000e-01 6.349000e-01 2.783000e-01 6.370000e-01 4.172000e-01 6.487000e-01 2.577000e-01 6.379000e-01 4.257000e-01 6.367000e-01 2.739000e-01 3.315000e-01 2.005000e-01 + 7.600000e-02 1.032000e+00 3.440000e-02 1.042800e+00 -1.570000e-02 8.486000e-01 4.830000e-02 1.026200e+00 9.170000e-02 7.007000e-01 -1.420000e-02 1.099300e+00 4.420000e-02 5.973000e-01 + 6.870000e-02 6.446000e-01 4.410000e-01 5.699000e-01 4.881000e-01 6.571000e-01 1.674200e+00 6.730000e-02 1.860000e-02 1.976100e+00 2.750000e-02 -3.180000e-02 2.199000e-01 3.772000e-01 + 1.838000e-01 9.222000e-01 1.668000e-01 8.231000e-01 1.093000e-01 6.645000e-01 1.597000e-01 8.303000e-01 1.604000e-01 5.899000e-01 1.905000e-01 7.935000e-01 9.970000e-02 5.155000e-01 + 5.985000e-01 2.114000e-01 8.599000e-01 2.747000e-01 8.155000e-01 4.715000e-01 1.754300e+00 -2.590000e-02 -1.890000e-02 2.017000e+00 -6.550000e-02 7.960000e-02 4.398000e-01 2.384000e-01 + 9.812000e-01 1.464000e-01 9.873000e-01 7.388000e-01 7.230000e-01 1.253800e+00 9.414000e-01 7.927000e-01 7.645000e-01 1.026800e+00 9.855000e-01 7.375000e-01 6.030000e-01 3.864000e-01 + 2.771000e-01 4.934000e-01 4.557000e-01 7.448000e-01 4.533000e-01 9.261000e-01 3.633000e-01 8.519000e-01 3.589000e-01 1.055700e+00 4.432000e-01 7.585000e-01 2.485000e-01 4.471000e-01 + 2.765000e-01 5.446000e-01 4.058000e-01 8.792000e-01 5.765000e-01 8.749000e-01 4.498000e-01 8.381000e-01 6.530000e-01 8.068000e-01 4.016000e-01 8.954000e-01 2.742000e-01 4.652000e-01 + 8.580000e-01 2.206000e-01 3.943000e-01 1.329800e+00 3.000000e-01 7.321000e-01 4.228000e-01 1.296500e+00 3.049000e-01 6.911000e-01 3.720000e-01 1.352200e+00 2.668000e-01 4.835000e-01 + 4.178000e-01 5.168000e-01 5.407000e-01 9.438000e-01 6.523000e-01 1.029500e+00 6.214000e-01 8.466000e-01 7.659000e-01 9.212000e-01 5.444000e-01 9.391000e-01 3.318000e-01 5.214000e-01 + 5.398000e-01 3.048000e-01 8.779000e-01 4.404000e-01 1.011700e+00 4.854000e-01 8.146000e-01 5.131000e-01 9.718000e-01 5.539000e-01 8.771000e-01 4.340000e-01 4.961000e-01 2.664000e-01 + 1.737000e-01 1.020000e-01 2.218000e-01 2.243000e-01 3.025000e-01 1.940000e-01 2.421000e-01 1.973000e-01 2.738000e-01 2.345000e-01 1.940000e-01 2.585000e-01 1.483000e-01 1.043000e-01 + 2.552000e-01 6.242000e-01 5.095000e-01 6.932000e-01 7.394000e-01 5.597000e-01 1.650500e+00 1.011000e-01 -2.090000e-02 2.025200e+00 4.510000e-02 -5.310000e-02 3.175000e-01 3.858000e-01 + 6.043000e-01 4.847000e-01 2.698000e-01 7.133000e-01 1.207000e-01 6.650000e-01 1.966000e-01 7.995000e-01 3.183000e-01 4.146000e-01 3.367000e-01 6.292000e-01 2.181000e-01 3.906000e-01 + 7.783000e-01 4.700000e-03 1.048700e+00 2.047000e-01 1.169700e+00 2.627000e-01 1.024000e+00 2.338000e-01 1.229200e+00 2.135000e-01 1.067300e+00 1.827000e-01 5.997000e-01 1.258000e-01 + 1.056900e+00 5.520000e-02 7.505000e-01 1.643000e-01 5.627000e-01 1.735000e-01 7.464000e-01 1.701000e-01 5.599000e-01 1.603000e-01 7.443000e-01 1.746000e-01 4.886000e-01 9.790000e-02 + 1.082000e+00 2.940000e-02 6.796000e-01 1.778000e-01 5.755000e-01 1.053000e-01 7.075000e-01 1.409000e-01 5.595000e-01 1.105000e-01 8.247000e-01 -2.300000e-03 5.124000e-01 3.890000e-02 + 5.710000e-02 1.050300e+00 -1.100000e-03 4.328000e-01 1.623000e-01 1.709000e-01 -3.400000e-02 4.719000e-01 -1.950000e-02 3.805000e-01 -9.330000e-02 5.434000e-01 3.350000e-02 2.892000e-01 + 1.309000e-01 3.577000e-01 1.534000e-01 6.240000e-01 1.369000e-01 7.509000e-01 2.430000e-01 5.122000e-01 2.289000e-01 6.538000e-01 2.045000e-01 5.573000e-01 1.022000e-01 3.412000e-01 + 3.363000e-01 4.877000e-01 6.284000e-01 5.175000e-01 6.639000e-01 6.105000e-01 1.634700e+00 1.160000e-01 -7.740000e-02 2.089800e+00 1.650000e-02 -2.060000e-02 3.415000e-01 3.319000e-01 + 8.430000e-02 3.520000e-01 5.280000e-02 6.516000e-01 6.420000e-02 7.334000e-01 5.900000e-02 6.430000e-01 6.910000e-02 7.387000e-01 1.051000e-01 5.891000e-01 3.390000e-02 3.710000e-01 + 2.419000e-01 2.554000e-01 3.898000e-01 3.872000e-01 4.288000e-01 4.575000e-01 3.504000e-01 4.362000e-01 4.903000e-01 3.988000e-01 3.969000e-01 3.784000e-01 2.171000e-01 2.331000e-01 + 6.307000e-01 4.740000e-01 9.798000e-01 5.770000e-01 9.152000e-01 7.924000e-01 1.912900e+00 -2.140000e-01 2.007500e+00 -1.050000e-02 -5.000000e-04 1.730100e+00 5.307000e-01 3.768000e-01 + 4.460000e-01 -1.960000e-02 7.009000e-01 -2.960000e-02 7.206000e-01 5.770000e-02 6.415000e-01 4.150000e-02 7.693000e-01 1.520000e-02 7.258000e-01 -5.950000e-02 3.754000e-01 1.790000e-02 + 1.226000e-01 8.072000e-01 5.387000e-01 6.823000e-01 6.626000e-01 6.829000e-01 1.689300e+00 4.540000e-02 -1.990000e-02 2.019300e+00 -3.810000e-02 4.740000e-02 2.932000e-01 4.304000e-01 + 7.825000e-01 9.250000e-02 1.127900e+00 2.562000e-01 1.325700e+00 2.437000e-01 1.183100e+00 1.954000e-01 1.290500e+00 3.130000e-01 1.240500e+00 1.253000e-01 6.916000e-01 1.009000e-01 + 4.689000e-01 1.481000e-01 7.357000e-01 2.329000e-01 7.442000e-01 3.792000e-01 6.927000e-01 2.833000e-01 8.208000e-01 3.054000e-01 7.793000e-01 1.818000e-01 4.031000e-01 1.613000e-01 + 6.073000e-01 1.505000e-01 8.918000e-01 3.078000e-01 1.123600e+00 2.238000e-01 8.878000e-01 3.141000e-01 1.023600e+00 3.635000e-01 8.919000e-01 3.122000e-01 4.959000e-01 2.008000e-01 + 1.084500e+00 2.560000e-02 1.255700e+00 4.653000e-01 9.001000e-01 1.065200e+00 1.240200e+00 4.783000e-01 9.773000e-01 7.541000e-01 1.280700e+00 4.358000e-01 6.714000e-01 3.143000e-01 + 1.122400e+00 -2.090000e-02 6.288000e-01 1.111500e+00 5.239000e-01 8.923000e-01 7.640000e-01 9.513000e-01 6.113000e-01 7.272000e-01 6.948000e-01 1.037500e+00 4.859000e-01 4.066000e-01 + 8.826000e-01 2.210000e-01 3.047000e-01 6.431000e-01 3.247000e-01 4.102000e-01 4.222000e-01 5.054000e-01 3.085000e-01 4.134000e-01 3.644000e-01 5.701000e-01 2.931000e-01 2.968000e-01 +-7.180000e-02 6.558000e-01 2.116000e-01 6.708000e-01 3.328000e-01 6.744000e-01 3.570000e-02 1.688300e+00 5.100000e-02 1.946700e+00 -7.030000e-02 8.540000e-02 1.075000e-01 4.049000e-01 + 3.613000e-01 7.365000e-01 2.189000e-01 5.576000e-01 1.782000e-01 4.514000e-01 2.020000e-01 5.775000e-01 1.855000e-01 4.275000e-01 2.536000e-01 5.144000e-01 1.511000e-01 3.725000e-01 + 1.074900e+00 4.140000e-02 9.121000e-01 1.876000e-01 7.667000e-01 5.760000e-02 9.258000e-01 1.722000e-01 7.291000e-01 8.450000e-02 8.942000e-01 2.083000e-01 5.681000e-01 8.440000e-02 + 2.760000e-02 1.086800e+00 -4.250000e-02 5.219000e-01 3.250000e-02 3.607000e-01 2.460000e-02 4.429000e-01 8.690000e-02 2.859000e-01 -1.100000e-03 4.726000e-01 3.560000e-02 3.120000e-01 +-4.950000e-02 5.970000e-02 5.990000e-02 1.809000e-01 1.120000e-02 3.660000e-01 1.340000e-02 -1.290000e-02 7.630000e-02 -9.200000e-02 5.320000e-02 -6.370000e-02 6.000000e-03 1.293000e-01 +-7.640000e-02 1.198800e+00 3.247000e-01 1.192000e+00 3.850000e-01 1.257300e+00 -8.280000e-02 1.832100e+00 4.500000e-03 1.991200e+00 -1.990000e-02 1.757700e+00 1.692000e-01 7.139000e-01 + 3.735000e-01 7.355000e-01 1.927000e-01 5.480000e-01 2.143000e-01 3.824000e-01 2.458000e-01 4.837000e-01 1.067000e-01 4.974000e-01 1.687000e-01 5.756000e-01 1.769000e-01 3.251000e-01 + 2.741000e-01 5.164000e-01 3.749000e-01 8.773000e-01 4.355000e-01 9.832000e-01 4.299000e-01 8.103000e-01 5.146000e-01 9.149000e-01 3.877000e-01 8.587000e-01 2.248000e-01 4.949000e-01 +-7.800000e-03 7.603000e-01 -4.110000e-02 1.110900e+00 -3.300000e-03 1.193900e+00 -7.110000e-02 1.819100e+00 6.250000e-02 1.926700e+00 4.510000e-02 -5.140000e-02 1.150000e-02 6.132000e-01 + 1.184000e+00 -9.540000e-02 4.044000e-01 6.970000e-01 2.919000e-01 5.550000e-01 5.266000e-01 5.560000e-01 3.209000e-01 5.018000e-01 3.657000e-01 7.450000e-01 3.112000e-01 3.396000e-01 + 4.732000e-01 3.046000e-01 6.683000e-01 4.455000e-01 8.095000e-01 4.224000e-01 1.736500e+00 1.300000e-03 3.190000e-02 1.964000e+00 -2.400000e-03 3.500000e-03 4.121000e-01 2.376000e-01 + 1.384000e-01 6.302000e-01 1.225000e-01 1.096600e+00 2.138000e-01 1.162500e+00 1.260000e-01 1.090700e+00 2.173000e-01 1.177400e+00 2.158000e-01 9.860000e-01 1.137000e-01 5.832000e-01 + 5.437000e-01 1.828000e-01 8.222000e-01 3.264000e-01 1.067400e+00 2.131000e-01 8.137000e-01 3.349000e-01 1.033500e+00 2.770000e-01 8.017000e-01 3.519000e-01 5.158000e-01 1.387000e-01 + 8.353000e-01 1.856000e-01 1.130800e+00 2.115000e-01 1.142400e+00 3.409000e-01 1.648800e+00 9.690000e-02 1.939400e+00 7.250000e-02 -3.960000e-02 4.650000e-02 6.589000e-01 1.338000e-01 + 5.665000e-01 4.000000e-01 8.040000e-01 4.914000e-01 9.725000e-01 4.351000e-01 1.770100e+00 -5.090000e-02 -6.530000e-02 2.077400e+00 8.140000e-02 -9.730000e-02 4.666000e-01 2.948000e-01 + 1.780000e-02 6.441000e-01 1.934000e-01 8.153000e-01 2.080000e-01 9.453000e-01 2.370000e-01 7.618000e-01 2.103000e-01 9.556000e-01 8.110000e-02 9.460000e-01 8.990000e-02 4.970000e-01 + 3.716000e-01 5.727000e-01 5.762000e-01 9.028000e-01 7.666000e-01 8.992000e-01 6.574000e-01 8.045000e-01 8.085000e-01 8.724000e-01 6.284000e-01 8.446000e-01 3.867000e-01 4.575000e-01 + 9.650000e-02 8.800000e-02 5.990000e-02 2.461000e-01 1.445000e-01 1.877000e-01 1.420000e-01 1.516000e-01 1.662000e-01 1.680000e-01 1.162000e-01 1.799000e-01 6.860000e-02 1.019000e-01 + 5.024000e-01 5.639000e-01 7.328000e-01 9.492000e-01 8.707000e-01 1.036400e+00 6.174000e-01 1.084500e+00 8.037000e-01 1.142800e+00 7.794000e-01 8.958000e-01 4.507000e-01 5.156000e-01 + 5.465000e-01 4.334000e-01 9.130000e-01 6.068000e-01 8.931000e-01 8.647000e-01 8.490000e-01 6.866000e-01 9.978000e-01 7.642000e-01 8.058000e-01 7.353000e-01 4.696000e-01 4.187000e-01 + 1.090000e-01 7.328000e-01 2.592000e-01 1.042400e+00 3.453000e-01 1.126100e+00 2.120000e-01 1.099000e+00 3.552000e-01 1.139400e+00 2.400000e-01 1.065500e+00 1.622000e-01 5.869000e-01 + 3.045000e-01 4.043000e-01 4.679000e-01 6.469000e-01 5.784000e-01 6.798000e-01 4.442000e-01 6.740000e-01 5.287000e-01 7.581000e-01 4.444000e-01 6.719000e-01 2.540000e-01 3.910000e-01 + 5.597000e-01 2.298000e-01 8.849000e-01 3.551000e-01 1.034300e+00 3.724000e-01 1.094000e+00 1.069000e-01 1.116900e+00 3.034000e-01 8.565000e-01 3.830000e-01 5.458000e-01 1.637000e-01 + 1.135400e+00 -3.660000e-02 9.930000e-01 7.328000e-01 7.396000e-01 4.829000e-01 1.096200e+00 6.153000e-01 7.750000e-01 4.016000e-01 1.057200e+00 6.568000e-01 6.032000e-01 2.289000e-01 + 2.453000e-01 4.808000e-01 4.855000e-01 6.343000e-01 6.267000e-01 6.316000e-01 5.035000e-01 6.100000e-01 6.372000e-01 6.430000e-01 4.382000e-01 6.838000e-01 2.897000e-01 3.544000e-01 + 1.590000e-01 9.624000e-01 5.616000e-01 1.167600e+00 7.067000e-01 1.198000e+00 1.750800e+00 -2.140000e-02 1.230000e-02 1.983100e+00 1.710000e-02 1.712500e+00 3.037000e-01 6.852000e-01 + 1.016200e+00 1.034000e-01 4.259000e-01 4.574000e-01 3.615000e-01 3.437000e-01 3.998000e-01 4.866000e-01 3.650000e-01 3.214000e-01 4.585000e-01 4.220000e-01 3.193000e-01 2.518000e-01 +-6.100000e-02 1.181100e+00 1.040000e-02 1.661100e+00 9.170000e-02 1.701800e+00 1.160000e-02 1.715900e+00 -5.730000e-02 2.068800e+00 -1.860000e-02 1.750300e+00 4.010000e-02 9.064000e-01 + 1.116900e+00 -1.100000e-02 1.668300e+00 7.910000e-02 1.079600e+00 2.620000e-02 1.753600e+00 -3.070000e-02 9.317000e-01 1.614000e-01 1.719400e+00 1.680000e-02 7.741000e-01 1.160000e-02 + 5.425000e-01 5.662000e-01 4.065000e-01 7.119000e-01 1.688000e-01 7.062000e-01 3.222000e-01 8.160000e-01 7.430000e-02 7.979000e-01 1.885000e-01 9.741000e-01 2.290000e-01 4.339000e-01 + 1.092100e+00 1.910000e-02 6.402000e-01 6.883000e-01 4.099000e-01 5.550000e-01 5.433000e-01 8.057000e-01 4.109000e-01 5.229000e-01 5.550000e-01 7.881000e-01 4.032000e-01 3.093000e-01 + 4.276000e-01 5.982000e-01 8.580000e-01 4.580000e-01 9.666000e-01 4.712000e-01 1.702400e+00 3.460000e-02 9.500000e-03 1.983900e+00 3.450000e-02 -4.050000e-02 4.654000e-01 3.163000e-01 + 1.115200e+00 -1.240000e-02 8.336000e-01 9.390000e-02 7.419000e-01 -2.350000e-02 7.926000e-01 1.436000e-01 6.870000e-01 2.140000e-02 8.857000e-01 3.260000e-02 5.579000e-01 2.920000e-02 + 6.695000e-01 2.592000e-01 1.030400e+00 1.922000e-01 1.123800e+00 2.317000e-01 1.701900e+00 3.900000e-02 1.941600e+00 6.780000e-02 -6.900000e-03 9.600000e-03 5.872000e-01 1.409000e-01 + 1.106000e+00 1.000000e-04 5.493000e-01 1.184000e+00 5.373000e-01 6.650000e-01 4.143000e-01 1.344100e+00 4.731000e-01 6.990000e-01 5.029000e-01 1.236600e+00 4.045000e-01 4.258000e-01 + 2.478000e-01 6.458000e-01 4.043000e-01 8.337000e-01 7.193000e-01 6.084000e-01 1.761800e+00 -2.940000e-02 3.350000e-02 1.959700e+00 -3.270000e-02 4.190000e-02 2.817000e-01 4.376000e-01 + 7.105000e-01 3.832000e-01 9.548000e-01 4.648000e-01 1.065400e+00 4.734000e-01 1.780600e+00 -5.460000e-02 1.931100e+00 7.980000e-02 -8.090000e-02 1.827600e+00 5.788000e-01 2.513000e-01 + 9.968000e-01 1.254000e-01 3.771000e-01 8.535000e-01 3.739000e-01 5.202000e-01 4.012000e-01 8.273000e-01 3.794000e-01 4.933000e-01 3.189000e-01 9.200000e-01 3.228000e-01 3.621000e-01 + 8.350000e-02 5.562000e-01 -2.360000e-02 9.990000e-01 1.625000e-01 8.986000e-01 -2.440000e-02 1.758700e+00 9.700000e-02 1.884700e+00 -6.000000e-03 8.400000e-03 4.000000e-03 5.609000e-01 + 7.489000e-01 2.026000e-01 1.055900e+00 4.588000e-01 1.076900e+00 6.637000e-01 9.010000e-01 6.377000e-01 1.106200e+00 6.628000e-01 9.712000e-01 5.537000e-01 5.625000e-01 3.189000e-01 + 6.852000e-01 4.216000e-01 2.922000e-01 1.446000e+00 2.703000e-01 1.491500e+00 3.944000e-01 1.328700e+00 2.530000e-01 1.362000e+00 3.464000e-01 1.382300e+00 2.611000e-01 7.005000e-01 + 1.096600e+00 1.160000e-02 5.389000e-01 6.236000e-01 4.291000e-01 4.441000e-01 4.939000e-01 6.776000e-01 5.015000e-01 3.356000e-01 5.481000e-01 6.099000e-01 3.832000e-01 2.869000e-01 + 4.674000e-01 3.366000e-01 6.300000e-01 6.509000e-01 8.707000e-01 5.648000e-01 8.028000e-01 4.547000e-01 7.896000e-01 6.800000e-01 6.882000e-01 5.842000e-01 3.931000e-01 3.419000e-01 + 3.740000e-02 3.977000e-01 1.968000e-01 5.588000e-01 3.630000e-01 5.027000e-01 5.700000e-03 1.723800e+00 5.880000e-02 -6.790000e-02 4.370000e-02 -5.240000e-02 9.980000e-02 3.309000e-01 +-8.640000e-02 7.606000e-01 1.386000e-01 8.285000e-01 1.761000e-01 9.310000e-01 1.890000e-02 1.707500e+00 8.900000e-02 1.899300e+00 -2.260000e-02 2.660000e-02 7.470000e-02 4.927000e-01 + 9.280000e-02 4.511000e-01 2.259000e-01 6.111000e-01 2.024000e-01 7.561000e-01 2.166000e-01 6.206000e-01 2.474000e-01 7.191000e-01 3.004000e-01 5.213000e-01 1.155000e-01 3.700000e-01 + 1.129200e+00 -2.850000e-02 1.839300e+00 -1.256000e-01 1.044800e+00 3.002000e-01 1.690200e+00 4.770000e-02 1.060200e+00 2.250000e-01 1.756400e+00 -2.550000e-02 7.967000e-01 7.160000e-02 + 4.576000e-01 6.481000e-01 8.842000e-01 6.654000e-01 8.400000e-01 8.585000e-01 1.776800e+00 -5.000000e-02 -2.600000e-02 2.027400e+00 -3.770000e-02 1.776600e+00 4.504000e-01 4.539000e-01 + 9.950000e-01 1.179000e-01 4.039000e-01 5.380000e-02 3.093000e-01 8.520000e-02 3.899000e-01 6.880000e-02 3.294000e-01 5.420000e-02 3.797000e-01 8.240000e-02 3.046000e-01 4.090000e-02 + 3.339000e-01 6.875000e-01 4.558000e-01 1.153400e+00 6.172000e-01 1.192000e+00 5.502000e-01 1.038800e+00 6.202000e-01 1.225200e+00 5.811000e-01 1.007600e+00 3.185000e-01 6.004000e-01 +-4.820000e-02 6.359000e-01 1.800000e-02 8.810000e-01 2.713000e-01 7.287000e-01 -7.900000e-03 1.742900e+00 -7.300000e-02 2.082400e+00 1.420000e-02 -1.830000e-02 5.610000e-02 4.595000e-01 + 2.457000e-01 8.729000e-01 5.142000e-01 1.241100e+00 6.712000e-01 1.218500e+00 1.593200e+00 1.650000e-01 -7.070000e-02 2.076500e+00 -4.660000e-02 1.782600e+00 3.449000e-01 6.335000e-01 + 1.229000e-01 4.821000e-01 1.893000e-01 7.599000e-01 1.586000e-01 9.281000e-01 1.126000e-01 8.487000e-01 1.617000e-01 9.455000e-01 1.438000e-01 8.120000e-01 8.710000e-02 4.641000e-01 + 1.070200e+00 4.060000e-02 9.576000e-01 7.759000e-01 7.389000e-01 6.740000e-01 9.694000e-01 7.624000e-01 6.454000e-01 7.210000e-01 8.924000e-01 8.504000e-01 5.947000e-01 3.001000e-01 + 1.169300e+00 -7.410000e-02 4.090000e-01 1.405000e-01 3.912000e-01 5.770000e-02 3.398000e-01 2.218000e-01 3.632000e-01 8.360000e-02 3.703000e-01 1.829000e-01 3.115000e-01 8.760000e-02 + 1.105400e+00 -1.500000e-03 8.807000e-01 8.586000e-01 7.397000e-01 7.915000e-01 8.299000e-01 9.165000e-01 6.437000e-01 8.200000e-01 8.560000e-01 8.872000e-01 5.826000e-01 3.420000e-01 + 7.364000e-01 3.606000e-01 3.379000e-01 1.395600e+00 3.809000e-01 6.602000e-01 3.742000e-01 1.349900e+00 2.766000e-01 7.495000e-01 3.453000e-01 1.385800e+00 2.625000e-01 5.014000e-01 + 2.111000e-01 4.445000e-01 3.171000e-01 7.100000e-01 2.735000e-01 9.087000e-01 2.507000e-01 7.881000e-01 3.714000e-01 8.123000e-01 3.026000e-01 7.286000e-01 1.915000e-01 3.995000e-01 + 1.103900e+00 1.500000e-03 1.563900e+00 1.408000e-01 1.042800e+00 2.625000e-01 1.342200e+00 4.042000e-01 9.119000e-01 3.649000e-01 1.385600e+00 3.508000e-01 7.253000e-01 1.404000e-01 + 4.195000e-01 8.830000e-02 7.209000e-01 9.850000e-02 9.006000e-01 3.060000e-02 1.774500e+00 -4.860000e-02 4.910000e-02 -5.700000e-02 -4.180000e-02 4.740000e-02 4.286000e-01 3.880000e-02 + 5.653000e-01 2.091000e-01 8.336000e-01 2.673000e-01 9.022000e-01 3.250000e-01 1.754900e+00 -2.750000e-02 -4.610000e-02 2.045700e+00 9.900000e-03 -1.070000e-02 4.766000e-01 1.708000e-01 + 1.073400e+00 3.650000e-02 1.302600e+00 4.334000e-01 9.583000e-01 1.296000e-01 1.387600e+00 3.254000e-01 9.933000e-01 4.790000e-02 1.318700e+00 4.104000e-01 7.018000e-01 7.740000e-02 + 1.012300e+00 9.170000e-02 1.274500e+00 4.656000e-01 1.394300e+00 5.197000e-01 1.596500e+00 1.580000e-01 1.963400e+00 4.470000e-02 -3.710000e-02 1.775800e+00 7.757000e-01 2.109000e-01 + 3.312000e-01 4.549000e-01 4.088000e-01 8.564000e-01 4.991000e-01 9.237000e-01 3.774000e-01 8.868000e-01 4.258000e-01 1.034300e+00 4.400000e-01 8.127000e-01 2.453000e-01 4.791000e-01 + 2.755000e-01 5.100000e-02 5.709000e-01 7.640000e-02 6.809000e-01 8.630000e-02 1.552000e+00 2.105000e-01 1.029000e-01 -1.244000e-01 4.730000e-02 -5.740000e-02 3.188000e-01 4.080000e-02 + 1.982000e-01 4.741000e-01 5.539000e-01 4.219000e-01 5.941000e-01 5.179000e-01 1.645400e+00 9.840000e-02 -2.010000e-02 2.023500e+00 2.800000e-03 -4.000000e-03 2.789000e-01 2.966000e-01 + 5.457000e-01 5.523000e-01 2.855000e-01 1.444900e+00 2.507000e-01 9.536000e-01 2.293000e-01 1.511200e+00 1.937000e-01 9.833000e-01 2.684000e-01 1.465500e+00 2.168000e-01 6.102000e-01 +-1.760000e-02 7.540000e-01 2.606000e-01 7.783000e-01 3.203000e-01 8.502000e-01 3.800000e-02 1.688500e+00 -8.700000e-03 2.010800e+00 1.600000e-03 -5.200000e-03 1.225000e-01 4.917000e-01 + 4.159000e-01 3.145000e-01 5.517000e-01 6.108000e-01 6.129000e-01 7.097000e-01 5.831000e-01 5.751000e-01 5.852000e-01 7.686000e-01 6.010000e-01 5.550000e-01 3.130000e-01 3.593000e-01 + 9.423000e-01 8.490000e-02 1.512500e+00 9.520000e-02 1.424700e+00 4.486000e-01 1.395900e+00 2.259000e-01 1.504800e+00 3.934000e-01 1.365400e+00 2.641000e-01 7.831000e-01 1.587000e-01 + 4.414000e-01 3.166000e-01 7.875000e-01 2.755000e-01 8.969000e-01 2.846000e-01 1.744600e+00 -1.050000e-02 2.500000e-02 1.969800e+00 6.880000e-02 -8.310000e-02 4.572000e-01 1.650000e-01 + 4.897000e-01 6.166000e-01 7.778000e-01 7.313000e-01 9.998000e-01 6.125000e-01 1.702800e+00 3.800000e-02 1.101000e-01 1.871500e+00 6.670000e-02 1.654300e+00 4.621000e-01 4.153000e-01 + 3.592000e-01 1.203000e-01 5.896000e-01 1.628000e-01 7.070000e-01 1.390000e-01 6.235000e-01 1.224000e-01 6.721000e-01 1.972000e-01 6.086000e-01 1.439000e-01 3.612000e-01 6.870000e-02 + 1.177800e+00 -8.270000e-02 1.425000e+00 2.890000e-02 1.075300e+00 -1.025000e-01 1.426900e+00 3.190000e-02 9.603000e-01 8.000000e-04 1.419000e+00 4.170000e-02 7.187000e-01 1.360000e-02 + 2.602000e-01 7.164000e-01 7.322000e-01 5.280000e-01 7.810000e-01 6.129000e-01 1.754400e+00 -2.570000e-02 -4.170000e-02 2.048600e+00 -7.420000e-02 8.850000e-02 3.622000e-01 3.902000e-01 + 9.853000e-01 1.293000e-01 4.345000e-01 2.152000e-01 3.437000e-01 1.954000e-01 4.099000e-01 2.471000e-01 3.284000e-01 2.032000e-01 4.200000e-01 2.330000e-01 3.041000e-01 1.577000e-01 + 9.117000e-01 1.948000e-01 1.158700e+00 4.074000e-01 1.335200e+00 3.364000e-01 1.714900e+00 1.760000e-02 2.016600e+00 -2.550000e-02 -1.041000e-01 1.856400e+00 7.200000e-01 1.807000e-01 + 1.140600e+00 -3.580000e-02 6.300000e-01 1.801000e-01 3.542000e-01 3.308000e-01 5.145000e-01 3.221000e-01 4.868000e-01 1.562000e-01 5.023000e-01 3.346000e-01 4.021000e-01 1.433000e-01 + 6.267000e-01 4.142000e-01 9.552000e-01 6.795000e-01 1.028300e+00 8.437000e-01 9.689000e-01 6.655000e-01 1.223900e+00 6.474000e-01 9.202000e-01 7.226000e-01 5.495000e-01 3.955000e-01 + 2.915000e-01 2.898000e-01 3.733000e-01 5.469000e-01 3.424000e-01 7.258000e-01 3.200000e-01 6.140000e-01 4.248000e-01 6.449000e-01 2.811000e-01 6.647000e-01 1.979000e-01 3.389000e-01 + 7.741000e-01 1.946000e-01 1.068400e+00 4.755000e-01 1.394300e+00 3.300000e-01 1.170700e+00 3.506000e-01 1.343000e+00 4.205000e-01 1.218300e+00 2.957000e-01 6.731000e-01 2.077000e-01 +-3.680000e-02 4.017000e-01 6.590000e-02 6.122000e-01 2.108000e-01 5.843000e-01 -7.320000e-02 1.821100e+00 -7.040000e-02 8.380000e-02 -1.620000e-02 1.770000e-02 6.800000e-02 3.089000e-01 + 3.511000e-01 8.330000e-02 6.243000e-01 4.320000e-02 6.950000e-01 7.220000e-02 6.273000e-01 3.880000e-02 8.073000e-01 -5.250000e-02 6.866000e-01 -2.930000e-02 3.668000e-01 1.780000e-02 +-2.200000e-03 4.328000e-01 9.620000e-02 5.566000e-01 1.032000e-01 6.397000e-01 7.780000e-02 5.799000e-01 1.418000e-01 6.060000e-01 1.540000e-02 6.526000e-01 4.010000e-02 3.400000e-01 + 1.157000e-01 3.932000e-01 2.233000e-01 5.684000e-01 3.136000e-01 5.740000e-01 2.221000e-01 5.673000e-01 1.697000e-01 7.595000e-01 1.936000e-01 6.002000e-01 1.201000e-01 3.379000e-01 + 3.699000e-01 7.311000e-01 7.281000e-01 9.934000e-01 6.734000e-01 1.311100e+00 1.734800e+00 -5.500000e-03 1.186000e-01 1.858800e+00 -1.470000e-02 1.746300e+00 3.813000e-01 6.166000e-01 + 3.891000e-01 2.215000e-01 6.282000e-01 3.252000e-01 7.111000e-01 3.700000e-01 6.102000e-01 3.434000e-01 7.287000e-01 3.707000e-01 5.959000e-01 3.620000e-01 3.536000e-01 1.964000e-01 + 1.167600e+00 -7.010000e-02 3.770000e-01 1.359000e+00 2.343000e-01 1.332700e+00 4.709000e-01 1.250900e+00 2.915000e-01 1.174700e+00 3.580000e-01 1.382700e+00 3.080000e-01 6.191000e-01 + 4.259000e-01 4.327000e-01 6.630000e-01 6.835000e-01 6.497000e-01 9.032000e-01 6.208000e-01 7.378000e-01 6.525000e-01 9.307000e-01 6.688000e-01 6.760000e-01 3.576000e-01 4.249000e-01 + 4.507000e-01 5.850000e-01 7.612000e-01 8.484000e-01 8.878000e-01 9.469000e-01 7.216000e-01 8.978000e-01 7.942000e-01 1.079400e+00 7.377000e-01 8.773000e-01 4.067000e-01 5.303000e-01 + 1.700000e-01 9.436000e-01 1.486000e-01 1.091900e+00 5.420000e-02 8.650000e-01 2.837000e-01 9.306000e-01 1.378000e-01 7.436000e-01 1.412000e-01 1.099300e+00 1.027000e-01 5.879000e-01 + 3.313000e-01 2.213000e-01 5.529000e-01 3.276000e-01 7.630000e-01 2.247000e-01 1.775100e+00 -4.960000e-02 -1.123000e-01 2.130400e+00 3.450000e-02 -4.120000e-02 3.109000e-01 1.995000e-01 +-3.970000e-02 8.653000e-01 2.490000e-02 1.106400e+00 1.539000e-01 1.077600e+00 2.170000e-02 1.703100e+00 -3.390000e-02 2.043400e+00 4.890000e-02 -5.940000e-02 9.900000e-03 6.578000e-01 + 4.160000e-02 2.492000e-01 3.630000e-02 5.778000e-01 2.428000e-01 4.730000e-01 -2.500000e-03 1.733400e+00 -5.520000e-02 6.750000e-02 4.880000e-02 -5.870000e-02 4.570000e-02 2.934000e-01 + 1.156100e+00 -6.180000e-02 4.096000e-01 1.040000e+00 4.179000e-01 5.648000e-01 4.842000e-01 9.480000e-01 3.375000e-01 6.273000e-01 3.995000e-01 1.051100e+00 3.377000e-01 3.912000e-01 + 1.151400e+00 -5.380000e-02 5.146000e-01 1.212200e+00 4.147000e-01 8.332000e-01 5.307000e-01 1.194600e+00 3.870000e-01 8.223000e-01 5.730000e-01 1.144800e+00 3.725000e-01 4.698000e-01 + 8.854000e-01 2.118000e-01 2.810000e-01 5.846000e-01 3.241000e-01 3.542000e-01 3.327000e-01 5.222000e-01 3.007000e-01 3.679000e-01 4.131000e-01 4.262000e-01 2.879000e-01 2.674000e-01 + 3.638000e-01 3.804000e-01 7.445000e-01 2.988000e-01 8.495000e-01 3.202000e-01 1.906100e+00 -2.026000e-01 0.000000e+00 2.001200e+00 -8.700000e-03 9.600000e-03 3.821000e-01 2.389000e-01 + 2.077000e-01 4.487000e-01 2.482000e-01 7.953000e-01 3.955000e-01 7.727000e-01 3.535000e-01 6.678000e-01 4.896000e-01 6.782000e-01 3.969000e-01 6.183000e-01 2.022000e-01 3.902000e-01 + 2.329000e-01 6.285000e-01 5.094000e-01 8.113000e-01 6.033000e-01 8.956000e-01 5.496000e-01 7.599000e-01 6.561000e-01 8.529000e-01 5.272000e-01 7.862000e-01 3.007000e-01 4.596000e-01 + 2.791000e-01 3.898000e-01 3.058000e-01 7.622000e-01 5.019000e-01 6.861000e-01 4.306000e-01 6.145000e-01 4.562000e-01 7.582000e-01 4.978000e-01 5.388000e-01 2.469000e-01 3.571000e-01 + 7.123000e-01 4.088000e-01 4.206000e-01 5.596000e-01 2.809000e-01 4.953000e-01 3.921000e-01 5.953000e-01 3.269000e-01 4.263000e-01 3.688000e-01 6.182000e-01 2.817000e-01 3.309000e-01 + 4.560000e-01 6.668000e-01 9.033000e-01 7.476000e-01 1.013600e+00 7.590000e-01 1.754300e+00 -2.050000e-02 1.967200e+00 3.600000e-02 -5.360000e-02 1.796700e+00 5.007000e-01 4.429000e-01 + 8.890000e-02 1.325000e-01 4.074000e-01 1.244000e-01 5.965000e-01 4.450000e-02 1.626600e+00 1.235000e-01 -4.290000e-02 5.310000e-02 4.770000e-02 -5.700000e-02 2.280000e-01 6.230000e-02 + 1.146600e+00 -4.920000e-02 7.559000e-01 2.360000e-02 5.875000e-01 5.060000e-02 7.465000e-01 3.910000e-02 6.143000e-01 2.300000e-03 8.245000e-01 -5.390000e-02 5.154000e-01 7.700000e-03 + 8.170000e-02 2.001000e-01 1.828000e-01 2.480000e-01 5.410000e-02 4.648000e-01 1.310000e-01 3.071000e-01 6.150000e-02 4.636000e-01 1.292000e-01 3.150000e-01 6.720000e-02 1.890000e-01 + 3.180000e-02 6.007000e-01 9.280000e-02 8.930000e-01 6.660000e-02 1.058400e+00 1.871000e-01 7.811000e-01 1.862000e-01 9.383000e-01 1.827000e-01 7.888000e-01 4.380000e-02 5.275000e-01 + 4.072000e-01 3.659000e-01 5.085000e-01 7.296000e-01 5.936000e-01 8.117000e-01 6.028000e-01 6.189000e-01 5.598000e-01 8.765000e-01 6.076000e-01 6.148000e-01 3.187000e-01 3.925000e-01 + 2.243000e-01 6.124000e-01 4.186000e-01 7.552000e-01 5.787000e-01 7.057000e-01 1.668500e+00 7.700000e-02 -8.110000e-02 2.092600e+00 -5.970000e-02 7.010000e-02 2.660000e-01 4.185000e-01 + 1.119900e+00 -1.730000e-02 7.295000e-01 1.010000e+00 6.374000e-01 6.944000e-01 7.769000e-01 9.545000e-01 7.149000e-01 5.439000e-01 7.168000e-01 1.024200e+00 5.167000e-01 3.524000e-01 + 4.520000e-01 4.722000e-01 6.902000e-01 7.582000e-01 9.625000e-01 6.599000e-01 7.994000e-01 6.382000e-01 8.570000e-01 8.108000e-01 7.531000e-01 6.950000e-01 4.221000e-01 4.123000e-01 + 1.081400e+00 2.580000e-02 7.381000e-01 8.548000e-01 5.455000e-01 4.843000e-01 8.089000e-01 7.721000e-01 6.447000e-01 3.352000e-01 7.806000e-01 8.049000e-01 5.102000e-01 2.349000e-01 +-3.530000e-02 9.139000e-01 3.109000e-01 8.767000e-01 3.875000e-01 9.291000e-01 1.760000e-02 1.708500e+00 5.190000e-02 1.937300e+00 -2.270000e-02 2.460000e-02 1.649000e-01 5.366000e-01 + 9.807000e-01 1.179000e-01 4.309000e-01 2.824000e-01 3.117000e-01 2.823000e-01 4.382000e-01 2.777000e-01 3.246000e-01 2.548000e-01 4.180000e-01 3.002000e-01 3.007000e-01 1.943000e-01 + 3.250000e-01 2.790000e-01 7.360000e-01 1.607000e-01 7.521000e-01 2.878000e-01 1.784500e+00 -6.330000e-02 -7.240000e-02 2.082300e+00 3.470000e-02 -4.150000e-02 3.443000e-01 1.906000e-01 + 5.688000e-01 9.040000e-02 8.446000e-01 1.970000e-01 9.281000e-01 2.649000e-01 8.492000e-01 1.929000e-01 8.796000e-01 3.376000e-01 7.574000e-01 3.030000e-01 4.745000e-01 1.294000e-01 + 3.120000e-01 4.004000e-01 6.671000e-01 3.566000e-01 5.774000e-01 6.072000e-01 1.690600e+00 5.000000e-02 1.290000e-02 1.987400e+00 -1.380000e-02 1.650000e-02 3.382000e-01 2.667000e-01 + 1.663000e-01 6.186000e-01 1.562000e-01 1.084300e+00 1.519000e-01 1.267800e+00 1.862000e-01 1.049100e+00 3.050000e-01 1.105300e+00 1.669000e-01 1.073600e+00 1.202000e-01 5.912000e-01 + 3.376000e-01 5.206000e-01 4.022000e-01 9.626000e-01 6.526000e-01 8.635000e-01 5.593000e-01 7.763000e-01 5.713000e-01 9.856000e-01 6.018000e-01 7.242000e-01 3.129000e-01 4.596000e-01 + 4.579000e-01 6.553000e-01 2.936000e-01 8.654000e-01 2.920000e-01 5.680000e-01 2.777000e-01 8.882000e-01 2.360000e-01 6.118000e-01 2.499000e-01 9.187000e-01 2.159000e-01 4.540000e-01 + 3.084000e-01 5.384000e-01 6.887000e-01 4.607000e-01 7.591000e-01 5.193000e-01 1.674400e+00 6.190000e-02 -2.330000e-02 2.028300e+00 -1.730000e-02 2.100000e-02 3.601000e-01 3.235000e-01 + 2.808000e-01 2.563000e-01 4.488000e-01 3.873000e-01 5.796000e-01 3.615000e-01 4.806000e-01 3.485000e-01 5.999000e-01 3.518000e-01 4.774000e-01 3.545000e-01 2.802000e-01 1.986000e-01 + 8.380000e-02 1.481000e-01 1.604000e-01 1.943000e-01 2.514000e-01 1.399000e-01 2.079000e-01 1.384000e-01 2.309000e-01 1.706000e-01 1.796000e-01 1.728000e-01 1.045000e-01 9.790000e-02 + 5.270000e-02 4.620000e-01 3.844000e-01 4.432000e-01 5.097000e-01 4.337000e-01 -2.350000e-02 1.761600e+00 -9.900000e-03 1.100000e-02 3.140000e-02 -3.780000e-02 1.804000e-01 3.003000e-01 + 4.784000e-01 -3.000000e-03 7.456000e-01 2.300000e-03 7.309000e-01 1.364000e-01 6.966000e-01 5.930000e-02 7.600000e-01 1.187000e-01 7.205000e-01 3.540000e-02 4.033000e-01 3.320000e-02 + 3.585000e-01 6.260000e-01 7.743000e-01 5.065000e-01 7.985000e-01 6.149000e-01 1.737500e+00 -1.090000e-02 4.900000e-02 1.945400e+00 -8.970000e-02 1.065000e-01 4.380000e-01 3.180000e-01 + 5.082000e-01 5.910000e-01 7.890000e-01 7.576000e-01 8.673000e-01 8.025000e-01 1.677600e+00 6.540000e-02 -5.020000e-02 2.058300e+00 8.400000e-02 1.632800e+00 4.487000e-01 4.481000e-01 + 3.315000e-01 5.409000e-01 6.587000e-01 5.250000e-01 6.879000e-01 6.287000e-01 1.730300e+00 0.000000e+00 8.930000e-02 1.898600e+00 4.910000e-02 -5.990000e-02 3.776000e-01 3.192000e-01 + 7.500000e-03 7.326000e-01 2.217000e-01 8.010000e-01 2.055000e-01 9.643000e-01 6.180000e-02 1.656800e+00 -5.870000e-02 2.070400e+00 -2.560000e-02 3.100000e-02 4.460000e-02 5.755000e-01 +-4.240000e-02 7.591000e-01 3.428000e-01 6.647000e-01 3.545000e-01 7.933000e-01 5.000000e-02 1.675500e+00 -9.690000e-02 2.112200e+00 -1.800000e-03 2.400000e-03 1.535000e-01 4.441000e-01 + 3.393000e-01 5.055000e-01 6.030000e-01 7.080000e-01 7.056000e-01 7.814000e-01 6.102000e-01 7.022000e-01 7.245000e-01 7.864000e-01 5.652000e-01 7.547000e-01 3.327000e-01 4.272000e-01 + 6.824000e-01 4.344000e-01 1.050700e+00 6.378000e-01 1.118400e+00 7.004000e-01 1.775100e+00 -5.320000e-02 1.875000e+00 1.442000e-01 3.940000e-02 1.686500e+00 6.244000e-01 3.308000e-01 + 3.648000e-01 5.643000e-01 7.458000e-01 4.867000e-01 8.342000e-01 5.187000e-01 1.737500e+00 -6.500000e-03 -9.810000e-02 2.117800e+00 -5.920000e-02 7.120000e-02 3.983000e-01 3.318000e-01 + 1.129600e+00 -3.020000e-02 6.534000e-01 1.097600e+00 6.021000e-01 4.781000e-01 7.221000e-01 1.015500e+00 6.298000e-01 4.144000e-01 7.729000e-01 9.528000e-01 5.045000e-01 2.708000e-01 + 4.052000e-01 3.131000e-01 6.783000e-01 4.436000e-01 7.699000e-01 5.071000e-01 6.499000e-01 4.775000e-01 8.500000e-01 4.314000e-01 7.182000e-01 3.959000e-01 4.053000e-01 2.391000e-01 + 1.131800e+00 -3.150000e-02 6.668000e-01 5.677000e-01 5.198000e-01 3.855000e-01 6.295000e-01 6.142000e-01 4.733000e-01 4.171000e-01 7.019000e-01 5.303000e-01 4.325000e-01 2.588000e-01 +-1.350000e-02 9.692000e-01 6.860000e-02 1.201600e+00 2.007000e-01 1.184700e+00 -1.011000e-01 1.850400e+00 2.550000e-02 1.973700e+00 6.100000e-02 -7.320000e-02 5.340000e-02 6.952000e-01 + 1.009400e+00 1.134000e-01 4.213000e-01 8.066000e-01 3.547000e-01 5.525000e-01 4.474000e-01 7.713000e-01 4.188000e-01 4.510000e-01 5.455000e-01 6.543000e-01 3.627000e-01 3.198000e-01 + 4.146000e-01 4.600000e-02 7.304000e-01 4.780000e-02 8.895000e-01 -9.000000e-04 1.766900e+00 -4.180000e-02 1.940000e-02 -2.560000e-02 1.600000e-03 -2.100000e-03 4.215000e-01 2.020000e-02 + 4.200000e-03 7.406000e-01 4.619000e-01 5.687000e-01 5.006000e-01 6.651000e-01 1.647100e+00 9.630000e-02 -8.380000e-02 2.093600e+00 1.680000e-02 -2.180000e-02 2.127000e-01 3.993000e-01 + 8.960000e-01 2.122000e-01 1.236100e+00 2.560000e-01 1.376200e+00 2.300000e-01 1.731100e+00 5.000000e-04 1.994500e+00 4.600000e-03 -2.330000e-02 1.759100e+00 7.241000e-01 1.468000e-01 + 1.125000e+00 -2.640000e-02 1.185100e+00 5.672000e-01 9.300000e-01 2.285000e-01 1.218800e+00 5.273000e-01 8.381000e-01 2.953000e-01 1.288500e+00 4.426000e-01 7.039000e-01 1.022000e-01 + 9.570000e-02 2.974000e-01 4.062000e-01 1.635000e-01 4.607000e-01 1.812000e-01 3.996000e-01 1.672000e-01 4.821000e-01 1.683000e-01 3.296000e-01 2.546000e-01 2.160000e-01 1.141000e-01 + 1.066400e+00 5.060000e-02 4.352000e-01 1.309700e+00 3.315000e-01 1.332100e+00 4.690000e-01 1.277800e+00 4.950000e-01 1.028800e+00 4.517000e-01 1.293800e+00 3.870000e-01 5.589000e-01 + 9.807000e-01 1.493000e-01 1.107100e+00 6.114000e-01 7.685000e-01 2.679000e-01 9.559000e-01 7.915000e-01 7.832000e-01 2.189000e-01 1.040300e+00 6.888000e-01 6.295000e-01 1.212000e-01 + 4.976000e-01 3.970000e-01 6.387000e-01 7.825000e-01 8.251000e-01 7.792000e-01 7.457000e-01 6.585000e-01 7.633000e-01 8.793000e-01 7.247000e-01 6.854000e-01 3.927000e-01 4.256000e-01 + 7.080000e-02 8.437000e-01 1.565000e-01 1.267600e+00 2.215000e-01 1.388900e+00 2.746000e-01 1.129200e+00 2.840000e-01 1.339000e+00 1.668000e-01 1.250800e+00 1.165000e-01 7.000000e-01 + 2.704000e-01 7.288000e-01 6.289000e-01 6.757000e-01 7.274000e-01 7.037000e-01 1.694200e+00 4.620000e-02 5.750000e-02 1.930000e+00 -1.150000e-02 1.200000e-02 3.791000e-01 3.871000e-01 + 3.198000e-01 7.852000e-01 6.013000e-01 1.135000e+00 7.234000e-01 1.198300e+00 1.628900e+00 1.248000e-01 -4.420000e-02 2.053300e+00 8.170000e-02 1.635800e+00 3.488000e-01 6.410000e-01 + 8.978000e-01 1.226000e-01 1.111100e+00 2.376000e-01 1.333900e+00 1.199000e-01 1.906700e+00 -2.029000e-01 1.955700e+00 4.500000e-02 1.680000e-02 -1.970000e-02 7.193000e-01 6.620000e-02 + 3.376000e-01 4.681000e-01 7.722000e-01 3.205000e-01 7.227000e-01 5.211000e-01 1.563600e+00 1.941000e-01 5.300000e-03 1.993600e+00 -4.300000e-02 5.260000e-02 3.808000e-01 2.757000e-01 +-1.780000e-02 9.781000e-01 2.480000e-01 1.004500e+00 2.617000e-01 1.130300e+00 -5.760000e-02 1.800700e+00 3.090000e-02 1.965300e+00 -6.770000e-02 7.990000e-02 9.110000e-02 6.573000e-01 + 3.065000e-01 4.802000e-01 5.440000e-01 6.745000e-01 5.246000e-01 8.795000e-01 4.652000e-01 7.694000e-01 5.639000e-01 8.570000e-01 4.735000e-01 7.587000e-01 2.864000e-01 4.227000e-01 + 7.427000e-01 7.390000e-02 1.189300e+00 8.300000e-02 1.414300e+00 2.050000e-02 1.250600e+00 6.000000e-03 1.441500e+00 1.240000e-02 1.193500e+00 7.420000e-02 7.161000e-01 1.110000e-02 + 4.666000e-01 6.226000e-01 8.800000e-01 7.976000e-01 9.467000e-01 9.688000e-01 6.725000e-01 1.039400e+00 8.872000e-01 1.067700e+00 7.361000e-01 9.614000e-01 4.290000e-01 5.515000e-01 + 3.063000e-01 6.264000e-01 7.307000e-01 4.934000e-01 8.177000e-01 5.290000e-01 1.775700e+00 -4.860000e-02 -1.500000e-02 2.013600e+00 -5.090000e-02 6.170000e-02 4.033000e-01 3.219000e-01 +-2.300000e-03 9.057000e-01 2.291000e-01 9.915000e-01 4.589000e-01 8.647000e-01 5.000000e-04 1.731800e+00 4.500000e-03 1.990400e+00 -5.000000e-03 5.300000e-03 1.490000e-01 5.679000e-01 + 7.854000e-01 3.098000e-01 2.999000e-01 6.876000e-01 2.856000e-01 4.767000e-01 3.644000e-01 6.130000e-01 2.626000e-01 4.918000e-01 3.000000e-01 6.889000e-01 2.698000e-01 3.384000e-01 + 4.141000e-01 2.904000e-01 6.394000e-01 3.938000e-01 8.197000e-01 3.237000e-01 1.624800e+00 1.227000e-01 -1.530000e-02 2.018500e+00 -3.490000e-02 4.160000e-02 4.051000e-01 1.915000e-01 + 6.392000e-01 4.688000e-01 9.039000e-01 8.416000e-01 1.061400e+00 8.376000e-01 1.598700e+00 1.593000e-01 1.972200e+00 3.310000e-02 -9.980000e-02 1.852400e+00 5.675000e-01 4.143000e-01 + 6.436000e-01 2.330000e-02 1.019000e+00 -4.770000e-02 9.992000e-01 1.174000e-01 1.605600e+00 1.474000e-01 2.067800e+00 -8.150000e-02 5.890000e-02 -6.990000e-02 5.617000e-01 1.030000e-02 + 1.056800e+00 5.610000e-02 9.277000e-01 3.840000e-01 7.862000e-01 1.421000e-01 9.047000e-01 4.088000e-01 7.154000e-01 1.977000e-01 9.725000e-01 3.286000e-01 5.982000e-01 1.060000e-01 + 2.863000e-01 2.397000e-01 3.017000e-01 5.512000e-01 4.118000e-01 5.414000e-01 4.823000e-01 3.335000e-01 2.984000e-01 6.918000e-01 4.641000e-01 3.558000e-01 2.224000e-01 2.593000e-01 + 4.731000e-01 8.560000e-02 7.429000e-01 1.325000e-01 9.045000e-01 8.170000e-02 8.110000e-01 4.620000e-02 8.391000e-01 1.784000e-01 7.406000e-01 1.352000e-01 4.483000e-01 5.340000e-02 + 1.335000e-01 9.647000e-01 4.820000e-02 1.051100e+00 1.408000e-01 6.785000e-01 3.630000e-02 1.068100e+00 7.970000e-02 7.308000e-01 1.641000e-01 9.179000e-01 4.960000e-02 6.001000e-01 + 4.580000e-02 9.044000e-01 3.932000e-01 8.614000e-01 5.322000e-01 8.431000e-01 1.675300e+00 6.230000e-02 3.500000e-02 1.957700e+00 -3.200000e-03 3.000000e-03 2.104000e-01 5.312000e-01 + 1.039000e+00 7.570000e-02 7.307000e-01 1.009800e+00 5.982000e-01 1.377000e+00 7.363000e-01 1.004300e+00 6.714000e-01 1.031500e+00 8.457000e-01 8.775000e-01 5.253000e-01 4.539000e-01 + 5.764000e-01 3.032000e-01 9.577000e-01 4.120000e-01 9.673000e-01 6.153000e-01 9.352000e-01 4.392000e-01 9.752000e-01 6.314000e-01 7.572000e-01 6.480000e-01 5.075000e-01 2.932000e-01 + 1.140700e+00 -4.550000e-02 9.417000e-01 7.776000e-01 7.075000e-01 5.760000e-01 8.199000e-01 9.228000e-01 6.980000e-01 5.391000e-01 9.480000e-01 7.681000e-01 5.668000e-01 2.880000e-01 +-2.900000e-02 6.733000e-01 3.202000e-01 6.315000e-01 4.043000e-01 6.685000e-01 -7.370000e-02 1.813700e+00 1.229000e-01 1.851200e+00 4.170000e-02 -5.200000e-02 1.569000e-01 3.977000e-01 + 3.265000e-01 4.894000e-01 6.227000e-01 5.130000e-01 7.186000e-01 5.417000e-01 1.765500e+00 -3.930000e-02 -4.840000e-02 2.053500e+00 9.800000e-03 -1.190000e-02 3.285000e-01 3.432000e-01 + 1.117100e+00 -8.600000e-03 7.302000e-01 3.616000e-01 6.130000e-01 2.139000e-01 6.646000e-01 4.410000e-01 6.113000e-01 1.987000e-01 7.062000e-01 3.899000e-01 4.921000e-01 1.567000e-01 + 1.141900e+00 -4.700000e-02 6.574000e-01 7.773000e-01 5.479000e-01 4.382000e-01 7.244000e-01 6.954000e-01 5.710000e-01 3.823000e-01 7.053000e-01 7.203000e-01 4.940000e-01 2.317000e-01 + 4.965000e-01 1.738000e-01 7.885000e-01 2.035000e-01 9.662000e-01 1.357000e-01 1.835300e+00 -1.154000e-01 -1.560000e-02 2.021700e+00 -1.450000e-02 1.650000e-02 4.759000e-01 9.810000e-02 + 4.544000e-01 3.766000e-01 8.270000e-01 4.520000e-01 7.125000e-01 7.868000e-01 6.932000e-01 6.157000e-01 7.669000e-01 7.423000e-01 5.673000e-01 7.614000e-01 3.870000e-01 3.691000e-01 + 4.055000e-01 3.654000e-01 7.823000e-01 3.933000e-01 7.759000e-01 5.908000e-01 7.125000e-01 4.798000e-01 9.026000e-01 4.538000e-01 7.434000e-01 4.432000e-01 4.013000e-01 2.881000e-01 + 5.728000e-01 2.989000e-01 8.306000e-01 3.644000e-01 9.304000e-01 3.902000e-01 1.696000e+00 4.100000e-02 -8.430000e-02 2.101000e+00 -7.130000e-02 8.580000e-02 5.000000e-01 2.026000e-01 + 3.376000e-01 7.775000e-01 3.464000e-01 1.362100e+00 2.232000e-01 9.473000e-01 2.115000e-01 1.521300e+00 2.551000e-01 8.705000e-01 2.435000e-01 1.486300e+00 1.729000e-01 6.418000e-01 + 1.090600e+00 1.860000e-02 6.090000e-01 1.122400e+00 4.901000e-01 5.531000e-01 5.431000e-01 1.201300e+00 5.042000e-01 5.040000e-01 5.107000e-01 1.237100e+00 4.253000e-01 3.304000e-01 + 1.114300e+00 -1.080000e-02 5.238000e-01 4.260000e-01 3.534000e-01 4.049000e-01 5.256000e-01 4.235000e-01 4.289000e-01 2.981000e-01 5.392000e-01 4.056000e-01 3.375000e-01 2.647000e-01 + 4.647000e-01 9.610000e-02 8.450000e-01 1.090000e-02 9.376000e-01 3.980000e-02 7.987000e-01 6.400000e-02 8.026000e-01 2.185000e-01 7.818000e-01 8.880000e-02 4.603000e-01 3.840000e-02 + 1.721000e-01 2.183000e-01 2.819000e-01 3.280000e-01 3.557000e-01 3.364000e-01 2.338000e-01 3.847000e-01 3.266000e-01 3.775000e-01 2.295000e-01 3.909000e-01 1.535000e-01 2.003000e-01 + 1.860000e-02 2.330000e-01 8.280000e-02 2.981000e-01 6.380000e-02 3.789000e-01 -1.660000e-02 4.184000e-01 1.610000e-01 2.707000e-01 8.580000e-02 2.988000e-01 3.820000e-02 1.848000e-01 + 4.782000e-01 4.641000e-01 8.946000e-01 3.435000e-01 8.573000e-01 5.357000e-01 1.875700e+00 -1.691000e-01 -6.900000e-03 2.014400e+00 2.090000e-02 -2.310000e-02 4.434000e-01 3.007000e-01 + 3.309000e-01 4.250000e-01 4.928000e-01 6.981000e-01 6.051000e-01 7.404000e-01 6.155000e-01 5.528000e-01 6.382000e-01 7.205000e-01 5.233000e-01 6.605000e-01 3.175000e-01 3.624000e-01 + 2.536000e-01 1.080000e-01 5.554000e-01 1.227000e-01 7.170000e-01 7.060000e-02 1.696900e+00 4.170000e-02 -1.440000e-02 1.760000e-02 -1.016000e-01 1.216000e-01 3.275000e-01 4.920000e-02 + 3.390000e-01 3.020000e-01 5.153000e-01 4.942000e-01 5.936000e-01 5.545000e-01 5.314000e-01 4.704000e-01 6.225000e-01 5.370000e-01 5.093000e-01 4.972000e-01 3.172000e-01 2.615000e-01 + 1.167900e+00 -7.670000e-02 1.151900e+00 5.657000e-01 8.234000e-01 1.057900e+00 1.061100e+00 6.778000e-01 6.292000e-01 1.053300e+00 1.050300e+00 6.942000e-01 6.490000e-01 3.211000e-01 + 4.128000e-01 6.886000e-01 7.720000e-01 6.333000e-01 8.191000e-01 7.276000e-01 1.696900e+00 4.170000e-02 -1.400000e-03 1.999100e+00 -2.770000e-02 1.763800e+00 4.395000e-01 3.880000e-01 + 2.401000e-01 8.456000e-01 2.801000e-01 1.440500e+00 3.156000e-01 1.645200e+00 4.983000e-01 1.182000e+00 4.511000e-01 1.513000e+00 4.349000e-01 1.260300e+00 2.223000e-01 7.598000e-01 + 3.190000e-02 2.327000e-01 1.329000e-01 4.503000e-01 1.820000e-01 5.382000e-01 1.320000e-01 1.579700e+00 6.880000e-02 -8.450000e-02 -1.080000e-02 1.390000e-02 8.020000e-02 2.422000e-01 + 4.039000e-01 3.573000e-01 6.585000e-01 5.334000e-01 6.388000e-01 7.320000e-01 6.293000e-01 5.603000e-01 6.803000e-01 7.047000e-01 5.667000e-01 6.358000e-01 3.484000e-01 3.435000e-01 + 1.027700e+00 8.760000e-02 5.360000e-01 1.174600e+00 3.294000e-01 7.892000e-01 3.724000e-01 1.366100e+00 2.953000e-01 7.905000e-01 4.678000e-01 1.253200e+00 3.132000e-01 4.753000e-01 + 6.950000e-02 4.076000e-01 1.708000e-01 5.636000e-01 1.306000e-01 7.205000e-01 -1.290000e-02 7.861000e-01 1.415000e-01 7.203000e-01 6.850000e-02 6.867000e-01 5.100000e-02 3.835000e-01 + 5.578000e-01 4.956000e-01 8.414000e-01 5.372000e-01 9.265000e-01 5.695000e-01 1.720000e+00 1.960000e-02 7.740000e-02 1.906200e+00 -9.720000e-02 1.850300e+00 4.662000e-01 3.445000e-01 + 2.043000e-01 1.528000e-01 3.941000e-01 1.483000e-01 4.314000e-01 1.899000e-01 4.244000e-01 1.136000e-01 5.053000e-01 1.130000e-01 4.428000e-01 8.950000e-02 2.396000e-01 7.130000e-02 + 3.115000e-01 6.206000e-01 5.196000e-01 9.349000e-01 4.160000e-01 1.275600e+00 4.773000e-01 9.806000e-01 5.755000e-01 1.103300e+00 3.650000e-01 1.116200e+00 2.662000e-01 5.800000e-01 + 1.036700e+00 8.220000e-02 8.868000e-01 8.528000e-01 7.260000e-01 4.464000e-01 1.059900e+00 6.539000e-01 6.681000e-01 4.769000e-01 9.639000e-01 7.606000e-01 5.826000e-01 2.310000e-01 + 1.069800e+00 4.080000e-02 4.507000e-01 2.553000e-01 3.520000e-01 2.347000e-01 4.403000e-01 2.715000e-01 3.898000e-01 1.776000e-01 4.880000e-01 2.127000e-01 3.393000e-01 1.490000e-01 + 9.398000e-01 9.490000e-02 1.514900e+00 9.620000e-02 1.657200e+00 1.844000e-01 1.592300e+00 4.300000e-03 1.785300e+00 7.150000e-02 1.490300e+00 1.233000e-01 8.673000e-01 6.500000e-02 + 2.916000e-01 8.154000e-01 2.496000e-01 7.557000e-01 7.570000e-02 7.302000e-01 2.293000e-01 7.797000e-01 1.223000e-01 6.540000e-01 1.639000e-01 8.590000e-01 1.455000e-01 4.776000e-01 + 1.027000e-01 1.016100e+00 1.556000e-01 1.566700e+00 1.183000e-01 9.786000e-01 1.365000e-01 1.589500e+00 1.133000e-01 9.559000e-01 1.248000e-01 1.605400e+00 8.940000e-02 6.949000e-01 + 1.103500e+00 3.500000e-03 5.619000e-01 1.930000e-01 4.270000e-01 1.961000e-01 5.009000e-01 2.624000e-01 4.615000e-01 1.391000e-01 5.809000e-01 1.712000e-01 3.970000e-01 1.167000e-01 + 1.000000e-03 9.440000e-01 3.068000e-01 9.555000e-01 4.642000e-01 9.027000e-01 -1.152000e-01 1.865900e+00 -8.080000e-02 2.098800e+00 -8.500000e-03 1.020000e-02 1.939000e-01 5.414000e-01 + 4.455000e-01 6.606000e-01 2.377000e-01 6.792000e-01 2.448000e-01 4.713000e-01 2.568000e-01 6.523000e-01 3.088000e-01 3.794000e-01 2.070000e-01 7.139000e-01 1.991000e-01 3.831000e-01 + 3.431000e-01 3.872000e-01 5.262000e-01 6.181000e-01 5.925000e-01 7.169000e-01 6.176000e-01 5.152000e-01 6.126000e-01 7.107000e-01 4.996000e-01 6.554000e-01 3.275000e-01 3.312000e-01 + 3.390000e-02 6.709000e-01 4.034000e-01 6.058000e-01 4.849000e-01 6.530000e-01 1.696000e+00 4.940000e-02 -7.460000e-02 2.088600e+00 -6.100000e-03 7.900000e-03 2.012000e-01 3.928000e-01 +-3.340000e-02 1.136700e+00 8.860000e-02 1.330500e+00 2.922000e-01 1.235300e+00 1.830000e-02 1.712800e+00 -8.160000e-02 2.100200e+00 3.500000e-03 1.726300e+00 7.930000e-02 7.498000e-01 + 1.158900e+00 -6.460000e-02 4.903000e-01 1.257400e+00 5.872000e-01 4.286000e-01 5.796000e-01 1.149900e+00 5.293000e-01 4.681000e-01 5.713000e-01 1.158500e+00 4.171000e-01 3.362000e-01 + 1.061400e+00 5.460000e-02 4.624000e-01 4.138000e-01 4.538000e-01 2.395000e-01 6.261000e-01 2.194000e-01 4.383000e-01 2.373000e-01 5.559000e-01 2.980000e-01 3.813000e-01 1.803000e-01 +-6.870000e-02 1.177000e+00 3.070000e-01 1.094200e+00 3.066000e-01 1.239500e+00 8.420000e-02 1.636600e+00 -2.100000e-03 1.999800e+00 -3.140000e-02 1.768700e+00 1.517000e-01 6.749000e-01 + 4.358000e-01 5.687000e-01 7.750000e-01 5.402000e-01 9.538000e-01 4.687000e-01 1.750700e+00 -2.560000e-02 1.563000e-01 1.817000e+00 6.030000e-02 -7.160000e-02 4.426000e-01 3.335000e-01 + 3.493000e-01 3.879000e-01 4.468000e-01 7.277000e-01 5.066000e-01 8.343000e-01 6.364000e-01 5.005000e-01 6.528000e-01 6.812000e-01 4.985000e-01 6.694000e-01 3.007000e-01 3.705000e-01 + 3.451000e-01 3.304000e-01 4.110000e-01 6.678000e-01 4.958000e-01 7.328000e-01 4.974000e-01 5.712000e-01 4.849000e-01 7.611000e-01 3.385000e-01 7.543000e-01 2.568000e-01 3.634000e-01 + 6.628000e-01 4.250000e-01 2.748000e-01 2.046000e-01 1.555000e-01 2.684000e-01 2.439000e-01 2.413000e-01 2.985000e-01 8.700000e-02 2.819000e-01 1.962000e-01 2.311000e-01 1.262000e-01 + 5.381000e-01 2.008000e-01 9.185000e-01 2.281000e-01 1.066400e+00 2.308000e-01 9.062000e-01 2.368000e-01 9.814000e-01 3.535000e-01 8.856000e-01 2.679000e-01 5.252000e-01 1.371000e-01 + 1.112900e+00 -9.100000e-03 8.964000e-01 8.369000e-01 6.871000e-01 7.348000e-01 8.873000e-01 8.529000e-01 7.308000e-01 6.121000e-01 8.880000e-01 8.495000e-01 5.774000e-01 3.174000e-01 + 9.408000e-01 1.572000e-01 1.180300e+00 3.368000e-01 1.296200e+00 3.425000e-01 1.684700e+00 5.910000e-02 2.001600e+00 2.300000e-03 4.320000e-02 1.685900e+00 7.190000e-01 1.595000e-01 + 1.581000e-01 3.713000e-01 3.274000e-01 4.837000e-01 2.731000e-01 6.740000e-01 4.187000e-01 3.731000e-01 3.721000e-01 5.650000e-01 3.401000e-01 4.673000e-01 1.892000e-01 2.795000e-01 + 3.393000e-01 1.388000e-01 2.235000e-01 3.140000e-02 7.320000e-02 1.648000e-01 2.143000e-01 4.110000e-02 1.466000e-01 7.950000e-02 1.840000e-01 7.610000e-02 1.639000e-01 4.260000e-02 + 2.449000e-01 1.485000e-01 1.748000e-01 3.990000e-02 7.900000e-02 1.229000e-01 1.639000e-01 5.250000e-02 1.908000e-01 -1.550000e-02 7.440000e-02 1.631000e-01 1.260000e-01 5.300000e-02 + 3.539000e-01 7.224000e-01 4.408000e-01 1.274400e+00 6.796000e-01 1.237900e+00 4.989000e-01 1.199600e+00 6.023000e-01 1.362300e+00 4.924000e-01 1.209500e+00 2.847000e-01 6.989000e-01 + 4.341000e-01 4.468000e-01 6.116000e-01 7.835000e-01 7.314000e-01 8.503000e-01 6.383000e-01 7.511000e-01 8.338000e-01 7.523000e-01 6.846000e-01 6.988000e-01 3.918000e-01 4.062000e-01 + 5.380000e-01 4.697000e-01 7.407000e-01 8.628000e-01 7.885000e-01 1.048800e+00 7.124000e-01 8.969000e-01 9.390000e-01 9.041000e-01 7.319000e-01 8.754000e-01 4.184000e-01 5.099000e-01 + 2.920000e-01 8.900000e-02 4.256000e-01 1.747000e-01 4.704000e-01 2.201000e-01 4.925000e-01 9.680000e-02 4.691000e-01 2.292000e-01 4.426000e-01 1.539000e-01 2.753000e-01 6.600000e-02 + 4.122000e-01 8.940000e-02 5.834000e-01 2.571000e-01 6.954000e-01 2.638000e-01 1.677400e+00 6.480000e-02 -2.700000e-03 2.000000e-03 1.870000e-02 -2.020000e-02 3.243000e-01 1.595000e-01 +-5.000000e-04 1.120800e+00 6.350000e-02 1.665100e+00 -5.980000e-02 1.582700e+00 -7.100000e-03 1.746900e+00 8.840000e-02 1.332500e+00 1.032000e-01 1.617900e+00 3.190000e-02 8.864000e-01 + 2.250000e-02 4.186000e-01 2.250000e-02 2.158000e-01 4.480000e-02 1.556000e-01 6.030000e-02 1.744000e-01 6.030000e-02 1.389000e-01 -2.940000e-02 2.825000e-01 1.240000e-02 1.834000e-01 + 1.856000e-01 3.467000e-01 6.820000e-02 2.194000e-01 4.480000e-02 2.059000e-01 9.490000e-02 1.882000e-01 3.840000e-02 2.095000e-01 5.270000e-02 2.388000e-01 7.130000e-02 1.545000e-01 + 5.003000e-01 2.536000e-01 1.001900e+00 1.310000e-01 1.054300e+00 2.531000e-01 8.935000e-01 2.608000e-01 1.017600e+00 3.172000e-01 9.822000e-01 1.566000e-01 5.335000e-01 1.295000e-01 + 4.205000e-01 6.878000e-01 8.430000e-02 1.679500e+00 1.585000e-01 9.973000e-01 2.664000e-01 1.464800e+00 1.663000e-01 9.523000e-01 3.439000e-01 1.375300e+00 1.936000e-01 6.086000e-01 + 7.230000e-01 2.897000e-01 1.061700e+00 5.399000e-01 1.291800e+00 5.118000e-01 1.153700e+00 4.307000e-01 1.208600e+00 6.427000e-01 9.571000e-01 6.620000e-01 6.198000e-01 3.023000e-01 + 2.124000e-01 6.169000e-01 4.337000e-01 8.492000e-01 4.279000e-01 1.041800e+00 4.311000e-01 8.475000e-01 4.554000e-01 1.033700e+00 3.460000e-01 9.528000e-01 2.282000e-01 5.165000e-01 + 1.087000e+00 2.050000e-02 3.814000e-01 9.184000e-01 3.311000e-01 6.046000e-01 3.696000e-01 9.346000e-01 3.396000e-01 5.683000e-01 4.533000e-01 8.328000e-01 3.132000e-01 3.887000e-01 + 1.122500e+00 -2.000000e-02 7.035000e-01 1.028900e+00 5.001000e-01 6.363000e-01 7.524000e-01 9.642000e-01 5.688000e-01 5.181000e-01 7.309000e-01 9.958000e-01 4.889000e-01 3.028000e-01 + 1.122700e+00 -2.120000e-02 7.345000e-01 1.126000e-01 5.620000e-01 1.254000e-01 7.846000e-01 5.580000e-02 5.239000e-01 1.548000e-01 7.256000e-01 1.218000e-01 4.972000e-01 5.870000e-02 + 2.307000e-01 7.521000e-01 4.215000e-01 8.968000e-01 4.895000e-01 9.583000e-01 1.706200e+00 3.010000e-02 2.510000e-02 1.976100e+00 5.130000e-02 -5.990000e-02 2.203000e-01 5.587000e-01 + 3.168000e-01 1.169000e-01 7.908000e-01 -7.260000e-02 8.958000e-01 -5.690000e-02 1.596400e+00 1.594000e-01 -4.350000e-02 5.030000e-02 -4.990000e-02 5.970000e-02 3.999000e-01 1.530000e-02 + 1.098600e+00 1.060000e-02 1.170200e+00 5.680000e-01 8.732000e-01 2.272000e-01 1.142800e+00 5.946000e-01 8.219000e-01 2.528000e-01 1.195700e+00 5.319000e-01 6.724000e-01 1.097000e-01 + 1.823000e-01 6.265000e-01 1.940000e-01 1.089600e+00 2.839000e-01 1.166500e+00 2.785000e-01 9.865000e-01 3.379000e-01 1.124600e+00 2.203000e-01 1.059100e+00 1.262000e-01 6.122000e-01 + 4.150000e-02 7.618000e-01 2.864000e-01 8.324000e-01 4.110000e-01 8.252000e-01 4.500000e-03 1.725100e+00 -2.590000e-02 2.027700e+00 -2.960000e-02 3.420000e-02 1.374000e-01 5.252000e-01 + 5.909000e-01 4.002000e-01 8.241000e-01 7.434000e-01 1.137400e+00 6.116000e-01 9.519000e-01 5.897000e-01 1.061400e+00 7.307000e-01 8.974000e-01 6.602000e-01 5.347000e-01 3.598000e-01 + 7.907000e-01 2.081000e-01 1.288000e+00 2.727000e-01 1.441700e+00 3.342000e-01 1.260200e+00 3.046000e-01 1.501900e+00 2.953000e-01 1.235200e+00 3.337000e-01 7.069000e-01 1.993000e-01 + 5.221000e-01 5.633000e-01 9.389000e-01 7.340000e-01 9.714000e-01 9.561000e-01 9.044000e-01 7.730000e-01 1.116400e+00 8.080000e-01 9.204000e-01 7.541000e-01 5.285000e-01 4.408000e-01 + 1.143700e+00 -4.720000e-02 5.848000e-01 9.845000e-01 6.145000e-01 3.859000e-01 6.835000e-01 8.577000e-01 5.857000e-01 3.928000e-01 6.892000e-01 8.599000e-01 4.808000e-01 2.604000e-01 + 1.652000e-01 5.220000e-01 7.564000e-01 1.959000e-01 7.352000e-01 3.619000e-01 1.798300e+00 -8.000000e-02 -6.110000e-02 2.072900e+00 -3.170000e-02 3.590000e-02 3.185000e-01 2.556000e-01 + 2.429000e-01 8.192000e-01 5.522000e-01 8.254000e-01 6.584000e-01 8.397000e-01 1.758500e+00 -2.970000e-02 1.117000e-01 1.871200e+00 2.010000e-02 1.704900e+00 3.055000e-01 5.041000e-01 + 1.027500e+00 9.290000e-02 7.770000e-01 9.647000e-01 7.027000e-01 6.231000e-01 8.090000e-01 9.281000e-01 7.108000e-01 5.548000e-01 9.417000e-01 7.737000e-01 5.562000e-01 3.125000e-01 + 2.098000e-01 4.697000e-01 2.148000e-01 8.691000e-01 2.709000e-01 9.603000e-01 2.065000e-01 8.809000e-01 1.751000e-01 1.090800e+00 1.634000e-01 9.365000e-01 1.426000e-01 4.818000e-01 + 8.182000e-01 2.860000e-01 1.228700e+00 2.121000e-01 1.275100e+00 2.980000e-01 1.731500e+00 2.500000e-03 2.029300e+00 -3.240000e-02 2.870000e-02 1.699900e+00 6.670000e-01 1.885000e-01 + 1.416000e-01 9.664000e-01 7.030000e-02 8.976000e-01 1.443000e-01 5.991000e-01 4.980000e-02 9.250000e-01 1.007000e-01 6.328000e-01 6.130000e-02 9.041000e-01 7.600000e-02 5.271000e-01 + 6.420000e-01 1.712000e-01 8.587000e-01 4.432000e-01 9.461000e-01 5.424000e-01 1.049100e+00 2.220000e-01 1.132600e+00 3.475000e-01 9.113000e-01 3.815000e-01 5.211000e-01 2.263000e-01 diff --git a/sourcecodes/bnt-master/netlab3.3/oilTst.dat b/sourcecodes/bnt-master/netlab3.3/oilTst.dat new file mode 100644 index 00000000..0fc51cc9 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/oilTst.dat @@ -0,0 +1,503 @@ + nin 12 + nout 2 + ndata 500 + 6.448000e-01 4.650000e-01 3.659000e-01 8.761000e-01 2.214000e-01 7.023000e-01 3.702000e-01 8.750000e-01 2.981000e-01 5.870000e-01 3.130000e-01 9.374000e-01 2.574000e-01 4.349000e-01 + 5.013000e-01 9.710000e-02 7.749000e-01 1.599000e-01 8.716000e-01 1.946000e-01 7.986000e-01 1.304000e-01 8.874000e-01 1.974000e-01 6.991000e-01 2.499000e-01 4.547000e-01 8.450000e-02 + 3.908000e-01 4.842000e-01 5.148000e-01 8.738000e-01 6.925000e-01 8.681000e-01 5.985000e-01 7.700000e-01 5.325000e-01 1.079000e+00 6.069000e-01 7.554000e-01 3.284000e-01 4.656000e-01 + 1.038400e+00 8.240000e-02 1.578400e+00 1.819000e-01 1.242100e+00 5.329000e-01 1.780600e+00 -5.680000e-02 1.239700e+00 3.690000e-01 1.656900e+00 8.240000e-02 8.512000e-01 1.109000e-01 +-9.100000e-03 6.477000e-01 3.510000e-02 9.081000e-01 3.810000e-02 1.031800e+00 6.780000e-02 1.656900e+00 3.950000e-02 1.950300e+00 -2.490000e-02 3.090000e-02 7.000000e-03 5.461000e-01 + 6.458000e-01 2.965000e-01 9.246000e-01 3.379000e-01 1.054200e+00 3.263000e-01 1.751900e+00 -2.230000e-02 2.070800e+00 -7.860000e-02 -1.010000e-02 1.030000e-02 5.419000e-01 2.027000e-01 + 4.951000e-01 5.899000e-01 9.069000e-01 7.700000e-01 8.367000e-01 1.101400e+00 7.952000e-01 9.014000e-01 9.185000e-01 1.039600e+00 7.024000e-01 1.008900e+00 4.378000e-01 5.457000e-01 + 1.136300e+00 -3.530000e-02 3.838000e-01 2.079000e-01 3.385000e-01 1.536000e-01 4.569000e-01 1.199000e-01 2.972000e-01 1.929000e-01 3.505000e-01 2.441000e-01 3.074000e-01 1.153000e-01 + 1.646000e-01 5.148000e-01 4.558000e-01 5.405000e-01 5.855000e-01 5.258000e-01 1.727000e+00 4.400000e-03 -1.250000e-02 2.019000e+00 3.960000e-02 -4.630000e-02 2.334000e-01 3.525000e-01 + 2.028000e-01 3.866000e-01 2.659000e-01 6.712000e-01 3.368000e-01 7.250000e-01 3.495000e-01 5.704000e-01 3.358000e-01 7.372000e-01 2.509000e-01 6.834000e-01 1.608000e-01 3.771000e-01 + 6.930000e-01 1.407000e-01 1.015300e+00 3.020000e-01 1.114100e+00 3.990000e-01 1.070600e+00 2.370000e-01 1.106800e+00 4.251000e-01 1.041400e+00 2.751000e-01 6.056000e-01 1.516000e-01 + 5.753000e-01 5.257000e-01 8.693000e-01 5.906000e-01 8.992000e-01 6.983000e-01 1.670000e+00 7.510000e-02 1.942200e+00 6.400000e-02 -2.000000e-02 1.753100e+00 5.033000e-01 3.513000e-01 + 8.400000e-03 5.567000e-01 2.480000e-02 8.560000e-01 1.377000e-01 8.414000e-01 8.530000e-02 7.866000e-01 -6.210000e-02 1.098600e+00 9.400000e-02 7.736000e-01 1.660000e-02 4.914000e-01 + 1.122000e-01 2.212000e-01 2.767000e-01 2.258000e-01 4.065000e-01 1.461000e-01 2.236000e-01 2.891000e-01 2.271000e-01 3.709000e-01 2.522000e-01 2.523000e-01 1.470000e-01 1.456000e-01 + 2.765000e-01 8.222000e-01 6.055000e-01 9.075000e-01 7.480000e-01 8.832000e-01 1.749700e+00 -2.060000e-02 4.540000e-02 1.950100e+00 4.800000e-02 1.672300e+00 3.080000e-01 5.781000e-01 + 3.000000e-01 2.843000e-01 5.036000e-01 4.081000e-01 6.104000e-01 4.229000e-01 4.810000e-01 4.380000e-01 5.600000e-01 5.012000e-01 4.791000e-01 4.364000e-01 2.898000e-01 2.373000e-01 + 1.142700e+00 -4.400000e-02 8.870000e-01 1.008000e-01 7.355000e-01 2.940000e-02 9.660000e-01 8.200000e-03 6.765000e-01 7.800000e-02 1.012500e+00 -4.360000e-02 5.768000e-01 3.400000e-02 + 2.788000e-01 1.947000e-01 4.153000e-01 3.330000e-01 4.493000e-01 4.033000e-01 3.633000e-01 3.906000e-01 4.544000e-01 4.106000e-01 3.182000e-01 4.397000e-01 2.318000e-01 1.993000e-01 + 4.448000e-01 6.422000e-01 8.192000e-01 5.634000e-01 8.853000e-01 6.292000e-01 1.754600e+00 -2.460000e-02 2.870000e-02 1.963900e+00 -1.710000e-02 1.754600e+00 4.858000e-01 3.275000e-01 + 1.169600e+00 -7.950000e-02 6.595000e-01 3.301000e-01 5.647000e-01 2.009000e-01 7.108000e-01 2.644000e-01 4.990000e-01 2.600000e-01 6.660000e-01 3.246000e-01 4.636000e-01 1.476000e-01 + 2.800000e-02 4.749000e-01 3.082000e-01 5.165000e-01 4.995000e-01 4.292000e-01 -6.680000e-02 1.809500e+00 -5.310000e-02 6.340000e-02 7.020000e-02 -8.250000e-02 1.794000e-01 2.888000e-01 +-2.050000e-02 1.080400e+00 1.990000e-01 1.171300e+00 2.044000e-01 1.301100e+00 -6.300000e-02 1.807000e+00 -1.090000e-01 2.131100e+00 9.110000e-02 1.625400e+00 1.084000e-01 6.975000e-01 +-1.990000e-02 1.129400e+00 6.400000e-03 1.475300e+00 1.195000e-01 1.479400e+00 -8.570000e-02 1.833300e+00 -5.250000e-02 2.061500e+00 8.600000e-02 1.630200e+00 1.760000e-02 8.479000e-01 + 7.734000e-01 3.108000e-01 1.078300e+00 6.418000e-01 1.267100e+00 6.887000e-01 1.212800e+00 4.844000e-01 1.260900e+00 7.241000e-01 1.197800e+00 5.027000e-01 6.789000e-01 3.063000e-01 + 4.740000e-01 6.379000e-01 2.710000e-01 7.720000e-01 2.148000e-01 5.921000e-01 2.197000e-01 8.294000e-01 2.198000e-01 5.664000e-01 2.387000e-01 8.073000e-01 2.159000e-01 4.153000e-01 + 1.412000e-01 9.814000e-01 1.210000e-01 5.891000e-01 5.280000e-02 5.381000e-01 1.400000e-01 5.603000e-01 1.442000e-01 4.171000e-01 1.087000e-01 6.031000e-01 1.059000e-01 3.814000e-01 + 8.093000e-01 1.447000e-01 1.019800e+00 2.690000e-01 1.202300e+00 2.014000e-01 1.642000e+00 1.033000e-01 2.117700e+00 -1.391000e-01 -8.490000e-02 1.015000e-01 6.179000e-01 1.411000e-01 + 6.051000e-01 3.129000e-01 1.013600e+00 4.145000e-01 1.062800e+00 5.765000e-01 9.327000e-01 5.089000e-01 1.094600e+00 5.687000e-01 9.348000e-01 5.066000e-01 5.441000e-01 2.873000e-01 + 1.044400e+00 7.250000e-02 7.841000e-01 6.701000e-01 5.830000e-01 4.092000e-01 7.340000e-01 7.407000e-01 4.950000e-01 4.886000e-01 7.533000e-01 7.148000e-01 4.912000e-01 2.421000e-01 + 6.847000e-01 3.897000e-01 9.817000e-01 4.101000e-01 1.045500e+00 4.734000e-01 1.734700e+00 -3.000000e-04 1.986600e+00 1.320000e-02 5.550000e-02 1.665800e+00 5.296000e-01 2.952000e-01 + 1.740000e-02 8.344000e-01 -7.070000e-02 1.251500e+00 4.490000e-02 1.250200e+00 -8.910000e-02 1.834800e+00 3.770000e-02 1.952000e+00 1.850000e-02 -2.100000e-02 2.190000e-02 6.670000e-01 + 2.622000e-01 8.393000e-01 1.888000e-01 7.998000e-01 2.062000e-01 5.559000e-01 2.330000e-01 7.465000e-01 1.065000e-01 6.558000e-01 1.107000e-01 8.962000e-01 1.222000e-01 4.926000e-01 + 6.600000e-03 1.097000e+00 4.751000e-01 9.095000e-01 5.517000e-01 9.640000e-01 1.777900e+00 -5.290000e-02 -2.350000e-02 2.028300e+00 5.520000e-02 1.666900e+00 2.132000e-01 6.103000e-01 + 4.735000e-01 2.950000e-02 6.149000e-01 2.003000e-01 7.815000e-01 1.288000e-01 6.322000e-01 1.794000e-01 6.874000e-01 2.575000e-01 6.046000e-01 2.099000e-01 3.779000e-01 8.710000e-02 + 6.480000e-02 5.514000e-01 3.730000e-02 9.392000e-01 1.505000e-01 9.401000e-01 2.350000e-02 9.564000e-01 1.424000e-01 9.696000e-01 3.140000e-02 9.441000e-01 3.910000e-02 5.217000e-01 + 1.271900e+00 -1.940000e-01 4.195000e-01 1.021300e+00 3.379000e-01 6.556000e-01 4.200000e-01 1.027200e+00 3.894000e-01 5.664000e-01 4.430000e-01 1.001100e+00 3.273000e-01 4.017000e-01 + 3.084000e-01 7.897000e-01 1.522000e-01 1.576900e+00 7.180000e-02 9.782000e-01 2.220000e-01 1.498500e+00 4.430000e-02 9.810000e-01 9.340000e-02 1.646600e+00 1.257000e-01 6.302000e-01 + 5.818000e-01 2.661000e-01 8.678000e-01 4.688000e-01 9.546000e-01 5.728000e-01 7.304000e-01 6.334000e-01 8.540000e-01 7.134000e-01 8.607000e-01 4.792000e-01 5.011000e-01 2.696000e-01 + 5.940000e-02 7.090000e-01 4.563000e-01 6.108000e-01 5.679000e-01 6.220000e-01 1.778700e+00 -5.690000e-02 3.970000e-02 1.952300e+00 3.590000e-02 -4.270000e-02 2.095000e-01 4.262000e-01 + 5.933000e-01 2.304000e-01 9.529000e-01 3.319000e-01 1.151400e+00 2.991000e-01 9.822000e-01 3.042000e-01 1.184000e+00 2.901000e-01 9.725000e-01 3.120000e-01 5.705000e-01 1.696000e-01 + 8.719000e-01 2.312000e-01 4.310000e-01 5.507000e-01 3.146000e-01 4.567000e-01 4.258000e-01 5.563000e-01 3.740000e-01 3.700000e-01 3.330000e-01 6.646000e-01 2.901000e-01 3.226000e-01 + 1.360000e-02 9.961000e-01 2.466000e-01 1.077500e+00 3.379000e-01 1.112000e+00 5.620000e-02 1.662300e+00 -4.080000e-02 2.046100e+00 -1.250000e-02 1.610000e-02 1.257000e-01 6.558000e-01 + 1.218000e+00 -1.338000e-01 6.016000e-01 1.136800e+00 4.434000e-01 7.737000e-01 6.493000e-01 1.074600e+00 5.333000e-01 6.233000e-01 5.916000e-01 1.146600e+00 4.410000e-01 3.835000e-01 + 7.636000e-01 1.110000e-01 1.172100e+00 4.000000e-04 1.307500e+00 -1.810000e-02 1.756100e+00 -3.150000e-02 1.876900e+00 1.464000e-01 4.910000e-02 -5.820000e-02 6.767000e-01 1.780000e-02 + 4.733000e-01 5.533000e-01 7.878000e-01 5.479000e-01 9.136000e-01 5.419000e-01 1.809900e+00 -8.920000e-02 -7.870000e-02 2.088800e+00 2.570000e-02 -3.140000e-02 4.780000e-01 3.072000e-01 + 2.005000e-01 5.829000e-01 6.213000e-01 4.544000e-01 6.185000e-01 6.031000e-01 1.674800e+00 6.370000e-02 4.690000e-02 1.944500e+00 4.540000e-02 -5.330000e-02 2.894000e-01 3.546000e-01 + 2.873000e-01 3.904000e-01 5.542000e-01 4.862000e-01 6.085000e-01 5.771000e-01 6.228000e-01 4.063000e-01 6.241000e-01 5.769000e-01 5.590000e-01 4.789000e-01 3.226000e-01 2.772000e-01 + 1.065800e+00 4.890000e-02 9.754000e-01 2.339000e-01 7.418000e-01 1.505000e-01 8.248000e-01 4.087000e-01 6.896000e-01 1.874000e-01 8.415000e-01 3.902000e-01 5.896000e-01 9.380000e-02 + 2.749000e-01 8.322000e-01 7.550000e-02 8.470000e-01 1.179000e-01 6.081000e-01 2.079000e-01 6.938000e-01 1.133000e-01 5.986000e-01 1.856000e-01 7.187000e-01 1.310000e-01 4.502000e-01 + 1.160400e+00 -7.070000e-02 3.873000e-01 4.790000e-01 3.702000e-01 3.135000e-01 3.731000e-01 4.934000e-01 4.204000e-01 2.415000e-01 3.768000e-01 4.884000e-01 3.431000e-01 2.132000e-01 + 7.850000e-01 3.097000e-01 1.039200e+00 6.899000e-01 1.007400e+00 9.179000e-01 1.606700e+00 1.494000e-01 2.023600e+00 -2.750000e-02 7.740000e-02 1.643000e+00 6.075000e-01 3.769000e-01 + 4.767000e-01 6.200000e-01 8.124000e-01 6.201000e-01 8.201000e-01 7.476000e-01 1.762100e+00 -3.200000e-02 2.850000e-02 1.962500e+00 -2.400000e-03 1.735200e+00 4.170000e-01 4.287000e-01 + 7.000000e-02 5.804000e-01 3.603000e-01 6.051000e-01 5.032000e-01 5.804000e-01 1.808000e+00 -9.050000e-02 2.450000e-02 1.971800e+00 5.380000e-02 -6.290000e-02 2.108000e-01 3.495000e-01 + 2.398000e-01 5.381000e-01 4.280000e-01 7.827000e-01 4.594000e-01 9.249000e-01 4.543000e-01 7.498000e-01 4.837000e-01 9.178000e-01 3.558000e-01 8.688000e-01 2.354000e-01 4.663000e-01 + 6.772000e-01 4.153000e-01 8.702000e-01 6.673000e-01 1.097600e+00 5.371000e-01 1.671400e+00 7.230000e-02 1.913400e+00 1.031000e-01 4.360000e-02 1.682100e+00 5.430000e-01 3.427000e-01 + 5.771000e-01 2.690000e-01 9.626000e-01 1.865000e-01 9.748000e-01 3.090000e-01 1.720800e+00 1.200000e-02 1.904000e+00 1.157000e-01 4.180000e-02 -5.090000e-02 5.076000e-01 1.776000e-01 + 9.678000e-01 1.205000e-01 4.169000e-01 7.365000e-01 2.957000e-01 5.802000e-01 4.317000e-01 7.145000e-01 3.142000e-01 5.355000e-01 3.243000e-01 8.496000e-01 2.935000e-01 3.748000e-01 + 3.039000e-01 8.250000e-02 5.032000e-01 1.001000e-01 6.313000e-01 4.200000e-02 5.443000e-01 4.810000e-02 6.187000e-01 7.040000e-02 5.802000e-01 5.800000e-03 3.000000e-01 4.560000e-02 + 1.066000e+00 4.740000e-02 8.390000e-01 3.301000e-01 6.501000e-01 2.295000e-01 8.427000e-01 3.239000e-01 6.770000e-01 1.715000e-01 7.632000e-01 4.213000e-01 5.597000e-01 1.096000e-01 + 4.445000e-01 4.328000e-01 7.354000e-01 6.306000e-01 7.351000e-01 8.423000e-01 7.142000e-01 6.597000e-01 8.077000e-01 7.814000e-01 6.867000e-01 6.933000e-01 3.992000e-01 3.965000e-01 + 1.113800e+00 -1.200000e-02 4.509000e-01 1.285400e+00 4.255000e-01 1.115600e+00 4.581000e-01 1.271400e+00 3.918000e-01 1.065600e+00 4.900000e-01 1.234000e+00 3.392000e-01 5.883000e-01 + 3.929000e-01 1.168000e-01 6.146000e-01 2.238000e-01 8.554000e-01 8.000000e-02 1.677700e+00 6.240000e-02 -6.710000e-02 7.830000e-02 3.840000e-02 -4.460000e-02 3.705000e-01 1.066000e-01 + 1.174900e+00 -7.820000e-02 5.318000e-01 1.208200e+00 4.090000e-01 8.032000e-01 6.121000e-01 1.106000e+00 4.824000e-01 6.767000e-01 5.083000e-01 1.234000e+00 3.942000e-01 4.315000e-01 + 1.112500e+00 -1.900000e-03 7.953000e-01 4.182000e-01 5.772000e-01 3.317000e-01 8.666000e-01 3.376000e-01 6.822000e-01 1.837000e-01 7.594000e-01 4.607000e-01 5.409000e-01 1.411000e-01 + 1.064300e+00 4.830000e-02 4.031000e-01 4.715000e-01 4.217000e-01 2.610000e-01 5.285000e-01 3.166000e-01 4.123000e-01 2.549000e-01 4.966000e-01 3.544000e-01 3.839000e-01 1.711000e-01 + 1.462000e-01 1.741000e-01 3.355000e-01 3.210000e-01 5.901000e-01 1.645000e-01 1.819900e+00 -1.075000e-01 4.480000e-02 -5.060000e-02 -1.540000e-02 1.900000e-02 1.997000e-01 1.640000e-01 + 1.082600e+00 2.900000e-02 5.759000e-01 1.143600e+00 3.587000e-01 1.001100e+00 3.984000e-01 1.351900e+00 3.697000e-01 9.311000e-01 4.609000e-01 1.280500e+00 3.813000e-01 4.929000e-01 + 1.749000e-01 9.337000e-01 4.971000e-01 9.799000e-01 5.158000e-01 1.106500e+00 1.858000e+00 -1.477000e-01 5.880000e-02 1.932100e+00 -9.480000e-02 1.840300e+00 2.799000e-01 5.831000e-01 + 2.320000e-02 4.472000e-01 2.880000e-02 7.578000e-01 1.640000e-02 8.930000e-01 -2.840000e-02 1.768200e+00 -4.300000e-03 5.000000e-03 1.630000e-02 -1.920000e-02 1.400000e-03 4.502000e-01 +-1.890000e-02 9.049000e-01 2.733000e-01 9.240000e-01 4.054000e-01 9.117000e-01 -1.630000e-02 1.749400e+00 8.600000e-03 1.989800e+00 1.650000e-02 -1.970000e-02 1.497000e-01 5.558000e-01 + 3.017000e-01 1.810000e-02 5.904000e-01 5.250000e-02 8.312000e-01 -9.490000e-02 1.793900e+00 -7.140000e-02 2.200000e-03 -2.400000e-03 1.700000e-03 -1.600000e-03 3.347000e-01 2.090000e-02 + 3.039000e-01 3.121000e-01 6.046000e-01 3.240000e-01 8.146000e-01 2.188000e-01 1.698900e+00 4.070000e-02 -9.370000e-02 2.104700e+00 -1.610000e-02 2.060000e-02 3.003000e-01 2.485000e-01 + 1.249000e-01 9.827000e-01 6.260000e-02 1.675200e+00 8.900000e-02 9.563000e-01 1.085000e-01 1.615700e+00 5.530000e-02 9.614000e-01 3.070000e-02 1.713800e+00 7.130000e-02 6.857000e-01 + 1.155700e+00 -5.860000e-02 3.913000e-01 1.361500e+00 4.194000e-01 8.176000e-01 4.429000e-01 1.295800e+00 4.218000e-01 7.674000e-01 4.723000e-01 1.270600e+00 3.738000e-01 4.626000e-01 + 2.459000e-01 7.784000e-01 5.481000e-01 7.902000e-01 6.902000e-01 7.715000e-01 1.765200e+00 -3.940000e-02 -1.560000e-02 2.020000e+00 4.010000e-02 -4.900000e-02 3.023000e-01 4.882000e-01 + 6.220000e-01 4.951000e-01 3.017000e-01 7.115000e-01 2.143000e-01 5.816000e-01 2.724000e-01 7.436000e-01 2.240000e-01 5.504000e-01 3.288000e-01 6.809000e-01 2.659000e-01 3.523000e-01 + 3.249000e-01 2.344000e-01 5.291000e-01 3.440000e-01 5.390000e-01 4.644000e-01 6.531000e-01 1.941000e-01 5.935000e-01 4.141000e-01 5.112000e-01 3.632000e-01 3.074000e-01 1.955000e-01 + 3.186000e-01 6.189000e-01 7.353000e-01 5.008000e-01 7.807000e-01 5.874000e-01 1.759200e+00 -3.670000e-02 1.700000e-02 1.983200e+00 -3.570000e-02 4.160000e-02 3.922000e-01 3.411000e-01 + 8.110000e-02 5.730000e-01 -5.400000e-03 1.059500e+00 4.620000e-02 1.141800e+00 9.510000e-02 9.377000e-01 4.330000e-02 1.157600e+00 4.870000e-02 9.916000e-01 3.510000e-02 5.644000e-01 + 2.594000e-01 3.892000e-01 5.220000e-01 4.511000e-01 6.600000e-01 4.274000e-01 1.716700e+00 1.820000e-02 -2.340000e-02 2.023900e+00 2.910000e-02 -3.230000e-02 3.178000e-01 2.422000e-01 + 5.516000e-01 4.323000e-01 9.537000e-01 3.289000e-01 1.019900e+00 3.933000e-01 1.765300e+00 -3.410000e-02 2.026400e+00 -2.980000e-02 1.780000e-02 -2.150000e-02 5.026000e-01 2.628000e-01 + 1.111500e+00 -6.900000e-03 5.701000e-01 1.157500e+00 4.381000e-01 1.535200e+00 4.560000e-01 1.290200e+00 4.360000e-01 1.382700e+00 5.761000e-01 1.156000e+00 4.032000e-01 5.885000e-01 + 4.961000e-01 1.890000e-01 7.077000e-01 3.780000e-01 8.067000e-01 4.309000e-01 7.211000e-01 3.645000e-01 8.028000e-01 4.556000e-01 7.352000e-01 3.442000e-01 3.970000e-01 2.335000e-01 + 1.133100e+00 -3.640000e-02 4.325000e-01 4.307000e-01 3.780000e-01 3.109000e-01 4.364000e-01 4.285000e-01 3.326000e-01 3.505000e-01 4.787000e-01 3.815000e-01 3.200000e-01 2.438000e-01 + 4.649000e-01 6.439000e-01 7.775000e-01 6.826000e-01 9.982000e-01 5.638000e-01 1.757300e+00 -2.250000e-02 -1.580000e-02 2.011300e+00 4.970000e-02 1.671300e+00 4.476000e-01 4.080000e-01 + 5.782000e-01 6.300000e-03 9.617000e-01 -6.070000e-02 1.034800e+00 4.300000e-03 9.053000e-01 7.600000e-03 1.051300e+00 2.400000e-03 1.007700e+00 -1.158000e-01 5.201000e-01 7.200000e-03 + 7.918000e-01 3.077000e-01 5.134000e-01 9.919000e-01 2.700000e-01 7.473000e-01 4.521000e-01 1.058100e+00 2.535000e-01 7.352000e-01 4.208000e-01 1.096100e+00 2.795000e-01 4.604000e-01 + 5.928000e-01 5.222000e-01 1.003600e+00 4.579000e-01 1.078100e+00 5.070000e-01 1.647000e+00 9.860000e-02 2.106100e+00 -1.167000e-01 3.710000e-02 1.683300e+00 5.517000e-01 3.079000e-01 + 5.960000e-02 7.629000e-01 3.239000e-01 8.181000e-01 4.259000e-01 8.440000e-01 1.670200e+00 7.190000e-02 -2.630000e-02 2.035400e+00 9.080000e-02 -1.085000e-01 1.982000e-01 4.708000e-01 + 1.049800e+00 6.410000e-02 5.919000e-01 2.174000e-01 3.979000e-01 2.718000e-01 4.788000e-01 3.486000e-01 4.119000e-01 2.361000e-01 5.073000e-01 3.169000e-01 3.661000e-01 1.799000e-01 + 1.155500e+00 -5.990000e-02 1.190600e+00 2.981000e-01 9.963000e-01 -1.430000e-02 1.078900e+00 4.294000e-01 8.709000e-01 1.026000e-01 1.261800e+00 2.111000e-01 6.923000e-01 4.150000e-02 + 6.215000e-01 2.120000e-01 1.045100e+00 2.494000e-01 1.139400e+00 3.451000e-01 1.084900e+00 1.967000e-01 1.175900e+00 3.235000e-01 1.139200e+00 1.358000e-01 6.034000e-01 1.435000e-01 + 2.349000e-01 1.442000e-01 2.265000e-01 3.929000e-01 2.736000e-01 4.296000e-01 2.931000e-01 3.112000e-01 2.363000e-01 4.802000e-01 2.454000e-01 3.726000e-01 1.406000e-01 2.151000e-01 + 3.019000e-01 8.005000e-01 2.399000e-01 1.477400e+00 1.681000e-01 1.178700e+00 1.256000e-01 1.612900e+00 1.998000e-01 1.085000e+00 1.193000e-01 1.621700e+00 1.418000e-01 7.335000e-01 + 1.147600e+00 -4.770000e-02 6.603000e-01 5.889000e-01 4.982000e-01 4.222000e-01 6.575000e-01 5.907000e-01 5.041000e-01 3.874000e-01 5.851000e-01 6.789000e-01 4.479000e-01 2.445000e-01 + 1.101200e+00 7.300000e-03 8.426000e-01 1.042000e-01 6.891000e-01 5.540000e-02 9.469000e-01 -1.900000e-02 7.682000e-01 -6.170000e-02 8.742000e-01 6.580000e-02 5.727000e-01 2.010000e-02 +-4.210000e-02 6.206000e-01 -8.900000e-03 9.080000e-01 4.440000e-02 9.645000e-01 2.560000e-02 8.678000e-01 3.390000e-02 9.927000e-01 1.381000e-01 7.345000e-01 9.100000e-03 5.054000e-01 + 3.173000e-01 1.439000e-01 4.652000e-01 2.672000e-01 4.718000e-01 3.732000e-01 4.224000e-01 3.186000e-01 5.422000e-01 2.998000e-01 4.661000e-01 2.660000e-01 2.607000e-01 1.635000e-01 + 3.098000e-01 5.266000e-01 7.139000e-01 4.210000e-01 7.526000e-01 5.123000e-01 1.844200e+00 -1.276000e-01 4.160000e-02 1.950200e+00 3.560000e-02 -4.230000e-02 3.870000e-01 2.836000e-01 + 1.113300e+00 -1.020000e-02 6.586000e-01 3.186000e-01 5.528000e-01 2.098000e-01 6.548000e-01 3.203000e-01 4.692000e-01 2.883000e-01 5.417000e-01 4.562000e-01 4.591000e-01 1.485000e-01 + 1.120500e+00 -1.250000e-02 4.273000e-01 1.305000e+00 4.001000e-01 1.569000e+00 5.607000e-01 1.152400e+00 4.132000e-01 1.340800e+00 3.841000e-01 1.354200e+00 3.303000e-01 6.559000e-01 + 6.990000e-01 3.968000e-01 1.746000e-01 6.364000e-01 2.241000e-01 4.211000e-01 3.194000e-01 4.643000e-01 2.155000e-01 4.168000e-01 4.041000e-01 3.630000e-01 2.536000e-01 2.708000e-01 + 4.087000e-01 1.144000e-01 6.518000e-01 1.959000e-01 7.540000e-01 2.240000e-01 1.739300e+00 -8.900000e-03 4.300000e-03 -5.200000e-03 -2.820000e-02 3.580000e-02 3.589000e-01 1.307000e-01 + 8.990000e-02 4.962000e-01 1.670000e-01 7.478000e-01 1.831000e-01 8.579000e-01 1.603000e-01 7.559000e-01 1.481000e-01 9.170000e-01 1.539000e-01 7.605000e-01 6.710000e-02 4.654000e-01 + 7.275000e-01 8.800000e-02 1.026300e+00 2.773000e-01 1.203800e+00 2.712000e-01 1.026200e+00 2.689000e-01 1.253600e+00 2.343000e-01 9.783000e-01 3.308000e-01 5.823000e-01 1.706000e-01 + 5.495000e-01 5.437000e-01 2.686000e-01 7.107000e-01 1.793000e-01 5.921000e-01 2.232000e-01 7.645000e-01 2.558000e-01 4.846000e-01 3.518000e-01 6.074000e-01 2.065000e-01 4.018000e-01 + 5.979000e-01 2.631000e-01 9.246000e-01 2.454000e-01 1.040700e+00 2.497000e-01 1.743200e+00 -1.210000e-02 2.035700e+00 -4.280000e-02 5.510000e-02 -6.860000e-02 5.446000e-01 1.450000e-01 + 4.053000e-01 3.193000e-01 7.128000e-01 4.065000e-01 6.600000e-01 6.450000e-01 6.435000e-01 4.922000e-01 7.326000e-01 5.784000e-01 6.610000e-01 4.686000e-01 3.683000e-01 2.864000e-01 + 3.101000e-01 7.882000e-01 2.058000e-01 6.524000e-01 1.578000e-01 5.337000e-01 1.578000e-01 7.100000e-01 1.407000e-01 5.390000e-01 1.659000e-01 7.029000e-01 1.362000e-01 4.268000e-01 + 3.448000e-01 7.733000e-01 2.226000e-01 3.745000e-01 1.856000e-01 3.102000e-01 2.002000e-01 3.992000e-01 2.583000e-01 2.151000e-01 1.775000e-01 4.266000e-01 1.738000e-01 2.538000e-01 + 1.163000e-01 6.281000e-01 6.240000e-02 1.128500e+00 1.577000e-01 1.181700e+00 8.570000e-02 1.103700e+00 1.171000e-01 1.252400e+00 2.270000e-01 9.335000e-01 7.730000e-02 6.036000e-01 + 1.841000e-01 9.089000e-01 9.620000e-02 1.631700e+00 3.060000e-02 1.153200e+00 4.420000e-02 1.694200e+00 1.440000e-02 1.136600e+00 9.730000e-02 1.630200e+00 6.780000e-02 7.479000e-01 + 2.871000e-01 4.391000e-01 5.545000e-01 5.586000e-01 5.610000e-01 7.182000e-01 6.155000e-01 4.853000e-01 6.982000e-01 5.763000e-01 5.284000e-01 5.929000e-01 3.306000e-01 3.102000e-01 + 6.266000e-01 4.869000e-01 8.659000e-01 8.900000e-01 1.030200e+00 9.553000e-01 1.955700e+00 -2.583000e-01 1.954300e+00 5.250000e-02 1.870000e-02 1.709700e+00 5.739000e-01 4.255000e-01 + 2.521000e-01 5.208000e-01 4.874000e-01 6.131000e-01 7.096000e-01 4.927000e-01 1.693000e+00 4.670000e-02 2.250000e-02 1.967700e+00 1.410000e-02 -1.790000e-02 2.880000e-01 3.549000e-01 + 1.115700e+00 -1.130000e-02 5.825000e-01 6.135000e-01 6.044000e-01 2.598000e-01 5.369000e-01 6.677000e-01 5.063000e-01 3.521000e-01 5.327000e-01 6.701000e-01 4.238000e-01 2.538000e-01 + 4.343000e-01 2.892000e-01 6.943000e-01 4.379000e-01 8.366000e-01 4.458000e-01 8.259000e-01 2.838000e-01 8.670000e-01 4.263000e-01 7.693000e-01 3.494000e-01 4.452000e-01 2.014000e-01 + 3.767000e-01 6.610000e-01 7.348000e-01 6.040000e-01 9.230000e-01 5.251000e-01 1.740300e+00 -4.500000e-03 -4.600000e-02 2.054200e+00 2.160000e-02 -2.470000e-02 4.012000e-01 3.929000e-01 + 1.776000e-01 9.447000e-01 6.292000e-01 9.101000e-01 7.055000e-01 9.575000e-01 1.797300e+00 -8.050000e-02 -8.980000e-02 2.111000e+00 -2.000000e-03 1.733600e+00 3.322000e-01 5.643000e-01 + 2.923000e-01 5.579000e-01 3.704000e-01 9.829000e-01 3.558000e-01 1.191800e+00 3.351000e-01 1.025100e+00 3.176000e-01 1.266500e+00 3.619000e-01 9.876000e-01 1.767000e-01 6.106000e-01 + 1.100500e+00 6.600000e-03 4.421000e-01 8.560000e-01 3.823000e-01 5.487000e-01 4.909000e-01 7.966000e-01 3.241000e-01 5.907000e-01 5.004000e-01 7.838000e-01 3.299000e-01 3.734000e-01 + 1.048000e+00 6.760000e-02 7.788000e-01 9.475000e-01 6.592000e-01 1.156800e+00 7.752000e-01 9.544000e-01 6.585000e-01 9.674000e-01 7.535000e-01 9.787000e-01 5.198000e-01 4.468000e-01 + 6.970000e-02 9.352000e-01 2.593000e-01 1.061000e+00 3.419000e-01 1.108000e+00 7.300000e-03 1.719100e+00 -7.600000e-03 2.004800e+00 3.520000e-02 -4.400000e-02 1.164000e-01 6.692000e-01 + 3.260000e-01 2.160000e-01 7.528000e-01 7.900000e-02 7.924000e-01 1.742000e-01 1.687000e+00 5.020000e-02 -3.940000e-02 4.670000e-02 -6.030000e-02 7.090000e-02 4.082000e-01 7.520000e-02 + 1.050000e-01 6.663000e-01 3.348000e-01 7.601000e-01 5.875000e-01 6.110000e-01 1.713200e+00 2.610000e-02 1.228000e-01 1.855500e+00 -2.320000e-02 2.860000e-02 2.201000e-01 4.189000e-01 + 1.002700e+00 1.252000e-01 6.166000e-01 4.814000e-01 4.798000e-01 3.617000e-01 7.088000e-01 3.755000e-01 5.636000e-01 2.377000e-01 6.239000e-01 4.772000e-01 4.345000e-01 2.159000e-01 + 4.496000e-01 2.169000e-01 6.392000e-01 3.630000e-01 8.200000e-01 2.899000e-01 1.770400e+00 -4.520000e-02 -1.100000e-02 2.011200e+00 1.760000e-02 -1.910000e-02 4.239000e-01 1.501000e-01 + 4.926000e-01 6.137000e-01 3.186000e-01 1.404600e+00 2.477000e-01 1.260500e+00 1.928000e-01 1.552100e+00 2.102000e-01 1.225200e+00 2.542000e-01 1.481800e+00 2.139000e-01 7.059000e-01 + 1.138700e+00 -4.270000e-02 8.879000e-01 1.301000e-01 6.693000e-01 1.269000e-01 8.105000e-01 2.219000e-01 6.306000e-01 1.497000e-01 8.711000e-01 1.517000e-01 5.312000e-01 9.750000e-02 + 9.711000e-01 1.022000e-01 1.494600e+00 2.016000e-01 1.763500e+00 1.583000e-01 1.554200e+00 1.290000e-01 1.834300e+00 1.014000e-01 1.488900e+00 2.066000e-01 8.892000e-01 8.380000e-02 + 2.105000e-01 1.522000e-01 5.753000e-01 9.360000e-02 6.696000e-01 1.234000e-01 1.763000e+00 -3.660000e-02 4.100000e-03 -4.000000e-03 -1.860000e-02 2.320000e-02 3.220000e-01 5.130000e-02 + 5.364000e-01 5.697000e-01 2.754000e-01 1.459900e+00 2.340000e-01 9.770000e-01 2.480000e-01 1.489200e+00 3.099000e-01 8.484000e-01 2.989000e-01 1.428300e+00 2.200000e-01 6.076000e-01 + 1.165300e+00 -7.410000e-02 4.147000e-01 6.831000e-01 3.322000e-01 5.073000e-01 4.404000e-01 6.515000e-01 3.882000e-01 4.203000e-01 3.910000e-01 7.064000e-01 3.222000e-01 3.256000e-01 + 5.929000e-01 4.753000e-01 7.672000e-01 9.424000e-01 8.572000e-01 1.086400e+00 7.978000e-01 8.958000e-01 9.469000e-01 1.005600e+00 7.537000e-01 9.506000e-01 4.475000e-01 5.341000e-01 + 1.982000e-01 7.611000e-01 4.901000e-01 7.844000e-01 6.065000e-01 7.851000e-01 1.734700e+00 -4.100000e-03 9.350000e-02 1.890800e+00 -3.790000e-02 4.530000e-02 2.528000e-01 5.018000e-01 + 1.546000e-01 4.171000e-01 2.119000e-01 6.868000e-01 2.017000e-01 8.355000e-01 3.120000e-01 5.704000e-01 2.865000e-01 7.473000e-01 2.101000e-01 6.932000e-01 1.401000e-01 3.778000e-01 + 5.030000e-02 4.674000e-01 7.340000e-02 7.643000e-01 2.045000e-01 7.558000e-01 -4.600000e-03 1.736500e+00 -6.460000e-02 7.550000e-02 -1.480000e-02 1.930000e-02 5.150000e-02 4.315000e-01 + 1.041000e-01 6.762000e-01 3.926000e-01 7.050000e-01 5.157000e-01 7.068000e-01 1.774500e+00 -5.130000e-02 3.810000e-02 1.958000e+00 -2.210000e-02 2.530000e-02 2.124000e-01 4.345000e-01 + 6.700000e-03 5.981000e-01 -6.650000e-02 1.033500e+00 -2.740000e-02 1.112100e+00 3.610000e-02 9.109000e-01 1.080000e-02 1.085600e+00 -6.430000e-02 1.026400e+00 6.900000e-03 5.412000e-01 + 2.601000e-01 8.426000e-01 1.748000e-01 5.536000e-01 1.488000e-01 4.436000e-01 2.177000e-01 4.987000e-01 1.135000e-01 4.778000e-01 1.067000e-01 6.321000e-01 1.217000e-01 3.789000e-01 + 7.495000e-01 3.638000e-01 3.087000e-01 7.132000e-01 2.289000e-01 5.704000e-01 4.013000e-01 5.987000e-01 4.147000e-01 3.338000e-01 3.795000e-01 6.300000e-01 2.800000e-01 3.398000e-01 + 2.536000e-01 3.032000e-01 3.890000e-01 4.836000e-01 4.285000e-01 5.635000e-01 4.099000e-01 4.558000e-01 5.144000e-01 4.786000e-01 3.665000e-01 5.107000e-01 2.354000e-01 2.655000e-01 + 1.108700e+00 -1.400000e-03 6.231000e-01 1.103400e+00 5.175000e-01 5.217000e-01 5.151000e-01 1.232300e+00 5.308000e-01 4.732000e-01 4.814000e-01 1.269600e+00 4.136000e-01 3.438000e-01 + 1.243000e-01 8.552000e-01 2.607000e-01 1.254300e+00 2.224000e-01 1.517600e+00 2.060000e-01 1.322800e+00 2.795000e-01 1.476000e+00 3.078000e-01 1.201100e+00 1.504000e-01 7.250000e-01 + 9.213000e-01 1.830000e-01 1.271100e+00 2.182000e-01 1.436000e+00 1.685000e-01 1.739100e+00 -6.500000e-03 2.118200e+00 -1.382000e-01 2.510000e-02 1.703800e+00 7.304000e-01 1.416000e-01 + 4.906000e-01 2.284000e-01 7.607000e-01 3.695000e-01 7.548000e-01 5.566000e-01 8.102000e-01 3.068000e-01 8.265000e-01 4.896000e-01 7.111000e-01 4.283000e-01 4.367000e-01 2.166000e-01 + 5.023000e-01 2.296000e-01 8.095000e-01 3.391000e-01 8.645000e-01 4.546000e-01 7.044000e-01 4.620000e-01 8.431000e-01 5.025000e-01 7.605000e-01 3.984000e-01 4.166000e-01 2.559000e-01 + 1.156500e+00 -5.760000e-02 6.427000e-01 1.085200e+00 4.490000e-01 9.378000e-01 5.651000e-01 1.172700e+00 4.125000e-01 9.192000e-01 6.713000e-01 1.048000e+00 4.359000e-01 4.478000e-01 + 3.471000e-01 4.969000e-01 5.672000e-01 6.026000e-01 6.693000e-01 6.248000e-01 1.642200e+00 1.036000e-01 2.180000e-02 1.976600e+00 -6.900000e-03 6.900000e-03 3.297000e-01 3.596000e-01 + 1.772000e-01 7.112000e-01 3.103000e-01 1.079600e+00 2.551000e-01 1.339400e+00 2.338000e-01 1.168600e+00 3.075000e-01 1.305000e+00 2.437000e-01 1.153100e+00 1.468000e-01 6.600000e-01 + 4.819000e-01 5.060000e-01 7.012000e-01 8.559000e-01 8.285000e-01 9.376000e-01 6.944000e-01 8.615000e-01 7.004000e-01 1.112700e+00 7.276000e-01 8.193000e-01 4.284000e-01 4.650000e-01 + 4.597000e-01 2.749000e-01 7.482000e-01 3.990000e-01 9.198000e-01 3.746000e-01 7.880000e-01 3.529000e-01 8.253000e-01 5.067000e-01 7.385000e-01 4.140000e-01 4.344000e-01 2.271000e-01 + 4.523000e-01 6.470000e-01 2.413000e-01 1.312200e+00 1.894000e-01 8.252000e-01 1.803000e-01 1.379300e+00 1.961000e-01 7.919000e-01 1.998000e-01 1.361800e+00 1.884000e-01 5.538000e-01 + 4.270000e-02 6.164000e-01 1.516000e-01 8.657000e-01 8.410000e-02 1.095600e+00 1.261000e-01 8.992000e-01 1.549000e-01 1.029100e+00 7.630000e-02 9.625000e-01 3.020000e-02 5.699000e-01 + 4.810000e-02 6.301000e-01 3.641000e-01 6.274000e-01 4.213000e-01 6.975000e-01 4.650000e-02 1.677500e+00 -2.970000e-02 2.037900e+00 1.500000e-03 -2.200000e-03 1.792000e-01 4.041000e-01 +-5.410000e-02 8.424000e-01 2.830000e-01 8.134000e-01 4.024000e-01 8.135000e-01 6.600000e-03 1.724300e+00 -8.130000e-02 2.099400e+00 -3.590000e-02 4.370000e-02 1.669000e-01 4.733000e-01 + 1.492000e-01 9.433000e-01 4.581000e-01 1.263300e+00 4.340000e-01 1.517700e+00 1.692600e+00 4.630000e-02 -5.270000e-02 2.056400e+00 -1.043000e-01 1.855000e+00 2.145000e-01 7.789000e-01 + 5.950000e-01 5.037000e-01 8.418000e-01 6.593000e-01 1.035600e+00 5.684000e-01 1.736200e+00 -4.000000e-03 1.998700e+00 2.900000e-03 7.450000e-02 1.641700e+00 5.081000e-01 3.627000e-01 + 1.041200e+00 7.690000e-02 3.429000e-01 5.131000e-01 4.083000e-01 2.563000e-01 4.457000e-01 3.872000e-01 3.686000e-01 2.875000e-01 4.201000e-01 4.196000e-01 3.165000e-01 2.346000e-01 + 5.750000e-01 9.830000e-02 7.359000e-01 3.524000e-01 9.226000e-01 3.027000e-01 7.922000e-01 2.849000e-01 9.758000e-01 2.588000e-01 8.202000e-01 2.514000e-01 4.557000e-01 1.661000e-01 + 2.356000e-01 2.608000e-01 3.491000e-01 4.312000e-01 5.015000e-01 3.656000e-01 3.259000e-01 4.589000e-01 3.623000e-01 5.480000e-01 3.415000e-01 4.391000e-01 2.121000e-01 2.362000e-01 +-4.400000e-03 5.750000e-02 2.999000e-01 5.590000e-02 3.436000e-01 1.495000e-01 6.220000e-02 -7.260000e-02 3.770000e-02 -4.620000e-02 -1.930000e-02 2.460000e-02 1.388000e-01 5.610000e-02 + 3.853000e-01 2.266000e-01 6.398000e-01 2.940000e-01 7.465000e-01 3.086000e-01 1.889800e+00 -1.804000e-01 2.080000e-02 1.977300e+00 -1.660000e-02 1.840000e-02 3.421000e-01 2.053000e-01 + 5.732000e-01 9.820000e-02 9.495000e-01 2.150000e-02 1.010600e+00 9.040000e-02 1.689700e+00 5.020000e-02 1.855100e+00 1.663000e-01 -1.440000e-02 1.560000e-02 5.282000e-01 4.370000e-02 + 5.975000e-01 9.590000e-02 8.709000e-01 2.277000e-01 9.448000e-01 3.096000e-01 8.984000e-01 1.909000e-01 1.040400e+00 2.144000e-01 8.695000e-01 2.280000e-01 4.989000e-01 1.341000e-01 + 1.161700e+00 -6.800000e-02 5.103000e-01 1.213400e+00 4.322000e-01 6.767000e-01 5.040000e-01 1.219600e+00 3.956000e-01 6.859000e-01 5.204000e-01 1.201400e+00 3.472000e-01 4.431000e-01 + 1.021700e+00 9.880000e-02 4.318000e-01 4.948000e-01 3.271000e-01 4.096000e-01 4.788000e-01 4.344000e-01 3.361000e-01 3.875000e-01 4.107000e-01 5.175000e-01 3.394000e-01 2.462000e-01 + 9.910000e-02 6.097000e-01 1.928000e-01 9.126000e-01 2.849000e-01 9.650000e-01 1.819000e-01 9.313000e-01 2.587000e-01 1.014800e+00 2.217000e-01 8.827000e-01 1.360000e-01 4.989000e-01 + 4.196000e-01 4.070000e-01 8.509000e-01 2.705000e-01 8.090000e-01 4.584000e-01 1.633500e+00 1.120000e-01 -6.650000e-02 2.080700e+00 1.660000e-02 -1.950000e-02 4.122000e-01 2.621000e-01 + 6.442000e-01 4.657000e-01 2.805000e-01 9.211000e-01 3.062000e-01 5.754000e-01 3.203000e-01 8.736000e-01 3.428000e-01 5.107000e-01 2.195000e-01 9.937000e-01 2.591000e-01 4.182000e-01 + 1.083000e-01 3.704000e-01 2.198000e-01 5.239000e-01 1.973000e-01 6.579000e-01 1.038000e-01 6.641000e-01 2.724000e-01 5.815000e-01 2.600000e-01 4.734000e-01 9.710000e-02 3.372000e-01 + 1.143400e+00 -4.700000e-02 6.394000e-01 1.084000e+00 5.072000e-01 8.219000e-01 7.274000e-01 9.770000e-01 4.117000e-01 8.792000e-01 6.133000e-01 1.115600e+00 4.261000e-01 4.439000e-01 + 4.894000e-01 2.322000e-01 9.444000e-01 1.599000e-01 1.136500e+00 1.100000e-01 9.970000e-01 9.400000e-02 1.083900e+00 1.925000e-01 9.986000e-01 9.700000e-02 5.573000e-01 7.900000e-02 + 6.940000e-02 4.496000e-01 3.861000e-01 4.431000e-01 5.083000e-01 4.420000e-01 1.722500e+00 8.000000e-03 -2.180000e-02 2.580000e-02 -1.260000e-02 1.530000e-02 2.012000e-01 2.778000e-01 + 4.709000e-01 1.637000e-01 7.540000e-01 2.368000e-01 9.168000e-01 1.987000e-01 7.055000e-01 2.919000e-01 8.799000e-01 2.597000e-01 6.879000e-01 3.119000e-01 4.364000e-01 1.345000e-01 + 3.695000e-01 7.475000e-01 7.178000e-01 7.092000e-01 8.749000e-01 6.676000e-01 1.720000e+00 6.400000e-03 -9.800000e-03 2.015000e+00 5.230000e-02 1.671600e+00 4.293000e-01 4.062000e-01 + 2.833000e-01 6.929000e-01 3.343000e-01 1.207300e+00 4.848000e-01 1.259600e+00 4.201000e-01 1.108400e+00 4.528000e-01 1.313000e+00 3.483000e-01 1.200200e+00 2.231000e-01 6.640000e-01 + 3.249000e-01 2.663000e-01 2.135000e-01 8.380000e-02 2.176000e-01 3.230000e-02 1.633000e-01 1.440000e-01 1.643000e-01 8.870000e-02 2.186000e-01 7.850000e-02 1.557000e-01 8.340000e-02 + 2.509000e-01 4.415000e-01 3.253000e-01 7.774000e-01 3.237000e-01 9.379000e-01 2.093000e-01 9.146000e-01 3.470000e-01 9.313000e-01 2.318000e-01 8.854000e-01 1.571000e-01 4.836000e-01 + 4.089000e-01 3.129000e-01 5.581000e-01 5.068000e-01 8.138000e-01 3.462000e-01 1.677900e+00 6.840000e-02 2.080000e-02 1.974000e+00 -2.290000e-02 2.710000e-02 3.926000e-01 2.173000e-01 + 6.643000e-01 4.440000e-01 4.709000e-01 3.021000e-01 3.044000e-01 3.373000e-01 3.318000e-01 4.683000e-01 2.892000e-01 3.421000e-01 3.202000e-01 4.828000e-01 2.603000e-01 2.726000e-01 + 1.089800e+00 1.820000e-02 4.522000e-01 1.280700e+00 2.793000e-01 9.139000e-01 4.674000e-01 1.263400e+00 4.811000e-01 6.379000e-01 5.040000e-01 1.218100e+00 3.527000e-01 4.602000e-01 + 7.746000e-01 2.773000e-01 1.262500e+00 3.768000e-01 1.368200e+00 5.105000e-01 1.176300e+00 4.776000e-01 1.362200e+00 5.502000e-01 1.244100e+00 4.004000e-01 7.037000e-01 2.478000e-01 + 1.214400e+00 -1.345000e-01 5.728000e-01 8.045000e-01 4.732000e-01 4.932000e-01 5.432000e-01 8.459000e-01 4.365000e-01 5.079000e-01 6.445000e-01 7.233000e-01 4.171000e-01 3.019000e-01 + 1.020300e+00 1.023000e-01 1.339900e+00 3.863000e-01 8.918000e-01 4.010000e-01 1.400000e+00 3.113000e-01 9.264000e-01 3.066000e-01 1.256800e+00 4.880000e-01 7.044000e-01 1.491000e-01 +-8.000000e-04 8.211000e-01 2.361000e-01 9.075000e-01 3.880000e-01 8.679000e-01 -2.060000e-02 1.757600e+00 -1.680000e-02 2.020600e+00 -5.400000e-02 6.580000e-02 1.555000e-01 5.119000e-01 + 8.590000e-02 5.093000e-01 1.094000e-01 8.284000e-01 1.888000e-01 8.672000e-01 2.009000e-01 7.242000e-01 3.171000e-01 7.291000e-01 2.109000e-01 7.078000e-01 1.055000e-01 4.283000e-01 + 5.585000e-01 4.104000e-01 9.091000e-01 6.040000e-01 9.648000e-01 7.640000e-01 8.639000e-01 6.560000e-01 9.755000e-01 7.808000e-01 9.222000e-01 5.776000e-01 4.978000e-01 3.786000e-01 + 6.952000e-01 9.790000e-02 1.119200e+00 1.273000e-01 1.299400e+00 1.131000e-01 1.206300e+00 2.280000e-02 1.367300e+00 5.920000e-02 1.061200e+00 1.961000e-01 6.713000e-01 4.250000e-02 +-2.900000e-02 6.100000e-01 1.768000e-01 7.117000e-01 2.572000e-01 7.548000e-01 -4.300000e-02 1.783800e+00 -1.134000e-01 2.131000e+00 2.780000e-02 -3.580000e-02 9.120000e-02 4.239000e-01 + 6.690000e-02 1.033000e+00 5.470000e-02 4.185000e-01 6.780000e-02 3.254000e-01 1.660000e-02 4.632000e-01 3.370000e-02 3.636000e-01 6.200000e-03 4.806000e-01 1.390000e-02 3.445000e-01 + 4.392000e-01 1.730000e-01 7.467000e-01 1.972000e-01 7.673000e-01 3.279000e-01 8.360000e-01 9.120000e-02 8.220000e-01 2.830000e-01 7.616000e-01 1.845000e-01 4.333000e-01 1.134000e-01 + 1.127900e+00 -2.750000e-02 1.157800e+00 5.609000e-01 8.199000e-01 7.000000e-01 1.071100e+00 6.639000e-01 8.206000e-01 6.127000e-01 1.003100e+00 7.467000e-01 6.514000e-01 2.694000e-01 + 8.172000e-01 2.907000e-01 3.781000e-01 1.356400e+00 3.831000e-01 1.575800e+00 5.063000e-01 1.203100e+00 3.232000e-01 1.481600e+00 5.606000e-01 1.139900e+00 2.864000e-01 7.047000e-01 + 3.522000e-01 3.816000e-01 6.513000e-01 4.030000e-01 8.796000e-01 2.741000e-01 1.853000e+00 -1.437000e-01 2.020000e-02 1.981500e+00 4.100000e-02 -4.790000e-02 4.054000e-01 2.034000e-01 + 6.243000e-01 3.136000e-01 9.404000e-01 5.338000e-01 1.054800e+00 6.177000e-01 1.003000e+00 4.583000e-01 1.125700e+00 5.655000e-01 8.917000e-01 5.895000e-01 5.389000e-01 3.107000e-01 + 1.032000e-01 2.145000e-01 2.228000e-01 2.618000e-01 2.334000e-01 3.197000e-01 2.795000e-01 1.928000e-01 2.514000e-01 3.102000e-01 2.510000e-01 2.285000e-01 1.353000e-01 1.438000e-01 + 9.120000e-02 4.763000e-01 8.260000e-02 8.195000e-01 1.323000e-01 8.828000e-01 1.208000e-01 7.711000e-01 1.502000e-01 8.749000e-01 8.230000e-02 8.164000e-01 4.590000e-02 4.733000e-01 + 8.400000e-03 5.233000e-01 3.114000e-01 5.303000e-01 3.548000e-01 6.212000e-01 -7.810000e-02 1.824600e+00 -1.500000e-02 1.630000e-02 1.900000e-02 -2.190000e-02 1.548000e-01 3.324000e-01 + 3.621000e-01 3.948000e-01 5.814000e-01 5.091000e-01 7.371000e-01 4.622000e-01 1.827200e+00 -1.137000e-01 2.880000e-02 1.964000e+00 -9.400000e-03 1.210000e-02 3.026000e-01 3.417000e-01 + 3.580000e-01 2.763000e-01 4.854000e-01 5.267000e-01 6.305000e-01 5.067000e-01 6.041000e-01 3.864000e-01 6.265000e-01 5.287000e-01 5.911000e-01 4.011000e-01 3.291000e-01 2.458000e-01 + 3.187000e-01 7.551000e-01 4.209000e-01 1.268700e+00 6.258000e-01 1.271600e+00 6.451000e-01 1.001000e+00 5.475000e-01 1.395100e+00 5.579000e-01 1.113300e+00 3.066000e-01 6.586000e-01 + 1.185000e-01 7.796000e-01 2.708000e-01 1.118500e+00 3.414000e-01 1.232300e+00 2.990000e-01 1.083900e+00 4.309000e-01 1.150900e+00 3.358000e-01 1.039400e+00 1.932000e-01 6.014000e-01 + 1.116200e+00 -1.300000e-02 5.206000e-01 2.610000e-02 4.651000e-01 -1.630000e-02 5.007000e-01 4.660000e-02 4.346000e-01 1.190000e-02 4.827000e-01 6.850000e-02 3.726000e-01 2.650000e-02 + 8.306000e-01 9.530000e-02 1.104800e+00 3.818000e-01 1.345200e+00 3.360000e-01 1.102500e+00 3.846000e-01 1.274300e+00 4.425000e-01 1.127500e+00 3.547000e-01 6.521000e-01 2.045000e-01 + 1.156300e+00 -6.370000e-02 9.582000e-01 7.839000e-01 8.017000e-01 5.624000e-01 1.087200e+00 6.377000e-01 7.981000e-01 5.080000e-01 9.966000e-01 7.431000e-01 6.323000e-01 2.479000e-01 + 1.113700e+00 -1.060000e-02 5.751000e-01 8.128000e-01 5.425000e-01 4.148000e-01 5.360000e-01 8.606000e-01 6.221000e-01 2.954000e-01 6.010000e-01 7.833000e-01 4.364000e-01 2.829000e-01 +-7.000000e-04 1.000200e+00 6.240000e-02 1.491200e+00 3.840000e-02 1.734700e+00 1.108000e-01 1.435400e+00 1.197000e-01 1.670000e+00 -2.700000e-03 1.567500e+00 3.070000e-02 8.685000e-01 + 1.240000e-01 9.872000e-01 1.695000e-01 5.126000e-01 1.271000e-01 4.343000e-01 1.380000e-02 6.953000e-01 2.270000e-02 5.462000e-01 1.374000e-01 5.482000e-01 7.740000e-02 4.041000e-01 + 4.141000e-01 6.719000e-01 2.314000e-01 1.497700e+00 2.280000e-01 9.137000e-01 2.178000e-01 1.504500e+00 1.092000e-01 1.019000e+00 1.247000e-01 1.614500e+00 1.445000e-01 6.616000e-01 + 6.177000e-01 1.427000e-01 9.623000e-01 1.017000e-01 1.103000e+00 7.800000e-02 1.600300e+00 1.562000e-01 1.871500e+00 1.531000e-01 2.210000e-02 -2.600000e-02 5.527000e-01 7.560000e-02 + 1.059300e+00 5.470000e-02 9.245000e-01 3.887000e-01 7.576000e-01 1.769000e-01 9.578000e-01 3.412000e-01 7.952000e-01 1.095000e-01 1.041500e+00 2.450000e-01 6.124000e-01 9.040000e-02 + 9.449000e-01 1.353000e-01 3.213000e-01 1.421500e+00 3.429000e-01 7.263000e-01 3.767000e-01 1.351400e+00 3.432000e-01 6.875000e-01 3.776000e-01 1.354700e+00 2.832000e-01 4.873000e-01 + 6.301000e-01 4.748000e-01 4.008000e-01 9.958000e-01 2.741000e-01 6.979000e-01 3.320000e-01 1.073400e+00 3.438000e-01 5.908000e-01 3.196000e-01 1.086800e+00 2.474000e-01 4.766000e-01 + 1.978000e-01 9.140000e-01 1.938000e-01 4.248000e-01 6.020000e-02 4.722000e-01 9.960000e-02 5.354000e-01 7.210000e-02 4.447000e-01 1.717000e-01 4.529000e-01 1.206000e-01 3.249000e-01 + 2.309000e-01 2.064000e-01 6.680000e-01 5.680000e-02 6.932000e-01 1.725000e-01 1.859700e+00 -1.525000e-01 -3.130000e-02 3.630000e-02 5.910000e-02 -7.020000e-02 3.368000e-01 8.280000e-02 + 2.211000e-01 7.943000e-01 6.555000e-01 6.537000e-01 6.771000e-01 7.673000e-01 1.808700e+00 -8.840000e-02 7.200000e-03 1.994200e+00 -1.680000e-02 1.970000e-02 3.111000e-01 4.701000e-01 + 2.125000e-01 6.560000e-01 4.932000e-01 6.953000e-01 6.084000e-01 6.982000e-01 1.701900e+00 3.800000e-02 -5.100000e-02 2.060000e+00 -4.300000e-03 2.300000e-03 2.587000e-01 4.444000e-01 + 5.390000e-02 1.055300e+00 1.167000e-01 5.124000e-01 -7.110000e-02 6.230000e-01 1.115000e-01 5.170000e-01 3.800000e-02 4.834000e-01 6.440000e-02 5.757000e-01 3.960000e-02 4.136000e-01 + 2.916000e-01 2.618000e-01 3.233000e-01 5.671000e-01 5.434000e-01 4.386000e-01 4.561000e-01 4.079000e-01 4.168000e-01 6.065000e-01 3.949000e-01 4.845000e-01 2.435000e-01 2.603000e-01 + 3.046000e-01 1.198000e-01 5.786000e-01 1.665000e-01 7.735000e-01 8.260000e-02 1.760100e+00 -3.140000e-02 2.880000e-02 -3.630000e-02 -2.370000e-02 2.720000e-02 3.292000e-01 9.370000e-02 + 1.812000e-01 5.192000e-01 4.632000e-01 5.597000e-01 6.344000e-01 4.976000e-01 1.709800e+00 2.520000e-02 -7.980000e-02 2.089700e+00 -4.440000e-02 5.320000e-02 2.372000e-01 3.645000e-01 + 3.179000e-01 4.891000e-01 7.155000e-01 3.883000e-01 7.770000e-01 4.547000e-01 1.566300e+00 1.949000e-01 5.830000e-02 1.927600e+00 3.480000e-02 -4.360000e-02 3.791000e-01 2.744000e-01 + 1.477000e-01 4.547000e-01 2.569000e-01 6.831000e-01 2.450000e-01 8.360000e-01 3.765000e-01 5.412000e-01 2.740000e-01 8.186000e-01 1.857000e-01 7.708000e-01 1.411000e-01 4.037000e-01 + 2.131000e-01 6.178000e-01 6.259000e-01 4.974000e-01 7.543000e-01 4.947000e-01 1.708600e+00 2.940000e-02 4.010000e-02 1.945600e+00 2.130000e-02 -2.470000e-02 3.206000e-01 3.490000e-01 + 1.050800e+00 6.520000e-02 4.232000e-01 8.030000e-02 3.003000e-01 1.362000e-01 3.865000e-01 1.220000e-01 3.645000e-01 4.690000e-02 4.060000e-01 9.850000e-02 3.132000e-01 6.030000e-02 + 2.090000e-02 7.060000e-01 2.470000e-01 8.011000e-01 3.740000e-01 7.966000e-01 -3.690000e-02 1.777300e+00 4.610000e-02 1.946200e+00 -2.460000e-02 2.910000e-02 1.503000e-01 4.630000e-01 + 4.137000e-01 2.892000e-01 4.493000e-01 6.976000e-01 7.307000e-01 5.271000e-01 5.331000e-01 5.940000e-01 5.688000e-01 7.422000e-01 5.595000e-01 5.605000e-01 3.348000e-01 3.115000e-01 + 5.298000e-01 6.420000e-02 6.824000e-01 2.788000e-01 7.870000e-01 3.054000e-01 6.489000e-01 3.206000e-01 8.537000e-01 2.453000e-01 7.167000e-01 2.387000e-01 4.121000e-01 1.402000e-01 + 4.221000e-01 3.664000e-01 7.573000e-01 4.597000e-01 8.641000e-01 5.222000e-01 7.522000e-01 4.671000e-01 8.513000e-01 5.595000e-01 7.478000e-01 4.688000e-01 4.424000e-01 2.608000e-01 + 4.622000e-01 1.290000e-01 7.078000e-01 2.222000e-01 7.751000e-01 2.817000e-01 7.012000e-01 2.305000e-01 8.362000e-01 2.308000e-01 6.095000e-01 3.373000e-01 3.959000e-01 1.415000e-01 + 6.361000e-01 8.160000e-02 9.489000e-01 1.881000e-01 1.033400e+00 2.630000e-01 9.601000e-01 1.715000e-01 1.003700e+00 3.191000e-01 1.146200e+00 -4.830000e-02 5.663000e-01 8.560000e-02 + 2.930000e-01 5.022000e-01 7.152000e-01 3.739000e-01 7.363000e-01 4.900000e-01 1.759500e+00 -3.130000e-02 -6.460000e-02 2.074000e+00 4.020000e-02 -4.750000e-02 3.524000e-01 2.979000e-01 + 4.311000e-01 1.810000e-01 8.205000e-01 9.240000e-02 1.006300e+00 1.160000e-02 1.695800e+00 3.640000e-02 -8.400000e-03 2.010900e+00 -5.130000e-02 6.280000e-02 4.713000e-01 5.760000e-02 + 3.465000e-01 1.600000e-01 6.633000e-01 1.569000e-01 7.640000e-01 1.793000e-01 1.622700e+00 1.281000e-01 1.860000e-02 -2.340000e-02 -2.300000e-03 2.000000e-03 3.697000e-01 1.015000e-01 + 9.872000e-01 -1.126000e-01 1.332000e+00 7.640000e-02 1.548500e+00 5.010000e-02 1.269100e+00 1.510000e-01 1.585700e+00 2.980000e-02 1.368100e+00 3.380000e-02 7.755000e-01 3.590000e-02 + 2.610000e-01 7.540000e-01 6.275000e-01 6.880000e-01 6.739000e-01 7.751000e-01 1.638600e+00 1.087000e-01 5.130000e-02 1.938700e+00 4.940000e-02 -5.850000e-02 3.284000e-01 4.529000e-01 + 2.109000e-01 6.045000e-01 2.995000e-01 9.821000e-01 5.221000e-01 9.051000e-01 3.581000e-01 9.091000e-01 4.162000e-01 1.046800e+00 3.561000e-01 9.154000e-01 1.993000e-01 5.354000e-01 + 5.272000e-01 1.323000e-01 1.047100e+00 -1.195000e-01 1.095700e+00 -3.110000e-02 1.817900e+00 -1.026000e-01 1.931500e+00 7.750000e-02 6.700000e-03 -8.200000e-03 5.379000e-01 1.850000e-02 + 2.795000e-01 5.918000e-01 4.382000e-01 9.261000e-01 5.119000e-01 1.035300e+00 4.693000e-01 8.914000e-01 5.577000e-01 1.006300e+00 3.827000e-01 9.897000e-01 2.396000e-01 5.500000e-01 + 7.918000e-01 -6.760000e-02 9.720000e-01 9.020000e-02 1.069100e+00 1.199000e-01 1.781500e+00 -5.860000e-02 1.962800e+00 4.390000e-02 2.400000e-02 -2.710000e-02 5.785000e-01 4.470000e-02 + 1.210900e+00 -1.261000e-01 5.953000e-01 1.149400e+00 5.685000e-01 9.512000e-01 6.746000e-01 1.050000e+00 5.015000e-01 9.448000e-01 6.418000e-01 1.093700e+00 4.564000e-01 4.655000e-01 + 4.166000e-01 -2.610000e-02 7.158000e-01 -8.600000e-03 8.091000e-01 2.500000e-02 1.774000e+00 -4.670000e-02 -6.200000e-03 6.800000e-03 -6.670000e-02 7.770000e-02 3.702000e-01 3.560000e-02 + 1.066400e+00 4.440000e-02 3.890000e-01 1.361000e+00 4.381000e-01 6.264000e-01 5.386000e-01 1.188200e+00 4.047000e-01 6.327000e-01 3.706000e-01 1.379200e+00 3.626000e-01 4.052000e-01 + 3.560000e-01 5.282000e-01 6.139000e-01 5.976000e-01 7.348000e-01 5.920000e-01 1.600500e+00 1.565000e-01 -3.680000e-02 2.045900e+00 2.100000e-03 -9.000000e-04 3.704000e-01 3.378000e-01 + 2.368000e-01 4.550000e-01 4.686000e-01 5.905000e-01 5.762000e-01 6.224000e-01 4.614000e-01 5.998000e-01 5.424000e-01 6.833000e-01 5.509000e-01 4.959000e-01 2.914000e-01 3.168000e-01 + 1.412000e-01 5.766000e-01 2.682000e-01 7.994000e-01 4.284000e-01 7.535000e-01 2.010000e-02 1.708900e+00 -5.190000e-02 2.063600e+00 -2.990000e-02 3.540000e-02 1.924000e-01 4.226000e-01 + 4.028000e-01 3.229000e-01 5.538000e-01 5.916000e-01 6.593000e-01 6.407000e-01 5.754000e-01 5.726000e-01 6.679000e-01 6.512000e-01 4.964000e-01 6.656000e-01 3.227000e-01 3.403000e-01 + 1.014100e+00 1.139000e-01 6.173000e-01 1.123200e+00 5.625000e-01 5.521000e-01 7.679000e-01 9.384000e-01 5.604000e-01 5.170000e-01 5.513000e-01 1.193400e+00 4.537000e-01 3.371000e-01 + 2.231000e-01 2.160000e-01 3.652000e-01 3.177000e-01 3.538000e-01 4.355000e-01 3.714000e-01 3.082000e-01 3.945000e-01 4.011000e-01 3.866000e-01 2.949000e-01 2.034000e-01 1.923000e-01 + 3.449000e-01 7.698000e-01 2.342000e-01 1.963000e-01 1.374000e-01 2.406000e-01 2.404000e-01 1.906000e-01 1.173000e-01 2.574000e-01 2.592000e-01 1.675000e-01 1.858000e-01 1.419000e-01 + 3.156000e-01 1.832000e-01 3.843000e-01 4.186000e-01 5.150000e-01 3.846000e-01 4.053000e-01 3.939000e-01 4.708000e-01 4.471000e-01 4.462000e-01 3.420000e-01 2.460000e-01 2.126000e-01 + 6.092000e-01 1.414000e-01 8.663000e-01 3.277000e-01 8.921000e-01 4.823000e-01 8.205000e-01 3.809000e-01 1.009700e+00 3.674000e-01 9.123000e-01 2.730000e-01 5.124000e-01 1.745000e-01 + 8.463000e-01 1.475000e-01 1.383300e+00 1.659000e-01 1.409400e+00 3.871000e-01 1.236000e+00 3.424000e-01 1.439400e+00 3.775000e-01 1.240400e+00 3.315000e-01 7.584000e-01 1.434000e-01 + 1.186000e-01 5.166000e-01 2.754000e-01 7.017000e-01 3.716000e-01 7.266000e-01 1.927000e-01 8.017000e-01 2.624000e-01 8.739000e-01 3.585000e-01 6.027000e-01 1.595000e-01 4.034000e-01 +-7.150000e-02 1.194100e+00 1.877000e-01 1.441100e+00 3.201000e-01 1.431200e+00 4.790000e-02 1.673200e+00 -5.740000e-02 2.074100e+00 1.340000e-02 1.714800e+00 9.210000e-02 8.434000e-01 + 1.061600e+00 5.140000e-02 8.015000e-01 9.388000e-01 6.061000e-01 6.110000e-01 9.143000e-01 7.975000e-01 6.790000e-01 4.857000e-01 7.059000e-01 1.043000e+00 5.455000e-01 2.809000e-01 + 7.813000e-01 2.707000e-01 1.166100e+00 1.809000e-01 1.340200e+00 1.161000e-01 1.784300e+00 -6.440000e-02 2.014600e+00 -1.000000e-02 4.940000e-02 -5.860000e-02 6.698000e-01 1.289000e-01 +-2.000000e-02 8.819000e-01 2.728000e-01 8.867000e-01 2.715000e-01 1.033800e+00 1.349000e-01 1.575400e+00 2.450000e-02 1.974500e+00 2.900000e-02 -3.400000e-02 1.103000e-01 5.812000e-01 + 1.124500e+00 -2.180000e-02 4.186000e-01 1.652000e-01 3.845000e-01 9.650000e-02 4.261000e-01 1.539000e-01 3.543000e-01 1.256000e-01 4.996000e-01 6.970000e-02 3.512000e-01 6.390000e-02 +-6.500000e-03 7.993000e-01 3.267000e-01 7.767000e-01 4.522000e-01 7.680000e-01 -2.820000e-02 1.766800e+00 4.970000e-02 1.942100e+00 -5.330000e-02 6.490000e-02 1.645000e-01 4.867000e-01 + 6.900000e-02 7.249000e-01 9.240000e-02 1.152900e+00 4.710000e-02 1.384400e+00 6.110000e-02 1.191900e+00 4.000000e-02 1.400800e+00 1.907000e-01 1.034100e+00 6.170000e-02 6.550000e-01 + 1.428000e-01 5.786000e-01 4.101000e-01 6.276000e-01 4.227000e-01 7.570000e-01 1.796700e+00 -7.820000e-02 1.590000e-02 1.975800e+00 3.080000e-02 -3.800000e-02 2.194000e-01 3.917000e-01 + 4.230000e-01 6.180000e-01 6.667000e-01 9.600000e-01 8.423000e-01 9.991000e-01 6.821000e-01 9.383000e-01 7.610000e-01 1.118300e+00 6.837000e-01 9.371000e-01 3.987000e-01 5.377000e-01 + 1.123200e+00 -1.980000e-02 4.842000e-01 1.256400e+00 4.733000e-01 5.613000e-01 6.713000e-01 1.041900e+00 4.869000e-01 5.181000e-01 6.174000e-01 1.101800e+00 3.947000e-01 3.597000e-01 + 3.235000e-01 4.008000e-01 6.078000e-01 4.450000e-01 7.643000e-01 3.984000e-01 1.743500e+00 -1.240000e-02 7.710000e-02 1.904800e+00 -4.670000e-02 5.660000e-02 3.336000e-01 2.813000e-01 + 8.635000e-01 2.416000e-01 3.268000e-01 7.437000e-01 3.194000e-01 4.958000e-01 4.484000e-01 5.974000e-01 3.312000e-01 4.618000e-01 3.342000e-01 7.332000e-01 2.896000e-01 3.473000e-01 + 3.989000e-01 3.064000e-01 7.602000e-01 3.139000e-01 7.549000e-01 4.913000e-01 6.895000e-01 3.998000e-01 6.527000e-01 6.348000e-01 6.298000e-01 4.730000e-01 3.961000e-01 2.339000e-01 +-5.620000e-02 6.196000e-01 6.960000e-02 8.052000e-01 1.929000e-01 8.025000e-01 2.680000e-02 1.702400e+00 5.080000e-02 1.938200e+00 3.790000e-02 -4.510000e-02 7.300000e-02 4.286000e-01 + 7.670000e-02 5.532000e-01 3.697000e-01 5.783000e-01 5.229000e-01 5.426000e-01 1.730300e+00 2.500000e-03 -5.700000e-03 2.006900e+00 4.780000e-02 -5.690000e-02 2.083000e-01 3.421000e-01 + 1.222200e+00 -1.385000e-01 5.873000e-01 1.141900e+00 4.907000e-01 8.950000e-01 4.724000e-01 1.276000e+00 4.236000e-01 9.068000e-01 5.406000e-01 1.195200e+00 4.016000e-01 4.845000e-01 + 2.563000e-01 6.222000e-01 3.700000e-01 1.011000e+00 4.776000e-01 1.086300e+00 4.206000e-01 9.524000e-01 4.446000e-01 1.143800e+00 3.690000e-01 1.014000e+00 2.464000e-01 5.458000e-01 + 1.840000e-01 6.729000e-01 4.951000e-01 6.792000e-01 5.736000e-01 7.248000e-01 1.651000e+00 9.460000e-02 -1.130000e-02 2.015000e+00 1.500000e-03 -2.300000e-03 2.611000e-01 4.318000e-01 + 1.102100e+00 3.900000e-03 1.025300e+00 7.218000e-01 8.517000e-01 8.106000e-01 1.152900e+00 5.722000e-01 9.200000e-01 6.121000e-01 1.224200e+00 4.924000e-01 6.570000e-01 2.914000e-01 + 4.650000e-01 1.452000e-01 7.837000e-01 1.378000e-01 9.267000e-01 1.065000e-01 1.763500e+00 -3.920000e-02 1.258000e-01 1.849400e+00 -1.710000e-02 2.010000e-02 4.704000e-01 5.990000e-02 + 1.125400e+00 -1.860000e-02 4.118000e-01 1.341000e+00 5.358000e-01 4.982000e-01 5.828000e-01 1.137600e+00 4.601000e-01 5.488000e-01 5.933000e-01 1.129500e+00 3.844000e-01 3.734000e-01 + 1.075400e+00 3.380000e-02 7.599000e-01 9.807000e-01 6.185000e-01 7.503000e-01 8.009000e-01 9.346000e-01 6.739000e-01 6.285000e-01 8.539000e-01 8.723000e-01 5.453000e-01 3.332000e-01 + 3.525000e-01 3.345000e-01 5.116000e-01 5.722000e-01 5.640000e-01 6.692000e-01 4.808000e-01 6.056000e-01 5.902000e-01 6.561000e-01 5.130000e-01 5.635000e-01 3.051000e-01 3.173000e-01 + 5.190000e-01 5.924000e-01 8.648000e-01 7.441000e-01 1.038200e+00 6.802000e-01 1.758000e+00 -3.610000e-02 1.986400e+00 1.070000e-02 -4.070000e-02 1.777700e+00 5.055000e-01 4.177000e-01 + 5.158000e-01 1.822000e-01 8.558000e-01 1.480000e-01 9.620000e-01 1.680000e-01 1.790400e+00 -7.240000e-02 -3.050000e-02 2.032800e+00 7.580000e-02 -9.170000e-02 4.772000e-01 1.143000e-01 + 5.610000e-01 2.925000e-01 7.260000e-01 4.631000e-01 9.444000e-01 3.525000e-01 1.723300e+00 7.400000e-03 3.060000e-02 1.958500e+00 3.750000e-02 -4.560000e-02 4.457000e-01 2.517000e-01 + 5.050000e-01 6.070000e-01 8.627000e-01 8.609000e-01 8.843000e-01 1.080400e+00 1.662900e+00 7.810000e-02 9.900000e-02 1.886300e+00 -5.900000e-02 1.799200e+00 4.807000e-01 5.152000e-01 + 4.083000e-01 6.277000e-01 5.078000e-01 1.137300e+00 5.682000e-01 1.305400e+00 6.229000e-01 1.002100e+00 6.980000e-01 1.180200e+00 6.487000e-01 9.710000e-01 3.243000e-01 6.188000e-01 + 5.610000e-01 1.063000e-01 7.032000e-01 3.773000e-01 7.404000e-01 4.993000e-01 7.263000e-01 3.493000e-01 8.452000e-01 3.924000e-01 6.268000e-01 4.698000e-01 4.199000e-01 2.013000e-01 +-4.600000e-03 4.540000e-01 1.915000e-01 5.706000e-01 3.021000e-01 5.829000e-01 -6.530000e-02 1.807800e+00 -4.130000e-02 4.920000e-02 9.000000e-03 -1.080000e-02 1.095000e-01 3.238000e-01 + 4.558000e-01 6.417000e-01 7.385000e-01 7.060000e-01 9.107000e-01 6.386000e-01 1.640500e+00 1.095000e-01 -1.430000e-02 2.021500e+00 -6.380000e-02 1.806700e+00 4.203000e-01 4.233000e-01 + 3.749000e-01 3.478000e-01 6.614000e-01 3.763000e-01 8.617000e-01 2.874000e-01 1.680900e+00 6.080000e-02 5.110000e-02 1.937900e+00 -4.430000e-02 5.130000e-02 3.498000e-01 2.637000e-01 + 1.172000e+00 -7.900000e-02 5.569000e-01 2.045000e-01 5.080000e-01 1.028000e-01 5.461000e-01 2.194000e-01 4.582000e-01 1.502000e-01 6.272000e-01 1.214000e-01 4.147000e-01 9.840000e-02 + 1.114400e+00 -1.480000e-02 6.321000e-01 8.080000e-02 5.397000e-01 4.140000e-02 5.880000e-01 1.360000e-01 4.192000e-01 1.711000e-01 5.523000e-01 1.767000e-01 4.362000e-01 5.740000e-02 + 4.172000e-01 4.878000e-01 7.567000e-01 4.570000e-01 8.219000e-01 5.137000e-01 1.698300e+00 4.280000e-02 -2.360000e-02 2.029800e+00 -3.000000e-04 2.000000e-03 4.055000e-01 3.136000e-01 + 3.340000e-02 9.060000e-02 1.528000e-01 2.821000e-01 2.778000e-01 2.702000e-01 3.840000e-02 1.689500e+00 8.330000e-02 -1.008000e-01 7.790000e-02 -9.420000e-02 6.010000e-02 1.798000e-01 + 1.125000e+00 -1.910000e-02 6.254000e-01 1.100600e+00 3.906000e-01 8.229000e-01 6.111000e-01 1.120500e+00 4.252000e-01 7.403000e-01 6.368000e-01 1.083800e+00 4.206000e-01 4.015000e-01 + 6.356000e-01 4.044000e-01 8.279000e-01 8.315000e-01 9.620000e-01 9.261000e-01 8.210000e-01 8.397000e-01 1.001200e+00 9.082000e-01 8.058000e-01 8.573000e-01 4.849000e-01 4.709000e-01 + 1.061500e+00 5.180000e-02 7.158000e-01 1.021300e+00 6.573000e-01 4.111000e-01 7.008000e-01 1.036400e+00 5.019000e-01 5.624000e-01 7.263000e-01 1.009700e+00 4.987000e-01 2.754000e-01 + 1.755000e-01 3.300000e-01 3.664000e-01 4.088000e-01 3.820000e-01 5.115000e-01 3.903000e-01 3.836000e-01 4.519000e-01 4.405000e-01 2.869000e-01 5.086000e-01 1.966000e-01 2.555000e-01 + 2.700000e-02 1.430000e-02 3.190000e-02 3.198000e-01 3.130000e-02 4.564000e-01 -1.280000e-02 1.590000e-02 -6.610000e-02 7.780000e-02 -8.400000e-03 8.100000e-03 1.590000e-02 1.756000e-01 + 4.663000e-01 -4.600000e-03 7.408000e-01 4.170000e-02 9.221000e-01 -2.920000e-02 1.735200e+00 -1.600000e-03 7.200000e-02 -8.600000e-02 3.510000e-02 -4.170000e-02 4.131000e-01 3.500000e-02 + 3.971000e-01 1.871000e-01 6.050000e-01 3.133000e-01 6.799000e-01 3.637000e-01 5.564000e-01 3.703000e-01 7.048000e-01 3.569000e-01 5.925000e-01 3.298000e-01 3.250000e-01 2.095000e-01 + 1.881000e-01 2.852000e-01 4.351000e-01 3.640000e-01 5.826000e-01 3.299000e-01 1.650900e+00 9.930000e-02 -7.760000e-02 9.370000e-02 3.610000e-02 -4.430000e-02 2.443000e-01 2.118000e-01 + 7.071000e-01 4.006000e-01 3.163000e-01 1.421300e+00 2.512000e-01 1.208400e+00 3.451000e-01 1.385900e+00 3.214000e-01 1.050100e+00 3.445000e-01 1.383400e+00 2.696000e-01 6.342000e-01 + 6.183000e-01 4.862000e-01 8.977000e-01 8.166000e-01 1.012400e+00 8.138000e-01 1.728200e+00 4.700000e-03 2.084100e+00 -9.930000e-02 -2.800000e-03 1.733200e+00 5.319000e-01 4.307000e-01 + 7.480000e-01 2.723000e-01 1.088900e+00 5.234000e-01 1.335700e+00 4.807000e-01 1.197500e+00 3.954000e-01 1.335600e+00 5.135000e-01 1.092100e+00 5.162000e-01 6.428000e-01 2.840000e-01 + 1.220000e-02 4.463000e-01 3.817000e-01 3.752000e-01 4.660000e-01 4.161000e-01 -3.900000e-02 1.775400e+00 2.450000e-02 -2.800000e-02 -2.000000e-02 2.500000e-02 1.909000e-01 2.436000e-01 + 4.191000e-01 7.018000e-01 7.914000e-01 6.675000e-01 1.036300e+00 5.181000e-01 1.751200e+00 -2.240000e-02 -1.037000e-01 2.116700e+00 0.000000e+00 1.728200e+00 4.685000e-01 3.836000e-01 + 5.673000e-01 5.564000e-01 3.051000e-01 8.024000e-01 2.870000e-01 5.490000e-01 3.614000e-01 7.325000e-01 2.592000e-01 5.638000e-01 3.773000e-01 7.175000e-01 2.555000e-01 3.958000e-01 + 5.016000e-01 1.766000e-01 7.704000e-01 2.323000e-01 9.845000e-01 1.148000e-01 1.770000e+00 -4.560000e-02 -1.452000e-01 2.174300e+00 2.120000e-02 -2.570000e-02 4.619000e-01 1.186000e-01 + 3.840000e-02 4.844000e-01 2.104000e-01 6.335000e-01 3.165000e-01 6.526000e-01 7.100000e-03 1.721900e+00 2.400000e-02 -3.070000e-02 -3.000000e-04 -1.300000e-03 1.120000e-01 3.751000e-01 + 6.022000e-01 4.851000e-01 9.271000e-01 4.647000e-01 1.181800e+00 3.103000e-01 1.728700e+00 6.000000e-04 2.143800e+00 -1.692000e-01 2.100000e-02 1.706200e+00 6.078000e-01 2.020000e-01 + 4.198000e-01 1.360000e-01 7.111000e-01 1.498000e-01 7.691000e-01 2.240000e-01 7.126000e-01 1.501000e-01 8.195000e-01 1.747000e-01 7.925000e-01 5.360000e-02 4.255000e-01 6.960000e-02 + 4.734000e-01 3.226000e-01 7.321000e-01 5.214000e-01 6.302000e-01 8.344000e-01 7.167000e-01 5.399000e-01 8.025000e-01 6.486000e-01 7.317000e-01 5.186000e-01 3.959000e-01 3.324000e-01 + 1.976000e-01 2.874000e-01 5.407000e-01 2.456000e-01 6.124000e-01 3.106000e-01 1.765600e+00 -4.640000e-02 -8.500000e-03 1.090000e-02 -3.050000e-02 3.470000e-02 2.834000e-01 1.711000e-01 + 2.472000e-01 3.830000e-01 4.762000e-01 4.897000e-01 5.658000e-01 5.331000e-01 4.445000e-01 5.271000e-01 4.262000e-01 7.066000e-01 5.670000e-01 3.801000e-01 2.861000e-01 2.700000e-01 + 5.640000e-02 8.770000e-01 4.094000e-01 8.304000e-01 4.598000e-01 9.081000e-01 1.685500e+00 5.740000e-02 -1.160000e-02 2.014600e+00 3.930000e-02 -4.600000e-02 2.180000e-01 5.141000e-01 + 5.933000e-01 5.124000e-01 3.009000e-01 4.766000e-01 2.757000e-01 3.484000e-01 2.430000e-01 5.403000e-01 2.414000e-01 3.768000e-01 3.809000e-01 3.783000e-01 2.467000e-01 2.736000e-01 + 4.738000e-01 1.604000e-01 6.897000e-01 3.143000e-01 7.454000e-01 4.065000e-01 7.160000e-01 2.829000e-01 7.983000e-01 3.611000e-01 6.483000e-01 3.632000e-01 4.115000e-01 1.662000e-01 + 3.263000e-01 2.228000e-01 5.195000e-01 3.371000e-01 6.499000e-01 3.145000e-01 6.013000e-01 2.391000e-01 5.882000e-01 3.997000e-01 4.504000e-01 4.183000e-01 2.802000e-01 2.177000e-01 + 2.952000e-01 3.011000e-01 3.907000e-01 5.610000e-01 5.623000e-01 4.951000e-01 3.930000e-01 5.548000e-01 5.071000e-01 5.777000e-01 3.762000e-01 5.773000e-01 2.368000e-01 3.101000e-01 + 8.571000e-01 2.373000e-01 3.476000e-01 9.596000e-01 2.949000e-01 6.430000e-01 3.605000e-01 9.372000e-01 2.396000e-01 6.787000e-01 4.688000e-01 8.115000e-01 2.814000e-01 4.219000e-01 +-8.300000e-02 7.175000e-01 2.840000e-02 8.961000e-01 4.980000e-02 1.014500e+00 -5.960000e-02 1.804600e+00 -2.070000e-02 2.025200e+00 1.300000e-03 -1.400000e-03 2.410000e-02 5.176000e-01 + 4.093000e-01 4.796000e-01 7.052000e-01 4.992000e-01 8.137000e-01 5.116000e-01 1.559200e+00 2.063000e-01 -9.400000e-03 2.012900e+00 -4.380000e-02 5.310000e-02 3.694000e-01 3.477000e-01 +-4.070000e-02 3.429000e-01 1.610000e-01 4.522000e-01 3.372000e-01 3.839000e-01 -1.471000e-01 1.905600e+00 2.170000e-02 -2.650000e-02 1.330000e-02 -1.820000e-02 1.011000e-01 2.359000e-01 + 4.655000e-01 3.234000e-01 6.666000e-01 5.896000e-01 7.814000e-01 6.414000e-01 7.481000e-01 4.871000e-01 8.391000e-01 5.939000e-01 7.590000e-01 4.738000e-01 4.179000e-01 2.995000e-01 + 3.631000e-01 3.763000e-01 5.393000e-01 6.265000e-01 6.059000e-01 7.227000e-01 5.750000e-01 5.816000e-01 5.516000e-01 8.031000e-01 5.457000e-01 6.142000e-01 3.099000e-01 3.612000e-01 + 4.637000e-01 4.033000e-01 7.539000e-01 5.969000e-01 7.829000e-01 7.668000e-01 6.929000e-01 6.707000e-01 9.046000e-01 6.521000e-01 7.372000e-01 6.145000e-01 4.322000e-01 3.487000e-01 + 1.090400e+00 1.660000e-02 6.835000e-01 4.016000e-01 5.539000e-01 2.774000e-01 6.936000e-01 3.948000e-01 5.649000e-01 2.433000e-01 7.215000e-01 3.609000e-01 4.784000e-01 1.675000e-01 + 1.758000e-01 4.663000e-01 4.616000e-01 5.046000e-01 5.843000e-01 4.959000e-01 1.676900e+00 5.550000e-02 1.330000e-02 1.984500e+00 2.270000e-02 -2.750000e-02 2.619000e-01 2.967000e-01 + 1.214600e+00 -1.319000e-01 5.935000e-01 5.697000e-01 5.128000e-01 3.593000e-01 5.345000e-01 6.402000e-01 4.997000e-01 3.476000e-01 5.162000e-01 6.670000e-01 4.160000e-01 2.552000e-01 + 1.620000e-02 -1.880000e-02 2.976000e-01 1.450000e-02 4.584000e-01 -3.510000e-02 -5.870000e-02 6.980000e-02 9.900000e-03 -9.800000e-03 -5.190000e-02 6.120000e-02 1.608000e-01 6.200000e-03 + 9.487000e-01 1.556000e-01 4.250000e-01 9.013000e-01 2.160000e-01 7.500000e-01 5.109000e-01 8.025000e-01 3.473000e-01 5.742000e-01 4.527000e-01 8.721000e-01 2.995000e-01 4.109000e-01 + 4.073000e-01 6.996000e-01 6.978000e-01 8.732000e-01 8.389000e-01 8.391000e-01 1.652000e+00 9.270000e-02 -9.850000e-02 2.120300e+00 4.640000e-02 1.671500e+00 4.081000e-01 4.970000e-01 + 1.179000e-01 3.772000e-01 1.543000e-01 6.233000e-01 2.983000e-01 5.636000e-01 2.670000e-01 4.901000e-01 2.800000e-01 6.010000e-01 1.791000e-01 5.917000e-01 1.150000e-01 3.288000e-01 + 3.808000e-01 3.059000e-01 7.125000e-01 2.866000e-01 8.360000e-01 2.762000e-01 1.762600e+00 -4.020000e-02 -1.060000e-02 2.014400e+00 -2.190000e-02 2.570000e-02 3.894000e-01 1.950000e-01 + 1.100700e+00 6.400000e-03 6.214000e-01 1.107200e+00 5.279000e-01 5.033000e-01 6.244000e-01 1.102300e+00 4.872000e-01 5.243000e-01 4.857000e-01 1.269100e+00 4.358000e-01 3.183000e-01 + 4.452000e-01 4.062000e-01 7.953000e-01 5.247000e-01 7.304000e-01 8.046000e-01 7.100000e-01 6.177000e-01 8.594000e-01 6.745000e-01 7.237000e-01 6.055000e-01 4.072000e-01 3.626000e-01 +-3.360000e-02 7.815000e-01 3.460000e-01 6.911000e-01 3.249000e-01 8.536000e-01 -7.280000e-02 1.825300e+00 1.340000e-02 1.986900e+00 1.620000e-02 -2.070000e-02 1.240000e-01 4.968000e-01 + 4.713000e-01 4.386000e-01 7.293000e-01 6.936000e-01 7.792000e-01 8.545000e-01 6.979000e-01 7.343000e-01 8.918000e-01 7.419000e-01 6.975000e-01 7.380000e-01 4.071000e-01 4.185000e-01 + 4.175000e-01 3.384000e-01 7.303000e-01 4.443000e-01 7.458000e-01 6.023000e-01 6.859000e-01 4.919000e-01 8.668000e-01 4.794000e-01 7.331000e-01 4.360000e-01 4.239000e-01 2.520000e-01 + 5.340000e-02 9.828000e-01 2.123000e-01 1.128600e+00 2.351000e-01 1.247100e+00 -7.900000e-02 1.824500e+00 -3.620000e-02 2.039300e+00 -1.320000e-02 1.450000e-02 8.350000e-02 7.143000e-01 + 3.388000e-01 3.895000e-01 4.675000e-01 6.855000e-01 4.922000e-01 8.274000e-01 5.061000e-01 6.373000e-01 6.460000e-01 6.687000e-01 4.679000e-01 6.876000e-01 2.774000e-01 3.879000e-01 + 3.247000e-01 5.262000e-01 6.185000e-01 5.524000e-01 7.928000e-01 4.811000e-01 1.705800e+00 3.450000e-02 -1.000000e-04 1.999800e+00 -2.430000e-02 2.690000e-02 3.642000e-01 3.209000e-01 + 4.667000e-01 2.944000e-01 8.114000e-01 3.655000e-01 8.536000e-01 4.975000e-01 7.835000e-01 3.983000e-01 9.240000e-01 4.428000e-01 7.512000e-01 4.366000e-01 4.402000e-01 2.459000e-01 + 1.345000e-01 6.435000e-01 3.909000e-01 7.931000e-01 5.475000e-01 7.823000e-01 3.952000e-01 7.846000e-01 3.832000e-01 9.931000e-01 3.469000e-01 8.406000e-01 2.234000e-01 4.592000e-01 + 1.940000e-01 4.096000e-01 2.225000e-01 7.397000e-01 2.921000e-01 7.992000e-01 3.331000e-01 6.114000e-01 3.428000e-01 7.537000e-01 2.425000e-01 7.173000e-01 1.560000e-01 3.954000e-01 + 5.959000e-01 1.939000e-01 9.863000e-01 2.432000e-01 1.230900e+00 1.453000e-01 1.001400e+00 2.181000e-01 1.243900e+00 1.572000e-01 9.501000e-01 2.823000e-01 5.876000e-01 1.168000e-01 + 3.336000e-01 3.599000e-01 5.074000e-01 5.793000e-01 6.561000e-01 5.657000e-01 6.266000e-01 4.343000e-01 6.604000e-01 5.796000e-01 6.106000e-01 4.549000e-01 3.467000e-01 2.701000e-01 + 4.059000e-01 3.798000e-01 7.394000e-01 3.555000e-01 8.615000e-01 3.500000e-01 1.760300e+00 -3.010000e-02 -1.800000e-02 2.019900e+00 -1.730000e-02 2.140000e-02 4.065000e-01 2.399000e-01 + 3.310000e-01 6.894000e-01 7.105000e-01 6.104000e-01 8.226000e-01 6.172000e-01 1.725000e+00 9.400000e-03 1.055000e-01 1.875200e+00 3.200000e-03 -3.000000e-03 4.023000e-01 3.763000e-01 + 7.199000e-01 3.784000e-01 2.954000e-01 7.870000e-01 2.970000e-01 5.207000e-01 2.539000e-01 8.341000e-01 2.433000e-01 5.660000e-01 3.386000e-01 7.365000e-01 2.611000e-01 3.805000e-01 + 3.351000e-01 4.388000e-01 4.206000e-01 8.098000e-01 5.920000e-01 7.924000e-01 4.709000e-01 7.497000e-01 6.010000e-01 8.065000e-01 4.283000e-01 8.027000e-01 2.862000e-01 4.179000e-01 + 2.682000e-01 2.294000e-01 5.970000e-01 1.434000e-01 7.460000e-01 8.930000e-02 5.686000e-01 1.835000e-01 6.593000e-01 2.042000e-01 6.265000e-01 1.115000e-01 3.488000e-01 8.000000e-02 + 1.968000e-01 5.983000e-01 4.334000e-01 6.884000e-01 6.950000e-01 5.214000e-01 1.668500e+00 7.710000e-02 -1.500000e-02 2.017100e+00 -9.400000e-03 1.200000e-02 2.899000e-01 3.599000e-01 + 2.743000e-01 2.707000e-01 5.182000e-01 3.226000e-01 6.576000e-01 2.817000e-01 4.334000e-01 4.222000e-01 5.786000e-01 3.969000e-01 5.792000e-01 2.490000e-01 3.026000e-01 1.813000e-01 + 1.044000e-01 8.482000e-01 2.786000e-01 1.196100e+00 3.187000e-01 1.361100e+00 2.881000e-01 1.186100e+00 2.551000e-01 1.462000e+00 2.166000e-01 1.271600e+00 1.215000e-01 7.374000e-01 + 7.940000e-02 7.657000e-01 3.944000e-01 7.673000e-01 4.722000e-01 8.130000e-01 1.623500e+00 1.296000e-01 1.566000e-01 1.815600e+00 8.910000e-02 -1.058000e-01 2.025000e-01 4.817000e-01 + 5.802000e-01 5.219000e-01 2.736000e-01 3.596000e-01 3.377000e-01 1.663000e-01 3.450000e-01 2.768000e-01 2.607000e-01 2.504000e-01 2.996000e-01 3.324000e-01 2.310000e-01 2.168000e-01 + 3.435000e-01 7.501000e-01 1.504000e-01 4.932000e-01 6.410000e-02 4.826000e-01 2.269000e-01 4.038000e-01 1.557000e-01 3.627000e-01 6.050000e-02 6.040000e-01 1.334000e-01 3.200000e-01 + 2.296000e-01 8.800000e-02 3.187000e-01 1.855000e-01 3.660000e-01 2.098000e-01 2.541000e-01 2.652000e-01 3.659000e-01 2.184000e-01 1.360000e-01 4.045000e-01 1.647000e-01 1.304000e-01 + 3.472000e-01 5.533000e-01 7.213000e-01 6.622000e-01 7.495000e-01 8.366000e-01 6.911000e-01 7.010000e-01 7.428000e-01 8.703000e-01 6.704000e-01 7.270000e-01 3.956000e-01 4.075000e-01 + 1.127300e+00 -2.910000e-02 5.484000e-01 3.926000e-01 4.000000e-01 3.540000e-01 4.938000e-01 4.571000e-01 3.574000e-01 3.831000e-01 3.600000e-01 6.188000e-01 3.658000e-01 2.323000e-01 + 3.116000e-01 2.006000e-01 6.378000e-01 1.844000e-01 8.251000e-01 1.021000e-01 1.731600e+00 -2.000000e-03 -5.400000e-03 5.500000e-03 6.160000e-02 -7.480000e-02 3.810000e-01 8.600000e-02 + 5.224000e-01 9.700000e-03 7.253000e-01 1.418000e-01 8.516000e-01 1.318000e-01 1.666600e+00 7.340000e-02 1.910000e-02 -1.910000e-02 -1.140000e-02 1.270000e-02 4.417000e-01 5.310000e-02 + 5.163000e-01 2.570000e-01 7.918000e-01 4.214000e-01 9.877000e-01 3.764000e-01 8.280000e-01 3.769000e-01 9.526000e-01 4.407000e-01 8.246000e-01 3.858000e-01 4.710000e-01 2.260000e-01 + 5.571000e-01 5.544000e-01 2.239000e-01 9.428000e-01 2.880000e-01 5.743000e-01 3.653000e-01 7.806000e-01 2.938000e-01 5.459000e-01 3.241000e-01 8.265000e-01 2.381000e-01 4.292000e-01 + 2.273000e-01 4.148000e-01 3.511000e-01 6.521000e-01 3.822000e-01 7.671000e-01 4.646000e-01 5.182000e-01 4.037000e-01 7.567000e-01 3.498000e-01 6.586000e-01 2.074000e-01 3.724000e-01 + 4.839000e-01 4.570000e-02 1.840000e-01 1.079000e-01 2.019000e-01 4.250000e-02 3.259000e-01 -6.090000e-02 2.704000e-01 -4.380000e-02 2.482000e-01 3.690000e-02 2.045000e-01 1.880000e-02 + 1.147700e+00 -5.040000e-02 1.158300e+00 1.214000e-01 9.293000e-01 -1.200000e-02 1.340100e+00 -9.310000e-02 8.479000e-01 5.720000e-02 1.220500e+00 5.150000e-02 6.836000e-01 1.350000e-02 + 6.989000e-01 3.994000e-01 1.083100e+00 3.312000e-01 1.074000e+00 4.911000e-01 1.831800e+00 -1.164000e-01 2.058700e+00 -6.400000e-02 3.980000e-02 1.682900e+00 5.699000e-01 2.715000e-01 + 3.769000e-01 4.515000e-01 1.789000e-01 1.933000e-01 1.166000e-01 2.087000e-01 1.752000e-01 1.987000e-01 1.617000e-01 1.490000e-01 1.450000e-01 2.362000e-01 1.476000e-01 1.407000e-01 + 4.429000e-01 2.770000e-01 6.559000e-01 4.770000e-01 7.533000e-01 5.333000e-01 6.636000e-01 4.679000e-01 9.272000e-01 3.484000e-01 6.942000e-01 4.315000e-01 4.358000e-01 2.080000e-01 + 2.678000e-01 5.396000e-01 5.706000e-01 5.481000e-01 7.094000e-01 5.272000e-01 1.753200e+00 -2.780000e-02 1.525000e-01 1.819500e+00 -8.100000e-03 9.300000e-03 3.374000e-01 3.192000e-01 + 5.565000e-01 2.985000e-01 9.732000e-01 1.751000e-01 1.013600e+00 2.741000e-01 1.663900e+00 7.960000e-02 1.972300e+00 3.250000e-02 -3.070000e-02 3.620000e-02 5.264000e-01 1.595000e-01 + 7.532000e-01 3.612000e-01 3.635000e-01 1.018900e+00 3.685000e-01 5.868000e-01 3.342000e-01 1.066600e+00 2.891000e-01 6.552000e-01 3.743000e-01 1.018400e+00 2.811000e-01 4.401000e-01 + 1.108600e+00 -5.300000e-03 4.635000e-01 1.283100e+00 4.084000e-01 1.442300e+00 4.793000e-01 1.265200e+00 4.490000e-01 1.193800e+00 6.262000e-01 1.087300e+00 3.890000e-01 5.791000e-01 + 4.608000e-01 2.314000e-01 8.328000e-01 2.275000e-01 8.639000e-01 3.633000e-01 8.046000e-01 2.602000e-01 8.485000e-01 3.997000e-01 7.342000e-01 3.473000e-01 4.069000e-01 2.202000e-01 + 4.410000e-02 5.566000e-01 2.636000e-01 6.394000e-01 3.755000e-01 6.360000e-01 2.398000e-01 6.638000e-01 2.537000e-01 7.974000e-01 2.936000e-01 6.030000e-01 1.496000e-01 3.718000e-01 + 5.005000e-01 3.034000e-01 7.548000e-01 5.121000e-01 1.041600e+00 3.657000e-01 8.597000e-01 3.826000e-01 1.028000e+00 4.066000e-01 7.794000e-01 4.771000e-01 4.800000e-01 2.417000e-01 + 3.198000e-01 7.901000e-01 1.934000e-01 7.084000e-01 1.126000e-01 6.119000e-01 1.950000e-01 7.055000e-01 1.826000e-01 5.124000e-01 2.665000e-01 6.217000e-01 1.639000e-01 4.130000e-01 + 5.197000e-01 2.097000e-01 8.513000e-01 1.889000e-01 9.604000e-01 2.034000e-01 1.725800e+00 3.100000e-03 -2.900000e-02 2.034500e+00 -4.180000e-02 4.840000e-02 4.906000e-01 1.207000e-01 + 2.273000e-01 7.301000e-01 3.601000e-01 1.140200e+00 3.853000e-01 1.328900e+00 3.072000e-01 1.205700e+00 4.551000e-01 1.270000e+00 3.043000e-01 1.206400e+00 2.025000e-01 6.645000e-01 +-4.580000e-02 5.081000e-01 -4.810000e-02 8.260000e-01 -1.860000e-02 9.120000e-01 4.690000e-02 1.675900e+00 -5.220000e-02 6.230000e-02 -4.100000e-03 6.400000e-03 2.600000e-03 4.344000e-01 + 1.116200e+00 -1.530000e-02 5.599000e-01 5.398000e-01 3.945000e-01 4.545000e-01 5.338000e-01 5.660000e-01 4.524000e-01 3.615000e-01 5.239000e-01 5.846000e-01 3.863000e-01 2.650000e-01 + 6.918000e-01 3.518000e-01 1.041500e+00 3.028000e-01 1.083500e+00 3.950000e-01 1.622800e+00 1.234000e-01 2.023100e+00 -2.900000e-02 6.140000e-02 -7.670000e-02 6.017000e-01 1.932000e-01 + 7.017000e-01 3.974000e-01 1.022200e+00 5.651000e-01 1.199500e+00 5.025000e-01 1.701000e+00 3.670000e-02 2.096700e+00 -1.106000e-01 6.400000e-03 1.724200e+00 5.794000e-01 3.400000e-01 + 3.025000e-01 4.500000e-01 6.046000e-01 4.666000e-01 7.225000e-01 4.639000e-01 1.802600e+00 -8.280000e-02 -4.240000e-02 2.049500e+00 -3.420000e-02 4.300000e-02 3.911000e-01 2.265000e-01 + 2.405000e-01 4.124000e-01 3.764000e-01 6.427000e-01 4.999000e-01 6.475000e-01 4.117000e-01 5.978000e-01 4.728000e-01 6.944000e-01 4.349000e-01 5.751000e-01 2.347000e-01 3.495000e-01 + 3.042000e-01 4.975000e-01 5.813000e-01 5.410000e-01 5.880000e-01 6.772000e-01 1.773000e+00 -4.970000e-02 4.010000e-02 1.949800e+00 5.640000e-02 -6.680000e-02 3.048000e-01 3.605000e-01 + 5.140000e-02 6.044000e-01 1.610000e-01 8.481000e-01 1.999000e-01 9.469000e-01 1.133000e-01 9.042000e-01 1.455000e-01 1.025900e+00 1.506000e-01 8.629000e-01 8.570000e-02 4.987000e-01 + 7.159000e-01 3.887000e-01 3.221000e-01 2.428000e-01 3.203000e-01 1.404000e-01 3.725000e-01 1.797000e-01 2.981000e-01 1.584000e-01 2.153000e-01 3.692000e-01 2.653000e-01 1.412000e-01 + 1.038500e+00 8.070000e-02 6.368000e-01 2.733000e-01 5.443000e-01 1.728000e-01 5.774000e-01 3.499000e-01 5.029000e-01 2.067000e-01 6.043000e-01 3.120000e-01 4.286000e-01 1.561000e-01 + 3.442000e-01 6.581000e-01 6.663000e-01 8.721000e-01 5.898000e-01 1.199100e+00 6.507000e-01 8.913000e-01 5.997000e-01 1.210200e+00 5.873000e-01 9.664000e-01 3.414000e-01 5.564000e-01 + 4.650000e-02 1.068000e+00 1.044000e-01 4.719000e-01 1.267000e-01 3.475000e-01 2.050000e-02 5.726000e-01 7.500000e-03 4.792000e-01 5.820000e-02 5.241000e-01 4.720000e-02 3.740000e-01 + 5.320000e-02 3.194000e-01 1.602000e-01 4.124000e-01 1.449000e-01 5.132000e-01 1.762000e-01 3.928000e-01 2.111000e-01 4.435000e-01 1.781000e-01 3.896000e-01 1.014000e-01 2.271000e-01 + 1.134400e+00 -3.280000e-02 3.935000e-01 4.392000e-01 3.808000e-01 2.835000e-01 4.294000e-01 3.987000e-01 3.470000e-01 3.056000e-01 3.493000e-01 4.929000e-01 3.117000e-01 2.354000e-01 + 3.821000e-01 6.817000e-01 6.292000e-01 7.613000e-01 8.703000e-01 6.141000e-01 1.673400e+00 6.200000e-02 5.200000e-03 1.990900e+00 -8.500000e-02 1.832600e+00 4.190000e-01 3.872000e-01 + 3.013000e-01 6.641000e-01 5.217000e-01 9.826000e-01 5.665000e-01 1.154200e+00 6.320000e-01 8.539000e-01 7.991000e-01 9.070000e-01 5.886000e-01 9.031000e-01 3.344000e-01 5.291000e-01 + 1.506000e-01 7.948000e-01 1.765000e-01 1.315600e+00 2.341000e-01 1.461900e+00 2.085000e-01 1.275200e+00 2.402000e-01 1.474500e+00 1.973000e-01 1.294000e+00 1.203000e-01 7.379000e-01 + 3.685000e-01 6.324000e-01 7.840000e-01 5.156000e-01 8.288000e-01 6.059000e-01 1.723100e+00 1.340000e-02 -3.700000e-02 2.044700e+00 -1.219000e-01 1.448000e-01 4.090000e-01 3.624000e-01 + 4.675000e-01 3.226000e-01 7.764000e-01 4.498000e-01 8.159000e-01 5.908000e-01 6.720000e-01 5.726000e-01 8.785000e-01 5.377000e-01 5.608000e-01 7.012000e-01 4.061000e-01 3.096000e-01 + 1.032100e+00 8.920000e-02 7.001000e-01 1.020800e+00 4.296000e-01 1.249300e+00 5.637000e-01 1.175500e+00 5.356000e-01 1.006600e+00 5.957000e-01 1.141200e+00 4.473000e-01 5.009000e-01 + 3.537000e-01 3.750000e-01 5.846000e-01 5.609000e-01 6.021000e-01 7.155000e-01 5.300000e-01 6.204000e-01 6.224000e-01 7.085000e-01 5.285000e-01 6.264000e-01 3.126000e-01 3.522000e-01 + 1.028400e+00 5.230000e-02 3.442000e-01 1.396500e+00 2.839000e-01 1.006800e+00 3.878000e-01 1.345500e+00 4.144000e-01 8.010000e-01 3.794000e-01 1.354100e+00 2.956000e-01 5.589000e-01 + 1.255500e+00 -1.783000e-01 9.172000e-01 1.170000e-02 6.671000e-01 7.930000e-02 8.712000e-01 6.710000e-02 6.812000e-01 4.210000e-02 9.521000e-01 -2.890000e-02 5.712000e-01 2.040000e-02 + 3.895000e-01 2.033000e-01 5.606000e-01 3.758000e-01 7.402000e-01 3.101000e-01 5.242000e-01 4.181000e-01 7.393000e-01 3.271000e-01 5.231000e-01 4.183000e-01 3.386000e-01 1.988000e-01 + 3.526000e-01 7.464000e-01 6.101000e-01 9.113000e-01 7.630000e-01 8.749000e-01 1.689900e+00 4.920000e-02 1.480000e-02 1.983400e+00 -1.200000e-02 1.745600e+00 3.587000e-01 5.248000e-01 +-1.300000e-03 6.041000e-01 4.347000e-01 4.496000e-01 4.004000e-01 6.332000e-01 2.100000e-03 1.727800e+00 -8.340000e-02 2.097300e+00 1.580000e-02 -1.760000e-02 1.446000e-01 3.869000e-01 + 3.791000e-01 2.530000e-01 8.544000e-01 6.310000e-02 8.729000e-01 1.838000e-01 1.780600e+00 -5.310000e-02 7.710000e-02 1.912700e+00 -3.610000e-02 4.320000e-02 4.118000e-01 1.349000e-01 + 1.102700e+00 1.900000e-03 4.964000e-01 5.009000e-01 3.975000e-01 3.810000e-01 5.711000e-01 4.126000e-01 3.578000e-01 4.157000e-01 6.194000e-01 3.533000e-01 3.989000e-01 2.126000e-01 + 3.882000e-01 4.172000e-01 6.311000e-01 6.264000e-01 8.014000e-01 6.151000e-01 7.290000e-01 5.097000e-01 8.256000e-01 6.037000e-01 7.664000e-01 4.626000e-01 4.112000e-01 3.055000e-01 + 1.168300e+00 -7.610000e-02 5.486000e-01 1.192800e+00 5.699000e-01 4.532000e-01 5.990000e-01 1.130500e+00 4.773000e-01 5.330000e-01 5.430000e-01 1.196900e+00 4.272000e-01 3.263000e-01 + 1.946000e-01 8.463000e-01 3.935000e-01 9.886000e-01 5.201000e-01 9.817000e-01 1.797000e+00 -7.150000e-02 -9.460000e-02 2.110900e+00 3.840000e-02 1.683800e+00 2.347000e-01 5.724000e-01 + 2.387000e-01 5.287000e-01 5.081000e-01 5.803000e-01 6.225000e-01 5.861000e-01 1.602700e+00 1.486000e-01 -3.100000e-03 2.002900e+00 -3.110000e-02 3.750000e-02 2.908000e-01 3.475000e-01 + 3.340000e-02 8.543000e-01 9.320000e-02 1.295600e+00 1.707000e-01 1.390800e+00 3.800000e-02 1.361300e+00 1.285000e-01 1.471100e+00 7.590000e-02 1.317000e+00 5.550000e-02 7.455000e-01 + 2.603000e-01 6.276000e-01 6.728000e-01 5.071000e-01 7.549000e-01 5.508000e-01 1.824600e+00 -1.058000e-01 -1.900000e-02 2.019000e+00 -7.000000e-04 4.000000e-04 3.338000e-01 3.718000e-01 + 5.957000e-01 4.387000e-01 8.091000e-01 8.394000e-01 9.161000e-01 9.635000e-01 7.718000e-01 8.814000e-01 1.003700e+00 8.898000e-01 7.043000e-01 9.604000e-01 4.712000e-01 4.792000e-01 + 1.171000e+00 -7.670000e-02 5.022000e-01 3.121000e-01 4.242000e-01 2.324000e-01 4.959000e-01 3.193000e-01 4.355000e-01 2.043000e-01 5.301000e-01 2.812000e-01 3.576000e-01 1.846000e-01 + 1.121500e+00 -2.200000e-02 5.811000e-01 1.162100e+00 5.564000e-01 5.510000e-01 6.498000e-01 1.077300e+00 4.685000e-01 6.165000e-01 5.802000e-01 1.157100e+00 4.429000e-01 3.451000e-01 + 1.084900e+00 2.450000e-02 3.846000e-01 1.362700e+00 4.707000e-01 7.384000e-01 4.256000e-01 1.313500e+00 4.017000e-01 7.778000e-01 4.385000e-01 1.299300e+00 3.552000e-01 4.773000e-01 + 1.153600e+00 -5.470000e-02 4.680000e-01 3.564000e-01 3.592000e-01 3.113000e-01 3.693000e-01 4.764000e-01 3.756000e-01 2.744000e-01 4.279000e-01 4.029000e-01 3.265000e-01 2.203000e-01 + 3.748000e-01 2.994000e-01 6.536000e-01 3.408000e-01 8.457000e-01 2.527000e-01 1.739800e+00 -9.200000e-03 -6.410000e-02 2.075800e+00 3.580000e-02 -4.420000e-02 4.033000e-01 1.715000e-01 + 4.363000e-01 6.118000e-01 7.460000e-01 6.139000e-01 8.352000e-01 6.512000e-01 1.741400e+00 -1.040000e-02 -3.700000e-02 2.039400e+00 4.330000e-02 -5.150000e-02 4.164000e-01 3.860000e-01 + 1.798000e-01 2.871000e-01 3.085000e-01 4.197000e-01 3.603000e-01 4.645000e-01 2.507000e-01 4.863000e-01 3.400000e-01 5.030000e-01 3.426000e-01 3.776000e-01 1.842000e-01 2.344000e-01 + 6.403000e-01 5.420000e-02 1.025400e+00 6.470000e-02 1.219600e+00 9.200000e-03 9.108000e-01 1.963000e-01 1.133800e+00 1.292000e-01 9.212000e-01 1.857000e-01 5.768000e-01 5.450000e-02 + 4.600000e-02 8.990000e-01 1.115000e-01 1.363600e+00 1.098000e-01 1.570600e+00 1.085000e-01 1.371400e+00 2.028000e-01 1.486500e+00 4.920000e-02 1.434900e+00 6.760000e-02 7.827000e-01 + 3.339000e-01 7.763000e-01 7.458000e-01 7.458000e-01 7.757000e-01 8.486000e-01 1.733500e+00 7.000000e-04 -8.480000e-02 2.099900e+00 -2.780000e-02 1.761200e+00 3.877000e-01 4.879000e-01 + 2.704000e-01 8.443000e-01 6.167000e-01 8.324000e-01 7.813000e-01 7.781000e-01 1.633800e+00 1.169000e-01 -2.190000e-02 2.026000e+00 -8.870000e-02 1.835800e+00 3.523000e-01 4.940000e-01 + 3.663000e-01 5.339000e-01 4.716000e-01 9.602000e-01 5.079000e-01 1.127100e+00 5.363000e-01 8.867000e-01 5.595000e-01 1.094100e+00 5.606000e-01 8.596000e-01 3.032000e-01 5.188000e-01 + 6.235000e-01 1.920000e-01 1.090900e+00 1.685000e-01 1.179400e+00 2.634000e-01 1.007500e+00 2.645000e-01 1.161100e+00 3.117000e-01 1.011000e+00 2.615000e-01 5.957000e-01 1.379000e-01 +-1.810000e-02 6.916000e-01 2.449000e-01 7.359000e-01 4.121000e-01 6.803000e-01 -6.350000e-02 1.805700e+00 7.400000e-02 1.915100e+00 2.490000e-02 -2.860000e-02 1.347000e-01 4.371000e-01 + 5.528000e-01 5.532000e-01 3.238000e-01 7.494000e-01 2.123000e-01 6.204000e-01 2.494000e-01 8.322000e-01 2.191000e-01 5.926000e-01 2.836000e-01 7.916000e-01 2.339000e-01 4.084000e-01 + 7.864000e-01 -2.360000e-02 1.092900e+00 1.320000e-01 1.351800e+00 2.660000e-02 1.164200e+00 4.800000e-02 1.409700e+00 -1.860000e-02 1.181300e+00 2.760000e-02 6.641000e-01 3.780000e-02 + 4.138000e-01 5.768000e-01 7.693000e-01 5.243000e-01 9.692000e-01 4.379000e-01 1.803000e+00 -8.240000e-02 6.050000e-02 1.930600e+00 -1.340000e-02 1.410000e-02 4.108000e-01 3.600000e-01 + 1.148100e+00 -5.410000e-02 7.842000e-01 2.590000e-01 6.844000e-01 1.110000e-01 8.148000e-01 2.220000e-01 7.089000e-01 6.410000e-02 8.862000e-01 1.437000e-01 5.566000e-01 7.150000e-02 + 2.915000e-01 6.952000e-01 5.862000e-01 7.160000e-01 8.323000e-01 5.673000e-01 1.671200e+00 7.010000e-02 1.290000e-02 1.985300e+00 7.770000e-02 -9.110000e-02 3.706000e-01 3.907000e-01 + 1.131700e+00 -3.170000e-02 9.419000e-01 7.896000e-01 7.905000e-01 4.900000e-01 9.017000e-01 8.375000e-01 7.738000e-01 4.528000e-01 9.162000e-01 8.146000e-01 5.810000e-01 2.747000e-01 + 3.090000e-02 9.172000e-01 3.080000e-02 1.457100e+00 1.922000e-01 1.471700e+00 4.110000e-02 1.445200e+00 1.063000e-01 1.599500e+00 7.490000e-02 1.411300e+00 2.740000e-02 8.309000e-01 + 6.018000e-01 2.622000e-01 8.873000e-01 2.966000e-01 1.037300e+00 2.592000e-01 1.817400e+00 -9.650000e-02 2.013200e+00 -1.730000e-02 9.150000e-02 -1.097000e-01 5.341000e-01 1.592000e-01 + 3.616000e-01 5.395000e-01 7.610000e-01 4.352000e-01 8.018000e-01 5.294000e-01 1.749900e+00 -2.180000e-02 6.930000e-02 1.918400e+00 -5.740000e-02 6.910000e-02 4.071000e-01 3.045000e-01 + 3.879000e-01 5.093000e-01 6.455000e-01 7.533000e-01 7.869000e-01 7.986000e-01 5.291000e-01 8.947000e-01 7.951000e-01 8.101000e-01 6.323000e-01 7.764000e-01 3.686000e-01 4.411000e-01 + 8.884000e-01 2.158000e-01 1.187600e+00 5.403000e-01 1.283500e+00 5.980000e-01 1.680100e+00 5.840000e-02 2.025300e+00 -3.460000e-02 4.970000e-02 1.670900e+00 6.889000e-01 2.914000e-01 + 1.108700e+00 2.100000e-03 1.015600e+00 7.153000e-01 7.320000e-01 1.086200e+00 9.141000e-01 8.361000e-01 8.051000e-01 8.140000e-01 9.557000e-01 7.831000e-01 6.136000e-01 3.523000e-01 + 2.780000e-02 6.130000e-02 2.001000e-01 2.144000e-01 3.815000e-01 1.446000e-01 1.435000e-01 1.563000e+00 -2.340000e-02 2.930000e-02 -1.340000e-02 1.620000e-02 1.320000e-01 8.730000e-02 + 1.968000e-01 8.896000e-01 4.827000e-01 9.247000e-01 6.167000e-01 9.017000e-01 1.671100e+00 6.720000e-02 -1.130000e-02 2.014000e+00 5.820000e-02 1.666900e+00 2.653000e-01 5.603000e-01 + 5.201000e-01 5.841000e-01 3.171000e-01 7.668000e-01 3.882000e-01 4.175000e-01 2.912000e-01 8.033000e-01 1.717000e-01 6.541000e-01 2.294000e-01 8.757000e-01 2.216000e-01 4.260000e-01 + 5.786000e-01 4.950000e-02 8.336000e-01 1.201000e-01 9.328000e-01 1.453000e-01 1.627900e+00 1.184000e-01 1.938700e+00 6.730000e-02 2.700000e-03 -2.800000e-03 5.267000e-01 1.920000e-02 + 2.881000e-01 5.880000e-01 6.085000e-01 5.808000e-01 7.821000e-01 5.208000e-01 1.687500e+00 5.650000e-02 -1.156000e-01 2.135100e+00 -2.450000e-02 2.660000e-02 3.434000e-01 3.586000e-01 + 3.163000e-01 4.248000e-01 6.331000e-01 4.197000e-01 7.968000e-01 3.696000e-01 1.673900e+00 6.850000e-02 4.840000e-02 1.946100e+00 1.170000e-02 -1.470000e-02 3.255000e-01 2.971000e-01 + 1.184100e+00 -9.100000e-02 5.882000e-01 6.830000e-01 4.091000e-01 5.274000e-01 6.264000e-01 6.381000e-01 4.739000e-01 4.277000e-01 5.825000e-01 6.909000e-01 3.942000e-01 3.051000e-01 + 2.379000e-01 4.085000e-01 3.863000e-01 6.242000e-01 3.834000e-01 7.741000e-01 3.794000e-01 6.333000e-01 3.210000e-01 8.707000e-01 3.651000e-01 6.482000e-01 2.093000e-01 3.762000e-01 + 1.124500e+00 -2.220000e-02 3.990000e-01 1.027600e+00 2.620000e-01 7.330000e-01 3.240000e-01 1.115300e+00 3.763000e-01 5.714000e-01 4.534000e-01 9.653000e-01 3.213000e-01 4.047000e-01 + 1.099400e+00 8.000000e-03 8.668000e-01 8.454000e-01 5.698000e-01 1.414000e+00 6.727000e-01 1.078300e+00 5.144000e-01 1.395700e+00 6.686000e-01 1.077000e+00 5.120000e-01 4.853000e-01 + 3.698000e-01 1.077000e-01 5.224000e-01 2.341000e-01 5.438000e-01 3.333000e-01 5.149000e-01 2.459000e-01 5.892000e-01 2.902000e-01 5.628000e-01 1.885000e-01 3.137000e-01 1.224000e-01 + 1.164700e+00 -6.970000e-02 3.553000e-01 1.395300e+00 3.349000e-01 1.135600e+00 4.702000e-01 1.253800e+00 3.487000e-01 1.043500e+00 4.655000e-01 1.262100e+00 3.359000e-01 5.706000e-01 + 3.183000e-01 3.051000e-01 7.323000e-01 2.050000e-01 6.892000e-01 4.029000e-01 4.636000e-01 5.233000e-01 6.157000e-01 5.086000e-01 5.282000e-01 4.456000e-01 3.188000e-01 2.398000e-01 + 3.521000e-01 3.844000e-01 5.984000e-01 5.504000e-01 6.438000e-01 6.688000e-01 6.586000e-01 4.784000e-01 6.964000e-01 6.264000e-01 5.053000e-01 6.583000e-01 3.198000e-01 3.473000e-01 + 5.951000e-01 4.877000e-01 8.186000e-01 8.973000e-01 8.961000e-01 1.057700e+00 6.305000e-01 1.118100e+00 8.322000e-01 1.168700e+00 7.579000e-01 9.702000e-01 4.368000e-01 5.585000e-01 + 3.389000e-01 -2.500000e-03 6.462000e-01 1.600000e-03 8.299000e-01 -7.180000e-02 1.751500e+00 -2.660000e-02 6.930000e-02 -8.170000e-02 -3.820000e-02 4.560000e-02 3.626000e-01 1.000000e-03 + 5.361000e-01 6.900000e-03 8.957000e-01 -5.900000e-02 8.035000e-01 1.913000e-01 6.853000e-01 1.933000e-01 8.260000e-01 1.836000e-01 8.115000e-01 4.050000e-02 4.454000e-01 5.170000e-02 + 1.890000e-02 9.513000e-01 1.228000e-01 1.177100e+00 3.517000e-01 1.045600e+00 6.800000e-03 1.724800e+00 -4.210000e-02 2.044000e+00 6.100000e-03 -7.700000e-03 1.020000e-01 6.578000e-01 + 9.760000e-02 1.367000e-01 1.787000e-01 1.816000e-01 9.870000e-02 3.317000e-01 7.080000e-02 3.092000e-01 1.655000e-01 2.588000e-01 1.027000e-01 2.720000e-01 6.920000e-02 1.454000e-01 + 8.847000e-01 -4.760000e-02 1.263500e+00 7.040000e-02 1.474600e+00 3.880000e-02 1.329900e+00 -7.300000e-03 1.572400e+00 -5.190000e-02 1.251700e+00 8.340000e-02 7.533000e-01 1.260000e-02 + 4.102000e-01 6.910000e-01 7.255000e-01 8.216000e-01 8.329000e-01 8.367000e-01 1.705600e+00 3.830000e-02 6.300000e-02 1.926100e+00 -7.250000e-02 1.816300e+00 3.804000e-01 5.212000e-01 + 1.200200e+00 -1.123000e-01 7.355000e-01 6.722000e-01 6.381000e-01 3.311000e-01 7.752000e-01 6.237000e-01 5.633000e-01 3.925000e-01 7.433000e-01 6.622000e-01 5.182000e-01 2.040000e-01 + 1.303000e-01 5.144000e-01 2.051000e-01 8.057000e-01 2.318000e-01 9.195000e-01 1.189000e-01 9.103000e-01 2.451000e-01 9.204000e-01 6.870000e-02 9.648000e-01 9.940000e-02 4.873000e-01 +-1.940000e-02 1.128400e+00 9.260000e-02 8.642000e-01 -7.870000e-02 8.619000e-01 1.940000e-02 9.523000e-01 -3.220000e-02 7.856000e-01 1.277000e-01 8.247000e-01 2.400000e-03 6.047000e-01 + 4.836000e-01 3.774000e-01 8.733000e-01 2.825000e-01 9.310000e-01 3.550000e-01 1.847400e+00 -1.350000e-01 -6.380000e-02 2.077500e+00 -3.200000e-03 5.300000e-03 4.776000e-01 2.108000e-01 + 3.880000e-01 7.093000e-01 2.357000e-01 1.496200e+00 1.546000e-01 1.279100e+00 2.154000e-01 1.517400e+00 1.865000e-01 1.174800e+00 2.110000e-01 1.518400e+00 1.589000e-01 7.401000e-01 + 9.033000e-01 1.708000e-01 1.453700e+00 2.186000e-01 1.625800e+00 2.814000e-01 1.394600e+00 2.859000e-01 1.638100e+00 3.015000e-01 1.507100e+00 1.582000e-01 8.207000e-01 1.479000e-01 + 7.640000e-02 9.569000e-01 1.070000e-02 1.348700e+00 -2.890000e-02 1.525800e+00 -6.690000e-02 1.814900e+00 5.600000e-02 1.930900e+00 -7.500000e-03 8.600000e-03 1.070000e-02 7.913000e-01 + 6.920000e-01 1.085000e-01 1.126700e+00 1.214000e-01 1.123300e+00 3.258000e-01 9.782000e-01 2.989000e-01 1.109100e+00 3.648000e-01 9.312000e-01 3.528000e-01 5.654000e-01 1.710000e-01 + 1.140900e+00 -4.250000e-02 5.855000e-01 1.156300e+00 5.839000e-01 5.531000e-01 6.872000e-01 1.039200e+00 5.730000e-01 5.251000e-01 6.184000e-01 1.117100e+00 4.544000e-01 3.476000e-01 + 2.265000e-01 6.849000e-01 5.588000e-01 6.636000e-01 5.984000e-01 7.636000e-01 1.767000e+00 -3.880000e-02 4.490000e-02 1.949400e+00 -5.390000e-02 6.500000e-02 3.104000e-01 4.117000e-01 + 1.136600e+00 -3.860000e-02 7.026000e-01 6.384000e-01 4.141000e-01 5.617000e-01 6.621000e-01 6.825000e-01 6.124000e-01 3.018000e-01 7.418000e-01 5.917000e-01 4.552000e-01 2.588000e-01 + 1.151400e+00 -5.220000e-02 7.812000e-01 9.582000e-01 6.172000e-01 8.571000e-01 8.638000e-01 8.509000e-01 6.028000e-01 8.008000e-01 8.650000e-01 8.518000e-01 5.288000e-01 3.810000e-01 + 3.512000e-01 7.155000e-01 4.879000e-01 1.195400e+00 6.452000e-01 1.257600e+00 5.178000e-01 1.157700e+00 5.995000e-01 1.341400e+00 4.382000e-01 1.253600e+00 2.947000e-01 6.734000e-01 + 3.928000e-01 2.647000e-01 5.870000e-01 4.481000e-01 6.574000e-01 5.253000e-01 6.692000e-01 3.489000e-01 7.849000e-01 3.914000e-01 6.414000e-01 3.851000e-01 3.603000e-01 2.330000e-01 + 9.810000e-02 5.790000e-01 2.610000e-02 1.055300e+00 1.204000e-01 1.094800e+00 1.193000e-01 9.473000e-01 1.539000e-01 1.071500e+00 1.487000e-01 9.104000e-01 5.580000e-02 5.622000e-01 + 3.960000e-01 5.710000e-01 6.352000e-01 8.757000e-01 6.999000e-01 1.032200e+00 6.124000e-01 9.060000e-01 7.728000e-01 9.632000e-01 6.477000e-01 8.664000e-01 3.643000e-01 5.100000e-01 + 1.056200e+00 5.920000e-02 5.754000e-01 5.725000e-01 3.995000e-01 4.736000e-01 4.606000e-01 7.063000e-01 4.667000e-01 3.719000e-01 5.426000e-01 6.059000e-01 3.744000e-01 2.928000e-01 + 1.544000e-01 3.905000e-01 1.708000e-01 6.948000e-01 1.166000e-01 8.799000e-01 1.731000e-01 6.905000e-01 2.379000e-01 7.522000e-01 1.968000e-01 6.632000e-01 1.038000e-01 3.948000e-01 + 7.474000e-01 3.670000e-01 4.144000e-01 1.308300e+00 3.011000e-01 7.628000e-01 4.681000e-01 1.247800e+00 2.852000e-01 7.465000e-01 2.409000e-01 1.516200e+00 2.805000e-01 4.849000e-01 + 5.245000e-01 1.796000e-01 8.253000e-01 2.766000e-01 9.779000e-01 2.720000e-01 8.928000e-01 1.989000e-01 1.056100e+00 1.957000e-01 8.944000e-01 1.973000e-01 4.657000e-01 1.730000e-01 + 2.587000e-01 3.300000e-01 3.558000e-01 5.741000e-01 4.228000e-01 6.357000e-01 3.226000e-01 6.190000e-01 4.366000e-01 6.308000e-01 3.088000e-01 6.295000e-01 2.096000e-01 3.275000e-01 + 4.381000e-01 3.545000e-01 7.058000e-01 5.334000e-01 7.713000e-01 6.482000e-01 6.476000e-01 6.045000e-01 8.761000e-01 5.459000e-01 7.537000e-01 4.779000e-01 4.071000e-01 3.084000e-01 + 1.175000e-01 6.417000e-01 2.160000e-02 3.414000e-01 -4.500000e-03 3.157000e-01 4.350000e-02 3.126000e-01 8.800000e-02 2.014000e-01 8.650000e-02 2.635000e-01 5.010000e-02 2.254000e-01 + 1.150900e+00 -5.330000e-02 8.148000e-01 1.712000e-01 6.775000e-01 8.840000e-02 8.537000e-01 1.243000e-01 6.137000e-01 1.453000e-01 7.788000e-01 2.154000e-01 5.315000e-01 8.070000e-02 +-5.630000e-02 9.848000e-01 1.351000e-01 1.084000e+00 6.650000e-02 1.297100e+00 8.910000e-02 1.623000e+00 6.430000e-02 1.925100e+00 4.000000e-04 -6.000000e-04 4.630000e-02 6.800000e-01 + 1.074000e+00 4.070000e-02 4.828000e-01 1.259400e+00 3.677000e-01 1.607300e+00 4.678000e-01 1.268900e+00 5.239000e-01 1.195600e+00 4.787000e-01 1.261600e+00 3.748000e-01 6.080000e-01 + 3.009000e-01 2.211000e-01 5.387000e-01 3.073000e-01 6.518000e-01 3.179000e-01 1.888000e+00 -1.786000e-01 1.250000e-02 -1.310000e-02 4.310000e-02 -5.170000e-02 3.051000e-01 1.811000e-01 + 4.833000e-01 5.855000e-01 6.725000e-01 1.015700e+00 8.943000e-01 1.002800e+00 7.816000e-01 8.851000e-01 9.070000e-01 1.016200e+00 7.665000e-01 9.063000e-01 4.491000e-01 5.144000e-01 + 2.202000e-01 1.536000e-01 3.610000e-01 2.209000e-01 5.806000e-01 5.000000e-02 3.611000e-01 2.241000e-01 4.184000e-01 2.550000e-01 4.442000e-01 1.251000e-01 2.334000e-01 9.850000e-02 + 5.666000e-01 5.338000e-01 2.767000e-01 1.301500e+00 2.398000e-01 7.728000e-01 2.772000e-01 1.298300e+00 2.390000e-01 7.463000e-01 3.174000e-01 1.256700e+00 2.205000e-01 5.236000e-01 + 3.376000e-01 7.419000e-01 4.848000e-01 1.221000e+00 6.335000e-01 1.291000e+00 5.921000e-01 1.088100e+00 6.010000e-01 1.356400e+00 4.958000e-01 1.201200e+00 3.003000e-01 6.781000e-01 + 9.463000e-01 2.020000e-02 1.380900e+00 1.522000e-01 1.676300e+00 5.780000e-02 1.385100e+00 1.532000e-01 1.648500e+00 1.135000e-01 1.339300e+00 2.017000e-01 8.296000e-01 5.060000e-02 + 3.907000e-01 6.294000e-01 5.879000e-01 7.718000e-01 8.172000e-01 6.377000e-01 1.697300e+00 4.520000e-02 1.263000e-01 1.845900e+00 -1.900000e-03 1.300000e-03 4.017000e-01 3.842000e-01 + 1.724000e-01 3.817000e-01 1.702000e-01 7.203000e-01 2.873000e-01 7.049000e-01 2.921000e-01 5.731000e-01 3.515000e-01 6.442000e-01 2.986000e-01 5.659000e-01 1.455000e-01 3.580000e-01 + 1.312000e-01 7.093000e-01 3.948000e-01 7.673000e-01 5.272000e-01 7.554000e-01 1.743300e+00 -1.250000e-02 2.350000e-02 1.975500e+00 2.500000e-03 -3.900000e-03 2.266000e-01 4.576000e-01 + 1.263000e-01 6.105000e-01 2.655000e-01 8.748000e-01 2.336000e-01 1.076800e+00 1.833000e-01 9.726000e-01 3.004000e-01 1.016500e+00 1.962000e-01 9.532000e-01 1.264000e-01 5.363000e-01 + 6.043000e-01 4.914000e-01 1.090000e+00 5.961000e-01 1.056900e+00 8.966000e-01 1.027900e+00 6.694000e-01 1.212700e+00 7.549000e-01 1.065400e+00 6.278000e-01 6.046000e-01 3.749000e-01 + 1.028400e+00 8.930000e-02 4.097000e-01 9.375000e-01 2.813000e-01 6.865000e-01 3.827000e-01 9.620000e-01 2.830000e-01 6.586000e-01 3.508000e-01 1.008600e+00 3.087000e-01 4.039000e-01 +-1.970000e-02 7.719000e-01 4.634000e-01 5.700000e-01 5.143000e-01 6.520000e-01 -2.580000e-02 1.762600e+00 -1.730000e-02 2.023200e+00 2.500000e-02 -2.650000e-02 1.832000e-01 4.379000e-01 diff --git a/sourcecodes/bnt-master/netlab3.3/olgd.m b/sourcecodes/bnt-master/netlab3.3/olgd.m new file mode 100644 index 00000000..20681cf0 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/olgd.m @@ -0,0 +1,181 @@ +function [net, options, errlog, pointlog] = olgd(net, options, x, t) +%OLGD On-line gradient descent optimization. +% +% Description +% [NET, OPTIONS, ERRLOG, POINTLOG] = OLGD(NET, OPTIONS, X, T) uses on- +% line gradient descent to find a local minimum of the error function +% for the network NET computed on the input data X and target values T. +% A log of the error values after each cycle is (optionally) returned +% in ERRLOG, and a log of the points visited is (optionally) returned +% in POINTLOG. Because the gradient is computed on-line (i.e. after +% each pattern) this can be quite inefficient in Matlab. +% +% The error function value at final weight vector is returned in +% OPTIONS(8). +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG, and the points visited in the +% return argument POINTSLOG. If OPTIONS(1) is set to 0, then only +% warning messages are displayed. If OPTIONS(1) is -1, then nothing is +% displayed. +% +% OPTIONS(2) is the precision required for the value of X at the +% solution. If the absolute difference between the values of X between +% two successive steps is less than OPTIONS(2), then this condition is +% satisfied. +% +% OPTIONS(3) is the precision required of the objective function at the +% solution. If the absolute difference between the error functions +% between two successive steps is less than OPTIONS(3), then this +% condition is satisfied. Both this and the previous condition must be +% satisfied for termination. Note that testing the function value at +% each iteration roughly halves the speed of the algorithm. +% +% OPTIONS(5) determines whether the patterns are sampled randomly with +% replacement. If it is 0 (the default), then patterns are sampled in +% order. +% +% OPTIONS(6) determines if the learning rate decays. If it is 1 then +% the learning rate decays at a rate of 1/T. If it is 0 (the default) +% then the learning rate is constant. +% +% OPTIONS(9) should be set to 1 to check the user defined gradient +% function. +% +% OPTIONS(10) returns the total number of function evaluations +% (including those in any line searches). +% +% OPTIONS(11) returns the total number of gradient evaluations. +% +% OPTIONS(14) is the maximum number of iterations (passes through the +% complete pattern set); default 100. +% +% OPTIONS(17) is the momentum; default 0.5. +% +% OPTIONS(18) is the learning rate; default 0.01. +% +% See also +% GRADDESC +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set up the options. +if length(options) < 18 + error('Options vector too short') +end + +if (options(14)) + niters = options(14); +else + niters = 100; +end + +% Learning rate: must be positive +if (options(18) > 0) + eta = options(18); +else + eta = 0.01; +end +% Save initial learning rate for annealing +lr = eta; +% Momentum term: allow zero momentum +if (options(17) >= 0) + mu = options(17); +else + mu = 0.5; +end + +pakstr = [net.type, 'pak']; +unpakstr = [net.type, 'unpak']; + +% Extract initial weights from the network +w = feval(pakstr, net); + +display = options(1); + +% Work out if we need to compute f at each iteration. +% Needed if display results or if termination +% criterion requires it. +fcneval = (display | options(3)); + +% Check gradients +if (options(9)) + feval('gradchek', w, 'neterr', 'netgrad', net, x, t); +end + +dwold = zeros(1, length(w)); +fold = 0; % Must be initialised so that termination test can be performed +ndata = size(x, 1); + +if fcneval + fnew = neterr(w, net, x, t); + options(10) = options(10) + 1; + fold = fnew; +end + +j = 1; +if nargout >= 3 + errlog(j, :) = fnew; + if nargout == 4 + pointlog(j, :) = w; + end +end + +% Main optimization loop. +while j <= niters + wold = w; + if options(5) + % Randomise order of pattern presentation: with replacement + pnum = ceil(rand(ndata, 1).*ndata); + else + pnum = 1:ndata; + end + for k = 1:ndata + grad = netgrad(w, net, x(pnum(k),:), t(pnum(k),:)); + if options(6) + % Let learning rate decrease as 1/t + lr = eta/((j-1)*ndata + k); + end + dw = mu*dwold - lr*grad; + w = w + dw; + dwold = dw; + end + options(11) = options(11) + 1; % Increment gradient evaluation count + if fcneval + fold = fnew; + fnew = neterr(w, net, x, t); + options(10) = options(10) + 1; + end + if display + fprintf(1, 'Iteration %5d Error %11.8f\n', j, fnew); + end + j = j + 1; + if nargout >= 3 + errlog(j) = fnew; + if nargout == 4 + pointlog(j, :) = w; + end + end + if (max(abs(w - wold)) < options(2) & abs(fnew - fold) < options(3)) + % Termination criteria are met + options(8) = fnew; + net = feval(unpakstr, net, w); + return; + end +end + +if fcneval + options(8) = fnew; +else + % Return error on entire dataset + options(8) = neterr(w, net, x, t); + options(10) = options(10) + 1; +end +if (options(1) >= 0) + disp(maxitmess); +end + +net = feval(unpakstr, net, w); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/pca.m b/sourcecodes/bnt-master/netlab3.3/pca.m new file mode 100644 index 00000000..4b7063d6 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/pca.m @@ -0,0 +1,42 @@ +function [PCcoeff, PCvec] = pca(data, N) +%PCA Principal Components Analysis +% +% Description +% PCCOEFF = PCA(DATA) computes the eigenvalues of the covariance +% matrix of the dataset DATA and returns them as PCCOEFF. These +% coefficients give the variance of DATA along the corresponding +% principal components. +% +% PCCOEFF = PCA(DATA, N) returns the largest N eigenvalues. +% +% [PCCOEFF, PCVEC] = PCA(DATA) returns the principal components as well +% as the coefficients. This is considerably more computationally +% demanding than just computing the eigenvalues. +% +% See also +% EIGDEC, GTMINIT, PPCA +% + +% Copyright (c) Ian T Nabney (1996-2001) + +if nargin == 1 + N = size(data, 2); +end + +if nargout == 1 + evals_only = logical(1); +else + evals_only = logical(0); +end + +if N ~= round(N) | N < 1 | N > size(data, 2) + error('Number of PCs must be integer, >0, < dim'); +end + +% Find the sorted eigenvalues of the data covariance matrix +if evals_only + PCcoeff = eigdec(cov(data), N); +else + [PCcoeff, PCvec] = eigdec(cov(data), N); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/plotmat.m b/sourcecodes/bnt-master/netlab3.3/plotmat.m new file mode 100644 index 00000000..64f82c2c --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/plotmat.m @@ -0,0 +1,44 @@ +function plotmat(matrix, textcolour, gridcolour, fontsize) +%PLOTMAT Display a matrix. +% +% Description +% PLOTMAT(MATRIX, TEXTCOLOUR, GRIDCOLOUR, FONTSIZE) displays the matrix +% MATRIX on the current figure. The TEXTCOLOUR and GRIDCOLOUR +% arguments control the colours of the numbers and grid labels +% respectively and should follow the usual Matlab specification. The +% parameter FONTSIZE should be an integer. +% +% See also +% CONFFIG, DEMMLP2 +% + +% Copyright (c) Ian T Nabney (1996-2001) + +[m,n]=size(matrix); +for rowCnt=1:m, + for colCnt=1:n, + numberString=num2str(matrix(rowCnt,colCnt)); + text(colCnt-.5,m-rowCnt+.5,numberString, ... + 'HorizontalAlignment','center', ... + 'Color', textcolour, ... + 'FontWeight','bold', ... + 'FontSize', fontsize); + end; +end; + +set(gca,'Box','on', ... + 'Visible','on', ... + 'xLim',[0 n], ... + 'xGrid','on', ... + 'xTickLabel',[], ... + 'xTick',0:n, ... + 'yGrid','on', ... + 'yLim',[0 m], ... + 'yTickLabel',[], ... + 'yTick',0:m, ... + 'DataAspectRatio',[1, 1, 1], ... + 'GridLineStyle',':', ... + 'LineWidth',3, ... + 'XColor',gridcolour, ... + 'YColor',gridcolour); + diff --git a/sourcecodes/bnt-master/netlab3.3/ppca.m b/sourcecodes/bnt-master/netlab3.3/ppca.m new file mode 100644 index 00000000..fd125b02 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/ppca.m @@ -0,0 +1,52 @@ +function [var, U, lambda] = ppca(x, ppca_dim) +%PPCA Probabilistic Principal Components Analysis +% +% Description +% [VAR, U, LAMBDA] = PPCA(X, PPCA_DIM) computes the principal +% component subspace U of dimension PPCA_DIM using a centred covariance +% matrix X. The variable VAR contains the off-subspace variance (which +% is assumed to be spherical), while the vector LAMBDA contains the +% variances of each of the principal components. This is computed +% using the eigenvalue and eigenvector decomposition of X. +% +% See also +% EIGDEC, PCA +% + +% Copyright (c) Ian T Nabney (1996-2001) + + +if ppca_dim ~= round(ppca_dim) | ppca_dim < 1 | ppca_dim > size(x, 2) + error('Number of PCs must be integer, >0, < dim'); +end + +[ndata, data_dim] = size(x); +% Assumes that x is centred and responsibility weighted +% covariance matrix +[l Utemp] = eigdec(x, data_dim); +% Zero any negative eigenvalues (caused by rounding) +l(l<0) = 0; +% Now compute the sigma squared values for all possible values +% of q +s2_temp = cumsum(l(end:-1:1))./[1:data_dim]'; +% If necessary, reduce the value of q so that var is at least +% eps * largest eigenvalue +q_temp = min([ppca_dim; data_dim-min(find(s2_temp/l(1) > eps))]); +if q_temp ~= ppca_dim + wstringpart = 'Covariance matrix ill-conditioned: extracted'; + wstring = sprintf('%s %d/%d PCs', ... + wstringpart, q_temp, ppca_dim); + warning(wstring); +end +if q_temp == 0 + % All the latent dimensions have disappeared, so we are + % just left with the noise model + var = l(1)/data_dim; + lambda = var*ones(1, ppca_dim); +else + var = mean(l(q_temp+1:end)); +end +U = Utemp(:, 1:q_temp); +lambda(1:q_temp) = l(1:q_temp); + + diff --git a/sourcecodes/bnt-master/netlab3.3/quasinew.m b/sourcecodes/bnt-master/netlab3.3/quasinew.m new file mode 100644 index 00000000..680e7d15 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/quasinew.m @@ -0,0 +1,176 @@ +function [x, options, flog, pointlog] = quasinew(f, x, options, gradf, ... + varargin) +%QUASINEW Quasi-Newton optimization. +% +% Description +% [X, OPTIONS, FLOG, POINTLOG] = QUASINEW(F, X, OPTIONS, GRADF) uses a +% quasi-Newton algorithm to find a local minimum of the function F(X) +% whose gradient is given by GRADF(X). Here X is a row vector and F +% returns a scalar value. The point at which F has a local minimum is +% returned as X. The function value at that point is returned in +% OPTIONS(8). A log of the function values after each cycle is +% (optionally) returned in FLOG, and a log of the points visited is +% (optionally) returned in POINTLOG. +% +% QUASINEW(F, X, OPTIONS, GRADF, P1, P2, ...) allows additional +% arguments to be passed to F() and GRADF(). +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG, and the points visited in the +% return argument POINTSLOG. If OPTIONS(1) is set to 0, then only +% warning messages are displayed. If OPTIONS(1) is -1, then nothing is +% displayed. +% +% OPTIONS(2) is a measure of the absolute precision required for the +% value of X at the solution. If the absolute difference between the +% values of X between two successive steps is less than OPTIONS(2), +% then this condition is satisfied. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. If the absolute difference between the +% objective function values between two successive steps is less than +% OPTIONS(3), then this condition is satisfied. Both this and the +% previous condition must be satisfied for termination. +% +% OPTIONS(9) should be set to 1 to check the user defined gradient +% function. +% +% OPTIONS(10) returns the total number of function evaluations +% (including those in any line searches). +% +% OPTIONS(11) returns the total number of gradient evaluations. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% OPTIONS(15) is the precision in parameter space of the line search; +% default 1E-2. +% +% See also +% CONJGRAD, GRADDESC, LINEMIN, MINBRACK, SCG +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set up the options. +if length(options) < 18 + error('Options vector too short') +end + +if(options(14)) + niters = options(14); +else + niters = 100; +end + +% Set up options for line search +line_options = foptions; +% Don't need a very precise line search +if options(15) > 0 + line_options(2) = options(15); +else + line_options(2) = 1e-2; % Default +end +% Minimal fractional change in f from Newton step: otherwise do a line search +min_frac_change = 1e-4; + +display = options(1); + +% Next two lines allow quasinew to work with expression strings +f = fcnchk(f, length(varargin)); +gradf = fcnchk(gradf, length(varargin)); + +% Check gradients +if (options(9)) + feval('gradchek', x, f, gradf, varargin{:}); +end + +nparams = length(x); +fnew = feval(f, x, varargin{:}); +options(10) = options(10) + 1; +gradnew = feval(gradf, x, varargin{:}); +options(11) = options(11) + 1; +p = -gradnew; % Search direction +hessinv = eye(nparams); % Initialise inverse Hessian to be identity matrix +j = 1; +if nargout >= 3 + flog(j, :) = fnew; + if nargout == 4 + pointlog(j, :) = x; + end +end + +while (j <= niters) + + xold = x; + fold = fnew; + gradold = gradnew; + + x = xold + p; + fnew = feval(f, x, varargin{:}); + options(10) = options(10) + 1; + + % This shouldn't occur, but rest of code depends on sd being downhill + if (gradnew*p' >= 0) + p = -p; + if options(1) >= 0 + warning('search direction uphill in quasinew'); + end + end + + % Does the Newton step reduce the function value sufficiently? + if (fnew >= fold + min_frac_change * (gradnew*p')) + % No it doesn't + % Minimize along current search direction: must be less than Newton step + [lmin, line_options] = feval('linemin', f, xold, p, fold, ... + line_options, varargin{:}); + options(10) = options(10) + line_options(10); + options(11) = options(11) + line_options(11); + % Correct x and fnew to be the actual search point we have found + x = xold + lmin * p; + p = x - xold; + fnew = line_options(8); + end + + % Check for termination + if (max(abs(x - xold)) < options(2) & max(abs(fnew - fold)) < options(3)) + options(8) = fnew; + return; + end + gradnew = feval(gradf, x, varargin{:}); + options(11) = options(11) + 1; + v = gradnew - gradold; + vdotp = v*p'; + + % Skip update to inverse Hessian if fac not sufficiently positive + if (vdotp*vdotp > eps*sum(v.^2)*sum(p.^2)) + Gv = (hessinv*v')'; + vGv = sum(v.*Gv); + u = p./vdotp - Gv./vGv; + % Use BFGS update rule + hessinv = hessinv + (p'*p)/vdotp - (Gv'*Gv)/vGv + vGv*(u'*u); + end + + p = -(hessinv * gradnew')'; + + if (display > 0) + fprintf(1, 'Cycle %4d Function %11.6f\n', j, fnew); + end + + j = j + 1; + if nargout >= 3 + flog(j, :) = fnew; + if nargout == 4 + pointlog(j, :) = x; + end + end +end + +% If we get here, then we haven't terminated in the given number of +% iterations. + +options(8) = fold; +if (options(1) >= 0) + disp(maxitmess); +end diff --git a/sourcecodes/bnt-master/netlab3.3/rbf.m b/sourcecodes/bnt-master/netlab3.3/rbf.m new file mode 100644 index 00000000..5c3b3096 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbf.m @@ -0,0 +1,123 @@ +function net = rbf(nin, nhidden, nout, rbfunc, outfunc, prior, beta) +%RBF Creates an RBF network with specified architecture +% +% Description +% NET = RBF(NIN, NHIDDEN, NOUT, RBFUNC) constructs and initialises a +% radial basis function network returning a data structure NET. The +% weights are all initialised with a zero mean, unit variance normal +% distribution, with the exception of the variances, which are set to +% one. This makes use of the Matlab function RANDN and so the seed for +% the random weight initialization can be set using RANDN('STATE', S) +% where S is the seed value. The activation functions are defined in +% terms of the distance between the data point and the corresponding +% centre. Note that the functions are computed to a convenient +% constant multiple: for example, the Gaussian is not normalised. +% (Normalisation is not needed as the function outputs are linearly +% combined in the next layer.) +% +% The fields in NET are +% type = 'rbf' +% nin = number of inputs +% nhidden = number of hidden units +% nout = number of outputs +% nwts = total number of weights and biases +% actfn = string defining hidden unit activation function: +% 'gaussian' for a radially symmetric Gaussian function. +% 'tps' for r^2 log r, the thin plate spline function. +% 'r4logr' for r^4 log r. +% outfn = string defining output error function: +% 'linear' for linear outputs (default) and SoS error. +% 'neuroscale' for Sammon stress measure. +% c = centres +% wi = squared widths (null for rlogr and tps) +% w2 = second layer weight matrix +% b2 = second layer bias vector +% +% NET = RBF(NIN, NHIDDEN, NOUT, RBFUND, OUTFUNC) allows the user to +% specify the type of error function to be used. The field OUTFN is +% set to the value of this string. Linear outputs (for regression +% problems) and Neuroscale outputs (for topographic mappings) are +% supported. +% +% NET = RBF(NIN, NHIDDEN, NOUT, RBFUNC, OUTFUNC, PRIOR, BETA), in which +% PRIOR is a scalar, allows the field NET.ALPHA in the data structure +% NET to be set, corresponding to a zero-mean isotropic Gaussian prior +% with inverse variance with value PRIOR. Alternatively, PRIOR can +% consist of a data structure with fields ALPHA and INDEX, allowing +% individual Gaussian priors to be set over groups of weights in the +% network. Here ALPHA is a column vector in which each element +% corresponds to a separate group of weights, which need not be +% mutually exclusive. The membership of the groups is defined by the +% matrix INDX in which the columns correspond to the elements of ALPHA. +% Each column has one element for each weight in the matrix, in the +% order defined by the function RBFPAK, and each element is 1 or 0 +% according to whether the weight is a member of the corresponding +% group or not. A utility function RBFPRIOR is provided to help in +% setting up the PRIOR data structure. +% +% NET = RBF(NIN, NHIDDEN, NOUT, FUNC, PRIOR, BETA) also sets the +% additional field NET.BETA in the data structure NET, where beta +% corresponds to the inverse noise variance. +% +% See also +% RBFERR, RBFFWD, RBFGRAD, RBFPAK, RBFTRAIN, RBFUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.type = 'rbf'; +net.nin = nin; +net.nhidden = nhidden; +net.nout = nout; + +% Check that function is an allowed type +actfns = {'gaussian', 'tps', 'r4logr'}; +outfns = {'linear', 'neuroscale'}; +if (strcmp(rbfunc, actfns)) == 0 + error('Undefined activation function.') +else + net.actfn = rbfunc; +end +if nargin <= 4 + net.outfn = outfns{1}; +elseif (strcmp(outfunc, outfns) == 0) + error('Undefined output function.') +else + net.outfn = outfunc; + end + +% Assume each function has a centre and a single width parameter, and that +% hidden layer to output weights include a bias. Only the Gaussian function +% requires a width +net.nwts = nin*nhidden + (nhidden + 1)*nout; +if strcmp(rbfunc, 'gaussian') + % Extra weights for width parameters + net.nwts = net.nwts + nhidden; +end + +if nargin > 5 + if isstruct(prior) + net.alpha = prior.alpha; + net.index = prior.index; + elseif size(prior) == [1 1] + net.alpha = prior; + else + error('prior must be a scalar or a structure'); + end + if nargin > 6 + net.beta = beta; + end +end + +w = randn(1, net.nwts); +net = rbfunpak(net, w); + +% Make widths equal to one +if strcmp(rbfunc, 'gaussian') + net.wi = ones(1, nhidden); +end + +if strcmp(net.outfn, 'neuroscale') + net.mask = rbfprior(rbfunc, nin, nhidden, nout); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/rbfbkp.m b/sourcecodes/bnt-master/netlab3.3/rbfbkp.m new file mode 100644 index 00000000..2d7ffe54 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfbkp.m @@ -0,0 +1,72 @@ +function g = rbfbkp(net, x, z, n2, deltas) +%RBFBKP Backpropagate gradient of error function for RBF network. +% +% Description +% G = RBFBKP(NET, X, Z, N2, DELTAS) takes a network data structure NET +% together with a matrix X of input vectors, a matrix Z of hidden unit +% activations, a matrix N2 of the squared distances between centres and +% inputs, and a matrix DELTAS of the gradient of the error function +% with respect to the values of the output units (i.e. the summed +% inputs to the output units, before the activation function is +% applied). The return value is the gradient G of the error function +% with respect to the network weights. Each row of X corresponds to one +% input vector. +% +% This function is provided so that the common backpropagation +% algorithm can be used by RBF network models to compute gradients for +% the output values (in RBFDERIV) as well as standard error functions. +% +% See also +% RBF, RBFGRAD, RBFDERIV +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Evaluate second-layer gradients. +gw2 = z'*deltas; +gb2 = sum(deltas); + +% Evaluate hidden unit gradients +delhid = deltas*net.w2'; + +gc = zeros(net.nhidden, net.nin); +ndata = size(x, 1); +t1 = ones(ndata, 1); +t2 = ones(1, net.nin); +% Switch on activation function type +switch net.actfn + +case 'gaussian' % Gaussian + delhid = (delhid.*z); + % A loop seems essential, so do it with the shortest index vector + if (net.nin < net.nhidden) + for i = 1:net.nin + gc(:,i) = (sum(((x(:,i)*ones(1, net.nhidden)) - ... + (ones(ndata, 1)*(net.c(:,i)'))).*delhid, 1)./net.wi)'; + end + else + for i = 1:net.nhidden + gc(i,:) = sum((x - (t1*(net.c(i,:)))./net.wi(i)).*(delhid(:,i)*t2), 1); + end + end + gwi = sum((n2.*delhid)./(2.*(ones(ndata, 1)*(net.wi.^2))), 1); + +case 'tps' % Thin plate spline activation function + delhid = delhid.*(1+log(n2+(n2==0))); + for i = 1:net.nhidden + gc(i,:) = sum(2.*((t1*(net.c(i,:)) - x)).*(delhid(:,i)*t2), 1); + end + % widths are not adjustable in this model + gwi = []; +case 'r4logr' % r^4 log r activation function + delhid = delhid.*(n2.*(1+2.*log(n2+(n2==0)))); + for i = 1:net.nhidden + gc(i,:) = sum(2.*((t1*(net.c(i,:)) - x)).*(delhid(:,i)*t2), 1); + end + % widths are not adjustable in this model + gwi = []; +otherwise + error('Unknown activation function in rbfgrad') +end + +g = [gc(:)', gwi, gw2(:)', gb2]; diff --git a/sourcecodes/bnt-master/netlab3.3/rbfderiv.m b/sourcecodes/bnt-master/netlab3.3/rbfderiv.m new file mode 100644 index 00000000..ac644aaa --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfderiv.m @@ -0,0 +1,53 @@ +function g = rbfderiv(net, x) +%RBFDERIV Evaluate derivatives of RBF network outputs with respect to weights. +% +% Description +% G = RBFDERIV(NET, X) takes a network data structure NET and a matrix +% of input vectors X and returns a three-index matrix G whose I, J, K +% element contains the derivative of network output K with respect to +% weight or bias parameter J for input pattern I. The ordering of the +% weight and bias parameters is defined by RBFUNPAK. This function +% also takes into account any mask in the network data structure. +% +% See also +% RBF, RBFPAK, RBFGRAD, RBFBKP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'rbf', x); +if ~isempty(errstring); + error(errstring); +end + +if ~strcmp(net.outfn, 'linear') + error('Function only implemented for linear outputs') +end + +[y, z, n2] = rbffwd(net, x); +ndata = size(x, 1); + +if isfield(net, 'mask') + nwts = size(find(net.mask), 1); + temp = zeros(1, net.nwts); +else + nwts = net.nwts; +end + +g = zeros(ndata, nwts, net.nout); +for k = 1 : net.nout + delta = zeros(1, net.nout); + delta(1, k) = 1; + for n = 1 : ndata + if isfield(net, 'mask') + temp = rbfbkp(net, x(n, :), z(n, :), n2(n, :), delta); + g(n, :, k) = temp(logical(net.mask)); + else + g(n, :, k) = rbfbkp(net, x(n, :), z(n, :), n2(n, :),... + delta); + end + end +end + + \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbferr.m b/sourcecodes/bnt-master/netlab3.3/rbferr.m new file mode 100644 index 00000000..8f5b6a39 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbferr.m @@ -0,0 +1,50 @@ +function [e, edata, eprior] = rbferr(net, x, t) +%RBFERR Evaluate error function for RBF network. +% +% Description +% E = RBFERR(NET, X, T) takes a network data structure NET together +% with a matrix X of input vectors and a matrix T of target vectors, +% and evaluates the appropriate error function E depending on +% NET.OUTFN. Each row of X corresponds to one input vector and each +% row of T contains the corresponding target vector. +% +% [E, EDATA, EPRIOR] = RBFERR(NET, X, T) additionally returns the data +% and prior components of the error, assuming a zero mean Gaussian +% prior on the weights with inverse variance parameters ALPHA and BETA +% taken from the network data structure NET. +% +% See also +% RBF, RBFFWD, RBFGRAD, RBFPAK, RBFTRAIN, RBFUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +switch net.outfn +case 'linear' + errstring = consist(net, 'rbf', x, t); +case 'neuroscale' + errstring = consist(net, 'rbf', x); +otherwise + error(['Unknown output function ', net.outfn]); +end +if ~isempty(errstring); + error(errstring); +end + +switch net.outfn +case 'linear' + y = rbffwd(net, x); + edata = 0.5*sum(sum((y - t).^2)); +case 'neuroscale' + y = rbffwd(net, x); + y_dist = sqrt(dist2(y, y)); + % Take t as target distance matrix + edata = 0.5.*(sum(sum((t-y_dist).^2))); +otherwise + error(['Unknown output function ', net.outfn]); +end + +% Compute Bayesian regularised error +[e, edata, eprior] = errbayes(net, edata); + diff --git a/sourcecodes/bnt-master/netlab3.3/rbfevfwd.m b/sourcecodes/bnt-master/netlab3.3/rbfevfwd.m new file mode 100644 index 00000000..5bafc732 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfevfwd.m @@ -0,0 +1,30 @@ +function [y, extra, invhess] = rbfevfwd(net, x, t, x_test, invhess) +%RBFEVFWD Forward propagation with evidence for RBF +% +% Description +% Y = RBFEVFWD(NET, X, T, X_TEST) takes a network data structure NET +% together with the input X and target T training data and input test +% data X_TEST. It returns the normal forward propagation through the +% network Y together with a matrix EXTRA which consists of error bars +% (variance) for a regression problem or moderated outputs for a +% classification problem. +% +% The optional argument (and return value) INVHESS is the inverse of +% the network Hessian computed on the training data inputs and targets. +% Passing it in avoids recomputing it, which can be a significant +% saving for large training sets. +% +% See also +% FEVBAYES +% + +% Copyright (c) Ian T Nabney (1996-2001) + +y = rbffwd(net, x_test); +% RBF outputs must be linear, so just pass them twice (second copy is +% not used +if nargin == 4 + [extra, invhess] = fevbayes(net, y, y, x, t, x_test); +else + [extra, invhess] = fevbayes(net, y, y, x, t, x_test, invhess); +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbffwd.m b/sourcecodes/bnt-master/netlab3.3/rbffwd.m new file mode 100644 index 00000000..af8a63e8 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbffwd.m @@ -0,0 +1,55 @@ +function [a, z, n2] = rbffwd(net, x) +%RBFFWD Forward propagation through RBF network with linear outputs. +% +% Description +% A = RBFFWD(NET, X) takes a network data structure NET and a matrix X +% of input vectors and forward propagates the inputs through the +% network to generate a matrix A of output vectors. Each row of X +% corresponds to one input vector and each row of A contains the +% corresponding output vector. The activation function that is used is +% determined by NET.ACTFN. +% +% [A, Z, N2] = RBFFWD(NET, X) also generates a matrix Z of the hidden +% unit activations where each row corresponds to one pattern. These +% hidden unit activations represent the design matrix for the RBF. The +% matrix N2 is the squared distances between each basis function centre +% and each pattern in which each row corresponds to a data point. +% +% See also +% RBF, RBFERR, RBFGRAD, RBFPAK, RBFTRAIN, RBFUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'rbf', x); +if ~isempty(errstring); + error(errstring); +end + +[ndata, data_dim] = size(x); + +% Calculate squared norm matrix, of dimension (ndata, ncentres) +n2 = dist2(x, net.c); + +% Switch on activation function type +switch net.actfn + + case 'gaussian' % Gaussian + % Calculate width factors: net.wi contains squared widths + wi2 = ones(ndata, 1) * (2 .* net.wi); + + % Now compute the activations + z = exp(-(n2./wi2)); + + case 'tps' % Thin plate spline + z = n2.*log(n2+(n2==0)); + + case 'r4logr' % r^4 log r + z = n2.*n2.*log(n2+(n2==0)); + + otherwise + error('Unknown activation function in rbffwd') +end + +a = z*net.w2 + ones(ndata, 1)*net.b2; \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbfgrad.m b/sourcecodes/bnt-master/netlab3.3/rbfgrad.m new file mode 100644 index 00000000..af6767ac --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfgrad.m @@ -0,0 +1,65 @@ +function [g, gdata, gprior] = rbfgrad(net, x, t) +%RBFGRAD Evaluate gradient of error function for RBF network. +% +% Description +% G = RBFGRAD(NET, X, T) takes a network data structure NET together +% with a matrix X of input vectors and a matrix T of target vectors, +% and evaluates the gradient G of the error function with respect to +% the network weights (i.e. including the hidden unit parameters). The +% error function is sum of squares. Each row of X corresponds to one +% input vector and each row of T contains the corresponding target +% vector. If the output function is 'NEUROSCALE' then the gradient is +% only computed for the output layer weights and biases. +% +% [G, GDATA, GPRIOR] = RBFGRAD(NET, X, T) also returns separately the +% data and prior contributions to the gradient. In the case of multiple +% groups in the prior, GPRIOR is a matrix with a row for each group and +% a column for each weight parameter. +% +% See also +% RBF, RBFFWD, RBFERR, RBFPAK, RBFUNPAK, RBFBKP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +switch net.outfn +case 'linear' + errstring = consist(net, 'rbf', x, t); +case 'neuroscale' + errstring = consist(net, 'rbf', x); +otherwise + error(['Unknown output function ', net.outfn]); +end +if ~isempty(errstring); + error(errstring); +end + +ndata = size(x, 1); + +[y, z, n2] = rbffwd(net, x); + +switch net.outfn +case 'linear' + + % Sum squared error at output units + delout = y - t; + + gdata = rbfbkp(net, x, z, n2, delout); + [g, gdata, gprior] = gbayes(net, gdata); + +case 'neuroscale' + % Compute the error gradient with respect to outputs + y_dist = sqrt(dist2(y, y)); + D = (t - y_dist)./(y_dist+diag(ones(ndata, 1))); + temp = y'; + gradient = 2.*sum(kron(D, ones(1, net.nout)) .* ... + (repmat(y, 1, ndata) - repmat((temp(:))', ndata, 1)), 1); + gradient = (reshape(gradient, net.nout, ndata))'; + % Compute the error gradient + gdata = rbfbkp(net, x, z, n2, gradient); + [g, gdata, gprior] = gbayes(net, gdata); +otherwise + error(['Unknown output function ', net.outfn]); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/rbfhess.m b/sourcecodes/bnt-master/netlab3.3/rbfhess.m new file mode 100644 index 00000000..440d3ac0 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfhess.m @@ -0,0 +1,91 @@ +function [h, hdata] = rbfhess(net, x, t, hdata) +%RBFHESS Evaluate the Hessian matrix for RBF network. +% +% Description +% H = RBFHESS(NET, X, T) takes an RBF network data structure NET, a +% matrix X of input values, and a matrix T of target values and returns +% the full Hessian matrix H corresponding to the second derivatives of +% the negative log posterior distribution, evaluated for the current +% weight and bias values as defined by NET. Currently, the +% implementation only computes the Hessian for the output layer +% weights. +% +% [H, HDATA] = RBFHESS(NET, X, T) returns both the Hessian matrix H and +% the contribution HDATA arising from the data dependent term in the +% Hessian. +% +% H = RBFHESS(NET, X, T, HDATA) takes a network data structure NET, a +% matrix X of input values, and a matrix T of target values, together +% with the contribution HDATA arising from the data dependent term in +% the Hessian, and returns the full Hessian matrix H corresponding to +% the second derivatives of the negative log posterior distribution. +% This version saves computation time if HDATA has already been +% evaluated for the current weight and bias values. +% +% See also +% MLPHESS, HESSCHEK, EVIDENCE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'rbf', x, t); +if ~isempty(errstring); + error(errstring); +end + +if nargin == 3 + % Data term in Hessian needs to be computed + [a, z] = rbffwd(net, x); + hdata = datahess(net, z, t); +end + +% Add in effect of regularisation +[h, hdata] = hbayes(net, hdata); + +% Sub-function to compute data part of Hessian +function hdata = datahess(net, z, t) + +% Only works for output layer Hessian currently +if (isfield(net, 'mask') & ~any(net.mask(... + 1:(net.nwts - net.nout*(net.nhidden+1))))) + hdata = zeros(net.nwts); + ndata = size(z, 1); + out_hess = [z ones(ndata, 1)]'*[z ones(ndata, 1)]; + for j = 1:net.nout + hdata = rearrange_hess(net, j, out_hess, hdata); + end +else + error('Output layer Hessian only.'); +end +return + +% Sub-function to rearrange Hessian matrix +function hdata = rearrange_hess(net, j, out_hess, hdata) + +% Because all the biases come after all the input weights, +% we have to rearrange the blocks that make up the network Hessian. +% This function assumes that we are on the jth output and that all outputs +% are independent. + +% Start of bias weights block +bb_start = net.nwts - net.nout + 1; +% Start of weight block for jth output +ob_start = net.nwts - net.nout*(net.nhidden+1) + (j-1)*net.nhidden... + + 1; +% End of weight block for jth output +ob_end = ob_start + net.nhidden - 1; +% Index of bias weight +b_index = bb_start+(j-1); +% Put input weight block in right place +hdata(ob_start:ob_end, ob_start:ob_end) = out_hess(1:net.nhidden, ... + 1:net.nhidden); +% Put second derivative of bias weight in right place +hdata(b_index, b_index) = out_hess(net.nhidden+1, net.nhidden+1); +% Put cross terms (input weight v bias weight) in right place +hdata(b_index, ob_start:ob_end) = out_hess(net.nhidden+1, ... + 1:net.nhidden); +hdata(ob_start:ob_end, b_index) = out_hess(1:net.nhidden, ... + net.nhidden+1); + +return \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbfjacob.m b/sourcecodes/bnt-master/netlab3.3/rbfjacob.m new file mode 100644 index 00000000..6a6ec73a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfjacob.m @@ -0,0 +1,49 @@ +function jac = rbfjacob(net, x) +%RBFJACOB Evaluate derivatives of RBF network outputs with respect to inputs. +% +% Description +% G = RBFJACOB(NET, X) takes a network data structure NET and a matrix +% of input vectors X and returns a three-index matrix G whose I, J, K +% element contains the derivative of network output K with respect to +% input parameter J for input pattern I. +% +% See also +% RBF, RBFGRAD, RBFBKP +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'rbf', x); +if ~isempty(errstring); + error(errstring); +end + +if ~strcmp(net.outfn, 'linear') + error('Function only implemented for linear outputs') +end + +[y, z, n2] = rbffwd(net, x); + +ndata = size(x, 1); +jac = zeros(ndata, net.nin, net.nout); +Psi = zeros(net.nin, net.nhidden); +% Calculate derivative of activations wrt n2 +switch net.actfn +case 'gaussian' + dz = -z./(ones(ndata, 1)*net.wi); +case 'tps' + dz = 2*(1 + log(n2+(n2==0))); +case 'r4logr' + dz = 2*(n2.*(1+2.*log(n2+(n2==0)))); +otherwise + error(['Unknown activation function ', net.actfn]); +end + +% Ignore biases as they cannot affect Jacobian +for n = 1:ndata + Psi = (ones(net.nin, 1)*dz(n, :)).* ... + (x(n, :)'*ones(1, net.nhidden) - net.c'); + % Now compute the Jacobian + jac(n, :, :) = Psi * net.w2; +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbfpak.m b/sourcecodes/bnt-master/netlab3.3/rbfpak.m new file mode 100644 index 00000000..a482f99b --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfpak.m @@ -0,0 +1,19 @@ +function w = rbfpak(net) +%RBFPAK Combines all the parameters in an RBF network into one weights vector. +% +% Description +% W = RBFPAK(NET) takes a network data structure NET and combines the +% component parameter matrices into a single row vector W. +% +% See also +% RBFUNPAK, RBF +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'rbf'); +if ~errstring + error(errstring); +end + +w = [net.c(:)', net.wi, net.w2(:)', net.b2]; diff --git a/sourcecodes/bnt-master/netlab3.3/rbfprior.m b/sourcecodes/bnt-master/netlab3.3/rbfprior.m new file mode 100644 index 00000000..c2dfaeb7 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfprior.m @@ -0,0 +1,54 @@ +function [mask, prior] = rbfprior(rbfunc, nin, nhidden, nout, aw2, ab2) +%RBFPRIOR Create Gaussian prior and output layer mask for RBF. +% +% Description +% [MASK, PRIOR] = RBFPRIOR(RBFUNC, NIN, NHIDDEN, NOUT, AW2, AB2) +% generates a vector MASK that selects only the output layer weights. +% This is because most uses of RBF networks in a Bayesian context have +% fixed basis functions with the output layer as the only adjustable +% parameters. In particular, the Neuroscale output error function is +% designed to work only with this mask. +% +% The return value PRIOR is a data structure, with fields PRIOR.ALPHA +% and PRIOR.INDEX, which specifies a Gaussian prior distribution for +% the network weights in an RBF network. The parameters AW2 and AB2 are +% all scalars and represent the regularization coefficients for two +% groups of parameters in the network corresponding to second-layer +% weights, and second-layer biases respectively. Then PRIOR.ALPHA +% represents a column vector of length 2 containing the parameters, and +% PRIOR.INDEX is a matrix specifying which weights belong in each +% group. Each column has one element for each weight in the matrix, +% using the standard ordering as defined in RBFPAK, and each element is +% 1 or 0 according to whether the weight is a member of the +% corresponding group or not. +% +% See also +% RBF, RBFERR, RBFGRAD, EVIDENCE +% + +% Copyright (c) Ian T Nabney (1996-2001) + +nwts_layer2 = nout + (nhidden *nout); +switch rbfunc +case 'gaussian' + nwts_layer1 = nin*nhidden + nhidden; +case {'tps', 'r4logr'} + nwts_layer1 = nin*nhidden; +otherwise + error('Undefined activation function'); +end +nwts = nwts_layer1 + nwts_layer2; + +% Make a mask only for output layer +mask = [zeros(nwts_layer1, 1); ones(nwts_layer2, 1)]; + +if nargout > 1 + % Construct prior + indx = zeros(nwts, 2); + mark2 = nwts_layer1 + (nhidden * nout); + indx(nwts_layer1 + 1:mark2, 1) = ones(nhidden * nout, 1); + indx(mark2 + 1:nwts, 2) = ones(nout, 1); + + prior.index = indx; + prior.alpha = [aw2, ab2]'; +end \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbfsetbf.m b/sourcecodes/bnt-master/netlab3.3/rbfsetbf.m new file mode 100644 index 00000000..9e8c3ca4 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfsetbf.m @@ -0,0 +1,39 @@ +function net = rbfsetbf(net, options, x) +%RBFSETBF Set basis functions of RBF from data. +% +% Description +% NET = RBFSETBF(NET, OPTIONS, X) sets the basis functions of the RBF +% network NET so that they model the unconditional density of the +% dataset X. This is done by training a GMM with spherical covariances +% using GMMEM. The OPTIONS vector is passed to GMMEM. The widths of +% the functions are set by a call to RBFSETFW. +% +% See also +% RBFTRAIN, RBFSETFW, GMMEM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'rbf', x); +if ~isempty(errstring) + error(errstring); +end + +% Create a spherical Gaussian mixture model +mix = gmm(net.nin, net.nhidden, 'spherical'); + +% Initialise the parameters from the input data +% Just use a small number of k means iterations +kmoptions = zeros(1, 18); +kmoptions(1) = -1; % Turn off warnings +kmoptions(14) = 5; % Just 5 iterations to get centres roughly right +mix = gmminit(mix, x, kmoptions); + +% Train mixture model using EM algorithm +[mix, options] = gmmem(mix, x, options); + +% Now set the centres of the RBF from the centres of the mixture model +net.c = mix.centres; + +% options(7) gives scale of function widths +net = rbfsetfw(net, options(7)); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rbfsetfw.m b/sourcecodes/bnt-master/netlab3.3/rbfsetfw.m new file mode 100644 index 00000000..8ed27605 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfsetfw.m @@ -0,0 +1,30 @@ +function net = rbfsetfw(net, scale) +%RBFSETFW Set basis function widths of RBF. +% +% Description +% NET = RBFSETFW(NET, SCALE) sets the widths of the basis functions of +% the RBF network NET. If Gaussian basis functions are used, then the +% variances are set to the largest squared distance between centres if +% SCALE is non-positive and SCALE times the mean distance of each +% centre to its nearest neighbour if SCALE is positive. Non-Gaussian +% basis functions do not have a width. +% +% See also +% RBFTRAIN, RBFSETBF, GMMEM +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set the variances to be the largest squared distance between centres +if strcmp(net.actfn, 'gaussian') + cdist = dist2(net.c, net.c); + if scale > 0.0 + % Set variance of basis to be scale times average + % distance to nearest neighbour + cdist = cdist + realmax*eye(net.nhidden); + widths = scale*mean(min(cdist)); + else + widths = max(max(cdist)); + end + net.wi = widths * ones(size(net.wi)); +end diff --git a/sourcecodes/bnt-master/netlab3.3/rbftrain.m b/sourcecodes/bnt-master/netlab3.3/rbftrain.m new file mode 100644 index 00000000..f2b68422 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbftrain.m @@ -0,0 +1,203 @@ +function [net, options] = rbftrain(net, options, x, t) +%RBFTRAIN Two stage training of RBF network. +% +% Description +% NET = RBFTRAIN(NET, OPTIONS, X, T) uses a two stage training +% algorithm to set the weights in the RBF model structure NET. Each row +% of X corresponds to one input vector and each row of T contains the +% corresponding target vector. The centres are determined by fitting a +% Gaussian mixture model with circular covariances using the EM +% algorithm through a call to RBFSETBF. (The mixture model is +% initialised using a small number of iterations of the K-means +% algorithm.) If the activation functions are Gaussians, then the basis +% function widths are then set to the maximum inter-centre squared +% distance. +% +% For linear outputs, the hidden to output weights that give rise to +% the least squares solution can then be determined using the pseudo- +% inverse. For neuroscale outputs, the hidden to output weights are +% determined using the iterative shadow targets algorithm. Although +% this two stage procedure may not give solutions with as low an error +% as using general purpose non-linear optimisers, it is much faster. +% +% The options vector may have two rows: if this is the case, then the +% second row is passed to RBFSETBF, which allows the user to specify a +% different number iterations for RBF and GMM training. The optional +% parameters to RBFTRAIN have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values during EM training. +% +% OPTIONS(2) is a measure of the precision required for the value of +% the weights W at the solution. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. Both this and the previous condition must +% be satisfied for termination. +% +% OPTIONS(5) is set to 1 if the basis functions parameters should +% remain unchanged; default 0. +% +% OPTIONS(6) is set to 1 if the output layer weights should be should +% set using PCA. This is only relevant for Neuroscale outputs; default +% 0. +% +% OPTIONS(14) is the maximum number of iterations for the shadow +% targets algorithm; default 100. +% +% See also +% RBF, RBFERR, RBFFWD, RBFGRAD, RBFPAK, RBFUNPAK, RBFSETBF +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +switch net.outfn +case 'linear' + errstring = consist(net, 'rbf', x, t); +case 'neuroscale' + errstring = consist(net, 'rbf', x); +otherwise + error(['Unknown output function ', net.outfn]); +end +if ~isempty(errstring) + error(errstring); +end + +% Allow options to have two rows: if this is the case, then the second row +% is passed to rbfsetbf +if size(options, 1) == 2 + setbfoptions = options(2, :); + options = options(1, :); +else + setbfoptions = options; +end + +if(~options(14)) + options(14) = 100; +end +% Do we need to test for termination? +test = (options(2) | options(3)); + +% Set up the basis function parameters to model the input data density +% unless options(5) is set. +if ~(logical(options(5))) + net = rbfsetbf(net, setbfoptions, x); +end + +% Compute the design (or activations) matrix +[y, act] = rbffwd(net, x); +ndata = size(x, 1); + +if strcmp(net.outfn, 'neuroscale') & options(6) + % Initialise output layer weights by projecting data with PCA + mu = mean(x); + [pcvals, pcvecs] = pca(x, net.nout); + xproj = (x - ones(ndata, 1)*mu)*pcvecs; + % Now use projected data as targets to compute output layer weights + temp = pinv([act ones(ndata, 1)]) * xproj; + net.w2 = temp(1:net.nhidden, :); + net.b2 = temp(net.nhidden+1, :); + % Propagate again to compute revised outputs + [y, act] = rbffwd(net, x); +end + +switch net.outfn +case 'linear' + % Sum of squares error function in regression model + % Solve for the weights and biases using pseudo-inverse from activations + Phi = [act ones(ndata, 1)]; + if ~isfield(net, 'alpha') + % Solve for the weights and biases using left matrix divide + temp = pinv(Phi)*t; + elseif size(net.alpha == [1 1]) + % Use normal form equation + hessian = Phi'*Phi + net.alpha*eye(net.nhidden+1); + temp = pinv(hessian)*(Phi'*t); + else + error('Only scalar alpha allowed'); + end + net.w2 = temp(1:net.nhidden, :); + net.b2 = temp(net.nhidden+1, :); + +case 'neuroscale' + % Use the shadow targets training algorithm + if nargin < 4 + % If optional input distances not passed in, then use + % Euclidean distance + x_dist = sqrt(dist2(x, x)); + else + x_dist = t; + end + Phi = [act, ones(ndata, 1)]; + % Compute the pseudo-inverse of Phi + PhiDag = pinv(Phi); + % Compute y_dist, distances between image points + y_dist = sqrt(dist2(y, y)); + + % Save old weights so that we can check the termination criterion + wold = netpak(net); + % Compute initial error (stress) value + errold = 0.5*(sum(sum((x_dist - y_dist).^2))); + + % Initial value for eta + eta = 0.1; + k_up = 1.2; + k_down = 0.1; + success = 1; % Force initial gradient calculation + + for j = 1:options(14) + if success + % Compute the negative error gradient with respect to network outputs + D = (x_dist - y_dist)./(y_dist+(y_dist==0)); + temp = y'; + neg_gradient = -2.*sum(kron(D, ones(1, net.nout)) .* ... + (repmat(y, 1, ndata) - repmat((temp(:))', ndata, 1)), 1); + neg_gradient = (reshape(neg_gradient, net.nout, ndata))'; + end + % Compute the shadow targets + t = y + eta*neg_gradient; + % Solve for the weights and biases + temp = PhiDag * t; + net.w2 = temp(1:net.nhidden, :); + net.b2 = temp(net.nhidden+1, :); + + % Do housekeeping and test for convergence + ynew = rbffwd(net, x); + y_distnew = sqrt(dist2(ynew, ynew)); + err = 0.5.*(sum(sum((x_dist-y_distnew).^2))); + if err > errold + success = 0; + % Restore previous weights + net = netunpak(net, wold); + err = errold; + eta = eta * k_down; + else + success = 1; + eta = eta * k_up; + errold = err; + y = ynew; + y_dist = y_distnew; + if test & j > 1 + w = netpak(net); + if (max(abs(w - wold)) < options(2) & abs(err-errold) < options(3)) + options(8) = err; + return; + end + end + wold = netpak(net); + end + if options(1) + fprintf(1, 'Cycle %4d Error %11.6f\n', j, err) + end + if nargout >= 3 + errlog(j) = err; + end + end + options(8) = errold; + if (options(1) >= 0) + disp('Warning: Maximum number of iterations has been exceeded'); + end +otherwise + error(['Unknown output function ', net.outfn]); + +end diff --git a/sourcecodes/bnt-master/netlab3.3/rbfunpak.m b/sourcecodes/bnt-master/netlab3.3/rbfunpak.m new file mode 100644 index 00000000..64b15a9a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rbfunpak.m @@ -0,0 +1,43 @@ +function net = rbfunpak(net, w) +%RBFUNPAK Separates a vector of RBF weights into its components. +% +% Description +% NET = RBFUNPAK(NET, W) takes an RBF network data structure NET and a +% weight vector W, and returns a network data structure identical to +% the input network, except that the centres C, the widths WI, the +% second-layer weight matrix W2 and the second-layer bias vector B2 +% have all been set to the corresponding elements of W. +% +% See also +% RBFPAK, RBF +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'rbf'); +if ~errstring + error(errstring); +end + +if net.nwts ~= length(w) + error('Invalid length of weight vector') +end + +nin = net.nin; +nhidden = net.nhidden; +nout = net.nout; + +mark1 = nin*nhidden; +net.c = reshape(w(1:mark1), nhidden, nin); +if strcmp(net.actfn, 'gaussian') + mark2 = mark1 + nhidden; + net.wi = reshape(w(mark1+1:mark2), 1, nhidden); +else + mark2 = mark1; + net.wi = []; +end +mark3 = mark2 + nhidden*nout; +net.w2 = reshape(w(mark2+1:mark3), nhidden, nout); +mark4 = mark3 + nout; +net.b2 = reshape(w(mark3+1:mark4), 1, nout); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/rosegrad.m b/sourcecodes/bnt-master/netlab3.3/rosegrad.m new file mode 100644 index 00000000..898e5d80 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rosegrad.m @@ -0,0 +1,20 @@ +function g = rosegrad(x) +%ROSEGRAD Calculate gradient of Rosenbrock's function. +% +% Description +% G = ROSEGRAD(X) computes the gradient of Rosenbrock's function at +% each row of X, which should have two columns. +% +% See also +% DEMOPT1, ROSEN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Return gradient of Rosenbrock's test function + +nrows = size(x, 1); +g = zeros(nrows,2); + +g(:,1) = -400 * (x(:,2) - x(:,1).^2) * x(:,1) - 2 * (1 - x(:,1)); +g(:,2) = 200 * (x(:,2) - x(:,1).^2); diff --git a/sourcecodes/bnt-master/netlab3.3/rosen.m b/sourcecodes/bnt-master/netlab3.3/rosen.m new file mode 100644 index 00000000..89332a52 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/rosen.m @@ -0,0 +1,16 @@ +function y = rosen(x) +%ROSEN Calculate Rosenbrock's function. +% +% Description +% Y = ROSEN(X) computes the value of Rosenbrock's function at each row +% of X, which should have two columns. +% +% See also +% DEMOPT1, ROSEGRAD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Calculate value of Rosenbrock's function: x should be nrows by 2 columns + +y = 100 * ((x(:,2) - x(:,1).^2).^2) + (1.0 - x(:,1)).^2; diff --git a/sourcecodes/bnt-master/netlab3.3/scg.m b/sourcecodes/bnt-master/netlab3.3/scg.m new file mode 100644 index 00000000..c248d9e4 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/scg.m @@ -0,0 +1,208 @@ +function [x, options, flog, pointlog, scalelog] = scg(f, x, options, gradf, varargin) +%SCG Scaled conjugate gradient optimization. +% +% Description +% [X, OPTIONS] = SCG(F, X, OPTIONS, GRADF) uses a scaled conjugate +% gradients algorithm to find a local minimum of the function F(X) +% whose gradient is given by GRADF(X). Here X is a row vector and F +% returns a scalar value. The point at which F has a local minimum is +% returned as X. The function value at that point is returned in +% OPTIONS(8). +% +% [X, OPTIONS, FLOG, POINTLOG, SCALELOG] = SCG(F, X, OPTIONS, GRADF) +% also returns (optionally) a log of the function values after each +% cycle in FLOG, a log of the points visited in POINTLOG, and a log of +% the scale values in the algorithm in SCALELOG. +% +% SCG(F, X, OPTIONS, GRADF, P1, P2, ...) allows additional arguments to +% be passed to F() and GRADF(). The optional parameters have the +% following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs error +% values in the return argument ERRLOG, and the points visited in the +% return argument POINTSLOG. If OPTIONS(1) is set to 0, then only +% warning messages are displayed. If OPTIONS(1) is -1, then nothing is +% displayed. +% +% OPTIONS(2) is a measure of the absolute precision required for the +% value of X at the solution. If the absolute difference between the +% values of X between two successive steps is less than OPTIONS(2), +% then this condition is satisfied. +% +% OPTIONS(3) is a measure of the precision required of the objective +% function at the solution. If the absolute difference between the +% objective function values between two successive steps is less than +% OPTIONS(3), then this condition is satisfied. Both this and the +% previous condition must be satisfied for termination. +% +% OPTIONS(9) is set to 1 to check the user defined gradient function. +% +% OPTIONS(10) returns the total number of function evaluations +% (including those in any line searches). +% +% OPTIONS(11) returns the total number of gradient evaluations. +% +% OPTIONS(14) is the maximum number of iterations; default 100. +% +% See also +% CONJGRAD, QUASINEW +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Set up the options. +if length(options) < 18 + error('Options vector too short') +end + +if(options(14)) + niters = options(14); +else + niters = 100; +end + +display = options(1); +gradcheck = options(9); + +% Set up strings for evaluating function and gradient +f = fcnchk(f, length(varargin)); +gradf = fcnchk(gradf, length(varargin)); + +nparams = length(x); + +% Check gradients +if (gradcheck) + feval('gradchek', x, f, gradf, varargin{:}); +end + +sigma0 = 1.0e-4; +fold = feval(f, x, varargin{:}); % Initial function value. +fnow = fold; +options(10) = options(10) + 1; % Increment function evaluation counter. +gradnew = feval(gradf, x, varargin{:}); % Initial gradient. +gradold = gradnew; +options(11) = options(11) + 1; % Increment gradient evaluation counter. +d = -gradnew; % Initial search direction. +success = 1; % Force calculation of directional derivs. +nsuccess = 0; % nsuccess counts number of successes. +beta = 1.0; % Initial scale parameter. +betamin = 1.0e-15; % Lower bound on scale. +betamax = 1.0e100; % Upper bound on scale. +j = 1; % j counts number of iterations. +if nargout >= 3 + flog(j, :) = fold; + if nargout == 4 + pointlog(j, :) = x; + end +end + +% Main optimization loop. +while (j <= niters) + + % Calculate first and second directional derivatives. + if (success == 1) + mu = d*gradnew'; + if (mu >= 0) + d = - gradnew; + mu = d*gradnew'; + end + kappa = d*d'; + if kappa < eps + options(8) = fnow; + return + end + sigma = sigma0/sqrt(kappa); + xplus = x + sigma*d; + gplus = feval(gradf, xplus, varargin{:}); + options(11) = options(11) + 1; + theta = (d*(gplus' - gradnew'))/sigma; + end + + % Increase effective curvature and evaluate step size alpha. + delta = theta + beta*kappa; + if (delta <= 0) + delta = beta*kappa; + beta = beta - theta/kappa; + end + alpha = - mu/delta; + + % Calculate the comparison ratio. + xnew = x + alpha*d; + fnew = feval(f, xnew, varargin{:}); + options(10) = options(10) + 1; + Delta = 2*(fnew - fold)/(alpha*mu); + if (Delta >= 0) + success = 1; + nsuccess = nsuccess + 1; + x = xnew; + fnow = fnew; + else + success = 0; + fnow = fold; + end + + if nargout >= 3 + % Store relevant variables + flog(j) = fnow; % Current function value + if nargout >= 4 + pointlog(j,:) = x; % Current position + if nargout >= 5 + scalelog(j) = beta; % Current scale parameter + end + end + end + if display > 0 + fprintf(1, 'Cycle %4d Error %11.6f Scale %e\n', j, fnow, beta); + end + + if (success == 1) + % Test for termination + + if (max(abs(alpha*d)) < options(2) & max(abs(fnew-fold)) < options(3)) + options(8) = fnew; + return; + + else + % Update variables for new position + fold = fnew; + gradold = gradnew; + gradnew = feval(gradf, x, varargin{:}); + options(11) = options(11) + 1; + % If the gradient is zero then we are done. + if (gradnew*gradnew' == 0) + options(8) = fnew; + return; + end + end + end + + % Adjust beta according to comparison ratio. + if (Delta < 0.25) + beta = min(4.0*beta, betamax); + end + if (Delta > 0.75) + beta = max(0.5*beta, betamin); + end + + % Update search direction using Polak-Ribiere formula, or re-start + % in direction of negative gradient after nparams steps. + if (nsuccess == nparams) + d = -gradnew; + nsuccess = 0; + else + if (success == 1) + gamma = (gradold - gradnew)*gradnew'/(mu); + d = gamma*d - gradnew; + end + end + j = j + 1; +end + +% If we get here, then we haven't terminated in the given number of +% iterations. + +options(8) = fold; +if (options(1) >= 0) + disp(maxitmess); +end + diff --git a/sourcecodes/bnt-master/netlab3.3/som.m b/sourcecodes/bnt-master/netlab3.3/som.m new file mode 100644 index 00000000..4c975c2f --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/som.m @@ -0,0 +1,55 @@ +function net = som(nin, map_size) +%SOM Creates a Self-Organising Map. +% +% Description +% NET = SOM(NIN, MAP_SIZE) creates a SOM NET with input dimension (i.e. +% data dimension) NIN and map dimensions MAP_SIZE. Only two- +% dimensional maps are currently implemented. +% +% The fields in NET are +% type = 'som' +% nin = number of inputs +% map_dim = dimension of map (constrained to be 2) +% map_size = grid size: number of nodes in each dimension +% num_nodes = number of nodes: the product of values in map_size +% map = map_dim+1 dimensional array containing nodes +% inode_dist = map of inter-node distances using Manhatten metric +% +% The map contains the node vectors arranged column-wise in the first +% dimension of the array. +% +% See also +% KMEANS, SOMFWD, SOMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +net.type = 'som'; +net.nin = nin; + +% Create Map of nodes +if round(map_size) ~= map_size | (map_size < 1) + error('SOM specification must contain positive integers'); +end + +net.map_dim = length(map_size); +if net.map_dim ~= 2 + error('SOM is a 2 dimensional map'); +end +net.num_nodes = prod(map_size); +% Centres are stored by column as first index of multi-dimensional array. +% This makes extracting them later more easy. +% Initialise with rand to create square grid +net.map = rand([nin, map_size]); +net.map_size = map_size; + +% Crude function to compute inter-node distances +net.inode_dist = zeros([map_size, net.num_nodes]); +for m = 1:net.num_nodes + node_loc = [1+fix((m-1)/map_size(2)), 1+rem((m-1),map_size(2))]; + for k = 1:map_size(1) + for l = 1:map_size(2) + net.inode_dist(k, l, m) = round(max(abs([k l] - node_loc))); + end + end +end diff --git a/sourcecodes/bnt-master/netlab3.3/somfwd.m b/sourcecodes/bnt-master/netlab3.3/somfwd.m new file mode 100644 index 00000000..0e980284 --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/somfwd.m @@ -0,0 +1,30 @@ +function [d2, win_nodes] = somfwd(net, x) +%SOMFWD Forward propagation through a Self-Organising Map. +% +% Description +% D2 = SOMFWD(NET, X) propagates the data matrix X through a SOM NET, +% returning the squared distance matrix D2 with dimension NIN by +% NUM_NODES. The $i$th row represents the squared Euclidean distance +% to each of the nodes of the SOM. +% +% [D2, WIN_NODES] = SOMFWD(NET, X) also returns the indices of the +% winning nodes for each pattern. +% +% See also +% SOM, SOMTRAIN +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check for consistency +errstring = consist(net, 'som', x); +if ~isempty(errstring) + error(errstring); +end + +% Turn nodes into matrix of centres +nodes = (reshape(net.map, net.nin, net.num_nodes))'; +% Compute squared distance matrix +d2 = dist2(x, nodes); +% Find winning node for each pattern: minimum value in each row +[w, win_nodes] = min(d2, [], 2); diff --git a/sourcecodes/bnt-master/netlab3.3/sompak.m b/sourcecodes/bnt-master/netlab3.3/sompak.m new file mode 100644 index 00000000..a1d7653a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/sompak.m @@ -0,0 +1,23 @@ +function [c] = sompak(net) +%SOMPAK Combines node weights into one weights matrix. +% +% Description +% C = SOMPAK(NET) takes a SOM data structure NET and combines the node +% weights into a matrix of centres C where each row represents the node +% vector. +% +% The ordering of the parameters in W is defined by the indexing of the +% multi-dimensional array NET.MAP. +% +% See also +% SOM, SOMUNPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'som'); +if ~isempty(errstring) + error(errstring); +end +% Returns map as a sequence of row vectors +c = (reshape(net.map, net.nin, net.num_nodes))'; diff --git a/sourcecodes/bnt-master/netlab3.3/somtrain.m b/sourcecodes/bnt-master/netlab3.3/somtrain.m new file mode 100644 index 00000000..47df11af --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/somtrain.m @@ -0,0 +1,157 @@ +function net = somtrain(net, options, x) +%SOMTRAIN Kohonen training algorithm for SOM. +% +% Description +% NET = SOMTRAIN{NET, OPTIONS, X) uses Kohonen's algorithm to train a +% SOM. Both on-line and batch algorithms are implemented. The learning +% rate (for on-line) and neighbourhood size decay linearly. There is no +% error function minimised during training (so there is no termination +% criterion other than the number of epochs), but the sum-of-squares +% is computed and returned in OPTIONS(8). +% +% The optional parameters have the following interpretations. +% +% OPTIONS(1) is set to 1 to display error values; also logs learning +% rate ALPHA and neighbourhood size NSIZE. Otherwise nothing is +% displayed. +% +% OPTIONS(5) determines whether the patterns are sampled randomly with +% replacement. If it is 0 (the default), then patterns are sampled in +% order. This is only relevant to the on-line algorithm. +% +% OPTIONS(6) determines if the on-line or batch algorithm is used. If +% it is 1 then the batch algorithm is used. If it is 0 (the default) +% then the on-line algorithm is used. +% +% OPTIONS(14) is the maximum number of iterations (passes through the +% complete pattern set); default 100. +% +% OPTIONS(15) is the final neighbourhood size; default value is the +% same as the initial neighbourhood size. +% +% OPTIONS(16) is the final learning rate; default value is the same as +% the initial learning rate. +% +% OPTIONS(17) is the initial neighbourhood size; default 0.5*maximum +% map size. +% +% OPTIONS(18) is the initial learning rate; default 0.9. This +% parameter must be positive. +% +% See also +% KMEANS, SOM, SOMFWD +% + +% Copyright (c) Ian T Nabney (1996-2001) + +% Check arguments for consistency +errstring = consist(net, 'som', x); +if ~isempty(errstring) + error(errstring); +end + +% Set number of iterations in convergence phase +if (~options(14)) + options(14) = 100; +end +niters = options(14); + +% Learning rate must be positive +if (options(18) > 0) + alpha_first = options(18); +else + alpha_first = 0.9; +end +% Final learning rate must be no greater than initial learning rate +if (options(16) > alpha_first | options(16) < 0) + alpha_last = alpha_first; +else + alpha_last = options(16); +end + +% Neighbourhood size +if (options(17) >= 0) + nsize_first = options(17); +else + nsize_first = max(net.map_dim)/2; +end +% Final neighbourhood size must be no greater than initial size +if (options(15) > nsize_first | options(15) < 0) + nsize_last = nsize_first; +else + nsize_last = options(15); +end + +ndata = size(x, 1); + +if options(6) + % Batch algorithm + H = zeros(ndata, net.num_nodes); +end +% Put weights into matrix form +tempw = sompak(net); + +% Then carry out training +j = 1; +while j <= niters + if options(6) + % Batch version of algorithm + alpha = 0.0; + frac_done = (niters - j)/niters; + % Compute neighbourhood + nsize = round((nsize_first - nsize_last)*frac_done + nsize_last); + + % Find winning node: put weights back into net so that we can + % call somunpak + net = somunpak(net, tempw); + [temp, bnode] = somfwd(net, x); + for k = 1:ndata + H(k, :) = reshape(net.inode_dist(:, :, bnode(k))<=nsize, ... + 1, net.num_nodes); + end + s = sum(H, 1); + for k = 1:net.num_nodes + if s(k) > 0 + tempw(k, :) = sum((H(:, k)*ones(1, net.nin)).*x, 1)/ ... + s(k); + end + end + else + % On-line version of algorithm + if options(5) + % Randomise order of pattern presentation: with replacement + pnum = ceil(rand(ndata, 1).*ndata); + else + pnum = 1:ndata; + end + % Cycle through dataset + for k = 1:ndata + % Fraction done + frac_done = (((niters+1)*ndata)-(j*ndata + k))/((niters+1)*ndata); + % Compute learning rate + alpha = (alpha_first - alpha_last)*frac_done + alpha_last; + % Compute neighbourhood + nsize = round((nsize_first - nsize_last)*frac_done + nsize_last); + % Find best node + pat_diff = ones(net.num_nodes, 1)*x(pnum(k), :) - tempw; + [temp, bnode] = min(sum(abs(pat_diff), 2)); + + % Now update neighbourhood + neighbourhood = (net.inode_dist(:, :, bnode) <= nsize); + tempw = tempw + ... + ((alpha*(neighbourhood(:)))*ones(1, net.nin)).*pat_diff; + end + end + if options(1) + % Print iteration information + fprintf(1, 'Iteration %d; alpha = %f, nsize = %f. ', j, alpha, ... + nsize); + % Print sum squared error to nearest node + d2 = dist2(tempw, x); + fprintf(1, 'Error = %f\n', sum(min(d2))); + end + j = j + 1; +end + +net = somunpak(net, tempw); +options(8) = sum(min(dist2(tempw, x))); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/somunpak.m b/sourcecodes/bnt-master/netlab3.3/somunpak.m new file mode 100644 index 00000000..e243820a --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/somunpak.m @@ -0,0 +1,23 @@ +function net = somunpak(net, w) +%SOMUNPAK Replaces node weights in SOM. +% +% Description +% NET = SOMUNPAK(NET, W) takes a SOM data structure NET and weight +% matrix W (each node represented by a row) and puts the nodes back +% into the multi-dimensional array NET.MAP. +% +% The ordering of the parameters in W is defined by the indexing of the +% multi-dimensional array NET.MAP. +% +% See also +% SOM, SOMPAK +% + +% Copyright (c) Ian T Nabney (1996-2001) + +errstring = consist(net, 'som'); +if ~isempty(errstring) + error(errstring); +end +% Put weights back into network data structure +net.map = reshape(w', [net.nin net.map_size]); \ No newline at end of file diff --git a/sourcecodes/bnt-master/netlab3.3/xor.dat b/sourcecodes/bnt-master/netlab3.3/xor.dat new file mode 100644 index 00000000..f0885cee --- /dev/null +++ b/sourcecodes/bnt-master/netlab3.3/xor.dat @@ -0,0 +1,15 @@ +nin 2 +nout 1 +ndata 12 +1 0 1 +0 1 1 +0 0 0 +1 1 0 +1 0 1 +0 1 1 +0 0 0 +1 1 0 +1 0 1 +0 1 1 +0 0 0 +1 1 0 |