1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
function [net, options, errlog, pointlog] = olgd(net, options, x, t)
%OLGD On-line gradient descent optimization.
%
% Description
% [NET, OPTIONS, ERRLOG, POINTLOG] = OLGD(NET, OPTIONS, X, T) uses on-
% line gradient descent to find a local minimum of the error function
% for the network NET computed on the input data X and target values T.
% A log of the error values after each cycle is (optionally) returned
% in ERRLOG, and a log of the points visited is (optionally) returned
% in POINTLOG. Because the gradient is computed on-line (i.e. after
% each pattern) this can be quite inefficient in Matlab.
%
% The error function value at final weight vector is returned in
% OPTIONS(8).
%
% The optional parameters have the following interpretations.
%
% OPTIONS(1) is set to 1 to display error values; also logs error
% values in the return argument ERRLOG, and the points visited in the
% return argument POINTSLOG. If OPTIONS(1) is set to 0, then only
% warning messages are displayed. If OPTIONS(1) is -1, then nothing is
% displayed.
%
% OPTIONS(2) is the precision required for the value of X at the
% solution. If the absolute difference between the values of X between
% two successive steps is less than OPTIONS(2), then this condition is
% satisfied.
%
% OPTIONS(3) is the precision required of the objective function at the
% solution. If the absolute difference between the error functions
% between two successive steps is less than OPTIONS(3), then this
% condition is satisfied. Both this and the previous condition must be
% satisfied for termination. Note that testing the function value at
% each iteration roughly halves the speed of the algorithm.
%
% OPTIONS(5) determines whether the patterns are sampled randomly with
% replacement. If it is 0 (the default), then patterns are sampled in
% order.
%
% OPTIONS(6) determines if the learning rate decays. If it is 1 then
% the learning rate decays at a rate of 1/T. If it is 0 (the default)
% then the learning rate is constant.
%
% OPTIONS(9) should be set to 1 to check the user defined gradient
% function.
%
% OPTIONS(10) returns the total number of function evaluations
% (including those in any line searches).
%
% OPTIONS(11) returns the total number of gradient evaluations.
%
% OPTIONS(14) is the maximum number of iterations (passes through the
% complete pattern set); default 100.
%
% OPTIONS(17) is the momentum; default 0.5.
%
% OPTIONS(18) is the learning rate; default 0.01.
%
% See also
% GRADDESC
%
% Copyright (c) Ian T Nabney (1996-2001)
% Set up the options.
if length(options) < 18
error('Options vector too short')
end
if (options(14))
niters = options(14);
else
niters = 100;
end
% Learning rate: must be positive
if (options(18) > 0)
eta = options(18);
else
eta = 0.01;
end
% Save initial learning rate for annealing
lr = eta;
% Momentum term: allow zero momentum
if (options(17) >= 0)
mu = options(17);
else
mu = 0.5;
end
pakstr = [net.type, 'pak'];
unpakstr = [net.type, 'unpak'];
% Extract initial weights from the network
w = feval(pakstr, net);
display = options(1);
% Work out if we need to compute f at each iteration.
% Needed if display results or if termination
% criterion requires it.
fcneval = (display | options(3));
% Check gradients
if (options(9))
feval('gradchek', w, 'neterr', 'netgrad', net, x, t);
end
dwold = zeros(1, length(w));
fold = 0; % Must be initialised so that termination test can be performed
ndata = size(x, 1);
if fcneval
fnew = neterr(w, net, x, t);
options(10) = options(10) + 1;
fold = fnew;
end
j = 1;
if nargout >= 3
errlog(j, :) = fnew;
if nargout == 4
pointlog(j, :) = w;
end
end
% Main optimization loop.
while j <= niters
wold = w;
if options(5)
% Randomise order of pattern presentation: with replacement
pnum = ceil(rand(ndata, 1).*ndata);
else
pnum = 1:ndata;
end
for k = 1:ndata
grad = netgrad(w, net, x(pnum(k),:), t(pnum(k),:));
if options(6)
% Let learning rate decrease as 1/t
lr = eta/((j-1)*ndata + k);
end
dw = mu*dwold - lr*grad;
w = w + dw;
dwold = dw;
end
options(11) = options(11) + 1; % Increment gradient evaluation count
if fcneval
fold = fnew;
fnew = neterr(w, net, x, t);
options(10) = options(10) + 1;
end
if display
fprintf(1, 'Iteration %5d Error %11.8f\n', j, fnew);
end
j = j + 1;
if nargout >= 3
errlog(j) = fnew;
if nargout == 4
pointlog(j, :) = w;
end
end
if (max(abs(w - wold)) < options(2) & abs(fnew - fold) < options(3))
% Termination criteria are met
options(8) = fnew;
net = feval(unpakstr, net, w);
return;
end
end
if fcneval
options(8) = fnew;
else
% Return error on entire dataset
options(8) = neterr(w, net, x, t);
options(10) = options(10) + 1;
end
if (options(1) >= 0)
disp(maxitmess);
end
net = feval(unpakstr, net, w);
|
