1
2
3
4 """ Training algorithms for feed-forward neural nets
5
6 Unless noted otherwise, algorithms and notation are taken from:
7 "Artificial Neural Networks: Theory and Applications",
8 Dan W. Patterson, Prentice Hall, 1996
9
10 """
11 from __future__ import print_function
12
13 import numpy
14
15
17 """ "virtual base class" for network trainers
18
19 """
20 pass
21
22
24 """implement back propagation (algorithm on pp 153-154 of Patterson)
25
26 I don't *think* that I've made any assumptions about the connectivity of
27 the net (i.e. full connectivity between layers is not required).
28
29 **NOTE:** this code is currently making the assumption that the activation
30 functions on the nodes in the network are capable of calculating their
31 derivatives using only their values (i.e. a DerivFromVal method should
32 exist). This shouldn't be too hard to change.
33
34 """
35
37 """ does a BackProp step based upon the example
38
39 **Arguments**
40
41 - example: a 2-tuple:
42 1) a list of variable values values
43 2) a list of result values (targets)
44
45 - net: a _Network_ (or something supporting the same API)
46
47 - resVect: if this is nonzero, then the network is not required to
48 classify the _example_
49
50 **Returns**
51
52 the backprop error from _network_ **before the update**
53
54 **Note**
55
56 In case it wasn't blindingly obvious, the weights in _network_ are modified
57 in the course of taking a backprop step.
58
59 """
60 totNumNodes = net.GetNumNodes()
61 if self.oldDeltaW is None:
62 self.oldDeltaW = numpy.zeros(totNumNodes, numpy.float64)
63 outputNodeList = net.GetOutputNodeList()
64 nOutput = len(outputNodeList)
65 targetVect = numpy.array(example[-nOutput:], numpy.float64)
66 trainVect = example[:-nOutput]
67 if resVect is None:
68
69 net.ClassifyExample(trainVect)
70 resVect = net.GetLastOutputs()
71 outputs = numpy.take(resVect, outputNodeList)
72 errVect = targetVect - outputs
73
74 delta = numpy.zeros(totNumNodes, numpy.float64)
75
76 for i in range(len(outputNodeList)):
77 idx = outputNodeList[i]
78 node = net.GetNode(idx)
79
80 delta[idx] = errVect[i] * node.actFunc.DerivFromVal(resVect[idx])
81
82 inputs = node.GetInputs()
83 weights = delta[idx] * node.GetWeights()
84 for j in range(len(inputs)):
85 idx2 = inputs[j]
86 delta[idx2] = delta[idx2] + weights[j]
87
88
89 for layer in range(net.GetNumHidden() - 1, -1, -1):
90 nodesInLayer = net.GetHiddenLayerNodeList(layer)
91 for idx in nodesInLayer:
92 node = net.GetNode(idx)
93
94 delta[idx] = delta[idx] * node.actFunc.DerivFromVal(resVect[idx])
95
96
97 if layer != 0:
98 inputs = node.GetInputs()
99 weights = delta[idx] * node.GetWeights()
100 for i in range(len(inputs)):
101 idx2 = inputs[i]
102 delta[idx2] = delta[idx2] + weights[i]
103
104
105
106 nHidden = net.GetNumHidden()
107 for layer in range(0, nHidden + 1):
108 if layer == nHidden:
109 idxList = net.GetOutputNodeList()
110 else:
111 idxList = net.GetHiddenLayerNodeList(layer)
112 for idx in idxList:
113 node = net.GetNode(idx)
114 dW = self.speed * delta[idx] * numpy.take(resVect, node.GetInputs())
115 newWeights = node.GetWeights() + dW
116 node.SetWeights(newWeights)
117
118
119 return numpy.sqrt(errVect * errVect)[0]
120
121 - def TrainOnLine(self, examples, net, maxIts=5000, errTol=0.1, useAvgErr=1, silent=0):
122 """ carries out online training of a neural net
123
124 The definition of online training is that the network is updated after
125 each example is presented.
126
127 **Arguments**
128
129 - examples: a list of 2-tuple:
130 1) a list of variable values values
131 2) a list of result values (targets)
132
133 - net: a _Network_ (or something supporting the same API)
134
135 - maxIts: the maximum number of *training epochs* (see below for definition) to be
136 run
137
138 - errTol: the tolerance for convergence
139
140 - useAvgErr: if this toggle is nonzero, then the error at each step will be
141 divided by the number of training examples for the purposes of checking
142 convergence.
143
144 - silent: controls the amount of visual noise produced as this runs.
145
146
147 **Note**
148
149 a *training epoch* is one complete pass through all the training examples
150
151 """
152 nExamples = len(examples)
153 converged = 0
154 cycle = 0
155
156 while (not converged) and (cycle < maxIts):
157 maxErr = 0
158 newErr = 0
159
160 for example in examples:
161 localErr = self.StepUpdate(example, net)
162 newErr += localErr
163 if localErr > maxErr:
164 maxErr = localErr
165 if useAvgErr == 1:
166 newErr = newErr / nExamples
167 else:
168 newErr = maxErr
169
170
171 if newErr <= errTol:
172 converged = 1
173
174
175 if not silent:
176 print('epoch %d, error: % 6.4f' % (cycle, newErr))
177
178 cycle = cycle + 1
179 if not silent:
180 if converged:
181 print('Converged after %d epochs.' % cycle)
182 else:
183 print('NOT Converged after %d epochs.' % cycle)
184 print('final error: % 6.4f' % newErr)
185
186 - def __init__(self, speed=0.5, momentum=0.7):
187 """ Constructor
188
189 **Arguments**
190
191 - speed: the speed parameter for back prop training
192
193 - momentum: the momentum term for back prop training
194 *Not currently used*
195
196 """
197 self.speed = speed
198 self.momentum = momentum
199 self.oldDeltaW = None
200
201 if __name__ == '__main__':
202 from rdkit.ML.Neural import Network
203
205 examples = [[[0, 0, 1], [0.1]], [[0, 1, 1], [.1]], [[1, 0, 1], [.1]], [[1, 1, 1], [.9]]]
206 net = Network.Network([3, 1])
207 t = BackProp()
208 t.TrainOnLine(examples, net)
209 return net
210
212 examples = [[[0, 0, 1], [0.1]], [[0, 1, 1], [.9]], [[1, 0, 1], [.9]], [[1, 1, 1], [.9]]]
213 net = Network.Network([3, 1])
214 t = BackProp()
215 t.TrainOnLine(examples, net, maxIts=1000, useAvgErr=0)
216 print('classifications:')
217 for example in examples:
218 res = net.ClassifyExample(example[0])
219 print('%f -> %f' % (example[1][0], res))
220
221 return net
222
224 examples = [[[0, 0, 1], [.1]], [[0, 1, 1], [.9]], [[1, 0, 1], [.9]], [[1, 1, 1], [.1]]]
225 net = Network.Network([3, 3, 1])
226
227 t = BackProp(speed=.8)
228 t.TrainOnLine(examples, net, errTol=0.2)
229 return net
230
232 examples = [
233 [.1, .1],
234 [.2, .2],
235 [.3, .3],
236 [.4, .4],
237 [.8, .8],
238 ]
239 net = Network.Network([1, 2, 1])
240 t = BackProp(speed=.8)
241 t.TrainOnLine(examples, net, errTol=0.1, useAvgErr=0)
242 print('classifications:')
243 for example in examples:
244 res = net.ClassifyExample(example[:-1])
245 print('%f -> %f' % (example[-1], res))
246
247 return net
248
250 import random
251 random.seed(23)
252 import profile
253 import pstats
254 datFile = '%s.prof.dat' % (command)
255 profile.run('%s()' % command, datFile)
256 stats = pstats.Stats(datFile)
257 stats.strip_dirs()
258 stats.sort_stats('time').print_stats()
259
260 if 0:
261 net = testXor()
262 print('Xor:', net)
263 from rdkit.six.moves import cPickle
264 outF = open('xornet.pkl', 'wb+')
265 cPickle.dump(net, outF)
266 outF.close()
267 else:
268
269 net = testLinear()
270
271