Optimizer visualization script.

docs/build-documentation.sh

@@ -61,6 +61,9 @@ rm -rf $docdir/doxygen-old;
 # Now add the author tags to the generated tutorials.
 python $doxygensrcdir/tutorial_author_annotator.py $docdir/*/tutorials/ $docdir/doxygen/
 
+# Now add the visualization to the optimizer tutorial.
+python $doxygensrcdir/optimizer_visualization_annotator.py $docdir/doxygen/optimizertutorial.html
+
 # Copy the man page results to the right directory.
 rm -rf $docdir/man/*
 cp share/man/*.gz $docdir/man/

docs/doxygen-src/optimizer_visualization_annotator.py

@@ -0,0 +1,44 @@
+#!/usr/bin/python
+#
+# Usage: python optimizer_visualization_annotator.py path/to/the/doxygen/optimizertutorial.html
+
+import re
+import glob
+import os
+import sys
+
+optimizerVisualizationFile = sys.argv[1]
+
+with open(optimizerVisualizationFile) as f: content = f.read()
+
+jsPattern = '''<script type="text/javascript" src="dynamic_tables.js"></script>'''
+
+jsAddition = '''
+<script src="../../../js/optimizer-visualization.js"></script>
+<script src="../../../js/functions.js"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/vis/4.21.0/vis.min.js"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.7.1/Chart.min.js"></script>
+<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/vis/4.21.0/vis.min.css"/>
+'''
+
+visPattern = "</p>"
+
+visAddition = '''
+<p>This visualization allows us to see how many popular optimizers perform on different optimization problems. Select a problem to optimize, then select an optimizer and tune its parameters, and see the steps that the optimizer takes plotted in pink. Note that you can zoom in and rotate the drawing.  Also, you can compare how different optimizers perform on a given problem in the second graph.  As you try a given problem with more optimizers, the objective function vs. the number of iterations is plotted for each optimizer you have tried.</p>
+
+<div style="position: relative; width:600px; margin:0 auto;"> <div id="function" style="position: absolute; left: 50px; top: -150px;"></div><div id="optimizer" style="position: absolute; left: 50px; top: -150px;"></div></div><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><div id="functionWrapper" class="wrapper" style="width:610px; margin:0 auto;"> <div class="box a"><img id="f0" src="../../../img/fig_0.jpg" width="60px" style="border-radius: 5px; border: 3px solid #CC9900;" onclick="functionSelection(this)" onmouseover="functionOver(this)" onmouseout="functionOut(this)"></div><div class="box a"><img id="f8" src="../../../img/fig_8.jpg" width="60px" style="border-radius: 5px; border: 3px solid #333333;" onclick="functionSelection(this)" onmouseover="functionOver(this)" onmouseout="functionOut(this)"></div><div class="box a"><img id="f7" src="../../../img/fig_7.jpg" width="60px" style="border-radius: 5px; border: 3px solid #333333;" onclick="functionSelection(this)" onmouseover="functionOver(this)" onmouseout="functionOut(this)"></div><div class="box a"><img id="f5" src="../../../img/fig_5.jpg" width="60px" style="border-radius: 5px; border: 3px solid #333333;" onclick="functionSelection(this)" onmouseover="functionOver(this)" onmouseout="functionOut(this)"></div><div class="box a"><img id="f3" src="../../../img/fig_3.jpg" width="60px" style="border-radius: 5px; border: 3px solid #333333;" onclick="functionSelection(this)" onmouseover="functionOver(this)" onmouseout="functionOut(this)"></div><div class="box a"><img id="f1" src="../../../img/fig_1.jpg" width="60px" style="border-radius: 5px; border: 3px solid #333333;" onclick="functionSelection(this)" onmouseover="functionOver(this)" onmouseout="functionOut(this)"></div><div class="box a"><img id="f2" src="../../../img/fig_2.jpg" width="60px" style="border-radius: 5px; border: 3px solid #333333;" onclick="functionSelection(this)" onmouseover="functionOver(this)" onmouseout="functionOut(this)"></div><div class="box a"><img id="f4" src="../../../img/fig_4.jpg" width="60px" style="border-radius: 5px; border: 3px solid #333333;" onclick="functionSelection(this)" onmouseover="functionOver(this)" onmouseout="functionOut(this)"></div><div class="box a"><img id="f6" src="../../../img/fig_6.jpg" width="60px" style="border-radius: 5px; border: 3px solid #333333;" onclick="functionSelection(this)" onmouseover="functionOver(this)" onmouseout="functionOut(this)"></div></div><div class="wrapperA" style="width:600px; margin:0 auto;"> <div class="a"> <br><input id="stepsizeSlider" value="0.01" type="range" min="0" max="1" step="0.0001" onchange="stepsizeSliderChanged(this.value)"/> <p><code style="font-size: 0.8rem;"><span id="stepSizeDesc">Step-size:</span> <span id="stepsizeValue">0.01</span></code></p></div><div class="a"> <br><input id="iterationsSlider" value="4000" type="range" min="1" max="10000" step="1" onchange="iterationsSliderChanged(this.value)"/> <p><code style="font-size: 0.8rem;">Iterations: <span id="iterationsValue">4000</span></code></p></div><div class="a" style="left: 0px; top: 200px;"> <p class="small" style="color:#F1F1F1; font-size: 0.8rem;"> The defaults here are not necessarily good for the given problem, so it is suggested that the values used be tailored to the task at hand. (Use the mouse to zoom/drag the function.) <select style="background-color: transparent;color: #CC9900; font-family: monospace,monospace; font-weight:bold; line-height: 100%; font-size: 1em;" onchange="optimizerSelection(this)"> <option value="1">&#9654; Optimizer: Adam</option> <option value="2">&#9654; Optimizer: RMSProp</option> <option value="3">&#9654; Optimizer: AdaDelta</option> <option value="4">&#9654; Optimizer: AdaGrad</option> <option value="5">&#9654; Optimizer: CNE</option> <option value="6">&#9654; Optimizer: SMORMS3</option> <option value="7">&#9654; Optimizer: IQN</option> <option value="8">&#9654; Optimizer: CMAES</option> <option value="9">&#9654; Optimizer: AdaMax</option> <option value="10">&#9654; Optimizer: AMSGrad</option> <option value="11">&#9654; Optimizer: Nadam</option> <option value="12">&#9654; Optimizer: SGD</option> <option value="13">&#9654; Optimizer: SGD + Momentum</option> <option value="14">&#9654; Optimizer: L-BFGS</option> <option value="15">&#9654; Optimizer: Gradient descent</option> <option value="16">&#9654; Optimizer: Simulated Annealing</option> <option value="17">&#9654; Optimizer: SPALeRA-SGD</option> <option value="18">&#9654; Optimizer: SVRG</option> <option value="19">&#9654; Optimizer: SVRG-BB</option> <option value="20">&#9654; Optimizer: Katyusha</option> <option value="21">&#9654; Optimizer: Katyusha-Proximal</option> <option value="22">&#9654; Optimizer: SARAH</option> <option value="23">&#9654; Optimizer: SARAH+</option> </select> <button onclick="OptimizerSettingsOpen()" style="position: relative; left: -5px; color: #CC9900; font-family: monospace,monospace; font-weight:bold; line-height: 100%; font-size: 1em; background-color: Transparent; background-repeat:no-repeat; border: none; cursor:pointer; overflow: hidden; outline:none;">&#9654; Settings</button></p></div></div><div> <div id="settingsB" class="expanded"> <div class="wrapperA" style="width:600px; margin:0 auto;"> <div class="a parameterContainerA"> <input id="parameterASlider" type="range" min="0" max="1" step="0.001" onchange="optimizerSettings(this)"/> <p><code style="font-size: 0.8rem;"><span id="parameterADesc">First moment coefficient:</span> <span id="parameterAValue">0.9</span></code></p></div><div class="a parameterContainerB"> <input id="parameterBSlider" type="range" min="0" max="1" step="0.001" onchange="optimizerSettings(this)"/> <p><code style="font-size: 0.8rem;"><span id="parameterBDesc">Second moment coefficient:</span> <span id="parameterBValue">0.999</span></code></p></div><div class="a parameterContainerC"> <input id="parameterCSlider" type="range" min="0" max="1" step="0.001" onchange="optimizerSettings(this)"/> <p><code style="font-size: 0.8rem;"><span id="parameterCDesc">Second moment coefficient:</span> <span id="parameterCValue">0.999</span></code></p></div><div class="a parameterContainerD"> <input id="parameterDSlider" type="range" min="0" max="1" step="0.001" onchange="optimizerSettings(this)"/> <p><code style="font-size: 0.8rem;"><span id="parameterDDesc">Second moment coefficient:</span> <span id="parameterDValue">0.999</span></code></p></div></div></div><p style="color:#F1F1F1;font-size: 0.8rem;">As intuition says, system has higher probability of staying in the states with a smaller stepsize. As the stepsize goes up, imbalance becomes stronger. When the stepsize is close to zero, the system stays in the state(s) with the highest cost.</p><div style="width:100%;"> <canvas id="canvas"></canvas> </div><div class="wrapperA" style="width:600px; margin:0 auto;"> <div class="a"> <br><input id="stepsizeSlider" value="0" type="range" min="0" max="10000" step="1" onchange="evaluationsSliderChanged(this.value)"/> <p><code style="font-size: 0.8rem;"><span id="evaluationDesc">Evaluations:</span> <span id="evaluationValue">Dynamic</span></code></p></div><div class="a"> <br><form> <input id="combine" checked="true" type="checkbox" name="vehicle1" value="Bike" style="position: relative; left: -5px; color: #CC9900; font-family: monospace,monospace; font-weight:bold; line-height: 100%; font-size: 1em; background-color: Transparent; background-repeat:no-repeat; border: none; cursor:pointer; overflow: hidden; outline:none;" onchange="combineSettings(this)"> <code style="font-size: 0.8rem;">Unique Optimizer </code><br><input id="restrict" type="checkbox" name="vehicle1" value="Bike" style="position: relative; left: -5px; color: #CC9900; font-family: monospace,monospace; font-weight:bold; line-height: 100%; font-size: 1em; background-color: Transparent; background-repeat:no-repeat; border: none; cursor:pointer; overflow: hidden; outline:none;" onchange="restricttSettings(this)"> <code style="font-size: 0.8rem;">Restrict Evaluations</code><br></form> </div><div class="a" style="left: 0px; top: 200px;"> <p class="small" style="color:#F1F1F1; font-size: 0.8rem;"> A plot of the cost reveals distinct properties for each optimizer with its own style of convergence. </div></div>
+<h1 ><a class="anchor" id="optimizer_optimizertut"></a>
+Optimizer</h1>
+'''
+
+p = content.find(jsPattern)
+
+if p > 0:
+  content = content[0:p+len(jsPattern)] + "\n" + jsAddition + "\n" + content[p+len(jsPattern):len(content)]
+
+  p = content.find(visPattern)
+  if p > 0:
+    content = content[0:p+len(visPattern)] + "\n" + visAddition + "\n" + content[p+len(visPattern):len(content)]
+
+  with open(optimizerVisualizationFile, "w") as f:
+    f.write(content)

js/functions.js

@@ -0,0 +1,238 @@
+/**
+ * @file functions.js
+ * @author Marcus Edel
+ */
+
+function BoothFunction()
+{
+  // Create and populate a data table with the Booth function.
+  var data = new vis.DataSet();
+
+  var counter = 0;
+
+  var axisStep = 1;
+  var xStart = -10;
+  var yStart = -10;
+  var axisMax = 10;
+
+  for (var x = xStart; x < axisMax; x += axisStep)
+  {
+      for (var y = yStart; y < axisMax; y += axisStep)
+      {
+          var value = Math.pow(x + 2 * y - 7, 2) + Math.pow(2 * x + y - 5, 2)
+          data.add({id:counter++, x:x, y:y, z:value, style:value});
+      }
+  }
+
+  return {data:data, xMin:xStart, yMin:yStart, xMax:axisMax, yMax:axisMax, zMin:0, zMax:2500}
+}
+
+function BukinFunction()
+{
+  // Create and populate a data table with the Bukin function.
+  var data = new vis.DataSet();
+
+  var counter = 0;
+
+  var axisStep = 0.18;
+  var xStart = -15;
+  var yStart = -3;
+  var xAxisMax = -5;
+  var yAxisMax = 3;
+
+  for (var x = xStart; x < xAxisMax; x += axisStep)
+  {
+      for (var y = yStart; y < yAxisMax; y += axisStep)
+      {
+          var value = 100 * Math.sqrt(Math.abs(y - 0.01 * Math.pow(x, 2))) +
+              0.01 * Math.abs(x + 10);
+          data.add({id:counter++, x:x, y:y, z:value, style:value});
+      }
+  }
+
+  return {data:data, xMin:xStart, yMin:yStart, xMax:xAxisMax, yMax:yAxisMax, zMin:0, zMax:250}
+}
+
+function DropWaveFunction()
+{
+  // Create and populate a data table with the Drop-Wave function.
+  var data = new vis.DataSet();
+
+  var counter = 0;
+
+  var axisStep = 0.25;
+  var xStart = -5.12;
+  var yStart = -5.12;
+  var xAxisMax = 5.12;
+  var yAxisMax = 5.12;
+
+  for (var x = xStart; x < xAxisMax; x += axisStep)
+  {
+      for (var y = yStart; y < yAxisMax; y += axisStep)
+      {
+          var frac0 = 1 + Math.cos(12 * Math.sqrt(Math.pow(x, 2) +
+              Math.pow(y, 2)));
+          var frac1 =  0.5 * (Math.pow(x, 2) + Math.pow(y, 2)) + 2;
+          var value = -frac0 / frac1;
+          data.add({id:counter++, x:x, y:y, z:value, style:value});
+      }
+  }
+
+  return {data:data, xMin:xStart, yMin:yStart, xMax:xAxisMax, yMax:yAxisMax, zMin:-1, zMax:0}
+}
+
+function SphereFunction()
+{
+  // Create and populate a data table with the Sphere function.
+  var data = new vis.DataSet();
+
+  var counter = 0;
+
+  var axisStep = 0.3;
+  var xStart = -5.12;
+  var yStart = -5.12;
+  var xAxisMax = 5.12;
+  var yAxisMax = 5.12;
+
+  for (var x = xStart; x < xAxisMax; x += axisStep)
+  {
+      for (var y = yStart; y < yAxisMax; y += axisStep)
+      {
+          var value = Math.pow(x, 2) + Math.pow(y, 2);
+          data.add({id:counter++, x:x, y:y, z:value, style:value});
+      }
+  }
+
+  return {data:data, xMin:xStart, yMin:yStart, xMax:xAxisMax, yMax:yAxisMax, zMin:0, zMax:60}
+}
+
+function RastriginFunction()
+{
+  // Create and populate a data table with the Rastrigin function.
+  var data = new vis.DataSet();
+
+  var counter = 0;
+
+  var axisStep = 0.19;
+  var xStart = -5.12;
+  var yStart = -5.12;
+  var xAxisMax = 5.12;
+  var yAxisMax = 5.12;
+
+  for (var x = xStart; x < xAxisMax; x += axisStep)
+  {
+      for (var y = yStart; y < yAxisMax; y += axisStep)
+      {
+          var value = 10 * 2.0 + ((Math.pow(x, 2) - 10 * Math.cos(2 * Math.PI * x)) +
+              (Math.pow(y, 2) - 10 * Math.cos(2 * Math.PI * y)));
+          data.add({id:counter++, x:x, y:y, z:value, style:value});
+      }
+  }
+
+  return {data:data, xMin:xStart, yMin:yStart, xMax:xAxisMax, yMax:yAxisMax, zMin:0, zMax:80}
+}
+
+function McCormickFunction()
+{
+  // Create and populate a data table with the Mc-Cormick function.
+  var data = new vis.DataSet();
+
+  var counter = 0;
+
+  var axisStep = 0.2;
+  var xStart = -2;
+  var yStart = -4;
+  var xAxisMax = 4;
+  var yAxisMax = 4;
+
+  for (var x = xStart; x < xAxisMax; x += axisStep)
+  {
+      for (var y = yStart; y < yAxisMax; y += axisStep)
+      {
+          var value = Math.sin(x + y) + Math.pow(x - y, 2) - 1.5 * x + 2.5 * y + 1;
+          data.add({id:counter++, x:x, y:y, z:value, style:value});
+      }
+  }
+
+  return {data:data, xMin:xStart, yMin:yStart, xMax:xAxisMax, yMax:yAxisMax, zMin:0, zMax:45}
+}
+
+function EasomFunction()
+{
+  // Create and populate a data table with the Easom function.
+  var data = new vis.DataSet();
+
+  var counter = 0;
+
+  var axisStep = 1.3;
+  var xStart = -50;
+  var yStart = -50;
+  var xAxisMax = 50;
+  var yAxisMax = 50;
+
+  for (var x = xStart; x < xAxisMax; x += axisStep)
+  {
+      for (var y = yStart; y < yAxisMax; y += axisStep)
+      {
+          var fact0 = -Math.cos(x) * Math.cos(y);
+          var fact1 = Math.exp(-Math.pow(x - Math.PI, 2) - Math.pow(y - Math.PI, 2));
+          var value = fact0 * fact1;
+          data.add({id:counter++, x:x, y:y, z:value, style:value});
+      }
+  }
+
+  return {data:data, xMin:xStart, yMin:yStart, xMax:xAxisMax, yMax:yAxisMax, zMin:-1, zMax:0.3}
+}
+
+function StyblinskiTangFunction()
+{
+  // Create and populate a data table with the Styblinski-Tang function.
+  var data = new vis.DataSet();
+
+  var counter = 0;
+
+  var axisStep = 0.2;
+  var xStart = -5;
+  var yStart = -5;
+  var xAxisMax = 5;
+  var yAxisMax = 5;
+
+  for (var x = xStart; x < xAxisMax; x += axisStep)
+  {
+      for (var y = yStart; y < yAxisMax; y += axisStep)
+      {
+          var value = 0.5 * ((Math.pow(x, 4) - 16 * Math.pow(x, 2) + 6 * x) +
+              (Math.pow(y, 4) - 16 * Math.pow(y, 2) + 6 * y));
+          data.add({id:counter++, x:x, y:y, z:value, style:value});
+      }
+  }
+
+  return {data:data, xMin:xStart, yMin:yStart, xMax:xAxisMax, yMax:yAxisMax, zMin:-100, zMax:240}
+}
+
+function EggholderFunction()
+{
+  // Create and populate a data table with the Eggholder function.
+  var data = new vis.DataSet();
+
+  var counter = 0;
+
+  var axisStep = 20;
+  var xStart = -512;
+  var yStart = -512;
+  var xAxisMax = 512;
+  var yAxisMax = 512;
+
+  for (var x = xStart; x < xAxisMax; x += axisStep)
+  {
+      for (var y = yStart; y < yAxisMax; y += axisStep)
+      {
+          var term0 = -(y + 47) * Math.sin(Math.sqrt(Math.abs(y + x / 2 + 47)));
+          var term1 = -x * Math.sin(Math.sqrt(Math.abs(x - (y + 47))));
+          var value = term0 + term1;
+          data.add({id:counter++, x:x, y:y, z:value, style:value});
+      }
+  }
+
+  return {data:data, xMin:xStart, yMin:yStart, xMax:xAxisMax, yMax:yAxisMax, zMin:-1000, zMax:1000}
+}