Reformatted code and added comments to TestAll

Author: Ben Evans

Diff for: TestAll.java

@@ -6,8 +6,15 @@
 
 
 /**
- * Created by ben on 18/04/17.
+ * Runs the four selection methods against a given
+ * dataset. This is a useful class for running all the
+ * code at once and checking the output.
+ *
+ * The only tests this actually does is checks
+ * the size of the subsets returned from the numFeatures
+ * methods is less than or equal to the specified size.
  */
+
 public class TestAll {
 
     // Since weka treats attributes and classes uniformly, must explicitly state class indiex
@@ -129,18 +136,29 @@ public void testSequentialFloatingBackwardSelectionNumFeatures() throws Exceptio
      */
 
     private FeatureSelection generateSelector(Selection method) throws Exception {
+        FeatureSelection selector = null;
         switch (method){
             case SBS:
-                return new SequentialBackwardSelection(FILE_NAME, CLASS_INDEX);
+                selector = new SequentialBackwardSelection(FILE_NAME, CLASS_INDEX);
+                break;
             case SFS:
-                return new SequentialForwardSelection(FILE_NAME, CLASS_INDEX);
+                selector = new SequentialForwardSelection(FILE_NAME, CLASS_INDEX);
+                break;
             case SFBS:
-                return new SequentialFloatingBackwardSelection(FILE_NAME, CLASS_INDEX);
+                selector = new SequentialFloatingBackwardSelection(FILE_NAME, CLASS_INDEX);
+                break;
             case SFFS:
-                return new SequentialFloatingForwardSelection(FILE_NAME, CLASS_INDEX);
-            default:
-                return null;
+                selector = new SequentialFloatingForwardSelection(FILE_NAME, CLASS_INDEX);
+                break;
+        }
+
+        // Special case for musk
+        if(FILE_NAME.equals("musk.arff")){
+            // There is a "giveaway" feature (molecule_name) which stores some class information
+            selector.removeAttribute(0);
         }
+
+        return selector;
     }
 
     private enum Selection {

Diff for: selection/Classifier.java

@@ -1,6 +1,6 @@
 package selection;
 
-import weka.classifiers.*;
+import weka.classifiers.Evaluation;
 import weka.classifiers.lazy.IBk;
 import weka.classifiers.meta.FilteredClassifier;
 import weka.core.Instances;
@@ -17,14 +17,14 @@
 import java.util.stream.IntStream;
 
 /**
- *  Reads isntances from file and splits them into
- *
- *  - Training: Used to train the model (classifier)
- *  - Validation: Used to check performance throughout, avoid overfitting to training
- *  - Testing: Used only at the end to evaluate learnt performance
- *
- *  This uses weka: http://www.cs.waikato.ac.nz/ml/weka/
- *  both for the classifier and instances.
+ * Reads isntances from file and splits them into
+ * <p>
+ * - Training: Used to train the model (classifier)
+ * - Validation: Used to check performance throughout, avoid overfitting to training
+ * - Testing: Used only at the end to evaluate learnt performance
+ * <p>
+ * This uses weka: http://www.cs.waikato.ac.nz/ml/weka/
+ * both for the classifier and instances.
  */
 public class Classifier {
 
@@ -40,7 +40,7 @@ public class Classifier {
     private int CLASS_INDEX;
 
     public Classifier(String fileName) throws Exception {
-        Instances instances =  readArffFile(fileName);
+        Instances instances = readArffFile(fileName);
         instances.randomize(new java.util.Random(123));
 
         int trainSize = (int) Math.round(instances.numInstances() * 0.6);
@@ -53,7 +53,7 @@ public Classifier(String fileName) throws Exception {
     }
 
     public Classifier(String trainingFileName, String testingFileName) throws Exception {
-        Instances instances =  readArffFile(trainingFileName);
+        Instances instances = readArffFile(trainingFileName);
         instances.randomize(new java.util.Random(0));
 
         int trainSize = (int) Math.round(instances.numInstances() * 0.8);
@@ -79,7 +79,7 @@ private Instances removeAttribute(int index, Instances instances) throws Excepti
         return Filter.useFilter(instances, remove);
     }
 
-    public void setClassIndex(int index){
+    public void setClassIndex(int index) {
         this.CLASS_INDEX = index;
 
         training.setClassIndex(CLASS_INDEX);
@@ -89,6 +89,7 @@ public void setClassIndex(int index){
 
     /**
      * The method of classification to use
+     *
      * @return
      * @throws Exception
      */
@@ -185,22 +186,22 @@ private double evaluateOnTesting(weka.classifiers.Classifier classifier) throws
      * @param toKeep
      * @return
      */
-    private int[] remove(Set<Integer> toKeep){
+    private int[] remove(Set<Integer> toKeep) {
 
         List<Integer> toRemove = new ArrayList<Integer>();
 
-        for(int i=0; i<training.numAttributes(); i++){
-            if (!toKeep.contains(i) && i != CLASS_INDEX){
+        for (int i = 0; i < training.numAttributes(); i++) {
+            if (!toKeep.contains(i) && i != CLASS_INDEX) {
                 toRemove.add(i);
             }
         }
 
         // Convert list to int[]
-        return toRemove.stream().mapToInt(i->i).toArray();
+        return toRemove.stream().mapToInt(i -> i).toArray();
     }
 
 
-    private Instances readArffFile (String fileName) throws IOException{
+    private Instances readArffFile(String fileName) throws IOException {
         BufferedReader reader = new BufferedReader(
                 new FileReader("src/res/" + fileName));
 
@@ -209,7 +210,7 @@ private Instances readArffFile (String fileName) throws IOException{
         return instances;
     }
 
-    public int getNumFeatures(){
+    public int getNumFeatures() {
         return training.numAttributes();
     }
 

Diff for: selection/FeatureSelection.java

@@ -14,24 +14,21 @@ public abstract class FeatureSelection {
 
     // The number of iterations to try if no improvement is made
     protected final int MAX_ITERATIONS_WITHOUT_PROGRESS = 10;
-
+    private final boolean DEBUG = true;
     // The wrapped classifier to use
     private Classifier classifier;
 
-    private final boolean DEBUG = true;
-
-    public FeatureSelection (String fileName, int classIndex) throws Exception {
+    public FeatureSelection(String fileName, int classIndex) throws Exception {
         this.classifier = new Classifier(fileName);
-        //TODO: Only needed for musk
-        this.classifier.removeAttribute(0);
         this.classifier.setClassIndex(classIndex);
     }
 
-    public FeatureSelection (String trainingFile, String testingFile, int classIndex) throws Exception {
+    public FeatureSelection(String trainingFile, String testingFile, int classIndex) throws Exception {
         this.classifier = new Classifier(trainingFile, testingFile);
         this.classifier.setClassIndex(classIndex);
     }
 
+
     /**
      * Returns a subset of only the most important features,
      * the parameter specifies the maximum number of features to select (m).
@@ -117,6 +114,17 @@ protected double objectiveFunction(Set<Integer> selectedFeatures) throws Excepti
         return classifier.classify(selectedFeatures);
     }
 
+    /**
+     * Removes the specified attribute, this is useful if the dataset
+     * has extra "information" variables that give away the class.
+     *
+     * @param index
+     * @throws Exception
+     */
+    public void removeAttribute(int index) throws Exception {
+        this.classifier.removeAttribute(0);
+    }
+
 
     /***
      * Uses the testing instances to check the performance
@@ -136,8 +144,8 @@ public void compareTestingAccuracy(Set<Integer> selectedIndices) throws Exceptio
      * @param size
      * @param accuracy
      */
-    protected void printAccuracy (int size, double accuracy){
-        if(DEBUG) System.out.println(size + ": " + accuracy);
+    protected void printAccuracy(int size, double accuracy) {
+        if (DEBUG) System.out.println(size + ": " + accuracy);
     }
 
     /***
@@ -161,19 +169,19 @@ protected Set<Integer> getAllFeatureIndices() {
      * ===============
      */
 
-    protected boolean greaterThan(double d1, double d2){
+    protected boolean greaterThan(double d1, double d2) {
         return Double.compare(d1, d2) > 0;
     }
 
-    protected boolean lessThan(double d1, double d2){
+    protected boolean lessThan(double d1, double d2) {
         return Double.compare(d1, d2) < 0;
     }
 
-    protected boolean lessThanOrEqualTo(double d1, double d2){
+    protected boolean lessThanOrEqualTo(double d1, double d2) {
         return Double.compare(d1, d2) <= 0;
     }
 
-    protected boolean equalTo(double d1, double d2){
+    protected boolean equalTo(double d1, double d2) {
         return Double.compare(d1, d2) == 0;
     }
 

Diff for: selection/SequentialBackwardSelection.java

@@ -5,7 +5,7 @@
 
 /**
  * Performs Sequential Backward Selection (SBS)
- *
+ * <p>
  * - Starts with full set of features
  * - Repeatedly removes the "worst" feature until
  * stopping criteria is met,
@@ -15,6 +15,7 @@ public class SequentialBackwardSelection extends FeatureSelection {
     public SequentialBackwardSelection(String file, int classIndex) throws Exception {
         super(file, classIndex);
     }
+
     public SequentialBackwardSelection(String training, String testing, int classIndex) throws Exception {
         super(training, testing, classIndex);
     }
@@ -92,6 +93,4 @@ private Set<Integer> select(Criteria criteria, int maxNumFeatures) throws Except
     }
 
 
-
-
 }

Diff for: selection/SequentialFloatingBackwardSelection.java

@@ -5,7 +5,7 @@
 
 /**
  * Performs Sequential Floating Backward Selection (SFBS)
- *
+ * <p>
  * - Starts with full set of features
  * - Removes the "worst" feature
  * - Performs SFS as long as the objective function increases
@@ -16,6 +16,7 @@ public class SequentialFloatingBackwardSelection extends FeatureSelection {
     public SequentialFloatingBackwardSelection(String file, int classIndex) throws Exception {
         super(file, classIndex);
     }
+
     public SequentialFloatingBackwardSelection(String training, String testing, int classIndex) throws Exception {
         super(training, testing, classIndex);
     }
@@ -109,7 +110,7 @@ private Set<Integer> select(Criteria criteria, int maxNumFeatures) throws Except
 
             // If the accuracy is higher than our previous best, or the same with less features and its a valid size (<= maxFeatures)
             if ((greaterThan(accuracy, highestAccuracy) || (equalTo(accuracy, highestAccuracy) && selectedFeatures.size() < bestSoFar.size()))
-                    &&  selectedFeatures.size() <= maxNumFeatures) {
+                    && selectedFeatures.size() <= maxNumFeatures) {
                 highestAccuracy = accuracy;
                 // Save our best set
                 bestSoFar = new HashSet<>(selectedFeatures);

Diff for: selection/SequentialFloatingForwardSelection.java

@@ -5,7 +5,7 @@
 
 /**
  * Performs Sequential Floating Forward Selection (SFFS)
- *
+ * <p>
  * - Starts with empty set of features
  * - Adds the "best" feature
  * - Performs SBS as long as the objective function increases
@@ -16,6 +16,7 @@ public class SequentialFloatingForwardSelection extends FeatureSelection {
     public SequentialFloatingForwardSelection(String file, int classIndex) throws Exception {
         super(file, classIndex);
     }
+
     public SequentialFloatingForwardSelection(String training, String testing, int classIndex) throws Exception {
         super(training, testing, classIndex);
     }

Diff for: selection/SequentialForwardSelection.java

@@ -5,7 +5,7 @@
 
 /**
  * Performs Sequential Floating Forward Selection (SFFS)
- *
+ * <p>
  * - Starts with empty set of features
  * - Adds the "best" feature until stopping criteria is met
  */
@@ -14,6 +14,7 @@ public class SequentialForwardSelection extends FeatureSelection {
     public SequentialForwardSelection(String file, int classIndex) throws Exception {
         super(file, classIndex);
     }
+
     public SequentialForwardSelection(String training, String testing, int classIndex) throws Exception {
         super(training, testing, classIndex);
     }