Changed stopping criteria to be no improvement over X iterations, instead of reaching some %

Ben Evans · Ben Evans · commit 7b491a6e492e · 2017-05-05T09:36:29.000+12:00
diff --git a/Classifier.java b/Classifier.java
@@ -8,7 +8,9 @@
 
 
 /**
- * K-NN used as a classifier.
+ * Classifier to use as the evaluation criteria
+ * for feature selection. K-Nearest neigbhour is
+ * implemented here.
  */
 public class Classifier {
 
@@ -35,24 +37,25 @@ public Classifier(Set<Instance> training, Set<Instance> testing){
     }
 
     /**
-     * Classifies and calculates the percentage
-     * of correct classifications in the testingSet
-     * against the training set.
+     * Classifies and returns the percentage
+     * of correct classifications using every feature
+     * in the instances.
+     *
      */
     public double classify(){
         Instance sampleInstance = training.iterator().next();
         int totalFeatures = sampleInstance.getNumFeatures();
 
-        // To begin with all features are selected
+        // We are using all features
         Set<Integer> allIndices =  IntStream.rangeClosed(0, totalFeatures - 1)
                 .boxed().collect(Collectors.toSet());
 
         return classify(allIndices);
     }
     /**
-     * Classifies and calculates the percentage
-     * of correct classifications in the testingSet
-     * against the training set.
+     * Classifies and returns the percentage
+     * of correct classifications using only the specified indices
+     * for the instances.
      */
     public double classify(Set<Integer> indices) {
         int correct = 0;
diff --git a/FeatureSelection.java b/FeatureSelection.java
@@ -11,6 +11,9 @@ public abstract class FeatureSelection {
     private Classifier classifier;
     protected Set<Instance> instances;
 
+    // The number of times to try extra subsets if no improvement is made
+    protected final int MAX_ITERATIONS_WITHOUT_PROGRESS = 5;
+
     public FeatureSelection(Set<Instance> instances){
         this.instances = instances;
         this.classifier = new Classifier(instances);
@@ -29,7 +32,7 @@ If a number of features is found (n), where n < m, with a higher accuracy,
     features. If numFeatures is >= original.size(), the original
     set is returned.
     */
-   public abstract Set<Integer> select(double minimumAccuracy);
+   public abstract Set<Integer> select();
 
     /**
      * Returns the feature in remaining features
@@ -86,7 +89,7 @@ protected double objectiveFunction(Set<Integer> selectedFeatures) {
         return classifier.classify(selectedFeatures);
     }
 
-    protected Set<Integer> getFeatures(){
+    protected Set<Integer> getAllFeatureIndices(){
         // Extract an instance to check the amount of features, assumes all instances have same # of features
         Instance sampleInstance = instances.iterator().next();
         int totalFeatures = sampleInstance.getNumFeatures();
@@ -96,7 +99,7 @@ protected Set<Integer> getFeatures(){
                 .boxed().collect(Collectors.toSet());
     }
 
-    public void compareAccuracy(Set<Integer> selectedIndices) {
+    public void compareTestingAccuracy(Set<Integer> selectedIndices) {
         System.out.println("Classification accuracy on testing set using all features: " + classifier.classify());
         System.out.println("Classification accuracy on testing set using features " + selectedIndices + ": " + classifier.classify(selectedIndices));
     }
diff --git a/SequentialBackwardsSelection.java b/SequentialBackwardsSelection.java
@@ -12,27 +12,37 @@ public SequentialBackwardsSelection(Set<Instance> instances){
 
     @Override
     public Set<Integer> select(int maxNumFeatures) {
-        return select((accuracy, size) -> size > maxNumFeatures);
+        // While we have too many features or the accuracy is still improving
+        return select((noImprovement, size) -> size > maxNumFeatures || noImprovement < MAX_ITERATIONS_WITHOUT_PROGRESS, maxNumFeatures);
     }
 
     @Override
-    public Set<Integer> select(double minimumAccuracy) {
-        return select((accuracy, size) -> accuracy < minimumAccuracy);
+    public Set<Integer> select() {
+        // While the accuracy is still improving
+        return select((noImprovement, size) -> noImprovement < MAX_ITERATIONS_WITHOUT_PROGRESS);
     }
 
     public Set<Integer> select(Criteria criteria) {
+        return select(criteria, instances.size());
+    }
+
+    public Set<Integer> select(Criteria criteria, int maxNumFeatures) {
         // In this case we have no data to use, so return the empty set
         if (instances == null || instances.isEmpty()) return new HashSet<Integer>();
 
         // To begin with all features are selected
-        Set<Integer> selectedFeatures = getFeatures();
+        Set<Integer> selectedFeatures = getAllFeatureIndices();
 
         // Keep track of the best solution, so we never get worse
         double highestAccuracy = 0;
         Set<Integer> bestSoFar = new HashSet<>();
         double accuracy = objectiveFunction(selectedFeatures);
+        double lastAccuracy = accuracy;
+
+        // Number of iterations with no improvement
+        double noImprovement = 0;
 
-        while (criteria.evaluate(accuracy, selectedFeatures.size())){
+        while (criteria.evaluate(noImprovement, selectedFeatures.size())){
             int feature = worst(selectedFeatures);
 
             // No more valid features
@@ -43,11 +53,19 @@ public Set<Integer> select(Criteria criteria) {
 
             accuracy = objectiveFunction(selectedFeatures);
 
-            if (accuracy > highestAccuracy) {
+            // If this is the highest so far, and also valid (i.e < number of features required)
+            if (accuracy > highestAccuracy && selectedFeatures.size() <= maxNumFeatures) {
                 highestAccuracy = accuracy;
                 // Make a copy, so we don't accidentally modify this
                 bestSoFar = new HashSet<>(selectedFeatures);
             }
+
+            if (Double.compare(accuracy, lastAccuracy) <= 0){
+                noImprovement++;
+            } else{
+                noImprovement = 0;
+            }
+            lastAccuracy = accuracy;
         }
 
         return bestSoFar;
diff --git a/SequentialFloatingForwardSelection.java b/SequentialFloatingForwardSelection.java
@@ -14,16 +14,16 @@ public Set<Integer> select(int maxNumFeatures) {
         return select((accuracy, size) -> size < maxNumFeatures);
     }
 
-    public Set<Integer> select(double minimumAccuracy) {
-        return select((accuracy, size) -> accuracy < minimumAccuracy);
+    public Set<Integer> select() {
+        return select((noImprovement, size) -> noImprovement < MAX_ITERATIONS_WITHOUT_PROGRESS);
     }
 
     public Set<Integer> select(Criteria criteria) {
         // In this case we have no data to use, so return the empty set
         if (instances == null || instances.isEmpty()) return new HashSet<Integer>();
 
         // To begin with no features are selected, so all the indices from 0..totalFeatures are remaining
-        Set<Integer> remainingFeatures = getFeatures();
+        Set<Integer> remainingFeatures = getAllFeatureIndices();
 
         // Subset of only selected features indices
         Set<Integer> selectedFeatures = new HashSet<>();
@@ -34,24 +34,34 @@ public Set<Integer> select(Criteria criteria) {
         double highestAccuracy = 0;
         Set<Integer> bestSoFar = new HashSet<>();
         double accuracy = objectiveFunction(selectedFeatures);
+        double lastAccuracy = accuracy;
+
+        // Number of iterations with no improvement
+        double noImprovement = 0;
 
         while (criteria.evaluate(accuracy, selectedFeatures.size())){
             int feature = best(selectedFeatures, remainingFeatures);
+
+            System.out.println("Selected features are:" + selectedFeatures);
+            System.out.println("Remaining features are:" + remainingFeatures);
+            System.out.println("Adding feature: " + feature);
+
             // No more valid features
             if (feature == -1) break;
 
             selectedFeatures.add(feature);
             // Remove the feature so we do not keep selecting the same one
             remainingFeatures.remove(feature);
 
-            double lastAccuracy = objectiveFunction(selectedFeatures);
+            double accuracyBeforeRemoval = objectiveFunction(selectedFeatures);
+
 
             // Now remove the worst features, while we are improving
             while(true){
                 int worstFeature = worst(selectedFeatures);
 
                 // No more valid features
-                if (feature == -1) break;
+                if (worstFeature == -1) break;
 
                 selectedFeatures.remove(worstFeature);
                 // Feature becomes available again
@@ -62,14 +72,14 @@ public Set<Integer> select(Criteria criteria) {
                 double newAccuracy = objectiveFunction(selectedFeatures);
 
                 // If the accuracy did not improve, undo this step and continue adding features
-                if (newAccuracy < lastAccuracy) {
+                if (newAccuracy < accuracyBeforeRemoval) {
                     selectedFeatures.add(worstFeature);
                     remainingFeatures.remove(worstFeature);
                     System.out.println("Accuracy did not improve, so undoing above step");
                     break;
                 }
 
-                lastAccuracy = newAccuracy;
+                accuracyBeforeRemoval = newAccuracy;
             }
 
             accuracy = objectiveFunction(selectedFeatures);
@@ -80,6 +90,13 @@ public Set<Integer> select(Criteria criteria) {
                 bestSoFar = new HashSet<>(selectedFeatures);
             }
 
+            if (Double.compare(accuracy, lastAccuracy) <= 0){
+                noImprovement++;
+            } else{
+                noImprovement = 0;
+            }
+            lastAccuracy = accuracy;
+
         }
 
         return bestSoFar;
diff --git a/SequentialForwardSelection.java b/SequentialForwardSelection.java
@@ -14,16 +14,16 @@ public Set<Integer> select(int maxNumFeatures) {
        return select((accuracy, size) -> size < maxNumFeatures);
     }
 
-    public Set<Integer> select(double minimumAccuracy) {
-        return select((accuracy, size) -> accuracy < minimumAccuracy);
+    public Set<Integer> select() {
+        return select((noImprovement, size) -> noImprovement < MAX_ITERATIONS_WITHOUT_PROGRESS);
     }
 
     public Set<Integer> select(Criteria criteria) {
         // In this case we have no data to use, so return the empty set
         if (instances == null || instances.isEmpty()) return new HashSet<Integer>();
 
         // To begin with no features are selected, so all the indices from 0..totalFeatures are remaining
-        Set<Integer> remainingFeatures = getFeatures();
+        Set<Integer> remainingFeatures = getAllFeatureIndices();
 
         // Subset of only selected features indices
         Set<Integer> selectedFeatures = new HashSet<>();
@@ -34,8 +34,12 @@ public Set<Integer> select(Criteria criteria) {
         double highestAccuracy = 0;
         Set<Integer> bestSoFar = new HashSet<>();
         double accuracy = objectiveFunction(selectedFeatures);
+        double lastAccuracy = accuracy;
 
-        while (criteria.evaluate(accuracy, selectedFeatures.size())){
+        // Number of iterations with no improvement
+        double noImprovement = 0;
+
+        while (criteria.evaluate(noImprovement, selectedFeatures.size())){
             int feature = best(selectedFeatures, remainingFeatures);
             // No more valid features
             if (feature == -1) break;
@@ -51,6 +55,13 @@ public Set<Integer> select(Criteria criteria) {
                 // Make a copy, so we don't accidentally modify this
                 bestSoFar = new HashSet<>(selectedFeatures);
             }
+
+            if (Double.compare(accuracy, lastAccuracy) <= 0){
+                noImprovement++;
+            } else{
+                noImprovement = 0;
+            }
+            lastAccuracy = accuracy;
         }
 
         return bestSoFar;
