Changed classifier to use Weka classifiers, as K-NN will be too slow for larger datasets. Added check to prevent loops in floating methods

Author: Ben Evans

TestAll.java

@@ -25,43 +25,43 @@ public TestAll() throws FileNotFoundException {
     }
 
     @org.junit.Test
-    public void testSequentialForwardSelection() {
+    public void testSequentialForwardSelection() throws Exception {
         System.out.println("-------------------");
         System.out.println("Sequential forward selection");
-        FeatureSelection selector = new SequentialForwardSelection(training, testing);
+        FeatureSelection selector = new SequentialForwardSelection("wine.arff");
         Set<Integer> selectedIndices = selector.select();
         selector.compareTestingAccuracy(selectedIndices);
         System.out.println("-------------------");
     }
 
     @org.junit.Test
-    public void testSequentialForwardSelectionNumfeatures() {
+    public void testSequentialForwardSelectionNumfeatures() throws Exception {
         int maxFeatures = 10;
         System.out.println("-------------------");
         System.out.println("Sequential forward selection for max " + maxFeatures + " features");
-        FeatureSelection selector = new SequentialForwardSelection(training, testing);
+        FeatureSelection selector = new SequentialForwardSelection("wine.arff");
         Set<Integer> selectedIndices = selector.select(maxFeatures);
         selector.compareTestingAccuracy(selectedIndices);
         System.out.println("-------------------");
         assertTrue(selectedIndices.size() <= maxFeatures);
     }
 
     @org.junit.Test
-    public void testSequentialBackwardSelection() {
+    public void testSequentialBackwardSelection() throws Exception {
         System.out.println("-------------------");
         System.out.println("Sequential backward selection");
-        FeatureSelection selector = new SequentialBackwardSelection(training, testing);
+        FeatureSelection selector = new SequentialBackwardSelection("wine.arff");
         Set<Integer> selectedIndices = selector.select();
         selector.compareTestingAccuracy(selectedIndices);
         System.out.println("-------------------");
     }
 
     @org.junit.Test
-    public void testSequentialBackwardSelectionNumfeatures() {
+    public void testSequentialBackwardSelectionNumfeatures() throws Exception {
         int maxFeatures = 10;
         System.out.println("-------------------");
         System.out.println("Sequential backward selection for max " + maxFeatures + " Features");
-        FeatureSelection selector = new SequentialBackwardSelection(training, testing);
+        FeatureSelection selector = new SequentialBackwardSelection("wine.arff");
         Set<Integer> selectedIndices = selector.select(maxFeatures);
         selector.compareTestingAccuracy(selectedIndices);
         System.out.println("-------------------");
@@ -75,21 +75,21 @@ public void testSequentialBackwardSelectionNumfeatures() {
      */
 
     @org.junit.Test
-    public void testSequentialFloatingForwardSelection() {
+    public void testSequentialFloatingForwardSelection() throws Exception {
         System.out.println("-------------------");
         System.out.println("Sequential floating forward selection");
-        FeatureSelection selector = new SequentialFloatingForwardSelection(training, testing);
+        FeatureSelection selector = new SequentialFloatingForwardSelection("wine.arff");
         Set<Integer> selectedIndices = selector.select();
         selector.compareTestingAccuracy(selectedIndices);
         System.out.println("-------------------");
     }
 
     @org.junit.Test
-    public void testSequentialFloatingForwardSelectionNumFeatures() {
+    public void testSequentialFloatingForwardSelectionNumFeatures() throws Exception {
         int maxFeatures = 5;
         System.out.println("-------------------");
         System.out.println("Sequential floating forward selection for " + maxFeatures + " features");
-        FeatureSelection selector = new SequentialFloatingForwardSelection(training, testing);
+        FeatureSelection selector = new SequentialFloatingForwardSelection("wine.arff");
         Set<Integer> selectedIndices = selector.select(maxFeatures);
         selector.compareTestingAccuracy(selectedIndices);
         System.out.println("-------------------");
@@ -98,27 +98,28 @@ public void testSequentialFloatingForwardSelectionNumFeatures() {
     }
 
     @org.junit.Test
-    public void testSequentialBackwardFloatingSelection() {
+    public void testSequentialBackwardFloatingSelection() throws Exception {
         System.out.println("-------------------");
         System.out.println("Sequential backward floating selection");
-        FeatureSelection selector = new SequentialFloatingBackwardSelection(training, testing);
+        FeatureSelection selector = new SequentialFloatingBackwardSelection("wine.arff");
         Set<Integer> selectedIndices = selector.select();
         selector.compareTestingAccuracy(selectedIndices);
         System.out.println("-------------------");
     }
 
     @org.junit.Test
-    public void testSequentialBackwardFloatingSelectionNumFeatures() {
+    public void testSequentialBackwardFloatingSelectionNumFeatures() throws Exception {
         int maxFeatures = 10;
         System.out.println("-------------------");
         System.out.println("Sequential backward floating selection for " + maxFeatures + " features");
-        FeatureSelection selector = new SequentialFloatingBackwardSelection(training, testing);
+        FeatureSelection selector = new SequentialFloatingBackwardSelection("wine.arff");
         Set<Integer> selectedIndices = selector.select(maxFeatures);
         selector.compareTestingAccuracy(selectedIndices);
         System.out.println("-------------------");
         assertTrue(selectedIndices.size() <= maxFeatures);
     }
 
+
     private void loadWineSet() throws FileNotFoundException {
         Scanner scanner = new Scanner(new File("src/res/wine.data"));
         List<Instance> instances = new ArrayList<>();
@@ -144,10 +145,6 @@ private void loadWineSet() throws FileNotFoundException {
     private void loadIsoletSet() throws FileNotFoundException {
         this.training = loadIsoletSet(true);
         this.testing = loadIsoletSet(false);
-
-        System.out.println("Training size: " + training.size());
-        System.out.println("Testing size: " + testing.size());
-
     }
 
     private List<Instance> loadIsoletSet(boolean training) throws FileNotFoundException {
@@ -156,9 +153,7 @@ private List<Instance> loadIsoletSet(boolean training) throws FileNotFoundExcept
 
         Scanner scanner = new Scanner(new File("src/res/isolet-" + file + ".data"));
 
-        int occurences = 0;
-
-        while (scanner.hasNext() && occurences++ < 500) {
+        while (scanner.hasNext()) {
             String line = scanner.nextLine();
             Instance isolet = createIsolet(line);
             instances.add(isolet);

clasification/WekaClassifier.java

@@ -2,8 +2,11 @@
 
 import weka.classifiers.*;
 import weka.classifiers.bayes.NaiveBayes;
+import weka.classifiers.functions.MultilayerPerceptron;
 import weka.classifiers.meta.FilteredClassifier;
+import weka.classifiers.pmml.consumer.SupportVectorMachineModel;
 import weka.classifiers.trees.J48;
+import weka.classifiers.trees.RandomForest;
 import weka.core.Instance;
 import weka.core.Instances;
 import weka.filters.unsupervised.attribute.Remove;
@@ -14,43 +17,77 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Random;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 /**
  * Created by ben on 15/05/17.
  */
 public class WekaClassifier {
 
+    // Data to learn model from
     Instances training;
-    Instances testing;
 
-    public final int CLASS_INDEX = 0;
+    // Used to check performance of learnt model
+    Instances validation;
+
+    // Only used for final evaluation
+    Instances testing;
 
-    // classifier
-    weka.classifiers.Classifier classifier = new NaiveBayes();
+    private int CLASS_INDEX;
+    private final int NUM_FOLDS = 3;
 
     public WekaClassifier (String fileName) throws Exception {
         Instances instances =  readArffFile(fileName);
 
-        int trainSize = (int) Math.round(instances.numInstances() * 0.7);
-        int testSize = instances.numInstances() - trainSize;
+        instances.randomize(new java.util.Random(0));
+
+        int trainSize = (int) Math.round(instances.numInstances() * 0.6);
+        int validationSize = (int) Math.round(instances.numInstances() * 0.2);
+        int testSize = instances.numInstances() - trainSize - validationSize;
 
         this.training = new Instances(instances, 0, trainSize);
-        this.testing = new Instances(instances, trainSize, testSize);
+        this.validation = new Instances(instances, trainSize, validationSize);
+        this.testing = new Instances(instances, trainSize + validationSize, testSize);
 
-        System.out.println("Training instances: " + training.size());
-        System.out.println("Testing instances: " + testing.size());
+        setClassIndex();
 
         classify();
     }
 
     public WekaClassifier (String trainingFileName, String testingFileName) throws Exception {
+
+        Instances instances =  readArffFile(trainingFileName);
+
+        // Split into training and validation
         this.training = readArffFile(trainingFileName);
+
+        instances.randomize(new java.util.Random(0));
+
+        int trainSize = (int) Math.round(instances.numInstances() * 0.8);
+        int validationSize = instances.numInstances() - trainSize;
+
+        this.training = new Instances(instances, 0, trainSize);
+        this.validation = new Instances(instances, trainSize, validationSize);
+
+
         this.testing = readArffFile(testingFileName);
 
+        setClassIndex();
+
         classify();
     }
 
+    public void setClassIndex(){
+        // this.CLASS_INDEX = training.numAttributes() - 1;
+        this.CLASS_INDEX = 0;
+        training.setClassIndex(CLASS_INDEX);
+        testing.setClassIndex(CLASS_INDEX);
+
+    }
+
     public static void main(String[] args) throws Exception {
         new WekaClassifier("wine.arff");
     }
@@ -67,12 +104,60 @@ private Instances readArffFile (String fileName) throws IOException{
     }
 
     public double classify() throws Exception {
+
+        // classifier
+        weka.classifiers.Classifier classifier = createClassifier();
+
         classifier.buildClassifier(training);
         Evaluation eval = new Evaluation(training);
 
-        eval.evaluateModel(classifier, training);
+        return evaluate(classifier);
+    }
+
+    public double testAccuracy() throws Exception {
+        // classifier
+        weka.classifiers.Classifier classifier = createClassifier();
+
+        classifier.buildClassifier(training);
+        Evaluation eval = new Evaluation(training);
+
+        eval.evaluateModel(classifier, testing);
+
+        System.out.println(eval.toSummaryString());
+
+        return eval.pctCorrect();
+    }
+
+    private weka.classifiers.Classifier createClassifier() throws Exception {
+        return new NaiveBayes();
+    }
+
+    public double testAccuracy(Set<Integer> indices) throws Exception {
+        weka.classifiers.Classifier classifier = createClassifier();
+
+        Remove rm = new Remove();
+
+        int[] remove = remove(indices);
+        rm.setAttributeIndicesArray(remove);
+
+        FilteredClassifier fc = new FilteredClassifier();
+        fc.setFilter(rm);
+
+        fc.setClassifier(classifier);
+        fc.buildClassifier(training);
+
+        Evaluation eval = new Evaluation(training);
+        eval.evaluateModel(fc, testing);
+
+        return eval.pctCorrect();
+    }
+
+    private double evaluate(weka.classifiers.Classifier classifier) throws Exception {
+        Evaluation eval = new Evaluation(training);
+
+        eval.evaluateModel(classifier, validation);
+        // eval.crossValidateModel(classifier, training, NUM_FOLDS, new Random(1));
 
-        System.out.println(eval.pctCorrect());
         return eval.pctCorrect();
     }
 
@@ -81,18 +166,18 @@ private int[] remove(Set<Integer> toKeep){
         List<Integer> toRemove = new ArrayList<Integer>();
 
         for(int i=0; i<training.numAttributes(); i++){
-            if (!toKeep.contains(i)){
+            if (!toKeep.contains(i) && i != CLASS_INDEX){
                 toRemove.add(i);
             }
         }
 
-        System.out.println("Removing features: " + toRemove);
-
         // Convert list to int[]
         return toRemove.stream().mapToInt(i->i).toArray();
     }
 
     public double classify(Set<Integer> indices) throws Exception {
+        weka.classifiers.Classifier classifier = createClassifier();
+
         Remove rm = new Remove();
 
         int[] remove = remove(indices);
@@ -105,12 +190,24 @@ public double classify(Set<Integer> indices) throws Exception {
         // train and make predictions
         fc.buildClassifier(training);
 
-        Evaluation eval = new Evaluation(training);
-        eval.evaluateModel(fc, training);
+        return evaluate(fc);
+    }
 
-        System.out.println(eval.pctCorrect());
+    public int getNumFeatures(){
+        return training.numAttributes();
+    }
 
-        return eval.pctCorrect();
+    public Set<Integer> getAllFeatureIndices() {
+        int totalFeatures = training.numAttributes();
+
+        Set<Integer> features = IntStream.rangeClosed(0, totalFeatures - 1)
+                .boxed().collect(Collectors.toSet());
+
+        // Class shouldnt be considered a feature
+        features.remove(CLASS_INDEX);
+
+        // Return a set from 0..totalFeatures
+        return features;
     }
 
 }