Removed a duplicate code listing.

Author: ashleydavis

listing-10.js

@@ -1,43 +1,36 @@
 //
-// This example imports a CSV file, filters out an entire column and exports a new CSV file.
+// Aggregate data from multiple files.
 //
 // This example uses Data-Forge.
 //
 
 'use strict';
 
-var dataForge = require('data-forge');
+var globby = require('globby');
+var importCsvFile = require('./toolkit/importCsvFile.js');
+var exportCsvFile = require('./toolkit/exportCsvFile.js');
 
-var inputFileName = './data/surveys.csv';
-var outputFileName = './output/surveys-aggregated.csv';
+var inputFileSpec = './data/by-country/*.csv';
+var outputFileName = './output/surveys-aggregated-from-separate-files.csv';
 
-function transformData (inputDataFrame) {
-    return inputDataFrame
-        .parseFloats("transects_length")
-        .groupBy(inputRow => inputRow.reef_name)
-        .select(group => {
-            return {
-                reef_name: group.first().reef_name,
-                transects_length: group
-                    .select(row => row.transects_length)
-                    .sum(),
-            };
-        })
-        .inflate();
-}
-
-dataForge.readFile(inputFileName)
-    .parseCSV()
-    .then(inputDataFrame => {
-        var outputDataFrame = transformData(inputDataFrame);
-        return outputDataFrame
-            .asCSV()
-            .writeFile(outputFileName);
+globby(inputFileSpec)
+    .then(paths => {
+        return paths.reduce((prevPromise, path) => {
+                return prevPromise.then(workingData => {
+                    return importCsvFile(path)
+                        .then(inputData => {
+                            return workingData.concat(inputData);
+                        });
+                });
+            }, Promise.resolve([]));
+    })
+    .then(aggregatedData => {
+        return exportCsvFile(outputFileName, aggregatedData);
     })
     .then(() => {
-        console.log('Done!');
+        console.log("Done!");
     })
     .catch(err => {
-        console.error('Error!');
-        console.error(err && err.stack || err);
-    });
+        console.error("An error occurred.");
+        console.error(err);
+    })

listing-11.js

@@ -1,36 +1,50 @@
 //
-// Aggregate data from multiple files.
+// Split a data file out to multiple data files where data is grouped by country.
 //
 // This example uses Data-Forge.
 //
 
 'use strict';
 
-var globby = require('globby');
-var importCsvFile = require('./toolkit/importCsvFile.js');
-var exportCsvFile = require('./toolkit/exportCsvFile.js');
-
-var inputFileSpec = './data/by-country/*.csv';
-var outputFileName = './output/surveys-aggregated-from-separate-files.csv';
-
-globby(inputFileSpec)
-    .then(paths => {
-        return paths.reduce((prevPromise, path) => {
-                return prevPromise.then(workingData => {
-                    return importCsvFile(path)
-                        .then(inputData => {
-                            return workingData.concat(inputData);
-                        });
-                });
-            }, Promise.resolve([]));
-    })
-    .then(aggregatedData => {
-        return exportCsvFile(outputFileName, aggregatedData);
-    })
+var dataForge = require('data-forge');
+
+var inputFileName = './data/surveys.csv';
+
+function filterRow (inputRow, country) {
+    return inputRow.country === country;
+}
+
+function transformData (inputDataFrame, country) {
+    return inputDataFrame.where(inputRow => filterRow(inputRow, country));
+}
+
+function getCountries (inputDataFrame) {
+    return inputDataFrame
+        .getSeries("country")
+        .distinct();    
+}
+
+function splitDataByCountry (inputDataFrame) {
+    return getCountries(inputDataFrame)
+        .aggregate(Promise.resolve(), (prevPromise, country) => {
+            return prevPromise.then(() => {
+                var outputDataFrame = transformData(inputDataFrame, country);
+                var outputFileName = './data/by-country/' + country + '.csv';
+                console.log('>> ' + outputFileName);
+                return outputDataFrame
+                    .asCSV()
+                    .writeFile(outputFileName);
+            });
+        });
+}
+
+dataForge.readFile(inputFileName)
+    .parseCSV()
+    .then(splitDataByCountry)
     .then(() => {
-        console.log("Done!");
+        console.log('Done!');
     })
     .catch(err => {
-        console.error("An error occurred.");
-        console.error(err);
-    })
+        console.error('Error!');
+        console.error(err && err.stack || err);
+    });

listing-12.js

@@ -1,50 +1,41 @@
 //
-// Split a data file out to multiple data files where data is grouped by country.
-//
-// This example uses Data-Forge.
+// Example Data-Forge data pipeline.
 //
 
+
 'use strict';
 
 var dataForge = require('data-forge');
+var moment = require('moment');
 
+var importDateFormat = "YYYY-MM-DD HH:mm:ss";
 var inputFileName = './data/surveys.csv';
-
-function filterRow (inputRow, country) {
-    return inputRow.country === country;
-}
-
-function transformData (inputDataFrame, country) {
-    return inputDataFrame.where(inputRow => filterRow(inputRow, country));
-}
-
-function getCountries (inputDataFrame) {
-    return inputDataFrame
-        .getSeries("country")
-        .distinct();    
-}
-
-function splitDataByCountry (inputDataFrame) {
-    return getCountries(inputDataFrame)
-        .aggregate(Promise.resolve(), (prevPromise, country) => {
-            return prevPromise.then(() => {
-                var outputDataFrame = transformData(inputDataFrame, country);
-                var outputFileName = './data/by-country/' + country + '.csv';
-                console.log('>> ' + outputFileName);
-                return outputDataFrame
-                    .asCSV()
-                    .writeFile(outputFileName);
-            });
-        });
-}
+var outputFileName = './output/data-pipeline-output.csv';
 
 dataForge.readFile(inputFileName)
     .parseCSV()
-    .then(splitDataByCountry)
+    .then(dataFrame => {
+        return dataFrame.dropSeries(["exp_id", "dive_observations", "obs_topography"])
+            .parseDates(["start_datetime", "end_datetime"], importDateFormat)
+            .where(row => moment(row.start_datetime).year() === 2014)
+            .parseFloats("dive_temperature")
+            .where(row => row.dive_temperature !== 0)
+            .groupBy(row => row.country)
+            .select(group => ({
+                country: group.first().country,
+                dive_temperature: group.select(row => row.dive_temperature).average()
+            }))
+            .inflate()
+            .asCSV()
+            .writeFile(outputFileName);
+    })
     .then(() => {
-        console.log('Done!');
+        console.log("Done!");
     })
     .catch(err => {
-        console.error('Error!');
-        console.error(err && err.stack || err);
+        console.error("An error occured");
+        console.error(err);
     });
+
+
+