boalang · giangnm58 · Apr 10, 2020 · Apr 10, 2020 · Apr 20, 2020 · Apr 20, 2020
diff --git a/.classpath b/.classpath
@@ -1,12 +1,16 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <classpath>
+	<classpathentry kind="src" path="src/antlr"/>
+	<classpathentry kind="src" path="src/java"/>
+	<classpathentry kind="src" path="src/test"/>
+	<classpathentry kind="src" path="src/compiled-proto"/>
+	<classpathentry kind="src" path="src/proto"/>
 	<classpathentry kind="lib" path="rtlib/antlr-runtime-4.5.jar"/>
 	<classpathentry kind="lib" path="rtlib/commons-cli-1.2.jar"/>
 	<classpathentry kind="lib" path="rtlib/log4j-1.2.15.jar"/>
 	<classpathentry kind="lib" path="rtlib/protobuf-java-2.5.0.jar"/>
 	<classpathentry kind="lib" path="rtlib/scannotation-1.0.3.jar"/>
 	<classpathentry kind="lib" path="rtlib/ST-4.0.8.jar"/>
-
 	<classpathentry kind="lib" path="lib/antlr-4.5-complete.jar"/>
 	<classpathentry kind="lib" path="lib/commons-lang-2.4.jar"/>
 	<classpathentry kind="lib" path="lib/commons-math-2.1.jar"/>
@@ -39,19 +43,57 @@
 	<classpathentry kind="lib" path="lib/datagen/sac.jar"/>
 	<classpathentry kind="lib" path="lib/datagen/trove-3.0.3.jar"/>
 	<classpathentry kind="lib" path="lib/datagen/xml-security-impl-1.0.jar"/>
-
-	<classpathentry kind="output" path="build/classes" />
-	<classpathentry kind="src" path="build/java" />
-	<classpathentry kind="src" path="src/java" />
-	<classpathentry kind="src" path="src/compiled-proto" />
-
 	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
-
-	<classpathentry kind="src" output="build/tests" path="src/test">
-		<attributes>
-			<attribute name="optional" value="true"/>
-			<attribute name="test" value="true"/>
-		</attributes>
-	</classpathentry>
-</classpath>
+	<classpathentry kind="lib" path="lib/datagen/app-1.3.200-v20130910-1609-javadoc.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/app-1.3.200-v20130910-1609-sources.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/app-1.3.200-v20130910-1609.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/commons-beanutils-1.8.3.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/commons-collections-3.2.1.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/commons-configuration-1.6.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/commons-logging-1.1.1.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/cssparser-0.9.23-sources.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/dom4j-2.0.2-sources.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/ezmorph-1.0.6.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/gson-gson-parent-2.7.zip"/>
+	<classpathentry kind="lib" path="lib/datagen/jena-core-2.7.3.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/jena-iri-0.9.3.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/jsoup-1.10.3-javadoc.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/jsoup-1.10.3-sources.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/log4j-1.2.16.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/maven-model-3.0-sources.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/org.eclipse.core.filesystem_1.6.0.v20160503-1608.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/org.eclipse.equinox.registry_3.5.100.v20110502.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/org.eclipse.equinox.registry.source_3.5.100.v20110502.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/org.eclipse.jgit-4.0.1.201506240215-r-javadoc.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/org.eclipse.jgit-4.0.1.201506240215-r-sources.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/org.eclipse.osgi.source_3.9.1.v20130814-1242.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/org.eclipse.php.core.parser.source_4.3.1.201703301037.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/org.eclipse.php.core.source_4.3.1.201703301037.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/org.eclipse.text-3.5.101-sources.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/org.eclipse.text-3.5.101.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/plexus-utils-2.0.5-sources.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/protobuf-2.5.0.zip"/>
+	<classpathentry kind="lib" path="lib/datagen/protobuf-java-2.5.0.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/rhino-1.7.7.1-sources.zip"/>
+	<classpathentry kind="lib" path="lib/datagen/sacjavadoc-1.3.zip"/>
+	<classpathentry kind="lib" path="lib/datagen/slf4j-api-1.4.3.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/slf4j-log4j12-1.4.3.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/svnkit-1.7.6.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/trove-3.0.3-src.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/xercesImpl.jar"/>
+	<classpathentry kind="lib" path="lib/datagen/xml-apis.jar"/>
+	<classpathentry kind="lib" path="lib/evaluator/com.springsource.org.codehaus.jackson-1.4.2.jar"/>
+	<classpathentry kind="lib" path="lib/evaluator/com.springsource.org.codehaus.jackson.mapper-1.4.2.jar"/>
+	<classpathentry kind="lib" path="lib/evaluator/commons-configuration-1.6.jar"/>
+	<classpathentry kind="lib" path="lib/evaluator/commons-httpclient-3.1.0.redhat-7.jar"/>
+	<classpathentry kind="lib" path="lib/evaluator/commons-io-2.4.jar"/>
+	<classpathentry kind="lib" path="lib/evaluator/jackson-mapper-asl-1.9.13.jar"/>
+	<classpathentry kind="lib" path="compile"/>
+	<classpathentry kind="lib" path="/home/tess/Downloads/arpack_combined.jar"/>
+	<classpathentry kind="lib" path="/home/tess/Downloads/core.jar"/>
+	<classpathentry kind="lib" path="/home/tess/Downloads/mtj.jar"/>
+	<classpathentry kind="lib" path="/home/tess/Downloads/weka-3-8-4/weka.jar"/>
+	<classpathentry kind="src" path="/boa2"/>
+	<classpathentry kind="output" path="build/classes"/>
+</classpath>
diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,5 @@
 /.settings/
 /junitvmwatcher*.properties
 .vscode/
+/model/
+Query.jar
diff --git a/src/antlr/Boa.g b/src/antlr/Boa.g
@@ -112,6 +112,7 @@ type returns [AbstractType ast]
 	| q=queueType    { $ast = $q.ast; }
 	| set=setType    { $ast = $set.ast; }
 	| e=enumType     { $ast = $e.ast; }
+	| model=modelType { $ast = $model.ast; }
 	| id=identifier  { $ast = $id.ast; }
 	;
 
@@ -202,7 +203,12 @@ outputType returns [OutputType ast]
 	@after { $ast.setPositions($l, $c, getEndLine(), getEndColumn()); }
 	: OUTPUT (tk=SET { $ast = new OutputType((Identifier)new Identifier($tk.text).setPositions(getStartLine($tk), getStartColumn($tk), getEndLine($tk), getEndColumn($tk))); } | id=identifier { $ast = new OutputType($id.ast); }) (LPAREN el=expressionList RPAREN { $ast.setArgs($el.list); })? (LBRACKET m=component RBRACKET { $ast.addIndice($m.ast); })* OF m=component { $ast.setType($m.ast); } (WEIGHT m=component { $ast.setWeight($m.ast); })? (FORMAT LPAREN el=expressionList RPAREN)?
 	;
-
+modelType returns [ModelType ast]
+	locals [int l, int c]
+	@init { $l = getStartLine(); $c = getStartColumn(); }
+	@after { $ast.setPositions($l, $c, getEndLine(), getEndColumn()); }
+	: id=identifier { $ast = new ModelType($id.ast); } OF m=component { $ast.setType($m.ast); }
+	;
 functionType returns [FunctionType ast]
 	locals [int l, int c]
 	@init {

diff --git a/src/java/boa/BoaTup.java b/src/java/boa/BoaTup.java
@@ -0,0 +1,11 @@
+package boa;
+
+import java.io.IOException;
+import java.lang.ClassNotFoundException;
+
+public interface BoaTup {
+	public String[] getValues();
+	public String[] getFieldNames();
+	public byte[] serialize(Object o) throws IOException;
+	public Object getValue(String f);
+}
diff --git a/src/java/boa/aggregators/Aggregator.java b/src/java/boa/aggregators/Aggregator.java
@@ -22,6 +22,7 @@
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Reducer.Context;
 
+import boa.BoaTup;
 import boa.functions.BoaCasts;
 import boa.io.EmitKey;
 import boa.io.EmitValue;
@@ -34,10 +35,12 @@
  */
 public abstract class Aggregator {
 	private long arg;
+	private String mlarg; //for ML
 	@SuppressWarnings("rawtypes")
 	private Context context;
 	private EmitKey key;
 	private boolean combining;
+	private int vectorSize; //for ML
 
 	/**
 	 * Construct an Aggregator.
@@ -60,6 +63,21 @@ public Aggregator(final long arg) {
 		this.arg = arg;
 	}
 
+	/**
+	 * Construct an Aggregator.
+	 *
+	 * @param arg
+	 *            A String containing the argument to the table
+	 *
+	 */
+	//for ML
+	public Aggregator(final String arg) {
+		this();
+
+		this.mlarg = arg;
+	}
+
+
 	/**
 	 * Reset this aggregator for a new key.
 	 * 
@@ -92,6 +110,8 @@ public void aggregate(final double data, final String metadata) throws IOExcepti
 	public void aggregate(final double data) throws IOException, InterruptedException, FinishedException {
 		this.aggregate(BoaCasts.doubleToString(data), null);
 	}
+
+
 
 	@SuppressWarnings("unchecked")
 	protected void collect(final String data, final String metadata) throws IOException, InterruptedException {
@@ -122,6 +142,13 @@ protected void collect(final double data, final String metadata) throws IOExcept
 	protected void collect(final double data) throws IOException, InterruptedException {
 		this.collect(BoaCasts.doubleToString(data), null);
 	}
+
+	public void aggregate(final BoaTup data, final String metadata) throws IOException, InterruptedException, FinishedException, IllegalAccessException {	
+	}
+
+	public void aggregate(final BoaTup data) throws IOException, InterruptedException, FinishedException, IllegalAccessException {
+		this.aggregate(data, null);
+	}
 
 	public void finish() throws IOException, InterruptedException {
 		// do nothing by default
@@ -155,4 +182,13 @@ public void setKey(final EmitKey key) {
 	public EmitKey getKey() {
 		return this.key;
 	}
+
+	//for ML
+	public int getVectorSize() {
+		return this.vectorSize;
+	}
+	//for ML
+	public void setVectorSize(int vectorSize) {
+		this.vectorSize = vectorSize;
+	}
 }
diff --git a/src/java/boa/aggregators/ml/LinearRegressionAggregator.java b/src/java/boa/aggregators/ml/LinearRegressionAggregator.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2014, Hridesh Rajan, Robert Dyer,
+ *                 and Iowa State University of Science and Technology
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package boa.aggregators.ml;
+
+import boa.BoaTup;
+import boa.aggregators.AggregatorSpec;
+import boa.aggregators.FinishedException;
+import weka.classifiers.functions.LinearRegression;
+
+import java.io.IOException;
+
+/**
+ * A Boa aggregator for training the model using LinearRegression.
+ *
+ * @author ankuraga
+ */
+@AggregatorSpec(name = "linearregression", formalParameters = {"string"})
+public class LinearRegressionAggregator extends MLAggregator {
+    private LinearRegression model;
+
+    public LinearRegressionAggregator() {
+        this.model = new LinearRegression();
+    }
+
+    public LinearRegressionAggregator(final String s) {
+        super(s);
+    }
+
+    @Override
+    public void aggregate(String data, String metadata) throws NumberFormatException, IOException, InterruptedException {
+        aggregate(data, metadata, "LinearRegression");
+    }
+
+    public void aggregate(final BoaTup data, final String metadata) throws IOException, InterruptedException, FinishedException, IllegalAccessException {
+        aggregate(data, metadata, "LinearRegression");
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public void finish() throws IOException, InterruptedException {
+        try {
+            System.out.println("Linearregression working now with: " + this.trainingSet.numInstances());
+            System.out.println(this.trainingSet);
+            this.model.buildClassifier(this.trainingSet);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        System.out.println("modeling done");
+        this.saveModel(this.model);
+//		this.saveTrainingSet(this.trainingSet);
+        this.collect(this.model.toString());
+    }
+}