Merge pull request #52 from IBMStreams/develop

1.4.0

Author: markheger

.gitignore

@@ -2,7 +2,7 @@
 
 # ignore directories generated by Streams compiler and runtime
 **/.apt_generated/
-**/.settings/
+#**/.settings/
 **/output/
 
 **/src-gen/

build.xml

@@ -50,6 +50,8 @@
         <arg value="Makefile" />
         <arg value="clean" />
      </exec>
+     <ant dir="test" target="clean">
+     </ant>
   </target>
 
 

com.ibm.streamsx.nlp/.externalToolBuilders/org.eclipse.jdt.core.javabuilder.launch

@@ -4,3 +4,4 @@
 /output/
 /data/
 /.pydevproject
+

com.ibm.streamsx.nlp/.gitignore

@@ -0,0 +1,2 @@
+eclipse.preferences.version=1
+line.separator=\n

com.ibm.streamsx.nlp/.settings/org.eclipse.core.runtime.prefs

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+<comment>SPL Build Options for Project</comment>
+<entry key="com.ibm.streams.studio.splproject:EXT_CLEAN_BUILD_CMD">make clean</entry>
+<entry key="com.ibm.streams.studio.splproject:REPLACE_ENVIRONMENT">F</entry>
+<entry key="com.ibm.streams.studio.splproject:EXT_FULL_BUILD_CMD">make all</entry>
+<entry key="com.ibm.streams.studio.splproject:OUTPUT_DIR">output</entry>
+<entry key="com.ibm.streams.studio.splproject:BUILDER_TYPE">1</entry>
+<entry key="com.ibm.streams.studio.splproject:DERIVED_OUTPUT_DIR">T</entry>
+<entry key="com.ibm.streams.studio.splproject:EXT_INCR_BUILD_CMD">make</entry>
+<entry key="com.ibm.streams.studio.splproject:DATA_DIR">data</entry>
+<entry key="com.ibm.streams.studio.splproject:EXT_BUILD_OUTPUT">T</entry>
+</properties>

com.ibm.streamsx.nlp/.settings/project.sploptions

@@ -32,8 +32,10 @@ ($;$)
 
 Options:
 		--pearFile <filename>					Specifies the PEAR file to be loaded.
-		--outputfile <filename>         			Filename as the output file name.  .spl will be appended.
-		--help                          			Print this message.
+		--outputfile <filename>         		Filename as the output file name.
+		--namespace <namespace>         		SPL namespace of the generated SPL files (optional)
+		--main <composite>						Name of the main composite to be generated (optional). File <composite>.spl will be created.
+		--help                          		Print this message.
 END
 if ($retCode == 0) {
     print STDOUT $usageString;
@@ -144,9 +146,20 @@ END
     return $prolog;
 }
 
+sub addNamespace($) {
+
+    my ($namespace) = @_;
+    my $prolog=<<END;
+namespace $namespace;
+
+END
+    return $prolog;
+}
+
 sub main() {
     my $pearFile;
     my $outfilename;
+    my $namespace;
     my $needHelp;
     my $outfilename="TypesGenerated.spl";
     my $createMain = 0;
@@ -155,6 +168,7 @@ ()
 
     GetOptions ("pearFile=s" => \$pearFile,
                 "outputfile=s", \$outfilename,
+                "namespace=s", \$namespace,
                 "main=s",\$mainParam,
                 "help|h|?",\$needHelp,
                 ) or usage(1);
@@ -192,9 +206,29 @@ ()
     if ($createMain) {
         my $annoType=getFirstAnnoType($outfilename);
         open (MAINFILE,">".$compositeName.".spl") or die "Could not create file ".$compositeName.".spl";
+        if (defined $namespace) {
+        	print MAINFILE addNamespace($namespace);
+        }        
         print MAINFILE generateComposite($compositeName,$pearFile,$annoType);
         close MAINFILE;
     }
+    
+    if (defined $namespace) {
+    	executeAndCapture("mkdir -p $namespace");
+    	if ($createMain) {
+			my $mainFile=$compositeName.".spl";
+    	   	executeAndCapture("mv $mainFile $namespace");
+    	}
+		open (DATA, "<".$outfilename) || die "could not open $outfilename\n";
+		my @body=<DATA>;
+		close(DATA);
+		# add namespace to top of the file
+		open(TYPESFILE,">".$namespace."/".$outfilename);
+		print TYPESFILE addNamespace($namespace);
+		print TYPESFILE @body;
+		close(TYPESFILE);    	
+    	executeAndCapture("rm -f $outfilename");    	
+    }
 }
 
 main();

com.ibm.streamsx.nlp/bin/createTypes.pl

@@ -10,6 +10,9 @@ use com.ibm.streamsx.nlp.utils::*;
 /**
  * This operator uses a previously trained model in order to find out the most likely intend of a text.
  *
+ * The ContentRanking operator should be used in a Streams release where SPL Python primitive support is not provided (e.g. Streams 3.2). It uses a **ShellPipe** operator to invoke Python scripts.
+ * When using a Streams release 4.2 or later, it is recommended to create a *SPL Python primitive operator* to invoke Python classes or functions.
+ * 
  * @input InStream
  * One tuple is one document.
  *
@@ -42,6 +45,9 @@ use com.ibm.streamsx.nlp.utils::*;
  * 
  * @param outStreamType
  * The OutStream (output port 0) schema of this operator. The schema must contain the schema defined by the resultType.
+ * 
+ * @param initOnFirstTuple
+ * The script is called on operator startup. If this parameter is set to true, then the script is called on first tuple.
  */
 public composite ContentRanking (
 	input InStream;
@@ -56,6 +62,7 @@ public composite ContentRanking (
 		expression<rstring> $kbNamesFile;
 		attribute $documentAttribute;
 		type $outStreamType;
+		expression<boolean> $initOnFirstTuple : false;
 
 	/**
 	 * OutStream must contain the resultType schema
@@ -77,6 +84,7 @@ public composite ContentRanking (
 				command: $pythonCommand + " -u " + $pythonScript + " " + $modelFile + " " + $kbIndexFile + " " +  $dictLemmasFile;
 				stdinAttribute: $documentAttribute;
 				stdoutAttribute: "crPythonScriptOutput";
+				initOnFirstTuple: $initOnFirstTuple;
 		}
 
 		(

com.ibm.streamsx.nlp/com.ibm.streamsx.nlp/ContentRanking.spl

@@ -10,6 +10,9 @@ use com.ibm.streamsx.nlp.utils::*;
 /**
  * This operator trains a model for ContentRanking based on training documents.
  *
+ * The ContentRankingModelBuilder operator should be used in a Streams release where SPL Python primitive support is not provided (e.g. Streams 3.2) . It uses a **ShellSource** operator to invoke Python scripts.
+ * When using a Streams release 4.2 or later, it is recommended to create a *SPL Python primitive operator* to invoke Python classes or functions.
+ * 
  * @param pythonCommand
  * The name of the python binary. The default name is `python`.
  * With this parameter you can change the version and the location of the python command according to your environment.
@@ -32,6 +35,59 @@ use com.ibm.streamsx.nlp.utils::*;
  */
 public composite ContentRankingModelBuilder
 (
+) {
+	param
+		expression<rstring> $pythonCommand: "python"; // Content ranking scripts need python 2.7 or above!
+		expression<rstring> $pythonScript: getToolkitPath()+"/etc/python/ContentRankingModelBuilder.py";
+		expression<rstring> $xRegFile;
+		expression<rstring> $yRegFile;
+		expression<rstring> $modelFilesDirectory;
+		expression<rstring> $nEpoch: "5";
+
+	graph 
+
+		(stream<rstring result> ResultStream) as CRMB = ContentRankingModelBuilderCore () {
+			param 
+				nEpoch: $nEpoch;
+				xRegFile: $xRegFile;
+				yRegFile: $yRegFile;
+				modelFilesDirectory: $modelFilesDirectory;
+				pythonCommand: $pythonCommand;
+				pythonScript: $pythonScript;
+		}
+}
+
+/**
+ * This operator trains a model for ContentRanking based on training documents.
+ *
+ * The ContentRankingModelBuilder operator should be used in a Streams release where SPL Python primitive support is not provided (e.g. Streams 3.2) . It uses a **ShellSource** operator to invoke Python scripts.
+ * When using a Streams release 4.2 or later, it is recommended to create a *SPL Python primitive operator* to invoke Python classes or functions.
+ * 
+ * @output ResultStream
+ * Result stream, window marker at end of building the model 
+ *
+ * @param pythonCommand
+ * The name of the python binary. The default name is `python`.
+ * With this parameter you can change the version and the location of the python command according to your environment.
+ * Content ranking scripts need python 2.7 or later!
+ *
+ * @param pythonScript
+ * The name of the python script. The default is `<toolkit_dir>/etc/python/ContentRankingModelBuilder.py`.
+ *
+ * @param xRegFile
+ * The name of the input file containing the X-register training data.
+ *
+ * @param yRegFile
+ * The name of the input file containing the Y-register training data.
+ * 
+ * @param modelFilesDirectory
+ * The name of the output directory where the model pkl files are created.
+ * 
+ * @param nEpoch
+ * The number of epochs used for neural network training. 
+ */
+composite ContentRankingModelBuilderCore
+( output ResultStream
 ) {
 	param
 		expression<rstring> $pythonCommand: "python"; // Content ranking scripts need python 2.7 or above!
@@ -70,12 +126,67 @@ public composite ContentRankingModelBuilder
 			}
 		}
 
-		() as WriterStat = Custom(StatStream as I) {
+		(stream<rstring result> ResultStream) as WriterStat = Custom(StatStream as I) {
 			logic onTuple I: {
 				if (exitCode != 0) 
 					appTrc(Trace.error, "Model Builder exit code: " + (rstring)exitCode + " reason: " + exitReason);
 				printStringLn("Model Builder exit code: " + (rstring)exitCode + " reason: " + exitReason);
+				submit(Sys.WindowMarker, ResultStream);
 			}
 		}
 
 }
+
+/**
+ * This operator trains a model for ContentRanking based on training documents.
+ *
+ * The ContentRankingModelBuilder operator should be used in a Streams release where SPL Python primitive support is not provided (e.g. Streams 3.2) . It uses a **ShellSource** operator to invoke Python scripts.
+ * When using a Streams release 4.2 or later, it is recommended to create a *SPL Python primitive operator* to invoke Python classes or functions.
+ * 
+ * @output ResultStream
+ * Result stream, window marker at end of building the model 
+ *
+ * @param pythonCommand
+ * The name of the python binary. The default name is `python`.
+ * With this parameter you can change the version and the location of the python command according to your environment.
+ * Content ranking scripts need python 2.7 or later!
+ *
+ * @param pythonScript
+ * The name of the python script. The default is `<toolkit_dir>/etc/python/ContentRankingModelBuilder.py`.
+ *
+ * @param xRegFile
+ * The name of the input file containing the X-register training data.
+ *
+ * @param yRegFile
+ * The name of the input file containing the Y-register training data.
+ * 
+ * @param modelFilesDirectory
+ * The name of the output directory where the model pkl files are created.
+ * 
+ * @param nEpoch
+ * The number of epochs used for neural network training. 
+ */
+public composite ContentRankingModelBuilder2
+( output ResultStream
+) {
+	param
+		expression<rstring> $pythonCommand: "python"; // Content ranking scripts need python 2.7 or above!
+		expression<rstring> $pythonScript: getToolkitPath()+"/etc/python/ContentRankingModelBuilder.py";
+		expression<rstring> $xRegFile;
+		expression<rstring> $yRegFile;
+		expression<rstring> $modelFilesDirectory;
+		expression<rstring> $nEpoch: "5";
+
+	graph 
+
+		(stream<rstring result> ResultStream) as CRMB = ContentRankingModelBuilderCore () {
+			param 
+				nEpoch: $nEpoch;
+				xRegFile: $xRegFile;
+				yRegFile: $yRegFile;
+				modelFilesDirectory: $modelFilesDirectory;
+				pythonCommand: $pythonCommand;
+				pythonScript: $pythonScript;
+		}
+
+}

com.ibm.streamsx.nlp/com.ibm.streamsx.nlp/ContentRankingModelBuilder.spl

@@ -23,7 +23,7 @@
             param
                 filterMode : DictionaryFilter.remove;
 		textAttribute : text;
-		dictionaryFile : "stopwords.csv";
+		dictionaryFile : "etc/stopwords.csv";
             output
                 ${outputStream} : ${outputAttribute} = ${value};
         }</template>
@@ -51,7 +51,7 @@ In case the filterMode `keep` is selected, then those words of the textAttribute
         <name>textAttribute</name>
         <description>
 <![CDATA[
-Specifies the dictionary file loaded on startup.
+Specifies the text attribute of the input stream.
 ]]></description>
         <optional>false</optional>
         <rewriteAllowed>false</rewriteAllowed>
@@ -63,7 +63,7 @@ Specifies the dictionary file loaded on startup.
         <name>dictionaryFile</name>
         <description>
 <![CDATA[
-Specifies the dictionary file loaded on startup.
+Specifies the dictionary file loaded on startup. If relative path is used, then root is application directory. It is recommended to store the file in etc directory.
 ]]></description>
         <optional>false</optional>
         <rewriteAllowed>false</rewriteAllowed>