Merge branch 'release/1.0.0-rc.6'

Author: paxtonhare

.gitignore

@@ -20,3 +20,4 @@ assetInstallTime.properties
 flow.properties
 node_modules/
 .tmp/
+*.sublime-*

CHANGELOG.md

@@ -1,7 +1,30 @@
 # Change Log
 
-## [v1.0.0-rc.5](https://github.com/marklogic/marklogic-data-hub/tree/v1.0.0-rc.5)
+## [v1.0.0-rc.6](https://github.com/marklogic/marklogic-data-hub/tree/v1.0.0-rc.6)
 
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.0.0-rc.5...v1.0.0-rc.6)
+
+**Fixed bugs:**
+
+- Deadlock when evaling from writer [\#231](https://github.com/marklogic/marklogic-data-hub/issues/231)
+
+**Closed issues:**
+
+- Update Trace UI in Trace server [\#302](https://github.com/marklogic/marklogic-data-hub/issues/302)
+- Trace enhancement: Not logging enough for error trace [\#301](https://github.com/marklogic/marklogic-data-hub/issues/301)
+- ArrayIndexOutOfBoundsException [\#298](https://github.com/marklogic/marklogic-data-hub/issues/298)
+- Getting Started Tutorial - Sample code for Acme Tech header plugin does not update 'latest' variable [\#297](https://github.com/marklogic/marklogic-data-hub/issues/297)
+- Not all data is processed in harmonize flow when thread count is greater than 1 [\#296](https://github.com/marklogic/marklogic-data-hub/issues/296)
+- New Mlcp Error grabbing has false positives on windows [\#295](https://github.com/marklogic/marklogic-data-hub/issues/295)
+- Clarify that ML DHF is FOSS and not supported MarkLogic product [\#294](https://github.com/marklogic/marklogic-data-hub/issues/294)
+- custom thread count throws parsing error [\#292](https://github.com/marklogic/marklogic-data-hub/issues/292)
+- RC5: the ingest steps in quick start gives a exception and is not runned [\#291](https://github.com/marklogic/marklogic-data-hub/issues/291)
+- fix gradle tasks for run flow [\#290](https://github.com/marklogic/marklogic-data-hub/issues/290)
+- When exception is thrown, not all flow traces are persisted. [\#278](https://github.com/marklogic/marklogic-data-hub/issues/278)
+- Add ability to clear the Staging or Final database [\#198](https://github.com/marklogic/marklogic-data-hub/issues/198)
+- mlcp\_transform needs profiling [\#162](https://github.com/marklogic/marklogic-data-hub/issues/162)
+
+## [v1.0.0-rc.5](https://github.com/marklogic/marklogic-data-hub/tree/v1.0.0-rc.5) (2016-09-27)
 [Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.0.0-rc.4...v1.0.0-rc.5)
 
 **Implemented enhancements:**

README.md

@@ -1,3 +1,5 @@
+<p style="font-style: italic; font-size:12px;">The MarkLogic Data Hub Framework is free and open source under the <a href="https://github.com/marklogic/marklogic-data-hub/blob/master/LICENSE">Apache 2 License</a> and is supported by the community of developers who build and contribute to it. Please note that Data Hub Framework is not a supported MarkLogic product.</p>
+
 OS | Status
 --- | --- | ---
 Linux/Mac | [![Build Status](https://travis-ci.org/marklogic/marklogic-data-hub.svg?branch=master)](https://travis-ci.org/marklogic/marklogic-data-hub)

build.gradle

@@ -1,6 +1,5 @@
 subprojects {
     apply plugin: 'java'
-    apply plugin: 'eclipse'
 
     sourceCompatibility = "1.8"
     targetCompatibility = "1.8"

examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/headers/headers.sjs

@@ -17,6 +17,7 @@ function createHeaders(id, content, options) {
     var date = xs.date(xdmp.parseDateTime('[M01]/[D01]/[Y0001]', history.effectiveDate));
     if (date.gt(latest)) {
       salary = history.salary;
+      latest = date;
     }
   }
 

examples/spring-batch/build.gradle

@@ -3,19 +3,17 @@ plugins {
     id 'eclipse'
     id 'idea'
     id 'application'
-    id 'net.saliman.properties' version '1.4.5'
-    id 'com.marklogic.ml-gradle' version '2.3.1'
-    id 'com.marklogic.ml-data-hub' version '1.0.0-rc.5'
+    id 'net.saliman.properties' version '1.4.6'
+    id 'com.marklogic.ml-data-hub' version '1.0.0-rc.6'
 }
 
 repositories {
-    mavenLocal()
     jcenter()
     maven {url 'http://developer.marklogic.com/maven2/'}
 }
 
 dependencies {
-    compile 'com.marklogic:marklogic-data-hub:1.0.0-rc.5'
+    compile 'com.marklogic:marklogic-data-hub:1.0.0-rc.6'
     compile "com.marklogic:marklogic-spring-batch-core:0.6.0"
     compile "com.marklogic:ml-javaclient-util:2.9.1"
     testCompile "com.marklogic:marklogic-spring-batch-test:0.6.0"

marklogic-data-hub/build.gradle

@@ -1,11 +1,10 @@
 plugins {
-    id 'net.saliman.properties' version '1.4.4'
+    id 'net.saliman.properties' version '1.4.6'
     id 'java'
-    id 'eclipse'
-    id 'idea'
     id 'maven-publish'
-    id 'com.jfrog.bintray' version '1.6'
+    id 'com.jfrog.bintray' version '1.7.2'
     id 'com.marklogic.ml-gradle' version '2.3.4'
+    id 'com.moowork.node' version '0.13'
 }
 
 repositories {
@@ -35,6 +34,22 @@ dependencies {
     testCompile 'org.easymock:easymock:3.4'
 }
 
+node {
+  // Version of node to use.
+  version = '6.8.1'
+
+  // Version of npm to use.
+  npmVersion = '3.10.8'
+
+  download = true
+
+  // Set the work directory for unpacking node
+  workDir = file("${project.buildDir}/nodejs")
+
+  // Set the work directory where node_modules should be located
+  nodeModulesDir = file("${project.projectDir}/src/trace-ui")
+}
+
 task javadocJar(type: Jar, dependsOn: javadoc) {
     classifier = 'javadoc'
     from javadoc.destinationDir
@@ -45,24 +60,29 @@ task sourcesJar(type: Jar, dependsOn: classes) {
     from sourceSets.main.allSource
 }
 
-task npmInstallTraceUI(type:Exec) {
-    workingDir './src/trace-ui'
-    commandLine 'npm', 'install'
+task cleanUI(type: Delete) {
+  delete 'src/main/resources/ml-modules/root/trace-ui'
 }
 
-task buildTraceUI(type:Exec) {
-    workingDir './src/trace-ui'
-    commandLine 'npm', 'run', 'build'
-    dependsOn tasks.npmInstallTraceUI
+task npmInstallUI(type: NpmTask) {
+    args = ['install']
+    dependsOn tasks.cleanUI
 }
 
-task watchTraceUI(type:Exec) {
-    workingDir './src/trace-ui'
-    commandLine 'npm', 'start'
-    dependsOn tasks.npmInstallTraceUI
+task buildUI(type: NpmTask) {
+    args = ['run', 'build.prod']
+    dependsOn tasks.npmInstallUI
 }
 
-build.dependsOn buildTraceUI
+task copyUIAssets(type: Copy) {
+    from 'src/trace-ui/dist'
+    into 'src/main/resources/ml-modules/root/trace-ui'
+    dependsOn tasks.buildUI
+}
+
+if (!gradle.startParameter.taskNames.contains('bootrun')) {
+  processResources.dependsOn copyUIAssets
+}
 
 javadoc {
    options.overview = 'src/main/resources/overview.html'
@@ -131,7 +151,6 @@ bintray {
         repo = 'maven'
         name = project.name
         desc = pomCustomizations.textdescription.text()
-        // userOrg = user
         licenses = ['Apache-2.0']
         websiteUrl = pomCustomizations.url
         vcsUrl = pomCustomizations.scm.url
@@ -142,11 +161,14 @@ bintray {
 ext {
     // mlAppConfig is an instance of com.marklogic.appdeployer.AppConfig
     mlAppConfig {
+
+        moduleTokensPropertiesSources = new ArrayList<>()
+
         // override some default values with our values
         modulesDatabaseName = mlModulesDbName
         triggersDatabaseName = mlTriggersDbName
         schemasDatabaseName = mlSchemasDbName
-        restPort = Integer.parseInt(mlStagingPort)
+        restPort = Integer.parseInt(mlJobPort)
 
         // Configure custom tokens for our json files
         customTokens.put("%%mlStagingAppserverName%%", mlStagingAppserverName)

marklogic-data-hub/gradle.properties

@@ -33,6 +33,9 @@ mlSchemasDbName=data-hub-SCHEMAS
 
 hubModulesPath=examples/hr-hub/plugins
 
-version=1.0.0-rc.5
+mlReplaceTokensInModules=true
+mlUseRoxyTokenPrefix=false
+
+version=1.0.0-rc.6
 
 

marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java

@@ -74,7 +74,8 @@ public class DataHub extends LoggingObject {
 
     private ManageConfig config;
     private ManageClient client;
-
+    private DatabaseManager databaseManager;
+    private ServerManager serverManager;
     private HubConfig hubConfig;
 
     private AdminManager adminManager;
@@ -95,6 +96,8 @@ private void init(HubConfig hubConfig) {
         this.hubConfig = hubConfig;
         config = new ManageConfig(hubConfig.host, 8002, hubConfig.username, hubConfig.password);
         client = new ManageClient(config);
+        databaseManager = new DatabaseManager(client);
+        serverManager = new ServerManager(client);
 
         AdminConfig adminConfig = new AdminConfig();
         adminConfig.setHost(hubConfig.host);
@@ -112,17 +115,14 @@ public void setAdminManager(AdminManager manager) {
      * @return true if installed, false otherwise
      */
     public boolean isInstalled() {
-        ServerManager sm = new ServerManager(client);
-        DatabaseManager dm = new DatabaseManager(client);
-
-        ResourcesFragment srf = sm.getAsXml();
+        ResourcesFragment srf = serverManager.getAsXml();
         boolean stagingAppServerExists = srf.resourceExists(hubConfig.stagingHttpName);
         boolean finalAppServerExists = srf.resourceExists(hubConfig.finalHttpName);
         boolean traceAppServerExists = srf.resourceExists(hubConfig.traceHttpName);
         boolean jobAppServerExists = srf.resourceExists(hubConfig.jobHttpName);
         boolean appserversOk = (stagingAppServerExists && finalAppServerExists && traceAppServerExists && jobAppServerExists);
 
-        ResourcesFragment drf = dm.getAsXml();
+        ResourcesFragment drf = databaseManager.getAsXml();
         boolean stagingDbExists = drf.resourceExists(hubConfig.stagingDbName);
         boolean finalDbExists = drf.resourceExists(hubConfig.finalDbName);
         boolean traceDbExists = drf.resourceExists(hubConfig.traceDbName);
@@ -139,28 +139,28 @@ public boolean isInstalled() {
         boolean jobIndexesOn = false;
 
         if (stagingDbExists) {
-            Fragment f = dm.getPropertiesAsXml(hubConfig.stagingDbName);
+            Fragment f = databaseManager.getPropertiesAsXml(hubConfig.stagingDbName);
             stagingIndexesOn = Boolean.parseBoolean(f.getElementValue("//m:triple-index"));
             stagingIndexesOn = stagingIndexesOn && Boolean.parseBoolean(f.getElementValue("//m:collection-lexicon"));
             stagingForestsExist = (f.getElements("//m:forest").size() == hubConfig.stagingForestsPerHost);
         }
 
         if (finalDbExists) {
-            Fragment f = dm.getPropertiesAsXml(hubConfig.finalDbName);
+            Fragment f = databaseManager.getPropertiesAsXml(hubConfig.finalDbName);
             finalIndexesOn = Boolean.parseBoolean(f.getElementValue("//m:triple-index"));
             finalIndexesOn = finalIndexesOn && Boolean.parseBoolean(f.getElementValue("//m:collection-lexicon"));
             finalForestsExist = (f.getElements("//m:forest").size() == hubConfig.finalForestsPerHost);
         }
 
         if (traceDbExists) {
             traceIndexesOn = true;
-            Fragment f = dm.getPropertiesAsXml(hubConfig.traceDbName);
+            Fragment f = databaseManager.getPropertiesAsXml(hubConfig.traceDbName);
             traceForestsExist = (f.getElements("//m:forest").size() == hubConfig.traceForestsPerHost);
         }
 
         if (jobDbExists) {
             jobIndexesOn = true;
-            Fragment f = dm.getPropertiesAsXml(hubConfig.jobDbName);
+            Fragment f = databaseManager.getPropertiesAsXml(hubConfig.jobDbName);
             jobForestsExist = (f.getElements("//m:forest").size() == hubConfig.jobForestsPerHost);
         }
 
@@ -301,6 +301,14 @@ public void clearUserModules() {
         }
     }
 
+    /**
+     * Clears out the content in the given database
+     * @param database - the database name to clear
+     */
+    public void clearContent(String database) {
+        databaseManager.clearDatabase(database);
+    }
+
     private EvalResultIterator runInDatabase(String query, String databaseName) {
         ServerEvaluationCall eval = getDatabaseClient(hubConfig.stagingPort).newServerEval();
         String xqy =

marklogic-data-hub/src/main/java/com/marklogic/hub/FlowManager.java

@@ -161,6 +161,7 @@ private JobParameters buildJobParameters(Flow flow, int batchSize, int threadCou
      * Runs a given flow
      * @param flow - the flow to run
      * @param batchSize - the size to use for batching transactions
+     * @param threadCount - the number of threads to use
      * @param statusListener - the callback to receive job status updates
      * @return a JobExecution instance
      */