Merge branch 'release/2.0.0-beta.1' into 2.0-master

Author: paxtonhare

.travis.yml

@@ -7,5 +7,4 @@ before_install:
  - ./.travis/download.sh
 
 script:
-  - docker-compose build
   - docker-compose run marklogic

.travis/Dockerfile-ml

@@ -1,13 +1,25 @@
 FROM centos:centos7
 
 ENV LANG C.UTF-8
-
+ENV JAVA_VERSION 8u131
+ENV BUILD_VERSION b11
+ENV SUM d54c1d3a095b4ff2b6607d096fa80163
 RUN yum -y install glibc.i686 \
     gdb.x86_64 redhat-lsb.x86_64 vim \
     bzip2 \
     unzip \
-    xz-utils \
-    java-1.8.0-openjdk-devel 2>&1 > /dev/null
+    wget \
+    xz-utils 2>&1 > /dev/null
+
+# Downloading Java
+RUN wget --no-cookies --no-check-certificate --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/$JAVA_VERSION-$BUILD_VERSION/$SUM/jdk-$JAVA_VERSION-linux-x64.rpm" -O /tmp/jdk-8-linux-x64.rpm
+RUN yum -y install /tmp/jdk-8-linux-x64.rpm
+
+RUN alternatives --install /usr/bin/java jar /usr/java/latest/bin/java 200000
+RUN alternatives --install /usr/bin/javaws javaws /usr/java/latest/bin/javaws 200000
+RUN alternatives --install /usr/bin/javac javac /usr/java/latest/bin/javac 200000
+
+ENV JAVA_HOME /usr/java/latest
 
 # Install MarkLogic
 COPY ./MarkLogic.rpm /tmp/MarkLogic.rpm
@@ -29,7 +41,7 @@ VOLUME /marklogic-data-hub
 ADD ./ /marklogic-data-hub
 WORKDIR /marklogic-data-hub
 
-RUN ./gradlew tasks 2>&1 > /dev/null
+RUN ./gradlew build -x test 2>&1 > /dev/null
 
 # Define default command (which avoids immediate shutdown)
 CMD /tmp/startml.sh && ./gradlew test

.travis/download.sh

@@ -1,34 +1,7 @@
 #!/bin/bash
 
-# runs command from parameters and exits with the eoror code of the command
-# if it fails
-function successOrExit {
-    "$@"
-    local status=$?
-    if [ $status -ne 0 ]; then
-        echo "$1 exited with error: $status"
-        exit $status
-    fi
-}
+echo "Logging in to Docker registry"
+docker login -u ${DOCKER_USER} -p ${DOCKER_PASSWORD}
 
-set | grep TRAVIS
-
-test $1 && arg1=$1
-
-ver=${ML_VERSION}
-fname=MarkLogic-RHEL7-${ver}.x86_64.rpm
-fnamedeb="marklogic_"
-fnamedeb=$fnamedeb$ver
-suff="_amd64.deb"
-fnamedeb=$fnamedeb$suff
-
-echo "Logging in for Download"
-curl -s -c cookies.txt --data "email=${MLBUILD_USER}&password=${MLBUILD_PASSWORD}" https://developer.marklogic.com/login > /dev/null 2>&1
-
-echo
-echo "Getting Download Link"
-dl_link=$(curl -s -b cookies.txt --data "download=/download/binaries/8.0/${fname}" https://developer.marklogic.com/get-download-url | perl -pe 's/.*"path":"([^"]+).*/\1/')
-url="https://developer.marklogic.com${dl_link}"
-
-echo "********* Downloading MarkLogic $ver"
-successOrExit curl -s -k -o ./MarkLogic.rpm $url
+echo "Getting Docker Image"
+docker pull marklogiccommunity/marklogic-datahub-2x

CHANGELOG.md

@@ -1,7 +1,108 @@
 # Change Log
 
-## [Unreleased](https://github.com/marklogic/marklogic-data-hub/tree/v2.0.0-alpha.2)
+## [Unreleased](https://github.com/marklogic/marklogic-data-hub/tree/v2.0.0-beta.2)
 
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.1.3...v2.0.0-beta.2)
+
+**Implemented enhancements:**
+
+- Add a code editor [\#418](https://github.com/marklogic/marklogic-data-hub/issues/418)
+
+**Fixed bugs:**
+
+- Entity properties starting with a capital generate templates with a preceding dash in var names [\#350](https://github.com/marklogic/marklogic-data-hub/issues/350)
+
+## [v1.1.3](https://github.com/marklogic/marklogic-data-hub/tree/v1.1.3) (2017-05-24)
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v2.0.0-alpha.4...v1.1.3)
+
+**Implemented enhancements:**
+
+- Expose the ability to pass custom properties via gradle [\#416](https://github.com/marklogic/marklogic-data-hub/issues/416)
+- Add gradle wrapper to scaffolded project [\#415](https://github.com/marklogic/marklogic-data-hub/issues/415)
+- Entity view: show "Loading entities..." rather than "You don't have any entities yet" [\#411](https://github.com/marklogic/marklogic-data-hub/issues/411)
+- Better default document format for input flow [\#410](https://github.com/marklogic/marklogic-data-hub/issues/410)
+- Add button to sync indexes [\#393](https://github.com/marklogic/marklogic-data-hub/issues/393)
+- ugly scrollbars appear on project list in quickstart [\#385](https://github.com/marklogic/marklogic-data-hub/issues/385)
+- Handle Batch Flow Errors [\#379](https://github.com/marklogic/marklogic-data-hub/issues/379)
+- Default the harmonize collector to only get items in a standard input collection [\#344](https://github.com/marklogic/marklogic-data-hub/issues/344)
+- Add ability to specify source/target database for a Harmonize flow [\#319](https://github.com/marklogic/marklogic-data-hub/issues/319)
+
+**Fixed bugs:**
+
+- MLCP options: Output URI Replace is not working as expected [\#414](https://github.com/marklogic/marklogic-data-hub/issues/414)
+- Can't do load on Windows [\#407](https://github.com/marklogic/marklogic-data-hub/issues/407)
+- Illegal/unsupported escape sequence in Windows 10 when creating entities [\#406](https://github.com/marklogic/marklogic-data-hub/issues/406)
+- Better feedback for client-side validation failures [\#398](https://github.com/marklogic/marklogic-data-hub/issues/398)
+- XQuery bug detected but not shown on QuickStart GUI [\#395](https://github.com/marklogic/marklogic-data-hub/issues/395)
+- Save Options in Input Flow doesn't save changes to 'Output URI Replace' [\#390](https://github.com/marklogic/marklogic-data-hub/issues/390)
+- Default forests are created/attached even with custom forest JSON definitions [\#389](https://github.com/marklogic/marklogic-data-hub/issues/389)
+- mlReloadModules may not work [\#386](https://github.com/marklogic/marklogic-data-hub/issues/386)
+- Can't run harmonize on 2.0.0-alpha.2 [\#382](https://github.com/marklogic/marklogic-data-hub/issues/382)
+- Compile issue [\#381](https://github.com/marklogic/marklogic-data-hub/issues/381)
+- When one item fails in a harmonize batch run, other items in the chunk do not get processed [\#279](https://github.com/marklogic/marklogic-data-hub/issues/279)
+
+**Closed issues:**
+
+- Need to update my forest location while setting up the datahub framework; [\#356](https://github.com/marklogic/marklogic-data-hub/issues/356)
+- Harmonization flow not hitting staging port defined in gradle.properties [\#342](https://github.com/marklogic/marklogic-data-hub/issues/342)
+- gradle commands for flow creation [\#339](https://github.com/marklogic/marklogic-data-hub/issues/339)
+- gradle hubInit failing [\#307](https://github.com/marklogic/marklogic-data-hub/issues/307)
+- QuickStart App data loaded to incorrect directory? [\#251](https://github.com/marklogic/marklogic-data-hub/issues/251)
+- Allow Ingest to feed directly to conform w/o storing data [\#186](https://github.com/marklogic/marklogic-data-hub/issues/186)
+- DataHub::isInstalled\(\) is not sufficient [\#165](https://github.com/marklogic/marklogic-data-hub/issues/165)
+- Auto Generate Indexes based on entity defs [\#105](https://github.com/marklogic/marklogic-data-hub/issues/105)
+- Allow Exploration on the staged, raw data [\#2](https://github.com/marklogic/marklogic-data-hub/issues/2)
+
+**Merged pull requests:**
+
+- display a "Loading" message while retrieving entities [\#412](https://github.com/marklogic/marklogic-data-hub/pull/412) ([dmcassel](https://github.com/dmcassel))
+- disabled frame options in quickstart so it can be run inside an iframe [\#364](https://github.com/marklogic/marklogic-data-hub/pull/364) ([ryan321](https://github.com/ryan321))
+
+## [v2.0.0-alpha.4](https://github.com/marklogic/marklogic-data-hub/tree/v2.0.0-alpha.4) (2017-05-12)
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v2.0.0-alpha.3...v2.0.0-alpha.4)
+
+## [v2.0.0-alpha.3](https://github.com/marklogic/marklogic-data-hub/tree/v2.0.0-alpha.3) (2017-05-08)
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.1.2...v2.0.0-alpha.3)
+
+## [v1.1.2](https://github.com/marklogic/marklogic-data-hub/tree/v1.1.2) (2017-04-12)
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.1.1...v1.1.2)
+
+**Fixed bugs:**
+
+- null pointer exception on logout [\#383](https://github.com/marklogic/marklogic-data-hub/issues/383)
+
+**Closed issues:**
+
+- gradle version check fails on multiple dots [\#384](https://github.com/marklogic/marklogic-data-hub/issues/384)
+- QuickStart Template Compile Error 2.0-alpha.2 [\#380](https://github.com/marklogic/marklogic-data-hub/issues/380)
+
+## [v1.1.1](https://github.com/marklogic/marklogic-data-hub/tree/v1.1.1) (2017-03-20)
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v2.0.0-alpha.2...v1.1.1)
+
+**Implemented enhancements:**
+
+- customize session name to avoid conflicts [\#369](https://github.com/marklogic/marklogic-data-hub/issues/369)
+- Change Java Client API dependency to stable build [\#358](https://github.com/marklogic/marklogic-data-hub/issues/358)
+- Update Hub to use latest ml-gradle [\#355](https://github.com/marklogic/marklogic-data-hub/issues/355)
+
+**Closed issues:**
+
+- Entities and custom modules fail to deploy - v1.1.0 [\#377](https://github.com/marklogic/marklogic-data-hub/issues/377)
+- See if we can bail if the gradle version is too low [\#376](https://github.com/marklogic/marklogic-data-hub/issues/376)
+- Unable to login marklogic using hub frame work received  "500 Internal Server Error"  [\#374](https://github.com/marklogic/marklogic-data-hub/issues/374)
+- gradle plugin gets confused about users [\#371](https://github.com/marklogic/marklogic-data-hub/issues/371)
+- can't put binary inside envelope [\#366](https://github.com/marklogic/marklogic-data-hub/issues/366)
+- MLCP can't find /etc/hadooop [\#365](https://github.com/marklogic/marklogic-data-hub/issues/365)
+- Add CLA requirement to CONTRIBUTING [\#361](https://github.com/marklogic/marklogic-data-hub/issues/361)
+- Bug: QuickStart Login screen: Long paths aren't completely visible in Chrome [\#354](https://github.com/marklogic/marklogic-data-hub/issues/354)
+
+**Merged pull requests:**
+
+- Adds CLA requirement \(\#361\) [\#363](https://github.com/marklogic/marklogic-data-hub/pull/363) ([jmakeig](https://github.com/jmakeig))
+- Adds CLA requirement \(\#361\) [\#362](https://github.com/marklogic/marklogic-data-hub/pull/362) ([jmakeig](https://github.com/jmakeig))
+- Updates Java Client API dependency to stable build [\#359](https://github.com/marklogic/marklogic-data-hub/pull/359) ([jmakeig](https://github.com/jmakeig))
+
+## [v2.0.0-alpha.2](https://github.com/marklogic/marklogic-data-hub/tree/v2.0.0-alpha.2) (2017-01-16)
 [Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.1.0...v2.0.0-alpha.2)
 
 **Implemented enhancements:**

CONTRIBUTING.md

@@ -71,7 +71,7 @@ plugins {
    ...
 
    // comment out this line. It pulls the version from the cloud
-   // id 'com.marklogic.ml-data-hub' version '2.0.0-alpha.4'
+   // id 'com.marklogic.ml-data-hub' version '2.0.0-beta.1'
 }
 
 // this tells gradle to apply the plugin you included above in the buildscript section

README.md

@@ -11,9 +11,9 @@ _The Data Hub Framework is free and open source under the [Apache 2 License](htt
 
 ## WARNING!!!! This is the the 2.0 branch of The MarkLogic Data Hub Framework. The 2.0 version is only compatible with MarkLogic 9.0 which is currently in Early Access and not generally available.
 
-Go from nothing to Enterprise Data Hub in a matter of minutes.  
+Go from nothing to Operational Data Hub in a matter of minutes.  
 
-This project allows you to deploy a skeleton Data Hub into MarkLogic. With some basic configuration you will be running an Enterprise Data Hub in no time.
+This project allows you to deploy a skeleton Data Hub into MarkLogic. With some basic configuration you will be running an Operational Data Hub in no time.
 
 # Getting Started
 
@@ -48,7 +48,7 @@ Alternatively you can include the jar file as a build dependency in your Java pr
 **Gradle**
 
 ```groovy
-compile('com.marklogic:marklogic-data-hub:2.0.0-alpha.4')
+compile('com.marklogic:marklogic-data-hub:2.0.0-beta.1')
 ```
 
 **Maven**
@@ -57,15 +57,15 @@ compile('com.marklogic:marklogic-data-hub:2.0.0-alpha.4')
 <dependency>
   <groupId>com.marklogic</groupId>
   <artifactId>marklogic-data-hub</artifactId>
-  <version>2.0.0-alpha.4</version>
+  <version>2.0.0-beta.1</version>
   <type>pom</type>
 </dependency>
 ```
 
 **Ivy**
 
 ```xml
-<dependency org='com.marklogic' name='marklogic-data-hub' rev='2.0.0-alpha.4'>
+<dependency org='com.marklogic' name='marklogic-data-hub' rev='2.0.0-beta.1'>
   <artifact name='$AID' ext='pom'></artifact>
 </dependency>
 ```
@@ -76,7 +76,7 @@ If you prefer to use gradle for all of your hub interactions then you can includ
 
 ```groovy
 plugins {
-    id 'com.marklogic.ml-data-hub' version '2.0.0-alpha.4'
+    id 'com.marklogic.ml-data-hub' version '2.0.0-beta.1'
 }
 ```
 

docker-compose.yml

@@ -1,8 +1,8 @@
 version: '2'
 services:
   marklogic:
-    build:
-      context: ./
-      dockerfile: .travis/Dockerfile-ml
+    image: marklogiccommunity/marklogic-datahub-2x
     ports:
       - "8000-8020"
+    volumes:
+      - .:/marklogic-data-hub

examples/spring-batch/build.gradle

@@ -1,28 +1,38 @@
 plugins {
     id 'java'
-    id 'eclipse'
-    id 'idea'
     id 'application'
     id 'net.saliman.properties' version '1.4.6'
-    id 'com.marklogic.ml-data-hub' version '2.0.0-alpha.4'
+    id 'com.marklogic.ml-data-hub' version '2.0.0-beta.1'
 }
 
 repositories {
+    mavenLocal()
     jcenter()
     maven {url 'http://developer.marklogic.com/maven2/'}
 }
 
 dependencies {
-    compile 'com.marklogic:marklogic-data-hub:2.0.0-alpha.4'
-    compile 'com.marklogic:marklogic-spring-batch-core:0.6.0'
-    compile 'com.marklogic:ml-javaclient-util:4.0.alpha2'
-    testCompile 'com.marklogic:marklogic-spring-batch-test:0.6.0'
-}
+    compile 'com.marklogic:marklogic-data-hub:2.0.0-beta.1'
+    compile "com.marklogic:marklogic-spring-batch-core:0.7.4"
+    compile 'com.marklogic:ml-javaclient-util:4.0.alpha4'
 
-distributions {
-    main {
-        baseName = 'baseJob'
-    }
+    testCompile "com.marklogic:marklogic-spring-batch-test:0.7.4"
+    runtime "com.marklogic:marklogic-spring-batch-core:0.7.2"
 }
 
-mainClassName = "com.marklogic.spring.batch.hub.HubJobRunner"
+mainClassName = "com.marklogic.spring.batch.Main"
+
+task importMonsters(type: JavaExec) {
+    classpath = sourceSets.main.runtimeClasspath
+    main = "com.marklogic.spring.batch.Main"
+    args = [
+            "--config", "example.LoadAndRunFlow",
+            "--project_dir", ".",
+            "--env", "local",
+            "--input_file_path", "./input",
+            "--input_file_pattern", ".*\\.xml",
+            "--entity_name", "Monster",
+            "--flow_name", "ingest-monster",
+            "--chunk", "100"
+    ]
+}

examples/spring-batch/gradle.properties

@@ -8,31 +8,65 @@
 #
 # ....
 mlHost=localhost
-mlAppName=data-hub
 
+# Your MarkLogic Username and Password
 mlUsername=admin
 mlPassword=admin
 
+# If specified, the manage username/password combo is used with the ML Management REST API for managing application
+# resources; this user must have the manage-admin and rest-admin roles.
+#
+# If these are not set, then mlUsername/mlPassword is used for managing application resources.
+# mlManageUsername=
+# mlManagePassword=
+
+# If specified, these values can override where the DHF thinks
+# MarkLogic default ports are at. You would only use this if you
+# have changed the ports on which MarkLogic listens
+#
+# mlAppServicesPort=8000
+# mlAdminPort=8001
+# mlManagePort=8002
+
+# If specified, the admin username/password combo is used with the ML Management REST API for creating users and roles. This
+# user must have the manage-admin or admin role. A good practice is to use your admin account here to create app-specific
+# users and roles, which can then be used as mlManageUsername/mlManagePassword and mlUsername/mlPassword.
+#
+# These properties are also used for connecting to the admin application on port 8001 - e.g. for initializing ML and for
+# waiting for ML to restart.
+#
+# If these properties are not set, then mlUsername/mlPassword will be used.
+# mlAdminUsername=
+# mlAdminPassword=
+
 mlStagingAppserverName=data-hub-STAGING
 mlStagingPort=8010
 mlStagingDbName=data-hub-STAGING
 mlStagingForestsPerHost=4
+mlStagingAuth=digest
 
 mlFinalAppserverName=data-hub-FINAL
 mlFinalPort=8011
 mlFinalDbName=data-hub-FINAL
 mlFinalForestsPerHost=4
+mlFinalAuth=digest
 
 mlTraceAppserverName=data-hub-TRACING
 mlTracePort=8012
 mlTraceDbName=data-hub-TRACING
 mlTraceForestsPerHost=1
+mlTraceAuth=digest
 
 mlJobAppserverName=data-hub-JOBS
 mlJobPort=8013
 mlJobDbName=data-hub-JOBS
 mlJobForestsPerHost=1
+mlJobAuth=digest
 
 mlModulesDbName=data-hub-MODULES
 mlTriggersDbName=data-hub-TRIGGERS
 mlSchemasDbName=data-hub-SCHEMAS
+
+# The name of the Role to create for Hub Access
+mlHubUserRole=data-hub-ROLE
+mlHubUserName=data-hub-user