More documentation and remove lombok

Author: Chris Li

README.md

@@ -32,21 +32,25 @@ If building the distribution with tests turned on:
 To contribute, please use submit Pull Request (PR) for committers to merge. 
 - Create your own fork on GitHub off the main repository
 - Clone your fork to your local computer
->- `git clone https://github.com/<<your-github-login>>/data-integration-library.git`
+    - `git clone https://github.com/<<your-github-login>>/data-integration-library.git`
 - Add upstream and verify
->- `git remote add upstream https://github.com/linkedin/data-integration-library.git`
->- `git remote -v`
+    - `git remote add upstream https://github.com/linkedin/data-integration-library.git`
+    - `git remote -v`
 - Change, test, commit, and push to your fork
->- `git status`
->- `git add .`
->- `git commit -m "comments"`
->- `git push origin master`
+    - `git status`
+    - `git add .`
+    - `git commit -m "comments"`
+    - `git push origin master`
 - Create Pull Request on GitHub with the following details
->- Title 
->- Detailed description
->- Document the tests done
->- Links to the updated documents
+    - Title 
+    - Detailed description
+    - Document the tests done
+    - Links to the updated documents
+- Publish to local Maven repository
+    - `./gradlew publishToMavenLocal`
 
 # Detailed Documents
 
-[Deprecated Job Properties](https://github.com/linkedin/data-integration-library/blob/master/docs/parameters/deprecated.md)
+- [Job Properties](https://github.com/linkedin/data-integration-library/blob/master/docs/parameters/summary.md)
+- [Job Properties by Category](https://github.com/linkedin/data-integration-library/blob/master/docs/parameters/categories.md)
+- [Deprecated Job Properties](https://github.com/linkedin/data-integration-library/blob/master/docs/parameters/deprecated.md)

build.gradle

@@ -44,11 +44,6 @@ allprojects {
 
 subprojects {
   dependencies {
-    // Gradle 5 compatibility
-    compileOnly externalDependency.lombok
-    testCompileOnly externalDependency.lombok
-    annotationProcessor externalDependency.lombok
-    testAnnotationProcessor externalDependency.lombok
   }
   project.buildDir = new File(project.rootProject.buildDir, project.name)
 }

cdi-core/build.gradle

@@ -31,7 +31,6 @@ dependencies {
   compile externalDependency.'awsUtils'
   compile externalDependency.'commonsValidator'
   compile externalDependency.'guava'
-  compile externalDependency.'lombok'
   compile externalDependency.'commonsLang3'
   compile externalDependency.'testng'
   compile externalDependency.'jhyde'

cdi-core/src/main/java/com/linkedin/cdi/configuration/MultistageProperties.java

@@ -8,10 +8,10 @@
 import com.google.gson.JsonArray;
 import com.google.gson.JsonObject;
 import com.linkedin.cdi.factory.DefaultConnectionClientFactory;
-import lombok.Getter;
-import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang.StringUtils;
 import org.apache.gobblin.configuration.State;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 /**
@@ -37,8 +37,6 @@
  *
  * @author chrli
  */
-@Slf4j
-@Getter
 @SuppressWarnings("unchecked")
 public enum MultistageProperties {
   /**
@@ -865,13 +863,22 @@ public <T> T getDefaultValue() {
   },
   DATA_PUBLISHER_FINAL_DIR("data.publisher.final.dir", String.class);
 
+  private static final Logger LOG = LoggerFactory.getLogger(MultistageProperties.class);
   final static private Gson GSON = new Gson();
   final static private String PROPERTY_SEPARATOR = ".";
 
   private final String config;
   private final Class<?> className;
   private final Object defaultValue;
 
+  public String getConfig() {
+    return config;
+  }
+
+  public Class<?> getClassName() {
+    return className;
+  }
+
   MultistageProperties(String config, Class<?> className) {
     this.config = config;
     this.className = className;

cdi-core/src/main/java/com/linkedin/cdi/connection/HdfsConnection.java

@@ -7,24 +7,22 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
-import java.io.InputStream;
-import java.net.URI;
-import java.util.List;
-import java.util.stream.Collectors;
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.gobblin.configuration.State;
 import com.linkedin.cdi.exception.RetriableAuthenticationException;
 import com.linkedin.cdi.keys.ExtractorKeys;
 import com.linkedin.cdi.keys.HdfsKeys;
 import com.linkedin.cdi.keys.JobKeys;
 import com.linkedin.cdi.util.InputStreamUtils;
 import com.linkedin.cdi.util.WorkUnitStatus;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.gobblin.configuration.State;
 import org.apache.gobblin.source.extractor.filebased.FileBasedHelperException;
 import org.apache.gobblin.source.extractor.filebased.TimestampAwareFileBasedHelper;
 import org.apache.gobblin.source.extractor.hadoop.HadoopFsHelper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 /**
@@ -33,12 +31,23 @@
  *
  * @author Chris Li
  */
-@Slf4j
 public class HdfsConnection extends MultistageConnection {
+  private static final Logger LOG = LoggerFactory.getLogger(HdfsConnection.class);
+
+  public HdfsKeys getHdfsKeys() {
+    return hdfsKeys;
+  }
+
+  public TimestampAwareFileBasedHelper getFsHelper() {
+    return fsHelper;
+  }
+
+  public void setFsHelper(TimestampAwareFileBasedHelper fsHelper) {
+    this.fsHelper = fsHelper;
+  }
+
   private final static String URI_REGEXP_PATTERN = "RE=";
-  @Getter
   final private HdfsKeys hdfsKeys;
-  @Setter (AccessLevel.PACKAGE)
   private TimestampAwareFileBasedHelper fsHelper;
 
   public HdfsConnection(State state, JobKeys jobKeys, ExtractorKeys extractorKeys) {
@@ -99,7 +108,7 @@ public boolean closeAll(String message) {
       fsHelper = null;
       return true;
     } catch (Exception e) {
-      log.error("Error closing file system connection", e);
+      LOG.error("Error closing file system connection", e);
       return false;
     }
   }
@@ -132,7 +141,7 @@ private List<String> readFileList(final String path, final String pattern) {
           .filter(fileName -> fileName.matches(pattern))
           .collect(Collectors.toList());
     } catch (FileBasedHelperException e) {
-      log.error("Not able to run ls command due to " + e.getMessage(), e);
+      LOG.error("Not able to run ls command due to " + e.getMessage(), e);
     }
     return Lists.newArrayList();
   }
@@ -143,11 +152,11 @@ private List<String> readFileList(final String path, final String pattern) {
    * @return the file content in an InputStream
    */
   private InputStream readSingleFile(final String path) {
-    log.info("Processing file: {}", path);
+    LOG.info("Processing file: {}", path);
     try {
       return fsHelper.getFileStream(path);
     } catch (FileBasedHelperException e) {
-      log.error("Not able to run getFileStream command due to " + e.getMessage(), e);
+      LOG.error("Not able to run getFileStream command due to " + e.getMessage(), e);
       return null;
     }
   }
@@ -159,7 +168,7 @@ TimestampAwareFileBasedHelper getHdfsClient() {
       fsHelper.connect();
       return fsHelper;
     } catch (Exception e) {
-      log.error("Failed to initialize HdfsSource", e);
+      LOG.error("Failed to initialize HdfsSource", e);
       return null;
     }
   }