Wdt#457 capture user override scripts (#466)

* Change attribute in domainInfo not found in dictionary from SEVERE to INFO to allow user customization

* Collect user override scripts into archive and install into the domain

Author: CarolynRountree

core/pom.xml

@@ -32,6 +32,7 @@
         <dependency>
             <groupId>org.python</groupId>
             <artifactId>jython</artifactId>
+            <version>2.2.1</version>
             <scope>provided</scope>
         </dependency>
         <dependency>

core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java

@@ -71,6 +71,10 @@ public class WLSDeployArchive {
      */
     public static final String ARCHIVE_CPLIB_TARGET_DIR = WLSDPLY_ARCHIVE_BINARY_DIR + "/classpathLibraries";
 
+    /**
+     * Top-level archive subdirectory where the $DOMAIN_HOME/bin scripts are stored.
+     */
+    public static final String ARCHIVE_DOM_BIN_TARGET_DIR = WLSDPLY_ARCHIVE_BINARY_DIR + "/domainBin";
     /**
      * Top-level archive subdirectory where the FileStore directories are stored and the subdirectory
      * to which they will be extracted.
@@ -714,6 +718,62 @@ public void extractDomainLibLibrary(String archivePath, File extractToLocation)
         LOGGER.exiting(CLASS, METHOD);
     }
 
+    /**
+     * Adds a $DOMAIN_HOME/bin script to the archive.  If a script with the same name already exists, this method
+     * assumes that the new one also needs to be added so it changes the name to prevent conflicts by adding a
+     * numeric value onto the file's basename (e.g., myscript(1).cmd, myscript(2).cmd).
+     *
+     * @param domainBinPath - File representing the actual path of the script file in the file system
+     * @return the relative path where the script is stored within the archive
+     * @throws WLSDeployArchiveIOException if an IOException occurred while reading or writing changes
+     * @throws IllegalArgumentException    if the file or directory passed in does not exist
+     */
+    public String addDomainBinScript(File domainBinPath) throws WLSDeployArchiveIOException {
+        final String METHOD = "addDomainBinScript";
+
+        LOGGER.entering(CLASS, METHOD, domainBinPath);
+        validateExistingFile(domainBinPath, "domainBinPath", getArchiveFileName(), METHOD);
+
+        String newName = addItemToZip(ARCHIVE_DOM_BIN_TARGET_DIR, domainBinPath);
+        LOGGER.exiting(CLASS, METHOD, newName);
+        return newName;
+    }
+
+    /**
+     * Get the list of $DOMAIN_HOME/bin script names in the archive.
+     *
+     * @return the list of $DOMAIN_HOME/bin script names
+     * @throws WLSDeployArchiveIOException if an error occurs reading the archive
+     */
+    public List<String> listDomainBinScripts() throws WLSDeployArchiveIOException {
+        final String METHOD = "listDomainBinScripts";
+
+        LOGGER.entering(CLASS, METHOD);
+        List<String> result = getZipFile().listZipEntries(ARCHIVE_DOM_BIN_TARGET_DIR + ZIP_SEP);
+        // Remove the top-level directory entry from the list...
+        result.remove(ARCHIVE_DOM_BIN_TARGET_DIR + ZIP_SEP);
+        LOGGER.exiting(CLASS, METHOD, result);
+        return result;
+    }
+
+    /**
+     * Extract the specified domain bin user script to the specified location (e.g., $DOMAIN_HOME/bin).
+     *
+     * @param archivePath       the path of the script within the archive
+     * @param extractToLocation the location to write the file
+     * @throws WLSDeployArchiveIOException if an IOException occurred while extracting or writing the file
+     * @throws IllegalArgumentException    if the file or directory passed in does not exist
+     */
+    public void extractDomainBinScript(String archivePath, File extractToLocation) throws WLSDeployArchiveIOException {
+        final String METHOD = "extractDomainBinScript";
+
+        LOGGER.entering(CLASS, METHOD, archivePath, extractToLocation);
+        validateNonEmptyString(archivePath, "archivePath", METHOD);
+        validateExistingDirectory(extractToLocation, "extractToLocation", getArchiveFileName(), METHOD);
+
+        extractFileFromZip(archivePath, ARCHIVE_DOM_BIN_TARGET_DIR, "", extractToLocation);
+        LOGGER.exiting(CLASS, METHOD);
+    }
     /**
      * This method adds a classpath library to the archive.  If a library with the same name already exists, this
      * method assumes that the new one also needs to be added so it changes the name to prevent conflicts by adding

core/src/main/python/wlsdeploy/aliases/alias_entries.py

@@ -181,6 +181,7 @@ class AliasEntries(object):
         'AdminUserName': 'string',
         'AdminPassword': 'password',
         'ServerStartMode': 'string',
+        'domainBin': 'list',
         'domainLibraries': 'list',
         # A map of Server Group names to the list of servers/clusters to which they should
         # be targeted.  The ServerGroup must appear in the domain typedef definition.  If

core/src/main/python/wlsdeploy/aliases/model_constants.py

@@ -94,6 +94,7 @@
 DOMAIN_NAME = 'Name'
 DOMAIN_INFO = 'domainInfo'
 DOMAIN_LIBRARIES = 'domainLibraries'
+DOMAIN_SCRIPTS = 'domainBin'
 DYNAMIC_SERVERS = 'DynamicServers'
 EMBEDDED_LDAP = 'EmbeddedLDAP'
 ERROR_DESTINATION = 'ErrorDestination'

core/src/main/python/wlsdeploy/tool/create/domain_creator.py

@@ -345,6 +345,7 @@ def __create_domain(self):
 
         self.library_helper.install_domain_libraries()
         self.library_helper.extract_classpath_libraries()
+        self.library_helper.install_domain_scripts()
         self.wlsroles_helper.process_roles()
         if os.environ.has_key('__WLSDEPLOY_STORE_MODEL__'):
             model_helper.persist_model(self.model_context, self.model)

core/src/main/python/wlsdeploy/tool/deploy/topology_updater.py

@@ -140,6 +140,7 @@ def update(self):
 
         self.library_helper.install_domain_libraries()
         self.library_helper.extract_classpath_libraries()
+        self.library_helper.install_domain_scripts()
 
     def _process_section(self, folder_dict, folder_list, key, location):
         if key in folder_dict:

core/src/main/python/wlsdeploy/tool/discover/domain_info_discoverer.py

@@ -2,11 +2,13 @@
 Copyright (c) 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
+import glob
 import os
 
 from java.io import File
 
 from oracle.weblogic.deploy.util import WLSDeployArchiveIOException
+from oracle.weblogic.deploy.util import FileUtils
 
 from wlsdeploy.aliases import model_constants
 from wlsdeploy.aliases.wlst_modes import WlstModes
@@ -40,6 +42,8 @@ def discover(self):
         _logger.entering(class_name=_class_name, method_name=_method_name)
         model_top_folder_name, result = self.get_domain_libs()
         discoverer.add_to_model_if_not_empty(self._dictionary, model_top_folder_name, result)
+        model_top_folder_name, result = self.get_user_env_scripts()
+        discoverer.add_to_model_if_not_empty(self._dictionary, model_top_folder_name, result)
         _logger.exiting(class_name=_class_name, method_name=_method_name)
         return self._dictionary
 
@@ -73,3 +77,35 @@ def get_domain_libs(self):
 
         _logger.exiting(class_name=_class_name, method_name=_method_name, result=entries)
         return model_constants.DOMAIN_LIBRARIES, entries
+
+    def get_user_env_scripts(self):
+        """
+        Look for the user overrides scripts run in setDomainEnv in the domain bin directory
+        :raise: DiscoverException: an unexpected exception occurred writing a jar file to the archive file
+        """
+        _method_name = 'get_user_env_scripts'
+        _logger.entering(class_name=_class_name, method_name=_method_name)
+        archive_file = self._model_context.get_archive_file()
+        domain_bin = self._convert_path('bin')
+        entries = []
+        if os.path.isdir(domain_bin):
+            search_directory = FileUtils.fixupFileSeparatorsForJython(os.path.join(domain_bin, "setUserOverrides*.*"))
+            _logger.finer('WLSDPLY-06425', search_directory, class_name=_class_name, method_name=_method_name)
+            file_list = glob.glob(search_directory)
+            if file_list:
+                _logger.finer('WLSDPLY-06423', domain_bin, class_name=_class_name, method_name=_method_name)
+                for entry in file_list:
+                    try:
+                        updated_name = archive_file.addDomainBinScript(File(entry))
+                    except WLSDeployArchiveIOException, wioe:
+                        de = exception_helper.create_discover_exception('WLSDPLY-06426', entry,
+                                                                        wioe.getLocalizedMessage())
+                        _logger.throwing(class_name=_class_name, method_name=_method_name, error=de)
+                        raise de
+
+                    entries.append(updated_name)
+                    _logger.finer('WLSDPLY-06424', entry, updated_name, class_name=_class_name,
+                                  method_name=_method_name)
+
+        _logger.exiting(class_name=_class_name, method_name=_method_name, result=entries)
+        return model_constants.DOMAIN_SCRIPTS, entries

core/src/main/python/wlsdeploy/tool/util/archive_helper.py

@@ -280,6 +280,32 @@ def extract_classpath_libraries(self):
         self.__logger.exiting(class_name=self.__class_name, method_name=_method_name, result=count)
         return count
 
+    def extract_domain_bin_script(self, script_path):
+        """
+        Extract the specified domain bin script to the $DOMAIN_HOME/bin directory.
+        :param script_path: the domain bin path and script into the archive file
+        :raises: BundleAwareException of the appropriate type: if an error occurs
+        """
+        _method_name = 'extract_domain_bin_script'
+
+        self.__logger.entering(script_path, class_name=self.__class_name, method_name=_method_name)
+        try:
+            archive = self._find_archive_for_path(script_path)
+            if archive is not None:
+                archive.extractDomainBinScript(script_path, File(self.__domain_home, 'bin'))
+            else:
+                ex = exception_helper.create_exception(self.__exception_type, 'WLSDPLY-19308',
+                                                       script_path, self.__archive_files_text)
+                self.__logger.throwing(ex, class_name=self.__class_name, method_name=_method_name)
+                raise ex
+        except (WLSDeployArchiveIOException, IllegalArgumentException), e:
+            ex = exception_helper.create_exception(self.__exception_type, 'WLSDPLY-19309', script_path,
+                                                   self.__archive_files_text, e.getLocalizedMessage(), error=e)
+            self.__logger.throwing(ex, class_name=self.__class_name, method_name=_method_name)
+            raise ex
+        self.__logger.exiting(class_name=self.__class_name, method_name=_method_name)
+        return
+
     def get_archive_entries(self):
         """
         Get the entries from all the archives.

core/src/main/python/wlsdeploy/tool/util/library_helper.py

@@ -6,6 +6,7 @@
 from oracle.weblogic.deploy.util import WLSDeployArchive
 from shutil import copy
 
+from wlsdeploy.aliases.model_constants import DOMAIN_SCRIPTS
 from wlsdeploy.aliases.model_constants import DOMAIN_LIBRARIES
 from wlsdeploy.exception import exception_helper
 from wlsdeploy.tool.util.alias_helper import AliasHelper
@@ -86,6 +87,37 @@ def extract_classpath_libraries(self):
         self.logger.exiting(class_name=self.__class_name, method_name=_method_name)
         return
 
+    def install_domain_scripts(self):
+        """
+        Extract the scripts from domain bin listed in the model, if any, to the <DOMAIN_HOME>/bin directory.
+        :raises: BundleAwareException of the specified type: if an error occurs
+        """
+        _method_name = 'install_domain_scripts'
+
+        self.logger.entering(self.domain_home, class_name=self.__class_name, method_name=_method_name)
+        domain_info_dict = self.model.get_model_domain_info()
+        if DOMAIN_SCRIPTS not in domain_info_dict or len(domain_info_dict[DOMAIN_SCRIPTS]) == 0:
+            self.logger.info('WLSDPLY-12241', class_name=self.__class_name, method_name=_method_name)
+        elif DOMAIN_SCRIPTS in domain_info_dict:
+            domain_scripts = dictionary_utils.get_dictionary_element(domain_info_dict, DOMAIN_SCRIPTS)
+            if self.archive_helper is None:
+                ex = exception_helper.create_create_exception('WLSDPLY-12250', domain_scripts)
+                self.logger.throwing(ex, class_name=self.__class_name, method_name=_method_name)
+                raise ex
+
+            for domain_script in domain_scripts:
+                if WLSDeployArchive.isPathIntoArchive(domain_script):
+                    self.logger.info('WLSDPLY-12251', domain_script, self.domain_home,
+                                     class_name=self.__class_name, method_name=_method_name)
+                    self.archive_helper.extract_domain_bin_script(domain_script)
+                else:
+                    self.logger.info('WLSDPLY-12252', domain_script, self.domain_home,
+                                     class_name=self.__class_name, method_name=_method_name)
+                    self._copy_domain_bin(domain_script)
+
+        self.logger.exiting(class_name=self.__class_name, method_name=_method_name)
+        return
+
     def _copy_domain_library(self, domain_lib):
         """
         Copy the specified domain library to the domain's lib directory.
@@ -102,3 +134,20 @@ def _copy_domain_library(self, domain_lib):
             ex = exception_helper.create_create_exception('WLSDPLY-12234', source_path, target_dir)
             self.logger.throwing(ex, class_name=self.__class_name, method_name=_method_name)
             raise ex
+
+    def _copy_domain_bin(self, domain_bin):
+        """
+        Copy the specified domain user script to the domain's bin directory.
+        :raises: BundleAwareException of the specified type: if an error occurs
+        """
+        _method_name = '_copy_domain_bin'
+
+        source_path = File(domain_bin).getAbsolutePath()
+        target_dir = File(self.domain_home, 'bin').getPath()
+
+        try:
+            copy(str(source_path), str(target_dir))
+        except IOError:
+            ex = exception_helper.create_create_exception('WLSDPLY-12253', source_path, target_dir)
+            self.logger.throwing(ex, class_name=self.__class_name, method_name=_method_name)
+            raise ex

core/src/main/python/wlsdeploy/tool/validate/validator.py

@@ -30,6 +30,7 @@
 from wlsdeploy.util.enum import Enum
 from wlsdeploy.util.weblogic_helper import WebLogicHelper
 
+from wlsdeploy.aliases.model_constants import DOMAIN_SCRIPTS
 from wlsdeploy.aliases.model_constants import DOMAIN_LIBRARIES
 from wlsdeploy.aliases.model_constants import MODEL_LIST_DELIMITER
 from wlsdeploy.aliases.model_constants import NAME
@@ -400,7 +401,7 @@ def __validate_domain_info_section(self, model_section_key, model_dict):
             if section_dict_key in valid_attr_infos:
                 # section_dict_key is an attribute under the model section, and
                 # also in valid_attr_infos.
-                if section_dict_key == DOMAIN_LIBRARIES:
+                if section_dict_key in [DOMAIN_LIBRARIES, DOMAIN_SCRIPTS]:
                     self.__validate_path_tokens_attribute(section_dict_key, section_dict_value, model_folder_path)
 
                 elif section_dict_key == SERVER_GROUP_TARGETING_LIMITS:
@@ -415,10 +416,10 @@ def __validate_domain_info_section(self, model_section_key, model_dict):
                                               path_tokens_attr_keys, model_folder_path, validation_location)
             else:
                 # section_dict_key is not an attribute allowed under the model
-                # section, so record this as a validate ERROR in the validate
-                # results.
-                self._logger.severe('WLSDPLY-05029', section_dict_key, model_folder_path, valid_attr_infos.keys(),
-                                    class_name=_class_name, method_name=_method_name)
+                # section. Do not record this as an ERROR as this could be custom
+                # information that the user has written a filter to extract
+                self._logger.info('WLSDPLY-05029', section_dict_key, model_folder_path, valid_attr_infos.keys(),
+                                  class_name=_class_name, method_name=_method_name)
 
     def __validate_model_section(self, model_section_key, model_dict, valid_section_folders):
         _method_name = '__validate_model_section'

core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties

@@ -609,6 +609,10 @@ WLSDPLY-06402=Add application {0} deployment plan {1} to archive file
 WLSDPLY-06420=Add the java archive files from the domain library {0} to the archive file
 WLSDPLY-06421=Unexpected exception occurred writing the domain library file {0} to the archive : {1}
 WLSDPLY-06422=Add domain lib file {0} to the archive file and entry {1] to the model domain info section
+WLSDPLY-06423=Add the user override env script files from the domain bin location {0} to the archive file
+WLSDPLY-06424=Add user override script {0} to the archive file and entry {1] to the model domain info section
+WLSDPLY-06425=Look for user env override scripts using search pattern {0}
+WLSDPLY-06426=Unexpected exception occurred writing the domain bin user overrides file {0} to the archive : {1}
 
 # global_resources_discoverer.py
 WLSDPLY-06440=Discover Global Resources from the domain
@@ -1121,9 +1125,9 @@ WLSDPLY-12235=Installing domain library {0} to lib directory of {1}
 WLSDPLY-12236=Target the JRF resources to the clusters with dynamic servers {0}
 WLSDPLY-12237=Target the resources for server groups {0} to cluster {1} with dynamic server members
 WLSDPLY-12238=Unable to target non-JRF template server groups for domain type {0} to dynamic cluster(s).
-
 WLSDPLY-12239=Found server groups {0} for cluster {1}
 WLSDPLY-12240=Server group targeting limits {0} found in model
+WLSDPLY-12241=The model did not specify any domain scripts to install
 WLSDPLY-12242=The assignment of servers to server groups map is {0}
 WLSDPLY-12243=Server group list found for {0} in domainInfo : {1}
 WLSDPLY-12244=Targeting JRF resources to a dynamic cluster(s), {0}, for a Restricted JRF domain type is not valid \
@@ -1134,6 +1138,10 @@ WLSDPLY-12247=WebLogic does not support targeting resources to dynamic servers.
   will be targeted to the dynamic cluster using the applyJRF function.
 WLSDPLY-12248=The server group(s) {0} will not be targeted to the Admin server or a configured manage server
 WLSDPLY-12249=RCUDBInfo section is specified in the model but the domain type used is not JRF, RCU processing skipped
+WLSDPLY-12250=Domain bin ({0}) specified in the model but the archive file name was not provided
+WLSDPLY-12251=Installing domain script {0} from archive to bin directory of {1}
+WLSDPLY-12252=Copying domain script {0} to bin directory of {1}
+WLSDPLY-12253=Unable to copy domain user script {0} to directory {1}
 
 # domain_typedef.py
 WLSDPLY-12300={0} got the domain type {1} but the domain type definition file {2} was not valid: {3}
@@ -1341,6 +1349,8 @@ WLSDPLY-19304=Unable to compute hash for entry {0} in archive file {1}: {2}
 WLSDPLY-19305=Failed to extract domain library {0} because it does not exist in archive file {1}
 WLSDPLY-19306=Unable to extract domain library {0} from archive file {1}: {2}
 WLSDPLY-19307=Unable to extract classpath libraries from archive file {0} to domain directory {1}: {2}
+WLSDPLY-19308=Failed to extract script to domain bin {0} because it does not exist in archive file {1}
+WLSDPLY-19309=Unable to extract from domain bin {0} from archive file {1}: {2}
 
 # wlsdeploy/tool/util/topology_helper.py
 WLSDPLY-19400=Creating placeholder for server template {0}