N°5139 - Let data that is null also be null at the other side-fix

.github/workflows/action.yml

@@ -0,0 +1,43 @@
+name: Add PRs to Combodo PRs Dashboard
+
+on:
+  pull_request_target:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add PR to Combodo Project
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if author is a member of the organization
+        id: check-membership
+        run: |
+          ORG="Combodo"
+          AUTHOR=$(jq -r .pull_request.user.login "$GITHUB_EVENT_PATH")
+          RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token ${{ secrets.PR_AUTOMATICALLY_ADD_TO_PROJECT }}" \
+            "https://api.github.com/orgs/$ORG/members/$AUTHOR")
+          if [ "$RESPONSE" == "404" ]; then
+            echo "project_url=https://github.com/orgs/Combodo/projects/5" >> $GITHUB_ENV
+            echo "is_member=false" >> $GITHUB_ENV
+          else
+            echo "project_url=https://github.com/orgs/Combodo/projects/4" >> $GITHUB_ENV
+            echo "is_member=true" >> $GITHUB_ENV
+
+          fi
+
+      - name: Add internal tag if member
+        if: env.is_member == 'true'
+        run: |
+          curl -X POST -H "Authorization: token ${{ secrets.PR_AUTOMATICALLY_ADD_TO_PROJECT }}" \
+            -H "Accept: application/vnd.github.v3+json" \
+            https://api.github.com/repos/Combodo/itop-data-collector-base/issues/${{ github.event.pull_request.number }}/labels \
+            -d '{"labels":["internal"]}'
+        env:
+          is_member: ${{ env.is_member }}
+
+      - name: Add PR to the appropriate project
+        uses: actions/add-to-project@v1.0.2
+        with:
+          project-url: ${{ env.project_url }}
+          github-token: ${{ secrets.PR_AUTOMATICALLY_ADD_TO_PROJECT }}

composer.json

@@ -1,6 +1,6 @@
 {
    "require": {
-       "php": "7.*",
+       "php": "7.* || 8.* <= 8.3",
        "ext-libxml": "*"
    },
    "require-dev": {

conf/params.distrib.xml

@@ -107,5 +107,5 @@
   <!-- iTop user set as allowed to run synchronization. It is highly recommended to use the same as itop_login. By default, it is the same as itop_login -->
   <synchro_user></synchro_user>
   <!-- date format in collected data -->
-  <date_format>Y-m-d</date_format>
+  <date_format>Y-m-d H:i:s</date_format>
 </parameters>

core/collector.class.inc.php

@@ -594,7 +594,7 @@ protected function AddRow($aRow)
 	{
 		$aData = array();
 		foreach ($this->aCSVHeaders as $sHeader) {
-			if (is_null($aRow[$sHeader]) && $this->AttributeIsNullified($sHeader)) {
+			if (strlen($aRow[$sHeader] ?? '')===0 && $this->AttributeIsNullified($sHeader)) {
 				$aData[] = NULL_VALUE;
 			} else {
 				$aData[] = $aRow[$sHeader];
@@ -692,7 +692,7 @@ public function Synchronize($iMaxChunkSize = 0)
 				'output' => $sOutput,
 				'csvdata' => file_get_contents($sDataFile),
 				'charset' => $this->GetCharset(),
-                'date_format' => Utils::GetConfigurationValue('date_format', 'Y-m-d')
+                'date_format' => Utils::GetConfigurationValue('date_format', 'Y-m-d H:i:s')
 			);
 
 			$sLoginform = Utils::GetLoginMode();
@@ -708,6 +708,8 @@ public function Synchronize($iMaxChunkSize = 0)
 				Utils::Log(LOG_ERR, $sTrimmedOutput);
 
 				return false;
+			} else {
+				Utils::Log(LOG_DEBUG, $sTrimmedOutput);
 			}
 		}
 

core/jsoncollector.class.inc.php

@@ -150,7 +150,11 @@ public function Prepare()
 			Utils::Log(LOG_DEBUG, 'Source sFileJson: '.$this->sFileJson);
 			Utils::Log(LOG_INFO, 'Synchro URL (target): '.Utils::GetConfigurationValue('itop_url', array()));
 		} else {
-			$this->sFileJson = file_get_contents($this->sFilePath);
+            $this->sFileJson = @file_get_contents($this->sFilePath);
+            if ($this->sFileJson === false) {
+                $this->sFilePath = APPROOT.$this->sFilePath;
+                $this->sFileJson = @file_get_contents($this->sFilePath);
+            }
 			Utils::Log(LOG_DEBUG, 'Source sFileJson: '.$this->sFileJson);
 			Utils::Log(LOG_INFO, 'Synchro  URL (target): '.Utils::GetConfigurationValue('itop_url', array()));
 		}

core/orchestrator.class.inc.php

@@ -17,7 +17,7 @@
 class Orchestrator
 {
 	static $aCollectors = array();
-	static $aMinVersions = array('PHP' => '5.3.0', 'simplexml' => '0.1', 'dom' => '1.0');
+	static $aMinVersions = array('PHP' => '7.0', 'simplexml' => '7.0', 'dom' => '1');
 
 	/**
 	 * Add a collector class to be run in the specified order
@@ -175,7 +175,7 @@ public function InitSynchroDataSources($aCollectors)
 				Utils::Log(LOG_ERR, "Unable to find the contact with email = '$sEmailToNotify'. No contact to notify will be defined.");
 			}
 		}
-		$sSynchroUser = Utils::GetConfigurationValue('synchro_user', Utils::GetConfigurationValue('itop_login', ''));
+		$sSynchroUser = Utils::GetConfigurationValue('synchro_user') ?: Utils::GetConfigurationValue('itop_login');
 		$aPlaceholders['$synchro_user$'] = 0;
 		if ($sSynchroUser != '') {
 			$oRestClient = new RestClient();

test/CollectorTest.php

@@ -109,15 +109,19 @@ public function testAttributeIsNullified($sCollectorXmlSubSection, $bExpectedIsN
 
 		$this->assertEquals($bExpectedIsNullified, $oCollector->AttributeIsNullified('phone'));
 
-		$oCollector->Collect(1);
+		$oCollector->Collect();
 		$sExpectedCsv = <<<CSV
 primary_key;first_name;name;org_id;phone;mobile_phone;employee_number;email;function
-1;isaac;asimov;Demo;;123456;9998877665544;issac.asimov@function.io;writer
+1;isaac;asimov_null;Demo;;123456;9998877665544;issac.asimov@function.io;writer
+2;isaac;asimov_empty;Demo;;123456;9998877665544;issac.asimov@function.io;writer
+3;isaac;asimov_notempty;Demo;"not empty";123456;9998877665544;issac.asimov@function.io;writer
 
 CSV;
 		$sExpectedNullifiedCsv = <<<CSV
 primary_key;first_name;name;org_id;phone;mobile_phone;employee_number;email;function
-1;isaac;asimov;Demo;<NULL>;123456;9998877665544;issac.asimov@function.io;writer
+1;isaac;asimov_null;Demo;<NULL>;123456;9998877665544;issac.asimov@function.io;writer
+2;isaac;asimov_empty;Demo;<NULL>;123456;9998877665544;issac.asimov@function.io;writer
+3;isaac;asimov_notempty;Demo;"not empty";123456;9998877665544;issac.asimov@function.io;writer
 
 CSV;
 
@@ -158,12 +162,12 @@ public function testUpdateSDSAttributes($aExpectedAttrDef, $aSynchroAttrDef, boo
 	    $oCollector = new iTopPersonCollector();
 	    $oMockClient = $this->CreateMock('RestClient');
 	    $oMockClient->expects($this->exactly($bWillUpdate ? 1 : 0))->method("Update")->willReturn(['code' => 0]);
-	    
+
 	    $bRet = $this->InvokeNonPublicMethod(get_class($oCollector), 'UpdateSDSAttributes', $oCollector, [$aExpectedAttrDef, $aSynchroAttrDef, '', $oMockClient]);
-	   
+
 	    $this->assertTrue($bRet);
 	}
-	
+
 	public function providerUpdateSDSAttributes()
 	{
 	    return [
@@ -271,7 +275,7 @@ public function InvokeNonPublicMethod($sObjectClass, $sMethodName, $oObject, $aA
 	    $class = new \ReflectionClass($sObjectClass);
 	    $method = $class->getMethod($sMethodName);
 	    $method->setAccessible(true);
-	    
+
 	    return $method->invokeArgs($oObject, $aArgs);
 	}
 }

test/JsonCollectorTest.php

@@ -108,7 +108,7 @@ public function testOrgCollector($sAdditionalDir = '')
 		$this->assertEquals($sExpected_content, file_get_contents(APPROOT."/data/ITopPersonJsonCollector-1.csv"));
 	}
 
-	public function OrgCollectorProvider()
+	public static function OrgCollectorProvider()
 	{
 		return [
 			"default_value" => [ "default_value" ],
@@ -119,6 +119,7 @@ public function OrgCollectorProvider()
 			"sort of object xpath parsing via an index" => [ "format_json_5" ],
 			"first row nullified function" => [ "nullified_json_1" ],
 			"another row nullified function" => [ "nullified_json_2" ],
+            "json file with relative path" => [ "json_file_with_relative_path" ],
 		];
 	}
 
@@ -167,7 +168,7 @@ public function testJsonErrors($sAdditionalDir, $sErrorMsg, $sExceptionMsg = fal
 		}
 	}
 
-	public function ErrorFileProvider()
+	public static function ErrorFileProvider()
 	{
 		return [
 			"error_json_1" => [

test/collector/attribute_isnullified/iTopPersonCollector.class.inc.php

@@ -2,29 +2,60 @@
 
 class iTopPersonCollector extends Collector
 {
-	private $bFetched;
+	private $i;
+	private $aCurrentData;
 
-	protected function Fetch()
-	{
-		if (! $this->bFetched) {
-			$this->bFetched = true;
-			return [
+	public function __construct() {
+		parent::__construct();
+
+		$this->i = 0;
+		$this->aCurrentData = [
+			[
 				'primary_key' => 1,
 				'first_name' => "isaac",
-				'name' => "asimov",
+				'name' => "asimov_null",
 				'org_id' => "Demo",
 				'phone' => null,
 				'mobile_phone' => "123456",
 				'employee_number' => "9998877665544",
 				'email' => "issac.asimov@function.io",
 				'function' => "writer",
 				'Status' => "Active",
-			];
-		}
+			],
+			[
+				'primary_key' => 2,
+				'first_name' => "isaac",
+				'name' => "asimov_empty",
+				'org_id' => "Demo",
+				'phone' => "",
+				'mobile_phone' => "123456",
+				'employee_number' => "9998877665544",
+				'email' => "issac.asimov@function.io",
+				'function' => "writer",
+				'Status' => "Active",
+			],
+			[
+				'primary_key' => 3,
+				'first_name' => "isaac",
+				'name' => "asimov_notempty",
+				'org_id' => "Demo",
+				'phone' => "not empty",
+				'mobile_phone' => "123456",
+				'employee_number' => "9998877665544",
+				'email' => "issac.asimov@function.io",
+				'function' => "writer",
+				'Status' => "Active",
+			],
+		];
+	}
+	protected function Fetch()
+	{
+		$res = $this->aCurrentData[$this->i] ?? null;
+		$this->i++;
 
-		return null;
+		return $res;
 	}
-	
+
 	/**
 	 * {@inheritDoc}
 	 * @see Collector::AttributeIsOptional()

test/single_json/json_file_with_relative_path/dataTest.json

@@ -0,0 +1,38 @@
+{
+    "objects": {
+        "Person::1": {
+            "key": "1",
+            "name": {"bob": "My last name"},
+            "status": "active",
+            "org_id": "Blala",
+            "email": "my.email@foo.org",
+            "phone": "+00 000 000 000",
+            "notify": "yes",
+            "function": "",
+            "first_nameyo": "My first name"
+        },
+        "Person::2": {
+            "key": "2",
+            "name": {"bob":  "Picasso"},
+            "status": "active",
+            "org_id": "Demo",
+            "email": "pablo@demo.com",
+            "phone": "",
+            "notify": "yes",
+            "function": "",
+            "first_nameyo": "Pablo"
+        },
+        "Person::3": {
+            "key": "3",
+            "name": {"bob":  "Dali"},
+            "status": "active",
+            "email": "dali@demo.com",
+            "phone": "",
+            "notify": "yes",
+            "function": "",
+            "first_nameyo": "Salvador"
+        }
+    },
+    "code": 0,
+    "message": "Found: 1"
+}

test/single_json/json_file_with_relative_path/expected_generated.csv

@@ -0,0 +1,4 @@
+primary_key;name;status;first_name;email;phone;function;org_id
+1;"My last name";active;"My first name";my.email@foo.org;"+00 000 000 000";;Blala
+2;Picasso;active;Pablo;pablo@demo.com;123456789;;Demo
+3;Dali;active;Salvador;dali@demo.com;123456789;;Demo

test/single_json/json_file_with_relative_path/params.distrib.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Default values for parameters. Do NOT alter this file, use params.local.xml in the conf folder instead -->
+<parameters>
+	<itoppersonjsoncollector>
+		<jsonfile>collectors/dataTest.json</jsonfile>
+		<path>objects/*</path>
+		<fields>
+			<primary_key>key</primary_key>
+			<name>name/bob</name>
+			<status>status</status>
+			<first_name>first_nameyo</first_name>
+			<email>email</email>
+			<phone>phone</phone>
+			<mobile_phone>mobile</mobile_phone>
+			<function>function</function>
+			<employee_number>employeenumber</employee_number>
+			<org_id>org_id</org_id>
+		</fields>
+		<defaults>
+			<org_id>Demo</org_id>
+			<status>active</status>
+			<phone>123456789</phone>
+		</defaults>
+	</itoppersonjsoncollector>
+	<json_placeholders>
+		<!-- For compatibility with the version 1.1.x of the collector, define the data table names as following:
+		<prefix></prefix>
+		<persons_data_table>synchro_data_PersonAD</persons_data_table>
+		<users_data_table></users_data_table>
+		 -->
+		<prefix>$prefix$</prefix>
+		<persons_data_table>synchro_data_person_1</persons_data_table>
+	</json_placeholders>
+</parameters>

test/single_json/nullified_json_1/expected_generated.csv

@@ -1,4 +1,4 @@
 primary_key;name;status;first_name;email;phone;org_id;function
 1;"My last name";active;"My first name";my.email@foo.org;"+00 000 000 000";1;<NULL>
-2;Picasso;active;Pablo;pablo@demo.com;;3;
-3;Dali;active;Salvador;dali@demo.com;;3;
+2;Picasso;active;Pablo;pablo@demo.com;;3;<NULL>
+3;Dali;active;Salvador;dali@demo.com;;3;<NULL>