Skip to content

Commit dc772ec

Browse files
Shubhangi-csShubhangi-cs
authored
Fixes addressing two issues identified during DTS transfer [1.6.9]
Fixes addressing two issues identified during DTS transfer
2 parents 05c4674 + 8ac4e61 commit dc772ec

File tree

6 files changed

+221
-51
lines changed

6 files changed

+221
-51
lines changed

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
<name>Salesforce plugins</name>
2323
<groupId>io.cdap.plugin</groupId>
2424
<artifactId>salesforce-plugins</artifactId>
25-
<version>1.6.8</version>
25+
<version>1.6.9</version>
2626
<packaging>jar</packaging>
2727
<description>Salesforce Plugins</description>
2828
<url>https://github.com/data-integrations/salesforce</url>

‎src/e2e-test/java/io/cdap/plugin/salesforcebatchsource/actions/SalesforcePropertiesPageActions.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,11 +127,11 @@ public static void verifyNoErrorsFoundSuccessMessage() {
127127
}
128128

129129
private static AuthenticatorCredentials setAuthenticationCredentialsOfAdminUser() {
130-
return new AuthenticatorCredentials(PluginPropertyUtils.pluginProp("admin.username"),
130+
return new AuthenticatorCredentials(null, PluginPropertyUtils.pluginProp("admin.username"),
131131
PluginPropertyUtils.pluginProp("admin.password"),
132132
PluginPropertyUtils.pluginProp("admin.consumer.key"),
133133
PluginPropertyUtils.pluginProp("admin.consumer.secret"),
134-
PluginPropertyUtils.pluginProp("login.url"), null, null, null
134+
PluginPropertyUtils.pluginProp("login.url"), null, null, null, null
135135
);
136136
}
137137

‎src/e2e-test/resources/pluginParameters.properties

Lines changed: 166 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -60,51 +60,170 @@ simple.query.schema=[{"key":"Id","value":"string"},{"key":"Name","value":"string
6060
ChildToParent.query.schema=[{"key":"Id","value":"string"},{"key":"Name","value":"string"},\
6161
{"key":"Account_Name","value":"string"}]
6262

63-
lead.schema=[{"key":"Id","value":"string"},{"key":"IsDeleted","value":"boolean"},\
64-
{"key":"MasterRecordId","value":"string"},{"key":"LastName","value":"string"},\
65-
{"key":"FirstName","value":"string"},{"key":"Salutation","value":"string"},\
66-
{"key":"MiddleName","value":"string"},{"key":"Suffix","value":"string"},{"key":"Name","value":"string"},\
67-
{"key":"Title","value":"string"},{"key":"Company","value":"string"},\
68-
{"key":"Street","value":"string"},\
69-
{"key":"City","value":"string"},{"key":"State","value":"string"},\
70-
{"key":"PostalCode","value":"string"},{"key":"Country","value":"string"},\
71-
{"key":"Latitude","value":"double"},{"key":"Longitude","value":"double"},\
72-
{"key":"GeocodeAccuracy","value":"string"},{"key":"Phone","value":"string"},\
73-
{"key":"MobilePhone","value":"string"},{"key":"Email","value":"string"},\
74-
{"key":"Website","value":"string"},{"key":"PhotoUrl","value":"string"},\
75-
{"key":"LeadSource","value":"string"},{"key":"Status","value":"string"},\
76-
{"key":"Industry","value":"string"},{"key":"Rating","value":"string"},\
77-
{"key":"NumberOfEmployees","value":"int"},{"key":"OwnerId","value":"string"},\
78-
{"key":"IsConverted","value":"boolean"},{"key":"ConvertedDate","value":"date"},\
79-
{"key":"ConvertedAccountId","value":"string"},\
80-
{"key":"ConvertedContactId","value":"string"},{"key":"ConvertedOpportunityId","value":"string"},\
81-
{"key":"IsUnreadByOwner","value":"boolean"},{"key":"CreatedDate","value":"timestamp"},\
82-
{"key":"CreatedById","value":"string"},{"key":"LastModifiedDate","value":"timestamp"},\
83-
{"key":"LastModifiedById","value":"string"},{"key":"SystemModstamp","value":"timestamp"},\
84-
{"key":"LastActivityDate","value":"date"},{"key":"LastViewedDate","value":"timestamp"},\
85-
{"key":"LastReferencedDate","value":"timestamp"},{"key":"Jigsaw","value":"string"},\
86-
{"key":"JigsawContactId","value":"string"},{"key":"EmailBouncedReason","value":"string"},\
87-
{"key":"EmailBouncedDate","value":"timestamp"}]
63+
lead.schema=[{"key":"Id","value":"string"},\
64+
{"key":"IsDeleted","value":"boolean"},\
65+
{"key":"MasterRecordId","value":"string"},\
66+
{"key":"LastName","value":"string"},\
67+
{"key":"FirstName","value":"string"},\
68+
{"key":"Salutation","value":"string"},\
69+
{"key":"Name","value":"string"},\
70+
{"key":"Title","value":"string"},\
71+
{"key":"Company","value":"string"},\
72+
{"key":"Street","value":"string"},\
73+
{"key":"City","value":"string"},\
74+
{"key":"State","value":"string"},\
75+
{"key":"PostalCode","value":"string"},\
76+
{"key":"Country","value":"string"},\
77+
{"key":"Latitude","value":"double"},\
78+
{"key":"Longitude","value":"double"},\
79+
{"key":"GeocodeAccuracy","value":"string"},\
80+
{"key":"Phone","value":"string"},\
81+
{"key":"MobilePhone","value":"string"},\
82+
{"key":"Fax","value":"string"},\
83+
{"key":"Email","value":"string"},\
84+
{"key":"Website","value":"string"},\
85+
{"key":"PhotoUrl","value":"string"},\
86+
{"key":"Description","value":"string"},\
87+
{"key":"LeadSource","value":"string"},\
88+
{"key":"Status","value":"string"},\
89+
{"key":"Industry","value":"string"},\
90+
{"key":"Rating","value":"string"},\
91+
{"key":"AnnualRevenue","value":"double"},\
92+
{"key":"NumberOfEmployees","value":"int"},\
93+
{"key":"OwnerId","value":"string"},\
94+
{"key":"IsConverted","value":"boolean"},\
95+
{"key":"ConvertedDate","value":"date"},\
96+
{"key":"ConvertedAccountId","value":"string"},\
97+
{"key":"ConvertedContactId","value":"string"},\
98+
{"key":"ConvertedOpportunityId","value":"string"},\
99+
{"key":"IsUnreadByOwner","value":"boolean"},\
100+
{"key":"CreatedDate","value":"timestamp"},\
101+
{"key":"CreatedById","value":"string"},\
102+
{"key":"LastModifiedDate","value":"timestamp"},\
103+
{"key":"LastModifiedById","value":"string"},\
104+
{"key":"SystemModstamp","value":"timestamp"},\
105+
{"key":"LastActivityDate","value":"date"},\
106+
{"key":"LastViewedDate","value":"timestamp"},\
107+
{"key":"LastReferencedDate","value":"timestamp"},\
108+
{"key":"Jigsaw","value":"string"},\
109+
{"key":"JigsawContactId","value":"string"},\
110+
{"key":"EmailBouncedReason","value":"string"},\
111+
{"key":"EmailBouncedDate","value":"timestamp"},\
112+
{"key":"IndividualId","value":"string"},\
113+
{"key":"Follow_Up__c","value":"boolean"},\
114+
{"key":"npe01__Preferred_Email__c","value":"string"},\
115+
{"key":"npe01__Preferred_Phone__c","value":"string"},\
116+
{"key":"npsp__Batch__c","value":"string"},\
117+
{"key":"npsp__CompanyCity__c","value":"string"},\
118+
{"key":"npsp__CompanyCountry__c","value":"string"},\
119+
{"key":"npsp__CompanyPostalCode__c","value":"string"},\
120+
{"key":"npsp__CompanyState__c","value":"string"},\
121+
{"key":"npsp__CompanyStreet__c","value":"string"}]
88122

89-
account.schema= [{"key":"Id","value":"string"},{"key":"IsDeleted","value":"boolean"},\
90-
{"key":"MasterRecordId","value":"string"},{"key":"Name","value":"string"},\
91-
{"key":"Type","value":"string"},\
92-
{"key":"ParentId","value":"string"},{"key":"BillingStreet","value":"string"},{"key":"BillingCity","value":"string"},\
93-
{"key":"BillingState","value":"string"},{"key":"BillingPostalCode","value":"string"},\
94-
{"key":"BillingCountry","value":"string"},\
95-
{"key":"BillingLatitude","value":"double"},{"key":"BillingLongitude","value":"double"},\
96-
{"key":"BillingGeocodeAccuracy","value":"string"},{"key":"ShippingStreet","value":"string"},\
97-
{"key":"ShippingCity","value":"string"},{"key":"ShippingState","value":"string"},\
98-
{"key":"ShippingPostalCode","value":"string"},{"key":"ShippingCountry","value":"string"},\
99-
{"key":"ShippingLatitude","value":"double"},{"key":"ShippingLongitude","value":"double"},\
100-
{"key":"ShippingGeocodeAccuracy","value":"string"},{"key":"Phone","value":"string"},\
101-
{"key":"Website","value":"string"},{"key":"PhotoUrl","value":"string"},\
102-
{"key":"Industry","value":"string"},\
103-
{"key":"NumberOfEmployees","value":"int"},{"key":"Description","value":"string"},{"key":"OwnerId","value":"string"},\
104-
{"key":"CreatedDate","value":"timestamp"},{"key":"CreatedById","value":"string"},\
105-
{"key":"LastModifiedDate","value":"timestamp"},\
106-
{"key":"LastModifiedById","value":"string"},{"key":"SystemModstamp","value":"timestamp"},\
107-
{"key":"LastActivityDate","value":"date"},{"key":"LastViewedDate","value":"timestamp"},\
108-
{"key":"LastReferencedDate","value":"timestamp"},{"key":"Jigsaw","value":"string"},\
109-
{"key":"JigsawCompanyId","value":"string"},{"key":"AccountSource","value":"string"},\
110-
{"key":"SicDesc","value":"string"}]
123+
account.schema= [{"key":"Id","value":"string"},\
124+
{"key":"IsDeleted","value":"boolean"},\
125+
{"key":"MasterRecordId","value":"string"},\
126+
{"key":"Name","value":"string"},\
127+
{"key":"Type","value":"string"},\
128+
{"key":"RecordTypeId","value":"string"},\
129+
{"key":"ParentId","value":"string"},\
130+
{"key":"BillingStreet","value":"string"},\
131+
{"key":"BillingCity","value":"string"},\
132+
{"key":"BillingState","value":"string"},\
133+
{"key":"BillingPostalCode","value":"string"},\
134+
{"key":"BillingCountry","value":"string"},\
135+
{"key":"BillingLatitude","value":"double"},\
136+
{"key":"BillingLongitude","value":"double"},\
137+
{"key":"BillingGeocodeAccuracy","value":"string"},\
138+
{"key":"ShippingStreet","value":"string"},\
139+
{"key":"ShippingCity","value":"string"},\
140+
{"key":"ShippingState","value":"string"},\
141+
{"key":"ShippingPostalCode","value":"string"},\
142+
{"key":"ShippingCountry","value":"string"},\
143+
{"key":"ShippingLatitude","value":"double"},\
144+
{"key":"ShippingLongitude","value":"double"},\
145+
{"key":"ShippingGeocodeAccuracy","value":"string"},\
146+
{"key":"Phone","value":"string"},\
147+
{"key":"Fax","value":"string"},\
148+
{"key":"AccountNumber","value":"string"},\
149+
{"key":"Website","value":"string"},\
150+
{"key":"PhotoUrl","value":"string"},\
151+
{"key":"Sic","value":"string"},\
152+
{"key":"Industry","value":"string"},\
153+
{"key":"AnnualRevenue","value":"double"},\
154+
{"key":"NumberOfEmployees","value":"int"},\
155+
{"key":"Ownership","value":"string"},\
156+
{"key":"TickerSymbol","value":"string"},\
157+
{"key":"Description","value":"string"},\
158+
{"key":"Rating","value":"string"},\
159+
{"key":"Site","value":"string"},\
160+
{"key":"OwnerId","value":"string"},\
161+
{"key":"CreatedDate","value":"timestamp"},\
162+
{"key":"CreatedById","value":"string"},\
163+
{"key":"LastModifiedDate","value":"timestamp"},\
164+
{"key":"LastModifiedById","value":"string"},\
165+
{"key":"SystemModstamp","value":"timestamp"},\
166+
{"key":"LastActivityDate","value":"date"},\
167+
{"key":"LastViewedDate","value":"timestamp"},\
168+
{"key":"LastReferencedDate","value":"timestamp"},\
169+
{"key":"Jigsaw","value":"string"},\
170+
{"key":"JigsawCompanyId","value":"string"},\
171+
{"key":"AccountSource","value":"string"},\
172+
{"key":"SicDesc","value":"string"},\
173+
{"key":"Copy_Billing_Address_to_Shipping_Address__c","value":"boolean"},\
174+
{"key":"Test_External_Id__c","value":"string"},\
175+
{"key":"custom_field__c","value":"string"},\
176+
{"key":"npe01__One2OneContact__c","value":"string"},\
177+
{"key":"npe01__SYSTEMIsIndividual__c","value":"boolean"},\
178+
{"key":"npe01__SYSTEM_AccountType__c","value":"string"},\
179+
{"key":"npo02__AverageAmount__c","value":"double"},\
180+
{"key":"npo02__Best_Gift_Year_Total__c","value":"double"},\
181+
{"key":"npo02__Best_Gift_Year__c","value":"string"},\
182+
{"key":"npo02__FirstCloseDate__c","value":"date"},\
183+
{"key":"npo02__Formal_Greeting__c","value":"string"},\
184+
{"key":"npo02__HouseholdPhone__c","value":"string"},\
185+
{"key":"npo02__Informal_Greeting__c","value":"string"},\
186+
{"key":"npo02__LargestAmount__c","value":"double"},\
187+
{"key":"npo02__LastCloseDate__c","value":"date"},\
188+
{"key":"npo02__LastMembershipAmount__c","value":"double"},\
189+
{"key":"npo02__LastMembershipDate__c","value":"date"},\
190+
{"key":"npo02__LastMembershipLevel__c","value":"string"},\
191+
{"key":"npo02__LastMembershipOrigin__c","value":"string"},\
192+
{"key":"npo02__LastOppAmount__c","value":"double"},\
193+
{"key":"npo02__MembershipEndDate__c","value":"date"},\
194+
{"key":"npo02__MembershipJoinDate__c","value":"date"},\
195+
{"key":"npo02__NumberOfClosedOpps__c","value":"double"},\
196+
{"key":"npo02__NumberOfMembershipOpps__c","value":"double"},\
197+
{"key":"npo02__OppAmount2YearsAgo__c","value":"double"},\
198+
{"key":"npo02__OppAmountLastNDays__c","value":"double"},\
199+
{"key":"npo02__OppAmountLastYear__c","value":"double"},\
200+
{"key":"npo02__OppAmountThisYear__c","value":"double"},\
201+
{"key":"npo02__OppsClosed2YearsAgo__c","value":"double"},\
202+
{"key":"npo02__OppsClosedLastNDays__c","value":"double"},\
203+
{"key":"npo02__OppsClosedLastYear__c","value":"double"},\
204+
{"key":"npo02__OppsClosedThisYear__c","value":"double"},\
205+
{"key":"npo02__SYSTEM_CUSTOM_NAMING__c","value":"string"},\
206+
{"key":"npo02__SmallestAmount__c","value":"double"},\
207+
{"key":"npo02__TotalMembershipOppAmount__c","value":"double"},\
208+
{"key":"npo02__TotalOppAmount__c","value":"double"},\
209+
{"key":"npsp__Membership_Span__c","value":"double"},\
210+
{"key":"npsp__Membership_Status__c","value":"string"},\
211+
{"key":"npsp__Sustainer__c","value":"string"},\
212+
{"key":"npsp__All_Members_Deceased__c","value":"boolean"},\
213+
{"key":"npsp__Batch__c","value":"string"},\
214+
{"key":"npsp__CustomizableRollups_UseSkewMode__c","value":"boolean"},\
215+
{"key":"npsp__Funding_Focus__c","value":"string"},\
216+
{"key":"npsp__Grantmaker__c","value":"boolean"},\
217+
{"key":"npsp__Matching_Gift_Administrator_Name__c","value":"string"},\
218+
{"key":"npsp__Matching_Gift_Amount_Max__c","value":"double"},\
219+
{"key":"npsp__Matching_Gift_Amount_Min__c","value":"double"},\
220+
{"key":"npsp__Matching_Gift_Annual_Employee_Max__c","value":"double"},\
221+
{"key":"npsp__Matching_Gift_Comments__c","value":"string"},\
222+
{"key":"npsp__Matching_Gift_Company__c","value":"boolean"},\
223+
{"key":"npsp__Matching_Gift_Email__c","value":"string"},\
224+
{"key":"npsp__Matching_Gift_Info_Updated__c","value":"date"},\
225+
{"key":"npsp__Matching_Gift_Percent__c","value":"double"},\
226+
{"key":"npsp__Matching_Gift_Phone__c","value":"string"},\
227+
{"key":"npsp__Matching_Gift_Request_Deadline__c","value":"string"},\
228+
{"key":"npsp__Number_of_Household_Members__c","value":"double"},\
229+
{"key":"npsp__Undeliverable_Address__c","value":"boolean"}]

‎src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/MapToRecordTransformer.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import io.cdap.plugin.salesforce.SalesforceSchemaUtil;
2323
import io.cdap.plugin.salesforce.SalesforceTransformUtil;
2424

25+
import java.math.BigDecimal;
2526
import java.util.List;
2627
import java.util.Map;
2728
import java.util.Objects;
@@ -78,7 +79,11 @@ private Object convertValue(String fieldName, Object value, Schema fieldSchema)
7879
case BOOLEAN:
7980
return Boolean.parseBoolean(castValue(value, fieldName, String.class));
8081
case INT:
81-
return Integer.parseInt(castValue(value, fieldName, String.class));
82+
// 'Number' field in 'Site' sObject is defined as INT in schema,
83+
// however, the actual data is received as a Decimal (e.g., 0.0)
84+
// Wrapping as BigDecimal to safely extract the int value and avoid cast exceptions.
85+
String data = castValue(value, fieldName, String.class);
86+
return new BigDecimal(data).intValue();
8287
case LONG:
8388
return Long.parseLong(castValue(value, fieldName, String.class));
8489
case FLOAT:

‎src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SoapRecordToMapTransformer.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.util.HashMap;
2424
import java.util.List;
2525
import java.util.Map;
26+
import java.util.Objects;
2627
import java.util.stream.Collectors;
2728
import java.util.stream.StreamSupport;
2829

@@ -53,6 +54,9 @@ public class SoapRecordToMapTransformer {
5354
Iterable<XmlObject> subValues = () -> child.getChildren(SUB_QUERY_FIELDS_PARENT);
5455

5556
List<Map<String, String>> subQueryValues = StreamSupport.stream(subValues.spliterator(), false)
57+
// Since null XmlObject instances were encountered in the customer transfer runs, a
58+
// validation check is now applied to exclude them during processing.
59+
.filter(Objects::nonNull)
5660
.map(subValue -> transformRowToMap(subValue, childSObjectDescriptor))
5761
.collect(Collectors.toList());
5862
result.put(childSObjectDescriptor.getName(), subQueryValues);

‎src/test/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBulkRecordReaderTest.java

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -562,4 +562,46 @@ public void testCaseSenstiveMultipleResults() throws Exception {
562562
assertRecordReaderOutputRecords(new String[] {csvString1, csvString2}, schema, expectedRecords);
563563
}
564564

565+
@Test
566+
public void testCaseIntTypeWithDecimalDataInResults() throws Exception {
567+
String csvString1 = "\"Id\",\"IsDeleted\",\"ExpectedRevenue\",\"TotalRevenue\"\n" +
568+
"\"0061i000003XNcBAAW\",\"false\",\"1500.0\",\"1234.56\"\n";
569+
String csvString2 = "\"Id\",\"IsDeleted\",\"ExpectedRevenue\",\"TotalRevenue\"\n" +
570+
"\"0061i000003XNcCAAW\",\"false\",\"112500.0\",\"1234.56\"\n" +
571+
"\"0061i000003XNcDAAW\",\"false\",\"220000.0\",\"1234.56\"\n";
572+
573+
Schema schema = Schema.recordOf("output",
574+
Schema.Field.of("id", Schema.of(Schema.Type.STRING)),
575+
Schema.Field.of("isDeleted", Schema.of(Schema.Type.BOOLEAN)),
576+
Schema.Field.of("ExpectedRevenue", Schema.of(Schema.Type.INT)),
577+
Schema.Field.of("TotalRevenue", Schema.of(Schema.Type.DOUBLE))
578+
);
579+
580+
List<Map<String, Object>> expectedRecords = new ImmutableList.Builder<Map<String, Object>>()
581+
.add(new ImmutableMap.Builder<String, Object>()
582+
.put("id", "0061i000003XNcBAAW")
583+
.put("isDeleted", false)
584+
.put("ExpectedRevenue", 1500)
585+
.put("TotalRevenue", 1234.56)
586+
.build()
587+
)
588+
.add(new ImmutableMap.Builder<String, Object>()
589+
.put("id", "0061i000003XNcCAAW")
590+
.put("isDeleted", false)
591+
.put("ExpectedRevenue", 112500)
592+
.put("TotalRevenue", 1234.56)
593+
.build()
594+
)
595+
.add(new ImmutableMap.Builder<String, Object>()
596+
.put("id", "0061i000003XNcDAAW")
597+
.put("isDeleted", false)
598+
.put("ExpectedRevenue", 220000)
599+
.put("TotalRevenue", 1234.56)
600+
.build()
601+
)
602+
.build();
603+
604+
assertRecordReaderOutputRecords(new String[] {csvString1, csvString2}, schema, expectedRecords);
605+
}
606+
565607
}

0 commit comments

Comments
 (0)