Skip to content

Commit 60812c8

Browse files
alexmcdermidalexmcdermid
committed
composite index for collection duplicates query
1 parent 3743e74 commit 60812c8

File tree

5 files changed

+62
-10
lines changed

5 files changed

+62
-10
lines changed

‎api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/controller/v1/CollectionController.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -121,12 +121,20 @@ public ResponseEntity<Void> deleteCollection(UUID collectionID) {
121121

122122
@Override
123123
public List<MonitorSdcDistrictCollection> getMonitorSdcDistrictCollectionResponse(UUID collectionId) {
124-
return this.collectionService.getMonitorSdcDistrictCollectionResponse(collectionId);
124+
long startTime = System.currentTimeMillis();
125+
log.info("getMonitorSdcDistrictCollectionResponse: Received request for collectionID :: {}", collectionId);
126+
List<MonitorSdcDistrictCollection> result = this.collectionService.getMonitorSdcDistrictCollectionResponse(collectionId);
127+
log.info("getMonitorSdcDistrictCollectionResponse: Completed in {} ms for collectionID :: {}. Returned {} district collections", System.currentTimeMillis() - startTime, collectionId, result.size());
128+
return result;
125129
}
126130

127131
@Override
128132
public MonitorIndySdcSchoolCollectionsResponse getMonitorIndySdcSchoolCollectionResponse(UUID collectionId) {
129-
return this.collectionService.getMonitorIndySdcSchoolCollectionResponse(collectionId);
133+
long startTime = System.currentTimeMillis();
134+
log.info("getMonitorIndySdcSchoolCollectionResponse: Received request for collectionID :: {}", collectionId);
135+
MonitorIndySdcSchoolCollectionsResponse result = this.collectionService.getMonitorIndySdcSchoolCollectionResponse(collectionId);
136+
log.info("getMonitorIndySdcSchoolCollectionResponse: Completed in {} ms for collectionID :: {}", System.currentTimeMillis() - startTime, collectionId);
137+
return result;
130138
}
131139

132140
@Override
@@ -162,7 +170,11 @@ public ResponseEntity<Void> generateProvinceDuplicates(UUID collectionID) {
162170

163171
@Override
164172
public List<SdcDuplicate> getProvinceDuplicates(UUID collectionID) {
165-
return this.sdcDuplicatesService.getAllProvincialDuplicatesByCollectionID(collectionID).stream().map(duplicateMapper::toSdcDuplicate).toList();
173+
long startTime = System.currentTimeMillis();
174+
log.info("getProvinceDuplicates: Received request for collectionID :: {}", collectionID);
175+
List<SdcDuplicate> result = this.sdcDuplicatesService.getAllProvincialDuplicatesByCollectionID(collectionID).stream().map(duplicateMapper::toSdcDuplicate).toList();
176+
log.info("getProvinceDuplicates: Completed in {} ms for collectionID :: {}. Returned {} duplicates", System.currentTimeMillis() - startTime, collectionID, result.size());
177+
return result;
166178
}
167179

168180
@Override

‎api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SdcSchoolCollectionStudentRepository.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -376,24 +376,24 @@ HAVING COUNT(innerStud.assignedStudentId) > 1)
376376
List<SdcSchoolCollectionStudentLightEntity> findAllInProvinceDuplicateStudentsInCollection(UUID collectionID);
377377

378378
@Query(value = """
379-
WITH sc_ids AS (
379+
WITH all_school_collections_in_collection AS (
380380
SELECT sc.SDC_SCHOOL_COLLECTION_ID
381381
FROM SDC_SCHOOL_COLLECTION sc
382382
WHERE sc.COLLECTION_ID = :collectionID
383383
),
384-
keys AS (
384+
province_wide_duplicate_assigned_ids AS (
385385
SELECT s.ASSIGNED_STUDENT_ID
386386
FROM SDC_SCHOOL_COLLECTION_STUDENT s
387-
WHERE s.SDC_SCHOOL_COLLECTION_ID IN (SELECT SDC_SCHOOL_COLLECTION_ID FROM sc_ids)
387+
WHERE s.SDC_SCHOOL_COLLECTION_ID IN (SELECT SDC_SCHOOL_COLLECTION_ID FROM all_school_collections_in_collection)
388388
AND s.ASSIGNED_STUDENT_ID IS NOT NULL
389389
AND s.SDC_SCHOOL_COLLECTION_STUDENT_STATUS_CODE <> 'DELETED'
390390
GROUP BY s.ASSIGNED_STUDENT_ID
391391
HAVING COUNT(*) > 1
392392
)
393393
SELECT s.SDC_SCHOOL_COLLECTION_STUDENT_ID
394394
FROM SDC_SCHOOL_COLLECTION_STUDENT s
395-
WHERE s.SDC_SCHOOL_COLLECTION_ID IN (SELECT SDC_SCHOOL_COLLECTION_ID FROM sc_ids)
396-
AND s.ASSIGNED_STUDENT_ID IN (SELECT ASSIGNED_STUDENT_ID FROM keys)
395+
WHERE s.SDC_SCHOOL_COLLECTION_ID IN (SELECT SDC_SCHOOL_COLLECTION_ID FROM all_school_collections_in_collection)
396+
AND s.ASSIGNED_STUDENT_ID IN (SELECT ASSIGNED_STUDENT_ID FROM province_wide_duplicate_assigned_ids)
397397
AND s.SDC_SCHOOL_COLLECTION_STUDENT_STATUS_CODE <> 'DELETED'
398398
""", nativeQuery = true)
399399
List<UUID> findDuplicateLightIds(UUID collectionID);

‎api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/service/v1/CollectionService.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,16 +81,22 @@ public void deleteCollection(UUID collectionID) {
8181
}
8282

8383
public List<MonitorSdcDistrictCollection> getMonitorSdcDistrictCollectionResponse(UUID collectionID) {
84+
long startTime = System.currentTimeMillis();
85+
log.info("getMonitorSdcDistrictCollectionResponse: Starting for collectionID :: {}", collectionID);
8486
CollectionEntity collection = collectionRepository.findById(collectionID).orElseThrow(() -> new EntityNotFoundException(CollectionEntity.class, "CollectionID", collectionID.toString()));
8587

88+
long monitorQueryStart = System.currentTimeMillis();
8689
List<MonitorSdcDistrictCollectionQueryResponse> monitorSdcDistrictCollectionsQueryResponses = sdcDistrictCollectionRepository.findAllSdcDistrictCollectionMonitoringByCollectionID(collectionID);
90+
log.info("getMonitorSdcDistrictCollectionResponse: DB query findAllSdcDistrictCollectionMonitoringByCollectionID took {} ms, found {} districts for collectionID :: {}", System.currentTimeMillis() - monitorQueryStart, monitorSdcDistrictCollectionsQueryResponses.size(), collectionID);
8791

8892
var isCollectionComplete = collection.getCollectionStatusCode().equalsIgnoreCase(CollectionStatus.COMPLETED.getCode());
8993
HashMap<UUID, Integer> countMap;
9094

9195
if(!isCollectionComplete) {
96+
long dupeStart = System.currentTimeMillis();
9297
var duplicates = sdcDuplicatesService.getAllProvincialDuplicatesByCollectionID(collectionID);
9398
countMap = getDistrictDuplicatesCountMap(duplicates);
99+
log.info("getMonitorSdcDistrictCollectionResponse: Provincial duplicates retrieval + count map took {} ms, found {} duplicates for collectionID :: {}", System.currentTimeMillis() - dupeStart, duplicates.size(), collectionID);
94100
} else {
95101
countMap = new HashMap<>();
96102
}
@@ -110,6 +116,7 @@ public List<MonitorSdcDistrictCollection> getMonitorSdcDistrictCollectionRespons
110116
monitorSdcDistrictCollections.add(monitorSdcDistrictCollection);
111117
});
112118

119+
log.info("getMonitorSdcDistrictCollectionResponse: Completed in {} ms for collectionID :: {}. Returning {} district collections", System.currentTimeMillis() - startTime, collectionID, monitorSdcDistrictCollections.size());
113120
return monitorSdcDistrictCollections;
114121
}
115122

@@ -161,15 +168,23 @@ private HashMap<UUID, Integer> getDistrictDuplicatesCountMap(List<SdcDuplicateEn
161168
}
162169

163170
public MonitorIndySdcSchoolCollectionsResponse getMonitorIndySdcSchoolCollectionResponse(UUID collectionID) {
171+
long startTime = System.currentTimeMillis();
172+
log.info("getMonitorIndySdcSchoolCollectionResponse: Starting for collectionID :: {}", collectionID);
164173
CollectionEntity collection = collectionRepository.findById(collectionID).orElseThrow(() -> new EntityNotFoundException(CollectionEntity.class, "CollectionID", collectionID.toString()));
174+
175+
long monitorQueryStart = System.currentTimeMillis();
165176
List<MonitorIndySdcSchoolCollectionQueryResponse> monitorSdcSchoolCollectionQueryResponses = sdcSchoolCollectionRepository.findAllIndySdcSchoolCollectionMonitoringBySdcCollectionId(collectionID);
177+
log.info("getMonitorIndySdcSchoolCollectionResponse: DB query findAllIndySdcSchoolCollectionMonitoringBySdcCollectionId took {} ms, found {} schools for collectionID :: {}", System.currentTimeMillis() - monitorQueryStart, monitorSdcSchoolCollectionQueryResponses.size(), collectionID);
178+
166179
List<MonitorIndySdcSchoolCollection> monitorSdcSchoolCollections = new ArrayList<>();
167180
var isCollectionComplete = collection.getCollectionStatusCode().equalsIgnoreCase(CollectionStatus.COMPLETED.getCode());
168181
HashMap<UUID, Integer> countMap;
169182

170183
if(!isCollectionComplete) {
184+
long dupeStart = System.currentTimeMillis();
171185
var duplicates = sdcDuplicatesService.getAllProvincialDuplicatesByCollectionID(collectionID);
172186
countMap = getSchoolDuplicatesCountMap(duplicates);
187+
log.info("getMonitorIndySdcSchoolCollectionResponse: Provincial duplicates retrieval + count map took {} ms, found {} duplicates for collectionID :: {}", System.currentTimeMillis() - dupeStart, duplicates.size(), collectionID);
173188
} else {
174189
countMap = new HashMap<>();
175190
}
@@ -203,6 +218,7 @@ public MonitorIndySdcSchoolCollectionsResponse getMonitorIndySdcSchoolCollection
203218
response.setSchoolsSubmitted(monitorSdcSchoolCollections.stream().filter(MonitorIndySdcSchoolCollection::isSubmittedToDistrict).count());
204219
response.setSchoolsWithData(monitorSdcSchoolCollections.stream().filter(coll -> coll.getUploadDate() != null).count());
205220
response.setTotalSchools(monitorSdcSchoolCollections.size());
221+
log.info("getMonitorIndySdcSchoolCollectionResponse: Completed in {} ms for collectionID :: {}. Returning {} school collections", System.currentTimeMillis() - startTime, collectionID, monitorSdcSchoolCollections.size());
206222
return response;
207223
}
208224

‎api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/service/v1/SdcDuplicatesService.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,13 +67,28 @@ public List<SdcSchoolCollectionStudentEntity> getAllSchoolCollectionDuplicates(U
6767
}
6868

6969
public List<SdcDuplicateEntity> getAllProvincialDuplicatesByCollectionID(UUID collectionID) {
70+
long startTime = System.currentTimeMillis();
71+
log.info("getAllProvincialDuplicatesByCollectionID: Starting province-wide duplicates retrieval for collectionID :: {}", collectionID);
7072
List<SdcSchoolCollectionStudentLightEntity> provinceDupes = findAllInProvinceDuplicateStudentsInCollection(collectionID);
71-
return generateFinalDuplicatesSet(provinceDupes, DuplicateLevelCode.PROVINCIAL);
73+
log.info("getAllProvincialDuplicatesByCollectionID: findAllInProvinceDuplicateStudentsInCollection took {} ms, found {} students for collectionID :: {}", System.currentTimeMillis() - startTime, provinceDupes.size(), collectionID);
74+
long generateStart = System.currentTimeMillis();
75+
var dupes = generateFinalDuplicatesSet(provinceDupes, DuplicateLevelCode.PROVINCIAL);
76+
log.info("getAllProvincialDuplicatesByCollectionID: generateFinalDuplicatesSet took {} ms, produced {} duplicates for collectionID :: {}", System.currentTimeMillis() - generateStart, dupes.size(), collectionID);
77+
log.info("getAllProvincialDuplicatesByCollectionID: Completed in {} ms for collectionID :: {}. Total duplicates found :: {}", System.currentTimeMillis() - startTime, collectionID, dupes.size());
78+
return dupes;
7279
}
7380

7481
public List<SdcSchoolCollectionStudentLightEntity> findAllInProvinceDuplicateStudentsInCollection(UUID collectionID) {
82+
long startTime = System.currentTimeMillis();
7583
List<UUID> ids = sdcSchoolCollectionStudentRepository.findDuplicateLightIds(collectionID);
76-
return ids.isEmpty() ? List.of() : sdcSchoolCollectionStudentLightRepository.findAllById(ids);
84+
log.info("findAllInProvinceDuplicateStudentsInCollection: DB query findDuplicateLightIds took {} ms, found {} IDs for collectionID :: {}", System.currentTimeMillis() - startTime, ids.size(), collectionID);
85+
if (ids.isEmpty()) {
86+
return List.of();
87+
}
88+
long entityQueryStart = System.currentTimeMillis();
89+
List<SdcSchoolCollectionStudentLightEntity> entities = sdcSchoolCollectionStudentLightRepository.findAllById(ids);
90+
log.info("findAllInProvinceDuplicateStudentsInCollection: DB query findAllById took {} ms, fetched {} light entities for collectionID :: {}", System.currentTimeMillis() - entityQueryStart, entities.size(), collectionID);
91+
return entities;
7792
}
7893

7994
public List<SdcDuplicateEntity> getAllProvincialDuplicatesBySdcDistrictCollectionID(UUID sdcDistrictCollectionID) {

‎api/src/main/resources/db/migration/V1.0.100__STUDENT_DATA_COLLECTION.API.sql‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
CREATE INDEX idx_student_collection_assigned_id_status
2+
ON SDC_SCHOOL_COLLECTION_STUDENT (
3+
SDC_SCHOOL_COLLECTION_ID,
4+
ASSIGNED_STUDENT_ID
5+
)
6+
WHERE SDC_SCHOOL_COLLECTION_STUDENT_STATUS_CODE <> 'DELETED'
7+
AND ASSIGNED_STUDENT_ID IS NOT NULL;
8+
9+

0 commit comments

Comments
 (0)