Generator: Update national stats to 1M patients

Author: kaicode

.gitignore

@@ -5,7 +5,7 @@ data
 release
 index
 snomed-index
-patient-data-for-import
+patient-data-for-import*
 *.zip
 *.sh
 cpt-codes

generator/src/main/java/org/snomed/heathanalytics/datageneration/DemoPatientDataGenerator.java

@@ -88,9 +88,12 @@ public void createPatients(int patientCount, File outputFile) throws IOException
 	public void createLongitudinalPatients(File longitudinalNdJsonFile) throws IOException, ServiceException {
 		long start = new Date().getTime();
 
-		int multiplier = 1_000;
+		int ratePerCount = 100_000;
+		int populationSize = 1_000_000;
+		int multiplier = populationSize / ratePerCount;
 
 		// Model the stats we want the data to reflect
+		// The numbers are incidence rate
 		Map<Integer, Map<Long, Integer>> yearConceptCountMap = new HashMap<>();
 		long asthma = 195967001;
 		fillYears(yearConceptCountMap, 2000, asthma, 375, 400, 375, 350, 425, 400, 425, 400, 350, 325, 325, 300, 275, 225, 250, 225, 225, 200, 175, 175, 150);
@@ -106,44 +109,42 @@ public void createLongitudinalPatients(File longitudinalNdJsonFile) throws IOExc
 		fillYears(yearConceptCountMap, 2000, diabetesType2, 225, 250, 250, 275, 275, 275, 275, 300, 325, 325, 350, 400, 350, 250, 225, 250, 275, 250, 250, 250, 275);
 
 		logger.info("******** Generating data for longitudinal patients ...");
+		
+		List<Patient> allPatients = new ArrayList<>();
+		for (long i = 0; i < populationSize; i++) {
+			Patient patient = new Patient(i + 1 + "");
+			int age = ThreadLocalRandom.current().nextInt(30, 85);
+			patient.setDob(DateUtil.dateOfBirthFromAge(age));
+			allPatients.add(patient);
+		}
 
+		Iterator<Patient> patientIterator = allPatients.iterator();
 		try (SequenceWriter patientWriter = objectMapper
 				.writerFor(Patient.class)
 				.withView(View.API.class)
 				.withRootValueSeparator("\n")
 				.writeValues(longitudinalNdJsonFile)) {
 
-			Map<Long, List<Patient>> disorderPatientPools = new HashMap<>();
-			int id = 1;
-			for (Map.Entry<Integer, Map<Long, Integer>> yearConceptCounts : yearConceptCountMap.entrySet()) {
-				Integer year = yearConceptCounts.getKey();
-				Map<Long, Integer> conceptCounts = yearConceptCounts.getValue();
-				for (Map.Entry<Long, Integer> conceptCount : conceptCounts.entrySet()) {
-					Long concept = conceptCount.getKey();
-					int count = conceptCount.getValue() * multiplier;
-					List<Patient> patientsWithDisorder = disorderPatientPools.computeIfAbsent(concept, i -> new ArrayList<>());
-					while (patientsWithDisorder.size() < count) {
-						Patient patient = new Patient(id++ + "");
-						int age = ThreadLocalRandom.current().nextInt(30, 85);
-						patient.setDob(DateUtil.dateOfBirthFromAge(age));
-						patientsWithDisorder.add(patient);
-					}
-					GregorianCalendar gregorianCalendar = new GregorianCalendar(year, 1, 1);
+			for (Map.Entry<Integer, Map<Long, Integer>> yearConceptIncidenceCounts : yearConceptCountMap.entrySet()) {
+				Integer year = yearConceptIncidenceCounts.getKey();
+				Map<Long, Integer> conceptIncidenceCounts = yearConceptIncidenceCounts.getValue();
+				for (Map.Entry<Long, Integer> conceptIncidenceCount : conceptIncidenceCounts.entrySet()) {
+					Long concept = conceptIncidenceCount.getKey();
+					int count = conceptIncidenceCount.getValue() * multiplier;
+					GregorianCalendar gregorianCalendar = new GregorianCalendar(year, Calendar.JANUARY, 1);
 					for (int i = 0; i < count; i++) {
-						patientsWithDisorder.get(i).addEncounter(new ClinicalEncounter(gregorianCalendar, concepts.selectRandomDescendantOf(concept.toString())));
+						patientIterator.next().addEncounter(new ClinicalEncounter(gregorianCalendar, concepts.selectRandomDescendantOf(concept.toString())));
 					}
 				}
 			}
-			long total = 0;
-			for (List<Patient> list : disorderPatientPools.values()) {
-				total += list.size();
-			}
-			logger.info("Saving {} patients", new DecimalFormat( "#,###,###" ).format(total));
-			for (List<Patient> patientSet : disorderPatientPools.values()) {
-				for (List<Patient> batch : Iterables.partition(patientSet, 10_000)) {
-					patientWriter.writeAll(batch);
-					System.out.print(".");
-				}
+//			long total = 0;
+//			for (List<Patient> list : disorderPatientPools.values()) {
+//				total += list.size();
+//			}
+			logger.info("Saving {} patients", new DecimalFormat( "#,###,###" ).format(allPatients.size()));
+			for (List<Patient> batch : Iterables.partition(allPatients, 10_000)) {
+				patientWriter.writeAll(batch);
+				System.out.print(".");
 			}
 			patientWriter.flush();
 			System.out.println();