Skip to content

Commit 22dcc1e

Browse files
Merge pull request #37930 from rosswhitfield/compress_events_no_sort
Implement CompressEvents method without sorting events - ornl-next
2 parents befc3b6 + df96f36 commit 22dcc1e

File tree

9 files changed

+327
-41
lines changed

9 files changed

+327
-41
lines changed

Framework/DataHandling/src/CompressEvents.cpp

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "MantidKernel/DateTimeValidator.h"
1515
#include "MantidKernel/EnumeratedString.h"
1616
#include "MantidKernel/ListValidator.h"
17+
#include "MantidKernel/VectorHelper.h"
1718

1819
#include "tbb/parallel_for.h"
1920

@@ -70,6 +71,9 @@ void CompressEvents::init() {
7071
"Binning behavior can be specified in the usual way through sign of tolerance and other properties "
7172
"('Default'); or can be set to one of the allowed binning modes. This will override all other "
7273
"specification or default behavior.");
74+
declareProperty("SortFirst", true,
75+
"If false a different method, that will not sort events first, will be used to compress events which "
76+
"is faster when you have a large number of events per compress tolerance");
7377
}
7478

7579
void CompressEvents::exec() {
@@ -79,6 +83,7 @@ void CompressEvents::exec() {
7983
double toleranceTof = getProperty("Tolerance");
8084
const double toleranceWallClock = getProperty("WallClockTolerance");
8185
const bool compressFat = !isEmpty(toleranceWallClock);
86+
const bool sortFirst = getProperty("SortFirst");
8287

8388
BINMODE mode = getPropertyValue("BinningMode");
8489
if (mode == BinningMode::LINEAR)
@@ -105,20 +110,33 @@ void CompressEvents::exec() {
105110

106111
// Sort the input workspace in-place by TOF. This can be faster if there are
107112
// few event lists. Compressing with wall clock does the sorting internally
108-
if (!compressFat) {
113+
if (!compressFat && sortFirst) {
109114
const auto timerStart = std::chrono::high_resolution_clock::now();
110115
inputWS->sortAll(TOF_SORT, &prog);
111116
addTimer("sortByTOF", timerStart, std::chrono::high_resolution_clock::now());
112117
}
113118

119+
// created required variables if using unsorted methdo
120+
auto histogram_bin_edges = std::make_shared<std::vector<double>>();
121+
size_t num_edges{0};
122+
if (!compressFat && !sortFirst && !(inputWS->getSortType() == TOF_SORT)) {
123+
// only initialize if needed
124+
double tof_min_fixed;
125+
double tof_max_fixed;
126+
inputWS->getEventXMinMax(tof_min_fixed, tof_max_fixed);
127+
Mantid::Kernel::VectorHelper::createAxisFromRebinParams(
128+
{tof_min_fixed, toleranceTof, (tof_max_fixed + std::abs(toleranceTof))}, *histogram_bin_edges, true, true);
129+
num_edges = histogram_bin_edges->size();
130+
}
131+
114132
// Are we making a copy of the input workspace?
115133
if (!inplace) {
116134
outputWS = create<EventWorkspace>(*inputWS, HistogramData::BinEdges(2));
117135
// We DONT copy the data though
118136
// Loop over the histograms (detector spectra)
119137
tbb::parallel_for(tbb::blocked_range<size_t>(0, noSpectra),
120-
[compressFat, toleranceTof, startTime, toleranceWallClock, &inputWS, &outputWS,
121-
&prog](const tbb::blocked_range<size_t> &range) {
138+
[compressFat, sortFirst, toleranceTof, startTime, toleranceWallClock, num_edges,
139+
&histogram_bin_edges, &inputWS, &outputWS, &prog](const tbb::blocked_range<size_t> &range) {
122140
for (size_t index = range.begin(); index < range.end(); ++index) {
123141
// The input event list
124142
EventList &input_el = inputWS->getSpectrum(index);
@@ -129,23 +147,27 @@ void CompressEvents::exec() {
129147
// The EventList method does the work.
130148
if (compressFat)
131149
input_el.compressFatEvents(toleranceTof, startTime, toleranceWallClock, &output_el);
132-
else
150+
else if (sortFirst || input_el.isSortedByTof() || input_el.getNumberEvents() <= num_edges)
133151
input_el.compressEvents(toleranceTof, &output_el);
152+
else
153+
input_el.compressEvents(toleranceTof, &output_el, histogram_bin_edges);
134154
prog.report("Compressing");
135155
}
136156
});
137157
} else { // inplace
138158
tbb::parallel_for(tbb::blocked_range<size_t>(0, noSpectra),
139-
[compressFat, toleranceTof, startTime, toleranceWallClock, &outputWS,
140-
&prog](const tbb::blocked_range<size_t> &range) {
159+
[compressFat, sortFirst, toleranceTof, startTime, toleranceWallClock, num_edges,
160+
&histogram_bin_edges, &outputWS, &prog](const tbb::blocked_range<size_t> &range) {
141161
for (size_t index = range.begin(); index < range.end(); ++index) {
142162
// The input (also output) event list
143163
auto &output_el = outputWS->getSpectrum(index);
144164
// The EventList method does the work.
145165
if (compressFat)
146166
output_el.compressFatEvents(toleranceTof, startTime, toleranceWallClock, &output_el);
147-
else
167+
else if (sortFirst || output_el.isSortedByTof() || output_el.getNumberEvents() <= num_edges)
148168
output_el.compressEvents(toleranceTof, &output_el);
169+
else
170+
output_el.compressEvents(toleranceTof, &output_el, histogram_bin_edges);
149171
prog.report("Compressing");
150172
}
151173
});

Framework/DataHandling/test/CompressEventsTest.h

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ using namespace Mantid::API;
2121
using namespace Mantid::Geometry;
2222
using namespace Mantid::DataObjects;
2323
using namespace Mantid::Types::Core;
24+
using namespace Mantid::Types::Event;
2425

2526
class CompressEventsTest : public CxxTest::TestSuite {
2627
public:
@@ -199,4 +200,86 @@ class CompressEventsTest : public CxxTest::TestSuite {
199200
void test_Logarithmic_binning_default() { doLogarithmicTest("Default", -1.); }
200201
void test_Logarithmic_binning_WithPulseTime() { doLogarithmicTest("Logarithmic", 1., 64); }
201202
void test_Logarithmic_binning_default_WithPulseTime() { doLogarithmicTest("Default", -1., 64); }
203+
204+
void test_unsorted_compression() {
205+
EventWorkspace_sptr input = WorkspaceCreationHelper::createEventWorkspace(1, 1, 0, 0, 1, 0);
206+
EventList &el = input->getSpectrum(0);
207+
el.addEventQuickly(TofEvent(2.8, 0));
208+
el.addEventQuickly(TofEvent(2.9, 0));
209+
el.addEventQuickly(TofEvent(3.0, 0));
210+
el.addEventQuickly(TofEvent(3.1, 0));
211+
el.addEventQuickly(TofEvent(3.2, 0));
212+
el.addEventQuickly(TofEvent(1.0, 0));
213+
214+
TS_ASSERT_EQUALS(input->getNumberEvents(), 6);
215+
216+
AnalysisDataService::Instance().addOrReplace("CompressEvents_input", input);
217+
218+
CompressEvents alg;
219+
alg.initialize();
220+
alg.setPropertyValue("InputWorkspace", "CompressEvents_input");
221+
alg.setPropertyValue("OutputWorkspace", "CompressEvents_output");
222+
alg.setProperty("Tolerance", 1.0);
223+
alg.setProperty("SortFirst", false);
224+
TS_ASSERT_THROWS_NOTHING(alg.execute());
225+
TS_ASSERT(alg.isExecuted());
226+
227+
// check that the input was not sorted
228+
TS_ASSERT_EQUALS(input->getSortType(), UNSORTED);
229+
230+
EventWorkspace_sptr output = AnalysisDataService::Instance().retrieveWS<EventWorkspace>("CompressEvents_output");
231+
TS_ASSERT_EQUALS(output->getNumberEvents(), 3);
232+
TS_ASSERT_EQUALS(output->getSortType(), TOF_SORT);
233+
TS_ASSERT_EQUALS(output->getEventType(), WEIGHTED_NOTIME);
234+
235+
EventList &output_el = output->getSpectrum(0);
236+
TS_ASSERT_DELTA(output_el.getEvent(0).tof(), 1, 1e-9);
237+
TS_ASSERT_DELTA(output_el.getEvent(0).weight(), 1.0, 1e-9);
238+
TS_ASSERT_DELTA(output_el.getEvent(0).errorSquared(), 1.0, 1e-9);
239+
TS_ASSERT_DELTA(output_el.getEvent(1).tof(), 2.85, 1e-9);
240+
TS_ASSERT_DELTA(output_el.getEvent(1).weight(), 2.0, 1e-9);
241+
TS_ASSERT_DELTA(output_el.getEvent(1).errorSquared(), 2.0, 1e-9);
242+
TS_ASSERT_DELTA(output_el.getEvent(2).tof(), 3.1, 1e-9);
243+
TS_ASSERT_DELTA(output_el.getEvent(2).weight(), 3.0, 1e-9);
244+
TS_ASSERT_DELTA(output_el.getEvent(2).errorSquared(), 3.0, 1e-9);
245+
}
246+
247+
void test_unsorted_compression_inplace() {
248+
EventWorkspace_sptr input = WorkspaceCreationHelper::createEventWorkspace(1, 1, 0, 0, 1, 0);
249+
EventList &el = input->getSpectrum(0);
250+
el.addEventQuickly(TofEvent(2.8, 0));
251+
el.addEventQuickly(TofEvent(2.9, 0));
252+
el.addEventQuickly(TofEvent(3.0, 0));
253+
el.addEventQuickly(TofEvent(3.1, 0));
254+
el.addEventQuickly(TofEvent(3.2, 0));
255+
el.addEventQuickly(TofEvent(1.0, 0));
256+
257+
TS_ASSERT_EQUALS(input->getNumberEvents(), 6);
258+
259+
AnalysisDataService::Instance().addOrReplace("CompressEvents_input", input);
260+
261+
CompressEvents alg;
262+
alg.initialize();
263+
alg.setChild(true);
264+
alg.setPropertyValue("InputWorkspace", "CompressEvents_input");
265+
alg.setPropertyValue("OutputWorkspace", "CompressEvents_input");
266+
alg.setProperty("Tolerance", 1.0);
267+
alg.setProperty("SortFirst", false);
268+
TS_ASSERT_THROWS_NOTHING(alg.execute());
269+
TS_ASSERT(alg.isExecuted());
270+
271+
// check that the input has been updated since this was done inplace
272+
TS_ASSERT_EQUALS(input->getNumberEvents(), 3);
273+
TS_ASSERT_EQUALS(input->getSortType(), TOF_SORT);
274+
TS_ASSERT_EQUALS(input->getEventType(), WEIGHTED_NOTIME);
275+
TS_ASSERT_DELTA(el.getEvent(0).tof(), 1, 1e-9);
276+
TS_ASSERT_DELTA(el.getEvent(0).weight(), 1.0, 1e-9);
277+
TS_ASSERT_DELTA(el.getEvent(0).errorSquared(), 1.0, 1e-9);
278+
TS_ASSERT_DELTA(el.getEvent(1).tof(), 2.85, 1e-9);
279+
TS_ASSERT_DELTA(el.getEvent(1).weight(), 2.0, 1e-9);
280+
TS_ASSERT_DELTA(el.getEvent(1).errorSquared(), 2.0, 1e-9);
281+
TS_ASSERT_DELTA(el.getEvent(2).tof(), 3.1, 1e-9);
282+
TS_ASSERT_DELTA(el.getEvent(2).weight(), 3.0, 1e-9);
283+
TS_ASSERT_DELTA(el.getEvent(2).errorSquared(), 3.0, 1e-9);
284+
}
202285
};

Framework/DataObjects/inc/MantidDataObjects/EventList.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,8 @@ class MANTID_DATAOBJECTS_DLL EventList : public Mantid::API::IEventList {
201201
virtual size_t histogram_size() const;
202202

203203
void compressEvents(double tolerance, EventList *destination);
204+
void compressEvents(double tolerance, EventList *destination,
205+
std::shared_ptr<std::vector<double>> histogram_bin_edges);
204206
void compressFatEvents(const double tolerance, const Types::Core::DateAndTime &timeStart, const double seconds,
205207
EventList *destination);
206208
// get EventType declaration
@@ -383,6 +385,15 @@ class MANTID_DATAOBJECTS_DLL EventList : public Mantid::API::IEventList {
383385
static void compressEventsHelper(const std::vector<T> &events, std::vector<WeightedEventNoTime> &out,
384386
double tolerance);
385387

388+
template <class T>
389+
static void createWeightedEvents(std::vector<WeightedEventNoTime> &out, const std::vector<double> &tof,
390+
const std::vector<T> &weight, const std::vector<T> &error);
391+
392+
template <class T>
393+
static void processWeightedEvents(const std::vector<T> &events, std::vector<WeightedEventNoTime> &out,
394+
const std::shared_ptr<std::vector<double>> histogram_bin_edges,
395+
struct FindBin findBin);
396+
386397
template <class T>
387398
static void compressFatEventsHelper(const std::vector<T> &events, std::vector<WeightedEvent> &out,
388399
const double tolerance, const Mantid::Types::Core::DateAndTime &timeStart,

Framework/DataObjects/src/EventList.cpp

Lines changed: 109 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,27 @@ struct comparePulseTimeTOFDelta {
139139
int64_t deltaNano;
140140
};
141141

142+
struct FindBin {
143+
double divisor;
144+
double offset;
145+
boost::optional<size_t> (*findBin)(const Mantid::MantidVec &, const double, const double, const double, const bool);
146+
FindBin(double step, double xmin) {
147+
if (step < 0) {
148+
findBin = Mantid::DataObjects::EventList::findLogBin;
149+
divisor = 1. / log1p(abs(step)); // use this to do change of base
150+
offset = log(xmin) * divisor;
151+
} else {
152+
findBin = Mantid::DataObjects::EventList::findLinearBin;
153+
divisor = 1. / step;
154+
offset = xmin * divisor;
155+
}
156+
}
157+
158+
boost::optional<size_t> operator()(const Mantid::MantidVec &X, const double tof, const bool findExact) {
159+
return findBin(X, tof, divisor, offset, findExact);
160+
}
161+
};
162+
142163
/// Constructor (empty)
143164
// EventWorkspace is always histogram data and so is thus EventList
144165
EventList::EventList(const EventType event_type)
@@ -1753,6 +1774,90 @@ void EventList::compressEvents(double tolerance, EventList *destination) {
17531774
destination->clearUnused();
17541775
}
17551776

1777+
template <class T>
1778+
inline void EventList::createWeightedEvents(std::vector<WeightedEventNoTime> &out, const std::vector<double> &tof,
1779+
const std::vector<T> &weight, const std::vector<T> &error) {
1780+
out.clear();
1781+
for (size_t i = 0; i < weight.size(); ++i) {
1782+
const auto errors = static_cast<float>(error[i]);
1783+
if (errors > 0)
1784+
out.emplace_back(tof[i], static_cast<float>(weight[i]), errors);
1785+
}
1786+
}
1787+
1788+
template <class T>
1789+
inline void EventList::processWeightedEvents(const std::vector<T> &events, std::vector<WeightedEventNoTime> &out,
1790+
const std::shared_ptr<std::vector<double>> histogram_bin_edges,
1791+
struct FindBin findBin) {
1792+
const auto NUM_BINS = histogram_bin_edges->size() - 1;
1793+
std::vector<double> tof(NUM_BINS, 0.);
1794+
std::vector<double> normalization(NUM_BINS, 0.);
1795+
std::vector<float> weight(NUM_BINS, 0.);
1796+
std::vector<float> error(NUM_BINS, 0.);
1797+
for (const auto &ev : events) {
1798+
const auto &bin_optional = findBin(*histogram_bin_edges.get(), ev.m_tof, false);
1799+
if (bin_optional) {
1800+
const auto bin = bin_optional.get();
1801+
const double norm = calcNorm(ev.m_errorSquared);
1802+
tof[bin] += ev.m_tof * norm;
1803+
normalization[bin] += norm;
1804+
weight[bin] += ev.m_weight;
1805+
error[bin] += ev.m_errorSquared;
1806+
}
1807+
}
1808+
1809+
// normalize TOFs
1810+
std::transform(tof.begin(), tof.end(), normalization.begin(), tof.begin(), std::divides<double>());
1811+
1812+
createWeightedEvents(out, tof, weight, error);
1813+
}
1814+
1815+
void EventList::compressEvents(double tolerance, EventList *destination,
1816+
std::shared_ptr<std::vector<double>> histogram_bin_edges) {
1817+
if (!this->empty()) {
1818+
const auto NUM_BINS = histogram_bin_edges->size() - 1;
1819+
const auto xmin = static_cast<double>(histogram_bin_edges->front());
1820+
1821+
auto findBin = FindBin(tolerance, xmin);
1822+
1823+
switch (eventType) {
1824+
case TOF: {
1825+
std::vector<double> tof(NUM_BINS, 0);
1826+
std::vector<uint32_t> count(NUM_BINS, 0);
1827+
for (const auto &ev : this->events) {
1828+
const auto &bin_optional = findBin(*histogram_bin_edges.get(), ev.m_tof, false);
1829+
if (bin_optional) {
1830+
const auto bin = bin_optional.get();
1831+
count[bin]++;
1832+
tof[bin] += ev.m_tof;
1833+
}
1834+
}
1835+
1836+
// average TOFs
1837+
std::transform(tof.begin(), tof.end(), count.begin(), tof.begin(), std::divides<double>());
1838+
1839+
createWeightedEvents(destination->weightedEventsNoTime, tof, count, count);
1840+
} break;
1841+
1842+
case WEIGHTED:
1843+
processWeightedEvents(this->weightedEvents, destination->weightedEventsNoTime, histogram_bin_edges, findBin);
1844+
break;
1845+
1846+
case WEIGHTED_NOTIME:
1847+
processWeightedEvents(this->weightedEventsNoTime, destination->weightedEventsNoTime, histogram_bin_edges,
1848+
findBin);
1849+
break;
1850+
}
1851+
}
1852+
1853+
// In all cases, you end up WEIGHTED_NOTIME.
1854+
destination->eventType = WEIGHTED_NOTIME;
1855+
// The result will be sorted
1856+
destination->order = TOF_SORT;
1857+
// Empty out storage for vectors that are now unused.
1858+
destination->clearUnused();
1859+
}
1860+
17561861
void EventList::compressFatEvents(const double tolerance, const Mantid::Types::Core::DateAndTime &timeStart,
17571862
const double seconds, EventList *destination) {
17581863

@@ -2002,27 +2107,14 @@ void EventList::histogramForWeightsHelper(const std::vector<T> &events, const do
20022107
const auto xmin = X.front();
20032108
const auto xmax = X.back();
20042109

2005-
double divisor, offset;
2006-
boost::optional<size_t> (*findBin)(const MantidVec &, const double, const double, const double,
2007-
const bool); // function pointer
2008-
2009-
// store 1/divisor because multiplication is less flops than division
2010-
if (step < 0) {
2011-
findBin = findLogBin;
2012-
divisor = 1. / log1p(abs(step)); // use this to do change of base
2013-
offset = log(xmin) * divisor;
2014-
} else {
2015-
findBin = findLinearBin;
2016-
divisor = 1. / step;
2017-
offset = xmin * divisor;
2018-
}
2110+
auto findBin = FindBin(step, xmin);
20192111

20202112
for (const T &ev : events) {
20212113
const double tof = ev.tof();
20222114
if (tof < xmin || tof >= xmax)
20232115
continue;
20242116

2025-
boost::optional<size_t> n_bin = findBin(X, tof, divisor, offset, true);
2117+
boost::optional<size_t> n_bin = findBin(X, tof, true);
20262118

20272119
if (n_bin) {
20282120
Y[n_bin.get()] += ev.weight();
@@ -2486,27 +2578,14 @@ void EventList::generateCountsHistogram(const double step, const MantidVec &X, M
24862578
const auto xmin = X.front();
24872579
const auto xmax = X.back();
24882580

2489-
double divisor, offset;
2490-
boost::optional<size_t> (*findBin)(const MantidVec &, const double, const double, const double,
2491-
const bool); // function pointer
2492-
2493-
// store 1/divisor because multiplication is less flops than division
2494-
if (step < 0) {
2495-
findBin = findLogBin;
2496-
divisor = 1. / log1p(abs(step)); // use this to do change of base
2497-
offset = log(xmin) * divisor;
2498-
} else {
2499-
findBin = findLinearBin;
2500-
divisor = 1. / step;
2501-
offset = xmin * divisor;
2502-
}
2581+
auto findBin = FindBin(step, xmin);
25032582

25042583
for (const TofEvent &ev : this->events) {
25052584
const double tof = ev.tof();
25062585
if (tof < xmin || tof >= xmax)
25072586
continue;
25082587

2509-
const boost::optional<size_t> n_bin = findBin(X, tof, divisor, offset, true);
2588+
const boost::optional<size_t> n_bin = findBin(X, tof, true);
25102589

25112590
if (n_bin)
25122591
Y[n_bin.get()]++;

0 commit comments

Comments
 (0)