Skip to content

Commit 510bc4d

Browse files
nksauterphyy-nx
nksauter
authored and
phyy-nx
committed
Debug case for non-working test.
The script tests/tst_memory_policy.py fails with a cuda illegal access. The intention is to get help from NESAP to get a functional test.
1 parent eae38fb commit 510bc4d

File tree

6 files changed

+664
-171
lines changed

6 files changed

+664
-171
lines changed

‎simtbx/kokkos/detector.cpp‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,5 +92,140 @@ namespace simtbx { namespace Kokkos {
9292
}
9393
}
9494

95+
template<>
96+
void kokkos_detector<small_whitelist_policy>::hello(){
97+
SCITBX_EXAMINE("small small small");
98+
}
99+
template<>
100+
void kokkos_detector<large_array_policy>::hello(){
101+
SCITBX_EXAMINE("large large large");
102+
}
103+
104+
template<> void
105+
kokkos_detector<large_array_policy>::each_image_allocate() {
106+
resize(m_rangemap, m_total_pixel_count);
107+
resize(m_omega_reduction, m_total_pixel_count);
108+
resize(m_max_I_x_reduction, m_total_pixel_count);
109+
resize(m_max_I_y_reduction, m_total_pixel_count);
110+
resize(m_floatimage, m_total_pixel_count);
111+
112+
resize(m_maskimage, m_total_pixel_count);
113+
kokkostbx::transfer_shared2kokkos(m_sdet_vector, metrology.sdet);
114+
kokkostbx::transfer_shared2kokkos(m_fdet_vector, metrology.fdet);
115+
kokkostbx::transfer_shared2kokkos(m_odet_vector, metrology.odet);
116+
kokkostbx::transfer_shared2kokkos(m_pix0_vector, metrology.pix0);
117+
kokkostbx::transfer_shared2kokkos(m_distance, metrology.dists);
118+
kokkostbx::transfer_shared2kokkos(m_Xbeam, metrology.Xbeam);
119+
kokkostbx::transfer_shared2kokkos(m_Ybeam, metrology.Ybeam);
120+
fence();
121+
122+
// metrology.show();
123+
124+
// printf(" rangemap size:%d\n", m_rangemap.span());
125+
// printf(" omega_reduction size:%d\n", m_omega_reduction.span());
126+
// printf(" max_I_x_reduction size:%d\n", m_max_I_x_reduction.span());
127+
// printf(" max_I_y_reduction size:%d\n", m_max_I_y_reduction.span());
128+
// printf(" maskimage size:%d\n", m_maskimage.span());
129+
// printf(" floatimage size:%d\n", m_floatimage.span());
130+
// printf(" sdet_vector size:%d\n", m_sdet_vector.span());
131+
// printf(" fdet_vector size:%d\n", m_fdet_vector.span());
132+
// printf(" odet_vector size:%d\n", m_odet_vector.span());
133+
// printf(" pix0_vector size:%d\n", m_pix0_vector.span());
134+
// printf(" distance size:%d\n", m_distance.span());
135+
// printf(" Xbeam size:%d\n", m_Xbeam.span());
136+
// printf(" Ybeam size:%d\n", m_Ybeam.span());
137+
138+
// print_view(m_fdet_vector);
139+
// print_view(m_odet_vector, 1, 3);
140+
141+
// printf("DONE.\n");
142+
}
143+
template<> void
144+
kokkos_detector<small_whitelist_policy>::each_image_allocate() {
145+
resize(m_maskimage, m_total_pixel_count);
146+
kokkostbx::transfer_shared2kokkos(m_sdet_vector, metrology.sdet);
147+
kokkostbx::transfer_shared2kokkos(m_fdet_vector, metrology.fdet);
148+
kokkostbx::transfer_shared2kokkos(m_odet_vector, metrology.odet);
149+
kokkostbx::transfer_shared2kokkos(m_pix0_vector, metrology.pix0);
150+
kokkostbx::transfer_shared2kokkos(m_distance, metrology.dists);
151+
kokkostbx::transfer_shared2kokkos(m_Xbeam, metrology.Xbeam);
152+
kokkostbx::transfer_shared2kokkos(m_Ybeam, metrology.Ybeam);
153+
fence();
154+
}
155+
156+
template<>
157+
void
158+
kokkos_detector<large_array_policy>::set_active_pixels_on_GPU(af::shared<std::size_t> active_pixel_list_value) {
159+
m_active_pixel_size = active_pixel_list_value.size();
160+
kokkostbx::transfer_shared2kokkos(m_active_pixel_list, active_pixel_list_value);
161+
active_pixel_list = active_pixel_list_value;
162+
}
163+
164+
template<>
165+
void
166+
kokkos_detector<small_whitelist_policy>::set_active_pixels_on_GPU(af::shared<std::size_t> active_pixel_list_value) {
167+
m_active_pixel_size = active_pixel_list_value.size();
168+
kokkostbx::transfer_shared2kokkos(m_active_pixel_list, active_pixel_list_value);
169+
active_pixel_list = active_pixel_list_value;
170+
resize(m_rangemap, m_active_pixel_size);
171+
resize(m_omega_reduction, m_active_pixel_size);
172+
resize(m_max_I_x_reduction, m_active_pixel_size);
173+
resize(m_max_I_y_reduction, m_active_pixel_size);
174+
resize(m_floatimage, m_active_pixel_size);
175+
resize(m_accumulate_floatimage, m_active_pixel_size);
176+
fence();
177+
}
178+
179+
template<> af::shared<double>
180+
kokkos_detector<large_array_policy>::get_whitelist_raw_pixels(af::shared<std::size_t> selection) {
181+
hello();
182+
//return the data array for the multipanel detector case, but only for whitelist pixels
183+
vector_size_t active_pixel_selection = vector_size_t("active_pixel_selection", selection.size());
184+
kokkostbx::transfer_shared2kokkos(active_pixel_selection, selection);
185+
186+
size_t output_pixel_size = selection.size();
187+
vector_cudareal_t active_pixel_results = vector_cudareal_t("active_pixel_results", output_pixel_size);
188+
189+
auto temp = m_accumulate_floatimage;
190+
191+
parallel_for("get_active_pixel_selection",
192+
range_policy(0, output_pixel_size),
193+
KOKKOS_LAMBDA (const int i) {
194+
size_t index = active_pixel_selection( i );
195+
active_pixel_results( i ) = temp( index );
196+
});
197+
198+
af::shared<double> output_array(output_pixel_size, af::init_functor_null<double>());
199+
kokkostbx::transfer_kokkos2shared(output_array, active_pixel_results);
200+
201+
SCITBX_ASSERT(output_array.size() == output_pixel_size);
202+
return output_array;
203+
}
204+
template<> af::shared<double>
205+
kokkos_detector<small_whitelist_policy>::get_whitelist_raw_pixels(af::shared<std::size_t> selection) {
206+
SCITBX_CHECK_POINT;
207+
hello();
208+
//return the data array for the multipanel detector case, but only for whitelist pixels
209+
210+
std::size_t output_pixel_size = selection.size();
211+
//vector_cudareal_t active_pixel_results = vector_cudareal_t("active_pixel_results", output_pixel_size);
212+
213+
//auto temp = m_accumulate_floatimage;
214+
215+
//parallel_for("get_active_pixel_selection2",
216+
// range_policy(0, output_pixel_size),
217+
// KOKKOS_LAMBDA (const int i) {
218+
// active_pixel_results( i ) = temp( i );
219+
//});
220+
221+
af::shared<double> output_array(output_pixel_size, af::init_functor_null<double>());
222+
SCITBX_CHECK_POINT;
223+
kokkostbx::transfer_kokkos2shared(output_array, m_accumulate_floatimage);//active_pixel_results);
224+
SCITBX_CHECK_POINT;
225+
226+
SCITBX_ASSERT(output_array.size() == output_pixel_size);
227+
return output_array;
228+
}
229+
95230
} // Kokkos
96231
} // simtbx

‎simtbx/kokkos/detector.h‎

Lines changed: 7 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,8 @@ struct packed_metrology{
4141
af::shared<double>Ybeam;
4242
};
4343

44-
struct large_array_policy {
45-
};
46-
struct small_whitelist_policy {
47-
};
44+
struct large_array_policy {};
45+
struct small_whitelist_policy {};
4846

4947
template <typename MemoryPolicy>
5048
struct kokkos_detector
@@ -63,7 +61,7 @@ struct kokkos_detector
6361
//void write_raw_pixels(simtbx::nanoBragg::nanoBragg&);
6462
//af::flex_double get_raw_pixels();
6563
//void set_active_pixels_on_GPU(af::shared<std::size_t>);
66-
//af::shared<double> get_whitelist_raw_pixels(af::shared<std::size_t>);
64+
af::shared<double> get_whitelist_raw_pixels(af::shared<std::size_t>);
6765
inline void each_image_free(){} //no op in Kokkos
6866
int h_deviceID;
6967

@@ -157,46 +155,7 @@ struct kokkos_detector
157155
return view_floatimage;
158156
};
159157

160-
inline void
161-
each_image_allocate() {
162-
resize(m_rangemap, m_total_pixel_count);
163-
resize(m_omega_reduction, m_total_pixel_count);
164-
resize(m_max_I_x_reduction, m_total_pixel_count);
165-
resize(m_max_I_y_reduction, m_total_pixel_count);
166-
167-
resize(m_maskimage, m_total_pixel_count);
168-
resize(m_floatimage, m_total_pixel_count);
169-
170-
kokkostbx::transfer_shared2kokkos(m_sdet_vector, metrology.sdet);
171-
kokkostbx::transfer_shared2kokkos(m_fdet_vector, metrology.fdet);
172-
kokkostbx::transfer_shared2kokkos(m_odet_vector, metrology.odet);
173-
kokkostbx::transfer_shared2kokkos(m_pix0_vector, metrology.pix0);
174-
kokkostbx::transfer_shared2kokkos(m_distance, metrology.dists);
175-
kokkostbx::transfer_shared2kokkos(m_Xbeam, metrology.Xbeam);
176-
kokkostbx::transfer_shared2kokkos(m_Ybeam, metrology.Ybeam);
177-
fence();
178-
179-
// metrology.show();
180-
181-
// printf(" rangemap size:%d\n", m_rangemap.span());
182-
// printf(" omega_reduction size:%d\n", m_omega_reduction.span());
183-
// printf(" max_I_x_reduction size:%d\n", m_max_I_x_reduction.span());
184-
// printf(" max_I_y_reduction size:%d\n", m_max_I_y_reduction.span());
185-
// printf(" maskimage size:%d\n", m_maskimage.span());
186-
// printf(" floatimage size:%d\n", m_floatimage.span());
187-
// printf(" sdet_vector size:%d\n", m_sdet_vector.span());
188-
// printf(" fdet_vector size:%d\n", m_fdet_vector.span());
189-
// printf(" odet_vector size:%d\n", m_odet_vector.span());
190-
// printf(" pix0_vector size:%d\n", m_pix0_vector.span());
191-
// printf(" distance size:%d\n", m_distance.span());
192-
// printf(" Xbeam size:%d\n", m_Xbeam.span());
193-
// printf(" Ybeam size:%d\n", m_Ybeam.span());
194-
195-
// print_view(m_fdet_vector);
196-
// print_view(m_odet_vector, 1, 3);
197-
198-
// printf("DONE.\n");
199-
}
158+
void each_image_allocate();
200159

201160
inline void
202161
scale_in_place(const double& factor){
@@ -206,6 +165,8 @@ struct kokkos_detector
206165
});
207166
}
208167

168+
void set_active_pixels_on_GPU(af::shared<std::size_t> active_pixel_list_value);
169+
209170
inline void
210171
write_raw_pixels(simtbx::nanoBragg::nanoBragg& nB) {
211172
//only implement the monolithic detector case, one panel
@@ -242,37 +203,8 @@ struct kokkos_detector
242203
return output_array;
243204
}
244205

245-
inline void
246-
set_active_pixels_on_GPU(af::shared<std::size_t> active_pixel_list_value) {
247-
m_active_pixel_size = active_pixel_list_value.size();
248-
kokkostbx::transfer_shared2kokkos(m_active_pixel_list, active_pixel_list_value);
249-
active_pixel_list = active_pixel_list_value;
250-
}
251-
252-
inline af::shared<double>
253-
get_whitelist_raw_pixels(af::shared<std::size_t> selection) {
254-
//return the data array for the multipanel detector case, but only for whitelist pixels
255-
vector_size_t active_pixel_selection = vector_size_t("active_pixel_selection", selection.size());
256-
kokkostbx::transfer_shared2kokkos(active_pixel_selection, selection);
206+
void hello();
257207

258-
size_t output_pixel_size = selection.size();
259-
vector_cudareal_t active_pixel_results = vector_cudareal_t("active_pixel_results", output_pixel_size);
260-
261-
auto temp = m_accumulate_floatimage;
262-
263-
parallel_for("get_active_pixel_selection",
264-
range_policy(0, output_pixel_size),
265-
KOKKOS_LAMBDA (const int i) {
266-
size_t index = active_pixel_selection( i );
267-
active_pixel_results( i ) = temp( index );
268-
});
269-
270-
af::shared<double> output_array(output_pixel_size, af::init_functor_null<double>());
271-
kokkostbx::transfer_kokkos2shared(output_array, active_pixel_results);
272-
273-
SCITBX_ASSERT(output_array.size() == output_pixel_size);
274-
return output_array;
275-
}
276208
};
277209

278210

0 commit comments

Comments
 (0)