boost/compute/algorithm/copy.hpp
//---------------------------------------------------------------------------// // Copyright (c) 2013 Kyle Lutz <kyle.r.lutz@gmail.com> // // Distributed under the Boost Software License, Version 1.0 // See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt // // See http://boostorg.github.com/compute for more information. //---------------------------------------------------------------------------// #ifndef BOOST_COMPUTE_ALGORITHM_COPY_HPP #define BOOST_COMPUTE_ALGORITHM_COPY_HPP #include <algorithm> #include <iterator> #include <boost/utility/enable_if.hpp> #include <boost/mpl/and.hpp> #include <boost/mpl/not.hpp> #include <boost/mpl/or.hpp> #include <boost/compute/buffer.hpp> #include <boost/compute/system.hpp> #include <boost/compute/command_queue.hpp> #include <boost/compute/algorithm/detail/copy_on_device.hpp> #include <boost/compute/algorithm/detail/copy_to_device.hpp> #include <boost/compute/algorithm/detail/copy_to_host.hpp> #include <boost/compute/async/future.hpp> #include <boost/compute/container/mapped_view.hpp> #include <boost/compute/detail/device_ptr.hpp> #include <boost/compute/detail/is_contiguous_iterator.hpp> #include <boost/compute/detail/iterator_range_size.hpp> #include <boost/compute/detail/parameter_cache.hpp> #include <boost/compute/iterator/buffer_iterator.hpp> #include <boost/compute/type_traits/type_name.hpp> #include <boost/compute/type_traits/is_device_iterator.hpp> namespace boost { namespace compute { namespace detail { namespace mpl = boost::mpl; // meta-function returning true if copy() between InputIterator and // OutputIterator can be implemented with clEnqueueCopyBuffer(). template<class InputIterator, class OutputIterator> struct can_copy_with_copy_buffer : mpl::and_< mpl::or_< boost::is_same< InputIterator, buffer_iterator<typename InputIterator::value_type> >, boost::is_same< InputIterator, detail::device_ptr<typename InputIterator::value_type> > >, mpl::or_< boost::is_same< OutputIterator, buffer_iterator<typename OutputIterator::value_type> >, boost::is_same< OutputIterator, detail::device_ptr<typename OutputIterator::value_type> > >, boost::is_same< typename InputIterator::value_type, typename OutputIterator::value_type > >::type {}; // meta-function returning true if value_types of HostIterator and // DeviceIterator are same template<class HostIterator, class DeviceIterator> struct is_same_value_type : boost::is_same< typename boost::remove_cv< typename std::iterator_traits<HostIterator>::value_type >::type, typename boost::remove_cv< typename DeviceIterator::value_type >::type >::type {}; // meta-function returning true if value_type of HostIterator is bool template<class HostIterator> struct is_bool_value_type : boost::is_same< typename boost::remove_cv< typename std::iterator_traits<HostIterator>::value_type >::type, bool >::type {}; // host -> device (async) template<class InputIterator, class OutputIterator> inline future<OutputIterator> dispatch_copy_async(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< mpl::not_< is_device_iterator<InputIterator> >, is_device_iterator<OutputIterator>, is_same_value_type<InputIterator, OutputIterator> > >::type* = 0) { BOOST_STATIC_ASSERT_MSG( is_contiguous_iterator<InputIterator>::value, "copy_async() is only supported for contiguous host iterators" ); return copy_to_device_async(first, last, result, queue, events); } // host -> device (async) // Type mismatch between InputIterator and OutputIterator value_types template<class InputIterator, class OutputIterator> inline future<OutputIterator> dispatch_copy_async(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< mpl::not_< is_device_iterator<InputIterator> >, is_device_iterator<OutputIterator>, mpl::not_< is_same_value_type<InputIterator, OutputIterator> > > >::type* = 0) { BOOST_STATIC_ASSERT_MSG( is_contiguous_iterator<InputIterator>::value, "copy_async() is only supported for contiguous host iterators" ); typedef typename std::iterator_traits<InputIterator>::value_type input_type; const context &context = queue.get_context(); size_t count = iterator_range_size(first, last); if(count < size_t(1)) { return future<OutputIterator>(); } // map [first; last) to device and run copy kernel // on device for copying & casting ::boost::compute::mapped_view<input_type> mapped_host( // make sure it's a pointer to constant data // to force read only mapping const_cast<const input_type*>( ::boost::addressof(*first) ), count, context ); return copy_on_device_async( mapped_host.begin(), mapped_host.end(), result, queue, events ); } // host -> device // InputIterator is a contiguous iterator template<class InputIterator, class OutputIterator> inline OutputIterator dispatch_copy(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< mpl::not_< is_device_iterator<InputIterator> >, is_device_iterator<OutputIterator>, is_same_value_type<InputIterator, OutputIterator>, is_contiguous_iterator<InputIterator> > >::type* = 0) { return copy_to_device(first, last, result, queue, events); } // host -> device // Type mismatch between InputIterator and OutputIterator value_types // InputIterator is a contiguous iterator template<class InputIterator, class OutputIterator> inline OutputIterator dispatch_copy(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< mpl::not_< is_device_iterator<InputIterator> >, is_device_iterator<OutputIterator>, mpl::not_< is_same_value_type<InputIterator, OutputIterator> >, is_contiguous_iterator<InputIterator> > >::type* = 0) { typedef typename OutputIterator::value_type output_type; typedef typename std::iterator_traits<InputIterator>::value_type input_type; const device &device = queue.get_device(); // loading parameters std::string cache_key = std::string("__boost_compute_copy_to_device_") + type_name<input_type>() + "_" + type_name<output_type>(); boost::shared_ptr<parameter_cache> parameters = detail::parameter_cache::get_global_cache(device); uint_ map_copy_threshold; uint_ direct_copy_threshold; // calculate default values of thresholds if (device.type() & device::gpu) { // GPUs map_copy_threshold = 524288; // 0.5 MB direct_copy_threshold = 52428800; // 50 MB } else { // CPUs and other devices map_copy_threshold = 134217728; // 128 MB direct_copy_threshold = 0; // it's never efficient for CPUs } // load thresholds map_copy_threshold = parameters->get( cache_key, "map_copy_threshold", map_copy_threshold ); direct_copy_threshold = parameters->get( cache_key, "direct_copy_threshold", direct_copy_threshold ); // select copy method based on thresholds & input_size_bytes size_t count = iterator_range_size(first, last); size_t input_size_bytes = count * sizeof(input_type); // [0; map_copy_threshold) -> copy_to_device_map() if(input_size_bytes < map_copy_threshold) { return copy_to_device_map(first, last, result, queue, events); } // [map_copy_threshold; direct_copy_threshold) -> convert [first; last) // on host and then perform copy_to_device() else if(input_size_bytes < direct_copy_threshold) { std::vector<output_type> vector(first, last); return copy_to_device( vector.begin(), vector.end(), result, queue, events ); } // [direct_copy_threshold; inf) -> map [first; last) to device and // run copy kernel on device for copying & casting // At this point we are sure that count > 1 (first != last). // Perform async copy to device, wait for it to be finished and // return the result. // At this point we are sure that count > 1 (first != last), so event // returned by dispatch_copy_async() must be valid. return dispatch_copy_async(first, last, result, queue, events).get(); } // host -> device // InputIterator is NOT a contiguous iterator template<class InputIterator, class OutputIterator> inline OutputIterator dispatch_copy(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< mpl::not_< is_device_iterator<InputIterator> >, is_device_iterator<OutputIterator>, mpl::not_< is_contiguous_iterator<InputIterator> > > >::type* = 0) { typedef typename OutputIterator::value_type output_type; typedef typename std::iterator_traits<InputIterator>::value_type input_type; const device &device = queue.get_device(); // loading parameters std::string cache_key = std::string("__boost_compute_copy_to_device_") + type_name<input_type>() + "_" + type_name<output_type>(); boost::shared_ptr<parameter_cache> parameters = detail::parameter_cache::get_global_cache(device); uint_ map_copy_threshold; uint_ direct_copy_threshold; // calculate default values of thresholds if (device.type() & device::gpu) { // GPUs map_copy_threshold = 524288; // 0.5 MB direct_copy_threshold = 52428800; // 50 MB } else { // CPUs and other devices map_copy_threshold = 134217728; // 128 MB direct_copy_threshold = 0; // it's never efficient for CPUs } // load thresholds map_copy_threshold = parameters->get( cache_key, "map_copy_threshold", map_copy_threshold ); direct_copy_threshold = parameters->get( cache_key, "direct_copy_threshold", direct_copy_threshold ); // select copy method based on thresholds & input_size_bytes size_t input_size = iterator_range_size(first, last); size_t input_size_bytes = input_size * sizeof(input_type); // [0; map_copy_threshold) -> copy_to_device_map() // // if direct_copy_threshold is less than map_copy_threshold // copy_to_device_map() is used for every input if(input_size_bytes < map_copy_threshold || direct_copy_threshold <= map_copy_threshold) { return copy_to_device_map(first, last, result, queue, events); } // [map_copy_threshold; inf) -> convert [first; last) // on host and then perform copy_to_device() std::vector<output_type> vector(first, last); return copy_to_device(vector.begin(), vector.end(), result, queue, events); } // device -> host (async) template<class InputIterator, class OutputIterator> inline future<OutputIterator> dispatch_copy_async(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< is_device_iterator<InputIterator>, mpl::not_< is_device_iterator<OutputIterator> >, is_same_value_type<OutputIterator, InputIterator> > >::type* = 0) { BOOST_STATIC_ASSERT_MSG( is_contiguous_iterator<OutputIterator>::value, "copy_async() is only supported for contiguous host iterators" ); return copy_to_host_async(first, last, result, queue, events); } // device -> host (async) // Type mismatch between InputIterator and OutputIterator value_types template<class InputIterator, class OutputIterator> inline future<OutputIterator> dispatch_copy_async(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< is_device_iterator<InputIterator>, mpl::not_< is_device_iterator<OutputIterator> >, mpl::not_< is_same_value_type<OutputIterator, InputIterator> > > >::type* = 0) { BOOST_STATIC_ASSERT_MSG( is_contiguous_iterator<OutputIterator>::value, "copy_async() is only supported for contiguous host iterators" ); typedef typename std::iterator_traits<OutputIterator>::value_type output_type; const context &context = queue.get_context(); size_t count = iterator_range_size(first, last); if(count < size_t(1)) { return future<OutputIterator>(); } // map host memory to device buffer mapped_host( context, count * sizeof(output_type), buffer::write_only | buffer::use_host_ptr, static_cast<void*>( ::boost::addressof(*result) ) ); // copy async on device ::boost::compute::future<buffer_iterator<output_type> > future = copy_on_device_async( first, last, make_buffer_iterator<output_type>(mapped_host), queue, events ); // update host memory asynchronously by maping and unmaping memory event map_event; void* ptr = queue.enqueue_map_buffer_async( mapped_host, CL_MAP_READ, 0, count * sizeof(output_type), map_event, future.get_event() ); event unmap_event = queue.enqueue_unmap_buffer(mapped_host, ptr, map_event); return make_future(result + count, unmap_event); } // device -> host // OutputIterator is a contiguous iterator template<class InputIterator, class OutputIterator> inline OutputIterator dispatch_copy(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< is_device_iterator<InputIterator>, mpl::not_< is_device_iterator<OutputIterator> >, is_same_value_type<OutputIterator, InputIterator>, is_contiguous_iterator<OutputIterator>, mpl::not_< is_bool_value_type<OutputIterator> > > >::type* = 0) { return copy_to_host(first, last, result, queue, events); } // device -> host // Type mismatch between InputIterator and OutputIterator value_types // OutputIterator is NOT a contiguous iterator or value_type of OutputIterator // is a boolean type. template<class InputIterator, class OutputIterator> inline OutputIterator dispatch_copy(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< is_device_iterator<InputIterator>, mpl::not_< is_device_iterator<OutputIterator> >, mpl::or_< mpl::not_< is_contiguous_iterator<OutputIterator> >, is_bool_value_type<OutputIterator> > > >::type* = 0) { typedef typename std::iterator_traits<OutputIterator>::value_type output_type; typedef typename InputIterator::value_type input_type; const device &device = queue.get_device(); // loading parameters std::string cache_key = std::string("__boost_compute_copy_to_host_") + type_name<input_type>() + "_" + type_name<output_type>(); boost::shared_ptr<parameter_cache> parameters = detail::parameter_cache::get_global_cache(device); uint_ map_copy_threshold; uint_ direct_copy_threshold; // calculate default values of thresholds if (device.type() & device::gpu) { // GPUs map_copy_threshold = 33554432; // 30 MB direct_copy_threshold = 0; // it's never efficient for GPUs } else { // CPUs and other devices map_copy_threshold = 134217728; // 128 MB direct_copy_threshold = 0; // it's never efficient for CPUs } // load thresholds map_copy_threshold = parameters->get( cache_key, "map_copy_threshold", map_copy_threshold ); direct_copy_threshold = parameters->get( cache_key, "direct_copy_threshold", direct_copy_threshold ); // select copy method based on thresholds & input_size_bytes size_t count = iterator_range_size(first, last); size_t input_size_bytes = count * sizeof(input_type); // [0; map_copy_threshold) -> copy_to_host_map() // // if direct_copy_threshold is less than map_copy_threshold // copy_to_host_map() is used for every input if(input_size_bytes < map_copy_threshold || direct_copy_threshold <= map_copy_threshold) { return copy_to_host_map(first, last, result, queue, events); } // [map_copy_threshold; inf) -> copy [first;last) to temporary vector // then copy (and convert) to result using std::copy() std::vector<input_type> vector(count); copy_to_host(first, last, vector.begin(), queue, events); return std::copy(vector.begin(), vector.end(), result); } // device -> host // Type mismatch between InputIterator and OutputIterator value_types // OutputIterator is a contiguous iterator // value_type of OutputIterator is NOT a boolean type template<class InputIterator, class OutputIterator> inline OutputIterator dispatch_copy(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< is_device_iterator<InputIterator>, mpl::not_< is_device_iterator<OutputIterator> >, mpl::not_< is_same_value_type<OutputIterator, InputIterator> >, is_contiguous_iterator<OutputIterator>, mpl::not_< is_bool_value_type<OutputIterator> > > >::type* = 0) { typedef typename std::iterator_traits<OutputIterator>::value_type output_type; typedef typename InputIterator::value_type input_type; const device &device = queue.get_device(); // loading parameters std::string cache_key = std::string("__boost_compute_copy_to_host_") + type_name<input_type>() + "_" + type_name<output_type>(); boost::shared_ptr<parameter_cache> parameters = detail::parameter_cache::get_global_cache(device); uint_ map_copy_threshold; uint_ direct_copy_threshold; // calculate default values of thresholds if (device.type() & device::gpu) { // GPUs map_copy_threshold = 524288; // 0.5 MB direct_copy_threshold = 52428800; // 50 MB } else { // CPUs and other devices map_copy_threshold = 134217728; // 128 MB direct_copy_threshold = 0; // it's never efficient for CPUs } // load thresholds map_copy_threshold = parameters->get( cache_key, "map_copy_threshold", map_copy_threshold ); direct_copy_threshold = parameters->get( cache_key, "direct_copy_threshold", direct_copy_threshold ); // select copy method based on thresholds & input_size_bytes size_t count = iterator_range_size(first, last); size_t input_size_bytes = count * sizeof(input_type); // [0; map_copy_threshold) -> copy_to_host_map() if(input_size_bytes < map_copy_threshold) { return copy_to_host_map(first, last, result, queue, events); } // [map_copy_threshold; direct_copy_threshold) -> copy [first;last) to // temporary vector then copy (and convert) to result using std::copy() else if(input_size_bytes < direct_copy_threshold) { std::vector<input_type> vector(count); copy_to_host(first, last, vector.begin(), queue, events); return std::copy(vector.begin(), vector.end(), result); } // [direct_copy_threshold; inf) -> map [result; result + input_size) to // device and run copy kernel on device for copying & casting // map host memory to device. // Perform async copy to host, wait for it to be finished and // return the result. // At this point we are sure that count > 1 (first != last), so event // returned by dispatch_copy_async() must be valid. return dispatch_copy_async(first, last, result, queue, events).get(); } // device -> device template<class InputIterator, class OutputIterator> inline OutputIterator dispatch_copy(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< is_device_iterator<InputIterator>, is_device_iterator<OutputIterator>, mpl::not_< can_copy_with_copy_buffer< InputIterator, OutputIterator > > > >::type* = 0) { return copy_on_device(first, last, result, queue, events); } // device -> device (specialization for buffer iterators) template<class InputIterator, class OutputIterator> inline OutputIterator dispatch_copy(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< is_device_iterator<InputIterator>, is_device_iterator<OutputIterator>, can_copy_with_copy_buffer< InputIterator, OutputIterator > > >::type* = 0) { typedef typename std::iterator_traits<InputIterator>::value_type value_type; typedef typename std::iterator_traits<InputIterator>::difference_type difference_type; difference_type n = std::distance(first, last); if(n < 1){ // nothing to copy return result; } queue.enqueue_copy_buffer(first.get_buffer(), result.get_buffer(), first.get_index() * sizeof(value_type), result.get_index() * sizeof(value_type), static_cast<size_t>(n) * sizeof(value_type), events); return result + n; } // device -> device (async) template<class InputIterator, class OutputIterator> inline future<OutputIterator> dispatch_copy_async(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< is_device_iterator<InputIterator>, is_device_iterator<OutputIterator>, mpl::not_< can_copy_with_copy_buffer< InputIterator, OutputIterator > > > >::type* = 0) { return copy_on_device_async(first, last, result, queue, events); } // device -> device (async, specialization for buffer iterators) template<class InputIterator, class OutputIterator> inline future<OutputIterator> dispatch_copy_async(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if< mpl::and_< is_device_iterator<InputIterator>, is_device_iterator<OutputIterator>, can_copy_with_copy_buffer< InputIterator, OutputIterator > > >::type* = 0) { typedef typename std::iterator_traits<InputIterator>::value_type value_type; typedef typename std::iterator_traits<InputIterator>::difference_type difference_type; difference_type n = std::distance(first, last); if(n < 1){ // nothing to copy return make_future(result, event()); } event event_ = queue.enqueue_copy_buffer( first.get_buffer(), result.get_buffer(), first.get_index() * sizeof(value_type), result.get_index() * sizeof(value_type), static_cast<size_t>(n) * sizeof(value_type), events ); return make_future(result + n, event_); } // host -> host template<class InputIterator, class OutputIterator> inline OutputIterator dispatch_copy(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue, const wait_list &events, typename boost::enable_if_c< !is_device_iterator<InputIterator>::value && !is_device_iterator<OutputIterator>::value >::type* = 0) { (void) queue; (void) events; return std::copy(first, last, result); } } // end detail namespace /// Copies the values in the range [\p first, \p last) to the range /// beginning at \p result. /// /// The generic copy() function can be used for a variety of data /// transfer tasks and provides a standard interface to the following /// OpenCL functions: /// /// \li \c clEnqueueReadBuffer() /// \li \c clEnqueueWriteBuffer() /// \li \c clEnqueueCopyBuffer() /// /// Unlike the aforementioned OpenCL functions, copy() will also work /// with non-contiguous data-structures (e.g. \c std::list<T>) as /// well as with "fancy" iterators (e.g. transform_iterator). /// /// \param first first element in the range to copy /// \param last last element in the range to copy /// \param result first element in the result range /// \param queue command queue to perform the operation /// /// \return \c OutputIterator to the end of the result range /// /// For example, to copy an array of \c int values on the host to a vector on /// the device: /// \code /// // array on the host /// int data[] = { 1, 2, 3, 4 }; /// /// // vector on the device /// boost::compute::vector<int> vec(4, context); /// /// // copy values to the device vector /// boost::compute::copy(data, data + 4, vec.begin(), queue); /// \endcode /// /// The copy algorithm can also be used with standard containers such as /// \c std::vector<T>: /// \code /// std::vector<int> host_vector = ... /// boost::compute::vector<int> device_vector = ... /// /// // copy from the host to the device /// boost::compute::copy( /// host_vector.begin(), host_vector.end(), device_vector.begin(), queue /// ); /// /// // copy from the device to the host /// boost::compute::copy( /// device_vector.begin(), device_vector.end(), host_vector.begin(), queue /// ); /// \endcode /// /// Space complexity: \Omega(1) /// /// \see copy_n(), copy_if(), copy_async() template<class InputIterator, class OutputIterator> inline OutputIterator copy(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue = system::default_queue(), const wait_list &events = wait_list()) { return detail::dispatch_copy(first, last, result, queue, events); } /// Copies the values in the range [\p first, \p last) to the range /// beginning at \p result. The copy is performed asynchronously. /// /// \see copy() template<class InputIterator, class OutputIterator> inline future<OutputIterator> copy_async(InputIterator first, InputIterator last, OutputIterator result, command_queue &queue = system::default_queue(), const wait_list &events = wait_list()) { return detail::dispatch_copy_async(first, last, result, queue, events); } } // end compute namespace } // end boost namespace #endif // BOOST_COMPUTE_ALGORITHM_COPY_HPP