boost/mpi/python/serialize.hpp
// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com> // Use, modification and distribution is subject to the Boost Software // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt) // Authors: Douglas Gregor /** @file serialize.hpp * * This file provides Boost.Serialization support for Python objects * within Boost.MPI. Python objects can be serialized in one of two * ways. The default serialization method involves using the Python * "pickle" module to pickle the Python objects, transmits the * pickled representation, and unpickles the result when * received. For C++ types that have been exposed to Python and * registered with register_serialized(), objects are directly * serialized for transmissing, skipping the pickling step. */ #ifndef BOOST_MPI_PYTHON_SERIALIZE_HPP #define BOOST_MPI_PYTHON_SERIALIZE_HPP #include <boost/mpi/python/config.hpp> #include <boost/python/object.hpp> #include <boost/python/str.hpp> #include <boost/python/extract.hpp> #include <map> #include <boost/function/function3.hpp> #include <boost/mpl/bool.hpp> #include <boost/mpl/if.hpp> #include <boost/serialization/split_free.hpp> #include <boost/serialization/array.hpp> #include <boost/serialization/array_wrapper.hpp> #include <boost/smart_ptr/scoped_array.hpp> #include <boost/assert.hpp> #include <boost/type_traits/is_fundamental.hpp> #define BOOST_MPI_PYTHON_FORWARD_ONLY #include <boost/mpi/python.hpp> #include "bytesobject.h" /************************************************************************ * Boost.Python Serialization Section * ************************************************************************/ #if !defined(BOOST_NO_SFINAE) && !defined(BOOST_NO_IS_CONVERTIBLE) /** * @brief Declare IArchive and OArchive as a Boost.Serialization * archives that can be used for Python objects. * * This macro can only be expanded from the global namespace. It only * requires that Archiver be forward-declared. IArchiver and OArchiver * will only support Serialization of Python objects by pickling * them. If the Archiver type should also support "direct" * serialization (for C++ types), use * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE instead. */ # define BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver) \ namespace boost { namespace python { namespace api { \ template<typename R, typename T> \ struct enable_binary< IArchiver , R, T> {}; \ \ template<typename R, typename T> \ struct enable_binary< OArchiver , R, T> {}; \ } } } # else # define BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver) #endif /** * @brief Declare IArchiver and OArchiver as a Boost.Serialization * archives that can be used for Python objects and C++ objects * wrapped in Python. * * This macro can only be expanded from the global namespace. It only * requires that IArchiver and OArchiver be forward-declared. However, * note that you will also need to write * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL(IArchiver, * OArchiver) in one of your translation units. DPG PICK UP HERE */ #define BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(IArchiver, OArchiver) \ BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver) \ namespace boost { namespace python { namespace detail { \ template<> \ BOOST_MPI_PYTHON_DECL direct_serialization_table< IArchiver , OArchiver >& \ get_direct_serialization_table< IArchiver , OArchiver >(); \ } \ \ template<> \ struct has_direct_serialization< IArchiver , OArchiver> : mpl::true_ { }; \ \ template<> \ struct output_archiver< IArchiver > { typedef OArchiver type; }; \ \ template<> \ struct input_archiver< OArchiver > { typedef IArchiver type; }; \ } } /** * @brief Define the implementation for Boost.Serialization archivers * that can be used for Python objects and C++ objects wrapped in * Python. * * This macro can only be expanded from the global namespace. It only * requires that IArchiver and OArchiver be forward-declared. Before * using this macro, you will need to declare IArchiver and OArchiver * as direct serialization archives with * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(IArchiver, OArchiver). */ #define BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL(IArchiver, OArchiver) \ namespace boost { namespace python { namespace detail { \ template \ class BOOST_MPI_PYTHON_DECL direct_serialization_table< IArchiver , OArchiver >; \ \ template<> \ BOOST_MPI_PYTHON_DECL \ direct_serialization_table< IArchiver , OArchiver >& \ get_direct_serialization_table< IArchiver , OArchiver >( ) \ { \ static direct_serialization_table< IArchiver, OArchiver > table; \ return table; \ } \ } } } namespace boost { namespace python { /** * INTERNAL ONLY * * Provides access to the Python "pickle" module from within C++. */ class BOOST_MPI_PYTHON_DECL pickle { struct data_t; public: static object dumps(object obj, int protocol = -1); static object loads(object s); private: static void initialize_data(); static data_t* data; }; /** * @brief Whether the input/output archiver pair has "direct" * serialization for C++ objects exposed in Python. * * Users do not typically need to specialize this trait, as it will be * specialized as part of the macro * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE. */ template<typename IArchiver, typename OArchiver> struct has_direct_serialization : mpl::false_ { }; /** * @brief A metafunction that determines the output archiver for the * given input archiver. * * Users do not typically need to specialize this trait, as it will be * specialized as part of the macro * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE. */ template<typename IArchiver> struct output_archiver { }; /** * @brief A metafunction that determines the input archiver for the * given output archiver. * * Users do not typically need to specialize this trait, as it will be * specialized as part of the macro * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE. * */ template<typename OArchiver> struct input_archiver { }; namespace detail { /** * INTERNAL ONLY * * This class contains the direct-serialization code for the given * IArchiver/OArchiver pair. It is intended to be used as a * singleton class, and will be accessed when (de-)serializing a * Boost.Python object with an archiver that supports direct * serializations. Do not create instances of this class directly: * instead, use get_direct_serialization_table. */ template<typename IArchiver, typename OArchiver> class BOOST_MPI_PYTHON_DECL direct_serialization_table { public: typedef boost::function3<void, OArchiver&, const object&, const unsigned int> saver_t; typedef boost::function3<void, IArchiver&, object&, const unsigned int> loader_t; typedef std::map<PyTypeObject*, std::pair<int, saver_t> > savers_t; typedef std::map<int, loader_t> loaders_t; /** * Retrieve the saver (serializer) associated with the Python * object @p obj. * * @param obj The object we want to save. Only its (Python) type * is important. * * @param descriptor The value of the descriptor associated to * the returned saver. Will be set to zero if no saver was found * for @p obj. * * @returns a function object that can be used to serialize this * object (and other objects of the same type), if possible. If * no saver can be found, returns an empty function object.. */ saver_t saver(const object& obj, int& descriptor) { typename savers_t::iterator pos = savers.find(obj.ptr()->ob_type); if (pos != savers.end()) { descriptor = pos->second.first; return pos->second.second; } else { descriptor = 0; return saver_t(); } } /** * Retrieve the loader (deserializer) associated with the given * descriptor. * * @param descriptor The descriptor number provided by saver() * when determining the saver for this type. * * @returns a function object that can be used to deserialize an * object whose type is the same as that corresponding to the * descriptor. If the descriptor is unknown, the return value * will be an empty function object. */ loader_t loader(int descriptor) { typename loaders_t::iterator pos = loaders.find(descriptor); if (pos != loaders.end()) return pos->second; else return loader_t(); } /** * Register the type T for direct serialization. * * @param value A sample value of the type @c T. This may be used * to compute the Python type associated with the C++ type @c T. * * @param type The Python type associated with the C++ type @c * T. If not provided, it will be computed from the same value @p * value. */ template<typename T> void register_type(const T& value = T(), PyTypeObject* type = 0) { // If the user did not provide us with a Python type, figure it // out for ourselves. if (!type) { object obj(value); type = obj.ptr()->ob_type; } register_type(default_saver<T>(), default_loader<T>(type), value, type); } /** * Register the type T for direct serialization. * * @param saver A function object that will serialize a * Boost.Python object (that represents a C++ object of type @c * T) to an @c OArchive. * * @param loader A function object that will deserialize from an * @c IArchive into a Boost.Python object that represents a C++ * object of type @c T. * * @param value A sample value of the type @c T. This may be used * to compute the Python type associated with the C++ type @c T. * * @param type The Python type associated with the C++ type @c * T. If not provided, it will be computed from the same value @p * value. */ template<typename T> void register_type(const saver_t& saver, const loader_t& loader, const T& value = T(), PyTypeObject* type = 0) { // If the user did not provide us with a Python type, figure it // out for ourselves. if (!type) { object obj(value); type = obj.ptr()->ob_type; } int descriptor = savers.size() + 1; if (savers.find(type) != savers.end()) return; savers[type] = std::make_pair(descriptor, saver); loaders[descriptor] = loader; } protected: template<typename T> struct default_saver { void operator()(OArchiver& ar, const object& obj, const unsigned int) { T value = extract<T>(obj)(); ar << value; } }; template<typename T> struct default_loader { default_loader(PyTypeObject* type) : type(type) { } void operator()(IArchiver& ar, object& obj, const unsigned int) { // If we can, extract the object in place. if (!is_fundamental<T>::value && obj && obj.ptr()->ob_type == type) { ar >> extract<T&>(obj)(); } else { T value; ar >> value; obj = object(value); } } private: PyTypeObject* type; }; savers_t savers; loaders_t loaders; }; /** * @brief Retrieve the direct-serialization table for an * IArchiver/OArchiver pair. * * This function is responsible for returning a reference to the * singleton direct-serialization table. Its primary template is * left undefined, to force the use of an explicit specialization * with a definition in a single translation unit. Use the macro * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL to define this * explicit specialization. */ template<typename IArchiver, typename OArchiver> direct_serialization_table<IArchiver, OArchiver>& get_direct_serialization_table(); } // end namespace detail /** * @brief Register the type T for direct serialization. * * The @c register_serialized function registers a C++ type for direct * serialization with the given @c IArchiver/@c OArchiver pair. Direct * serialization elides the use of the Python @c pickle package when * serializing Python objects that represent C++ values. Direct * serialization can be beneficial both to improve serialization * performance (Python pickling can be very inefficient) and to permit * serialization for Python-wrapped C++ objects that do not support * pickling. * * @param value A sample value of the type @c T. This may be used * to compute the Python type associated with the C++ type @c T. * * @param type The Python type associated with the C++ type @c * T. If not provided, it will be computed from the same value @p * value. */ template<typename IArchiver, typename OArchiver, typename T> void register_serialized(const T& value = T(), PyTypeObject* type = 0) { detail::direct_serialization_table<IArchiver, OArchiver>& table = detail::get_direct_serialization_table<IArchiver, OArchiver>(); table.register_type(value, type); } namespace detail { /// Save a Python object by pickling it. template<typename Archiver> void save_impl(Archiver& ar, const boost::python::object& obj, const unsigned int /*version*/, mpl::false_ /*has_direct_serialization*/) { boost::python::object bytes = boost::python::pickle::dumps(obj); int sz = PyBytes_Size(bytes.ptr()); char *data = PyBytes_AsString(bytes.ptr()); ar << sz << boost::serialization::make_array(data, sz); } /// Try to save a Python object by directly serializing it; fall back /// on pickling if required. template<typename Archiver> void save_impl(Archiver& ar, const boost::python::object& obj, const unsigned int version, mpl::true_ /*has_direct_serialization*/) { typedef Archiver OArchiver; typedef typename input_archiver<OArchiver>::type IArchiver; typedef typename direct_serialization_table<IArchiver, OArchiver>::saver_t saver_t; direct_serialization_table<IArchiver, OArchiver>& table = get_direct_serialization_table<IArchiver, OArchiver>(); int descriptor = 0; if (saver_t saver = table.saver(obj, descriptor)) { ar << descriptor; saver(ar, obj, version); } else { // Pickle it ar << descriptor; detail::save_impl(ar, obj, version, mpl::false_()); } } /// Load a Python object by unpickling it template<typename Archiver> void load_impl(Archiver& ar, boost::python::object& obj, const unsigned int /*version*/, mpl::false_ /*has_direct_serialization*/) { int len; ar >> len; boost::scoped_array<char> data(new char[len]); ar >> boost::serialization::make_array(data.get(), len); boost::python::object bytes(boost::python::handle<>(PyBytes_FromStringAndSize(data.get(), len))); obj = boost::python::pickle::loads(bytes); } /// Try to load a Python object by directly deserializing it; fall back /// on unpickling if required. template<typename Archiver> void load_impl(Archiver& ar, boost::python::object& obj, const unsigned int version, mpl::true_ /*has_direct_serialization*/) { typedef Archiver IArchiver; typedef typename output_archiver<IArchiver>::type OArchiver; typedef typename direct_serialization_table<IArchiver, OArchiver>::loader_t loader_t; direct_serialization_table<IArchiver, OArchiver>& table = get_direct_serialization_table<IArchiver, OArchiver>(); int descriptor; ar >> descriptor; if (descriptor) { loader_t loader = table.loader(descriptor); BOOST_ASSERT(loader); loader(ar, obj, version); } else { // Unpickle it detail::load_impl(ar, obj, version, mpl::false_()); } } } // end namespace detail template<typename Archiver> void save(Archiver& ar, const boost::python::object& obj, const unsigned int version) { typedef Archiver OArchiver; typedef typename input_archiver<OArchiver>::type IArchiver; detail::save_impl(ar, obj, version, has_direct_serialization<IArchiver, OArchiver>()); } template<typename Archiver> void load(Archiver& ar, boost::python::object& obj, const unsigned int version) { typedef Archiver IArchiver; typedef typename output_archiver<IArchiver>::type OArchiver; detail::load_impl(ar, obj, version, has_direct_serialization<IArchiver, OArchiver>()); } template<typename Archive> inline void serialize(Archive& ar, boost::python::object& obj, const unsigned int version) { boost::serialization::split_free(ar, obj, version); } } } // end namespace boost::python /************************************************************************ * Boost.MPI-Specific Section * ************************************************************************/ namespace boost { namespace mpi { class packed_iarchive; class packed_oarchive; } } // end namespace boost::mpi BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE( ::boost::mpi::packed_iarchive, ::boost::mpi::packed_oarchive) namespace boost { namespace mpi { namespace python { template<typename T> void register_serialized(const T& value, PyTypeObject* type) { using boost::python::register_serialized; register_serialized<packed_iarchive, packed_oarchive>(value, type); } } } } // end namespace boost::mpi::python #endif // BOOST_MPI_PYTHON_SERIALIZE_HPP