lunchbox::PersistentMap Class Reference

Unified interface to save key-value pairs in a persistent store. More...

#include <persistentMap.h>

Inheritance diagram for lunchbox::PersistentMap:

Collaboration diagram for lunchbox::PersistentMap:

Public Member Functions
LUNCHBOX_API	PersistentMap (const std::string &uri=std::string())
	Construct a new persistent map. More...
LUNCHBOX_API	PersistentMap (const servus::URI &uri)
	Construct a persistent map using an URI. More...
LUNCHBOX_API	~PersistentMap ()
	Destruct the persistent map. More...
LUNCHBOX_API size_t	setQueueDepth (const size_t depth)
	Set the maximum number of asynchronous outstanding write operations. More...
template<class V >
bool	insert (const std::string &key, const V &value)
	Insert or update a value in the database. More...
template<class V >
bool	insert (const std::string &key, const std::vector< V > &values)
	Insert or update a vector of values in the database. More...
template<class V >
bool	insert (const std::string &key, const std::set< V > &values)
	Insert or update a set of values in the database. More...
LUNCHBOX_API std::string	operator[] (const std::string &key) const
	Retrieve a value for a key. More...
template<class V >
V	get (const std::string &key) const
	Retrieve a value for a key. More...
template<class V >
std::vector< V >	getVector (const std::string &key) const
	Retrieve a value as a vector for a key. More...
template<class V >
std::set< V >	getSet (const std::string &key) const
	Retrieve a value as a set for a key. More...
LUNCHBOX_API bool	fetch (const std::string &key, size_t sizeHint=0) const
	Asynchronously retrieve a value which to be read later. More...
LUNCHBOX_API bool	contains (const std::string &key) const
LUNCHBOX_API bool	flush ()
	Flush outstanding operations to the backend storage. More...
LUNCHBOX_API void	setByteswap (const bool swap)
	Enable or disable endianness conversion on reads. More...
template<>
bool	_insert (const std::string &k, const std::string &v, const boost::false_type &)

Static Public Member Functions
static LUNCHBOX_API bool	handles (const servus::URI &uri)

Detailed Description

Unified interface to save key-value pairs in a persistent store.

Example:

/* Copyright (c) 2014-2015, Stefan.Eilemann@epfl.ch
 *
 * This library is free software; you can redistribute it and/or modify it under
 * the terms of the GNU Lesser General Public License version 2.1 as published
 * by the Free Software Foundation.
 *
 * This library is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
 * details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this library; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#define TEST_RUNTIME 240 //seconds
#include <test.h>
#include <lunchbox/clock.h>
#include <lunchbox/os.h>
#include <lunchbox/persistentMap.h>
#include <lunchbox/rng.h>
#ifdef LUNCHBOX_USE_LEVELDB
#  include <leveldb/db.h>
#endif
#ifdef LUNCHBOX_USE_SKV
#  include <FxLogger/FxLogger.hpp>
#endif
#include <boost/format.hpp>
#include <stdexcept>

using lunchbox::PersistentMap;

const int ints[] = { 17, 53, 42, 65535, 32768 };
const size_t numInts = sizeof( ints ) / sizeof( int );
const int64_t loopTime = 1000;
bool perfTest = false;

template< class T > void insertVector( PersistentMap& map )
{
    std::vector< T > vector;
    for( size_t i = 0; i < numInts; ++i )
        vector.push_back( T( ints[ i ] ));
    TEST( map.insert( typeid( vector ).name(), vector ));
}

template< class T > void readVector( const PersistentMap& map )
{
    const std::vector< T >& vector =
        map.getVector< T >( typeid( vector ).name( ));
    TESTINFO( vector.size() ==  numInts, vector.size() << " != " << numInts );
    for( size_t i = 0; i < numInts; ++i )
        TEST( vector[ i ] == T( ints[i] ));
}

template< class T > void insertVector( PersistentMap& map, const size_t elems )
{
    std::vector< T > vector;
    for( size_t i = 0; i < elems; ++i )
        vector.push_back( i );
    TEST( map.insert( std::string( "bulk" ) + typeid( vector ).name(), vector ));
}

template< class T > void readVector( const PersistentMap& map, const size_t elems )
{
    const std::vector< T >& vector =
        map.getVector< T >( std::string( "bulk" ) + typeid( vector ).name());
    TESTINFO( vector.size() ==  elems, vector.size() << " != " << elems );
    for( size_t i = 0; i < numInts; ++i )
        TESTINFO( vector[ i ] == T( i ), vector[ i ] << " != " << i );
}

void read( const PersistentMap& map )
{
    const std::set< uint32_t >& bigSet =
        map.getSet< uint32_t >( "std::set< uint32_t >" );
    TEST( bigSet.size() == 1000 );
    for( uint32_t i = 1; i <= 1000; ++i )
        TEST( bigSet.find( i ) != bigSet.end( ));

    TEST( map[ "foo" ] == "bar" );
    TEST( map[ "bar" ].empty( ));
    TEST( map.get< bool >( "bValue" ) == true );
    TEST( map.get< int >( "iValue" ) == 42 );

    readVector< int >( map );
    readVector< uint16_t >( map );

    const std::set< int >& set = map.getSet< int >( "std::set< int >" );
    TESTINFO( set.size() ==  numInts, set.size() << " != " << numInts );
    for( size_t i = 0; i < numInts; ++i )
        TESTINFO( set.find( ints[i] ) != set.end(),
                  ints[i] << " not found in set" );
}

void read( const std::string& uri )
{
    PersistentMap map( uri );
    read( map );
}

void setup( const std::string& uri )
{
    PersistentMap map( uri );
    TEST( map.insert( "foo", "bar" ));
    TEST( map.contains( "foo" ));
    TESTINFO( map[ "foo" ] == "bar",
              map[ "foo" ] << " length " << map[ "foo" ].length( ));
    TEST( map[ "bar" ].empty( ));

    TEST( map.insert( "the quick brown fox", "jumped over something" ));
    TESTINFO( map[ "the quick brown fox" ] == "jumped over something",
              map[ "the quick brown fox" ] );

    TEST( map.insert( "hans", std::string( "dampf" )));
    TESTINFO( map[ "hans" ] == "dampf", map[ "hans" ] );

    const bool bValue = true;
    TEST( map.insert( "bValue", bValue ));
    TEST( map.get< bool >( "bValue" ) == bValue );

    const int iValue = 42;
    TEST( map.insert( "iValue", iValue ));
    TEST( map.get< int >( "iValue" ) == iValue );

    TEST( map.insert( "coffee", 0xC0FFEE ));
    map.setByteswap( true );
    TEST( map.get< unsigned >( "coffee" ) == 0xEEFFC000u );
    map.setByteswap( false );
    TEST( map.get< int >( "coffee" ) == 0xC0FFEE );

    insertVector< int >( map );
    insertVector< uint16_t >( map );
    readVector< int >( map );
    readVector< uint16_t >( map );

    insertVector< int >( map, LB_128KB );
    insertVector< uint16_t >( map, LB_128KB );
    map.fetch( std::string( "bulk" ) + typeid( std::vector< int > ).name( ));
    map.fetch( std::string( "bulk" ) + typeid( std::vector<uint16_t> ).name( ));
    readVector< int >( map, LB_128KB );
    readVector< uint16_t >( map, LB_128KB );

    std::set< int > set( ints, ints + numInts );
    TEST( map.insert( "std::set< int >", set ));

    std::set< uint32_t > bigSet;
    for( uint32_t i = 1; i <= 1000; ++i )
        bigSet.insert( i );
    TEST( map.insert( "std::set< uint32_t >", bigSet ));

    read( map );
}

void benchmark( const std::string& uri, const uint64_t queueDepth,
                const size_t valueSize )
{
    static std::string lastURI;
    if( uri != lastURI )
    {
        std::cout << uri << std::endl;
        lastURI = uri;
    }

    PersistentMap map( uri );
    map.setQueueDepth( queueDepth );

    // Prepare keys and value
    lunchbox::Strings keys;
    keys.resize( queueDepth + 1 );
    for( uint64_t i = 0; i <= queueDepth; ++i )
        keys[i].assign( reinterpret_cast< char* >( &i ), 8 );

    std::string value( valueSize, '*' );
    lunchbox::RNG rng;
    for( size_t i = 0; i < valueSize; ++i )
        value[i] = rng.get<char>();

    // write performance
    lunchbox::Clock clock;
    uint64_t i = 0;
    while( clock.getTime64() < loopTime )
    {
        map.insert( keys[ i % (queueDepth+1) ], value );
        ++i;
    }
    map.flush();
    const float writeTime = clock.getTimef() / 1000.f;
    const uint64_t wOps = i;
    TEST( i > queueDepth );

    // read performance
    clock.reset();
    if( queueDepth == 0 ) // sync read
    {
        for( i = 0; i < wOps && clock.getTime64() < loopTime; ++i ) // read keys
            map[ keys[ i % (queueDepth+1) ]];
    }
    else // fetch + async read
    {
        for( i = 0; i < queueDepth; ++i ) // prefetch queueDepth keys
            TEST( map.fetch( keys[ i % (queueDepth+1) ], valueSize ) );

        for( ; i < wOps && clock.getTime64() < loopTime; ++i ) // read keys
        {
            map[ keys[ (i - queueDepth) % (queueDepth+1) ] ];
            TEST( map.fetch( keys[ i % (queueDepth+1) ], valueSize ));
        }

        for( uint64_t j = i - queueDepth; j <= i; ++j ) // drain fetched keys
            map[ keys[ j % (queueDepth+1) ]];
    }

    const float readTime = clock.getTimef() / 1000.f;
    const size_t rOps = i;

    std::cout << boost::format( "%6i, %6i, %9.2f, %9.2f, %9.2f, %9.2f")
        // cppcheck-suppress zerodivcond
        % queueDepth % valueSize % (rOps/readTime) % (wOps/writeTime)
        % (rOps/1024.f/1024.f*valueSize/readTime)
        % (wOps/1024.f/1024.f*valueSize/writeTime) << std::endl;


    if( !perfTest )
    {
        // check contents of store (not all to save time on bigger tests)
        for( uint64_t j = 0; j < wOps && clock.getTime64() < loopTime; ++j )
        {
            const std::string& val = map[ keys[ j % (queueDepth+1) ]];
            TESTINFO( val.size() == valueSize,
                      val.size() << " != " << valueSize );
            TEST( val == value );
        }
    }

    // try to make sure there's nothing outstanding if we messed up in our test
    map.flush();
}

void testGenericFailures()
{
    try
    {
        setup( "foobar://" );
    }
    catch( const std::runtime_error& )
    {
        return;
    }
    TESTINFO( false, "Missing exception" );
}

void testLevelDBFailures()
{
#ifdef LUNCHBOX_USE_LEVELDB
    try
    {
        setup( "leveldb:///doesnotexist/deadbeef/coffee" );
    }
    catch( const std::runtime_error& )
    {
        return;
    }
    TESTINFO( false, "Missing exception" );
#endif
}

int main( int, char* argv[] )
{
    perfTest = std::string( argv[0] ).find( "perf-" ) != std::string::npos;
    if( perfTest )
        std::cout
            << " async,  value,   reads/s,  writes/s, read MB/s, write MB/s"
            << std::endl;
    try
    {
#ifdef LUNCHBOX_USE_LEVELDB
        setup( "" );
        setup( "leveldb://" );
        setup( "leveldb://persistentMap2.leveldb" );
        read( "" );
        read( "leveldb://" );
        read( "leveldb://persistentMap2.leveldb" );
        if( perfTest )
            for( size_t i=1; i <= 65536; i = i<<2 )
                benchmark( "leveldb://", 0, i );
#endif
#ifdef LUNCHBOX_USE_SKV
        FxLogger_Init( argv[0] );
        setup( "skv://" );
        read( "skv://" );
        if( perfTest )
        {
            benchmark( "skv://", 0, 64 );
            for( size_t i=1; i <= 65536; i = i<<1 )
                benchmark( "skv://", i, 64 );
            for( size_t i=1; i <= 65536; i = i<<2 )
                benchmark( "skv://", 65536, i );
        }
#endif
    }
#ifdef LUNCHBOX_USE_LEVELDB
    catch( const leveldb::Status& status )
    {
        TESTINFO( !"exception", status.ToString( ));
    }
#endif
    catch( const std::runtime_error& error )
    {
#ifdef LUNCHBOX_USE_SKV
        if( error.what() !=
            std::string( "skv init failed: SKV_ERRNO_CONN_FAILED" ))
#endif
        {
            TESTINFO( !"exception", error.what( ));
        }
    }

    testGenericFailures();
    testLevelDBFailures();

    return EXIT_SUCCESS;
}

Definition at line 48 of file persistentMap.h.

Constructor & Destructor Documentation

LUNCHBOX_API lunchbox::PersistentMap::PersistentMap ( const std::string &  uri = std::string() )

explicit

Construct a new persistent map.

Depending on the URI scheme an implementation backend is chosen. If no URI is given, a default one is selected. Available implementations are:

• leveldb://path (if LUNCHBOX_USE_LEVELDB is defined)
• skv://path_to_config#pdsname (if LUNCHBOX_USE_SKV is defined)

Parameters

uri	the storage backend and destination.

Exceptions

std::runtime_error	if no suitable implementation is found.
std::runtime_error	if opening the leveldb failed.

Version

1.9.2

LUNCHBOX_API lunchbox::PersistentMap::PersistentMap ( const servus::URI &  uri )

explicit

Construct a persistent map using an URI.

See other ctor for details.

Version

1.9.2

LUNCHBOX_API lunchbox::PersistentMap::~PersistentMap

(

)

Destruct the persistent map.

Version

1.9.2

Member Function Documentation

LUNCHBOX_API bool lunchbox::PersistentMap::contains

(

const std::string &

key

)

const

Returns

true if the key exists.

Version

1.9.2

LUNCHBOX_API bool lunchbox::PersistentMap::fetch	(	const std::string &	key,
		size_t	sizeHint = 0
	)		const

Asynchronously retrieve a value which to be read later.

Might be implemented as a 'NOP' by backend implementations.

Parameters

key	the key to retrieve.
sizeHint	the size of the value, may be ignored by implementation.

Returns

false on error, true otherwise.

Version

1.11

LUNCHBOX_API bool lunchbox::PersistentMap::flush

(

)

Flush outstanding operations to the backend storage.

Version

1.11

template<class V >

V lunchbox::PersistentMap::get ( const std::string &  key ) const

inline

Retrieve a value for a key.

Parameters

key	the key to retrieve.

Returns

the value, or an empty string if the key is not available.

Version

1.11

Definition at line 150 of file persistentMap.h.

template<class V >

std::set< V > lunchbox::PersistentMap::getSet ( const std::string &  key ) const

inline

Retrieve a value as a set for a key.

Parameters

key	the key to retrieve.

Returns

the values, or an empty set if the key is not available.

Version

1.9.2

Definition at line 270 of file persistentMap.h.

References lunchbox::byteswap().

Here is the call graph for this function:

template<class V >

std::vector< V > lunchbox::PersistentMap::getVector ( const std::string &  key ) const

inline

Retrieve a value as a vector for a key.

Parameters

key	the key to retrieve.

Returns

the values, or an empty vector if the key is not available.

Version

1.9.2

Definition at line 256 of file persistentMap.h.

References lunchbox::byteswap().

Here is the call graph for this function:

static LUNCHBOX_API bool lunchbox::PersistentMap::handles ( const servus::URI &  uri )

static

Returns

true if an implementation for the given URI is available.

Version

1.9.2

template<class V >

bool lunchbox::PersistentMap::insert ( const std::string &  key, const V &  value  )

inline

Insert or update a value in the database.

Parameters

key	the key to store the value.
value	the value stored at the key.

Returns

true on success, false otherwise

Exceptions

std::runtime_error

if the value is not copyable

Version

1.9.2

Definition at line 105 of file persistentMap.h.

Referenced by insert().

Here is the caller graph for this function:

template<class V >

bool lunchbox::PersistentMap::insert ( const std::string &  key, const std::vector< V > &  values  )

inline

Insert or update a vector of values in the database.

Parameters

key	the key to store the value.
values	the values stored at the key.

Returns

true on success, false otherwise

Exceptions

std::runtime_error

if the vector values are not copyable

Version

1.9.2

Definition at line 118 of file persistentMap.h.

template<class V >

bool lunchbox::PersistentMap::insert ( const std::string &  key, const std::set< V > &  values  )

inline

Insert or update a set of values in the database.

Parameters

key	the key to store the value.
values	the values stored at the key.

Returns

true on success, false otherwise

Exceptions

std::runtime_error

if the set values are not copyable

Version

1.9.2

Definition at line 131 of file persistentMap.h.

References insert().

Here is the call graph for this function:

LUNCHBOX_API std::string lunchbox::PersistentMap::operator[]

(

const std::string &

key

)

const

Retrieve a value for a key.

Parameters

key	the key to retrieve.

Returns

the value, or an empty string if the key is not available.

Version

1.9.2

LUNCHBOX_API void lunchbox::PersistentMap::setByteswap

(

const bool

swap

)

Enable or disable endianness conversion on reads.

Version

1.11

LUNCHBOX_API size_t lunchbox::PersistentMap::setQueueDepth

(

const size_t

depth

)

Set the maximum number of asynchronous outstanding write operations.

Some backend implementations support asynchronous writes, which can be enabled by setting a non-zero queue depth. Applications then need to quarantee that the inserted values stay valid until 'depth' other elements have been inserted or flush() has been called. Implementations which do not support asynchronous writes return 0.

Returns

the queue depth chosen by the implementation, smaller or equal to the given depth.

Version

1.11

---

The documentation for this class was generated from the following file:

• persistentMap.h