Lunchbox  1.11.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
lunchbox::PersistentMap Class Reference

Unified interface to save key-value pairs in a persistent store. More...

#include <persistentMap.h>

+ Inheritance diagram for lunchbox::PersistentMap:
+ Collaboration diagram for lunchbox::PersistentMap:

Public Member Functions

LUNCHBOX_API PersistentMap (const std::string &uri=std::string())
 Construct a new persistent map. More...
 
LUNCHBOX_API PersistentMap (const servus::URI &uri)
 Construct a persistent map using an URI. More...
 
LUNCHBOX_API ~PersistentMap ()
 Destruct the persistent map. More...
 
LUNCHBOX_API size_t setQueueDepth (const size_t depth)
 Set the maximum number of asynchronous outstanding write operations. More...
 
template<class V >
bool insert (const std::string &key, const V &value)
 Insert or update a value in the database. More...
 
template<class V >
bool insert (const std::string &key, const std::vector< V > &values)
 Insert or update a vector of values in the database. More...
 
template<class V >
bool insert (const std::string &key, const std::set< V > &values)
 Insert or update a set of values in the database. More...
 
LUNCHBOX_API std::string operator[] (const std::string &key) const
 Retrieve a value for a key. More...
 
template<class V >
get (const std::string &key) const
 Retrieve a value for a key. More...
 
template<class V >
std::vector< V > getVector (const std::string &key) const
 Retrieve a value as a vector for a key. More...
 
template<class V >
std::set< V > getSet (const std::string &key) const
 Retrieve a value as a set for a key. More...
 
LUNCHBOX_API bool fetch (const std::string &key, size_t sizeHint=0) const
 Asynchronously retrieve a value which to be read later. More...
 
LUNCHBOX_API bool contains (const std::string &key) const
 
LUNCHBOX_API bool flush ()
 Flush outstanding operations to the backend storage. More...
 
LUNCHBOX_API void setByteswap (const bool swap)
 Enable or disable endianness conversion on reads. More...
 
template<>
bool _insert (const std::string &k, const std::string &v, const boost::false_type &)
 

Static Public Member Functions

static LUNCHBOX_API bool handles (const servus::URI &uri)
 

Detailed Description

Unified interface to save key-value pairs in a persistent store.

Example:

/* Copyright (c) 2014-2015, Stefan.Eilemann@epfl.ch
*
* This library is free software; you can redistribute it and/or modify it under
* the terms of the GNU Lesser General Public License version 2.1 as published
* by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
* details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this library; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#define TEST_RUNTIME 240 //seconds
#include <test.h>
#include <lunchbox/clock.h>
#include <lunchbox/os.h>
#include <lunchbox/persistentMap.h>
#ifdef LUNCHBOX_USE_LEVELDB
# include <leveldb/db.h>
#endif
#ifdef LUNCHBOX_USE_SKV
# include <FxLogger/FxLogger.hpp>
#endif
#include <boost/format.hpp>
#include <stdexcept>
const int ints[] = { 17, 53, 42, 65535, 32768 };
const size_t numInts = sizeof( ints ) / sizeof( int );
const int64_t loopTime = 1000;
template< class T > void insertVector( PersistentMap& map )
{
std::vector< T > vector;
for( size_t i = 0; i < numInts; ++i )
vector.push_back( T( ints[ i ] ));
TEST( map.insert( typeid( vector ).name(), vector ));
}
template< class T > void readVector( const PersistentMap& map )
{
const std::vector< T >& vector =
map.getVector< T >( typeid( vector ).name( ));
TESTINFO( vector.size() == numInts, vector.size() << " != " << numInts );
for( size_t i = 0; i < numInts; ++i )
TEST( vector[ i ] == T( ints[i] ));
}
template< class T > void insertVector( PersistentMap& map, const size_t elems )
{
std::vector< T > vector;
for( size_t i = 0; i < elems; ++i )
vector.push_back( i );
TEST( map.insert( std::string( "bulk" ) + typeid( vector ).name(), vector ));
}
template< class T > void readVector( const PersistentMap& map, const size_t elems )
{
const std::vector< T >& vector =
map.getVector< T >( std::string( "bulk" ) + typeid( vector ).name());
TESTINFO( vector.size() == elems, vector.size() << " != " << elems );
for( size_t i = 0; i < numInts; ++i )
TESTINFO( vector[ i ] == T( i ), vector[ i ] << " != " << i );
}
void read( const PersistentMap& map )
{
const std::set< uint32_t >& bigSet =
map.getSet< uint32_t >( "std::set< uint32_t >" );
TEST( bigSet.size() == 1000 );
for( uint32_t i = 1; i <= 1000; ++i )
TEST( bigSet.find( i ) != bigSet.end( ));
TEST( map[ "foo" ] == "bar" );
TEST( map[ "bar" ].empty( ));
TEST( map.get< bool >( "bValue" ) == true );
TEST( map.get< int >( "iValue" ) == 42 );
readVector< int >( map );
readVector< uint16_t >( map );
const std::set< int >& set = map.getSet< int >( "std::set< int >" );
TESTINFO( set.size() == numInts, set.size() << " != " << numInts );
for( size_t i = 0; i < numInts; ++i )
TESTINFO( set.find( ints[i] ) != set.end(),
ints[i] << " not found in set" );
}
void read( const std::string& uri )
{
PersistentMap map( uri );
read( map );
}
void setup( const std::string& uri )
{
PersistentMap map( uri );
TEST( map.insert( "foo", "bar" ));
TEST( map.contains( "foo" ));
TESTINFO( map[ "foo" ] == "bar",
map[ "foo" ] << " length " << map[ "foo" ].length( ));
TEST( map[ "bar" ].empty( ));
TEST( map.insert( "the quick brown fox", "jumped over something" ));
TESTINFO( map[ "the quick brown fox" ] == "jumped over something",
map[ "the quick brown fox" ] );
TEST( map.insert( "hans", std::string( "dampf" )));
TESTINFO( map[ "hans" ] == "dampf", map[ "hans" ] );
const bool bValue = true;
TEST( map.insert( "bValue", bValue ));
TEST( map.get< bool >( "bValue" ) == bValue );
const int iValue = 42;
TEST( map.insert( "iValue", iValue ));
TEST( map.get< int >( "iValue" ) == iValue );
TEST( map.insert( "coffee", 0xC0FFEE ));
map.setByteswap( true );
TEST( map.get< unsigned >( "coffee" ) == 0xEEFFC000u );
map.setByteswap( false );
TEST( map.get< int >( "coffee" ) == 0xC0FFEE );
insertVector< int >( map );
insertVector< uint16_t >( map );
readVector< int >( map );
readVector< uint16_t >( map );
insertVector< int >( map, LB_128KB );
insertVector< uint16_t >( map, LB_128KB );
map.fetch( std::string( "bulk" ) + typeid( std::vector< int > ).name( ));
map.fetch( std::string( "bulk" ) + typeid( std::vector<uint16_t> ).name( ));
readVector< int >( map, LB_128KB );
readVector< uint16_t >( map, LB_128KB );
std::set< int > set( ints, ints + numInts );
TEST( map.insert( "std::set< int >", set ));
std::set< uint32_t > bigSet;
for( uint32_t i = 1; i <= 1000; ++i )
bigSet.insert( i );
TEST( map.insert( "std::set< uint32_t >", bigSet ));
read( map );
}
void benchmark( const std::string& uri, const uint64_t queueDepth )
{
PersistentMap map( uri );
map.setQueueDepth( queueDepth );
// Prepare keys
keys.resize( queueDepth + 1 );
for( uint64_t i = 0; i <= queueDepth; ++i )
keys[i].assign( reinterpret_cast< char* >( &i ), 8 );
// write performance
uint64_t i = 0;
while( clock.getTime64() < loopTime )
{
std::string& key = keys[ i % (queueDepth+1) ];
*reinterpret_cast< uint64_t* >( &key[0] ) = i;
map.insert( key, key );
++i;
}
map.flush();
const float insertTime = clock.getTimef();
const uint64_t wOps = i;
TEST( i > queueDepth );
// read performance
std::string key;
key.assign( reinterpret_cast< char* >( &i ), 8 );
clock.reset();
if( queueDepth == 0 ) // sync read
{
for( i = 0; i < wOps && clock.getTime64() < loopTime; ++i ) // read keys
{
*reinterpret_cast< uint64_t* >( &key[0] ) = i - queueDepth;
map[ key ];
}
}
else // fetch + async read
{
for( i = 0; i < queueDepth; ++i ) // prefetch queueDepth keys
{
*reinterpret_cast< uint64_t* >( &key[0] ) = i;
TEST( map.fetch( key, 8 ) );
}
for( ; i < wOps && clock.getTime64() < loopTime; ++i ) // read keys
{
*reinterpret_cast< uint64_t* >( &key[0] ) = i - queueDepth;
map[ key ];
*reinterpret_cast< uint64_t* >( &key[0] ) = i;
TEST( map.fetch( key, 8 ));
}
for( uint64_t j = i - queueDepth; j <= i; ++j ) // drain fetched keys
{
*reinterpret_cast< uint64_t* >( &key[0] ) = j;
map[ key ];
}
}
const float readTime = clock.getTimef();
const size_t rOps = i;
// fetch performance
clock.reset();
for( i = 0; i < wOps && clock.getTime64() < loopTime && i < LB_128KB; ++i )
{
*reinterpret_cast< uint64_t* >( &key[0] ) = i;
TEST( map.fetch( key, 8 ) );
}
const float fetchTime = clock.getTimef();
const size_t fOps = i;
std::cout << boost::format( "%7.2f, %7.2f, %7.2f, %6i")
// cppcheck-suppress zerodivcond
% queueDepth % (rOps/insertTime) % (wOps/readTime) % (fOps/fetchTime)
<< std::endl;
// check contents of store (not all to save time on bigger tests)
for( uint64_t j = 0; j < wOps && clock.getTime64() < loopTime; ++j )
{
*reinterpret_cast< uint64_t* >( &key[0] ) = j;
TESTINFO( map.get< uint64_t >( key ) == j,
j << " = " << map.get< uint64_t >( key ));
}
// try to make sure there's nothing outstanding if we messed up i our test.
map.flush();
}
void testGenericFailures()
{
try
{
setup( "foobar://" );
}
catch( const std::runtime_error& )
{
return;
}
TESTINFO( false, "Missing exception" );
}
void testLevelDBFailures()
{
#ifdef LUNCHBOX_USE_LEVELDB
try
{
setup( "leveldb:///doesnotexist/deadbeef/coffee" );
}
catch( const std::runtime_error& )
{
return;
}
TESTINFO( false, "Missing exception" );
#endif
}
int main( int, char* argv[] )
{
const bool perfTest LB_UNUSED
= std::string( argv[0] ).find( "perf_" ) != std::string::npos;
if( perfTest )
std::cout << " async, read, write, fetch" << std::endl;
try
{
#ifdef LUNCHBOX_USE_LEVELDB
setup( "" );
setup( "leveldb://" );
setup( "leveldb://persistentMap2.leveldb" );
read( "" );
read( "leveldb://" );
read( "leveldb://persistentMap2.leveldb" );
if( perfTest )
benchmark( "leveldb://", 0 );
#endif
#ifdef LUNCHBOX_USE_SKV
FxLogger_Init( argv[0] );
setup( "skv://" );
read( "skv://" );
if( perfTest )
{
benchmark( "skv://", 0 );
for( size_t i=1; i < 100000; i = i<<1 )
benchmark( "skv://", i );
}
#endif
}
#ifdef LUNCHBOX_USE_LEVELDB
catch( const leveldb::Status& status )
{
TESTINFO( !"exception", status.ToString( ));
}
#endif
catch( const std::runtime_error& error )
{
#ifdef LUNCHBOX_USE_SKV
if( error.what() !=
std::string( "skv init failed: SKV_ERRNO_CONN_FAILED" ))
#endif
{
TESTINFO( !"exception", error.what( ));
}
}
testGenericFailures();
testLevelDBFailures();
return EXIT_SUCCESS;
}

Definition at line 48 of file persistentMap.h.

Constructor & Destructor Documentation

LUNCHBOX_API lunchbox::PersistentMap::PersistentMap ( const std::string &  uri = std::string())

Construct a new persistent map.

Depending on the URI scheme an implementation backend is chosen. If no URI is given, a default one is selected. Available implementations are:

  • leveldb://path (if LUNCHBOX_USE_LEVELDB is defined)
  • skv://path_to_config#pdsname (if LUNCHBOX_USE_SKV is defined)
Parameters
urithe storage backend and destination.
Exceptions
std::runtime_errorif no suitable implementation is found.
std::runtime_errorif opening the leveldb failed.
Version
1.9.2
LUNCHBOX_API lunchbox::PersistentMap::PersistentMap ( const servus::URI &  uri)
explicit

Construct a persistent map using an URI.

See other ctor for details.

Version
1.9.2
LUNCHBOX_API lunchbox::PersistentMap::~PersistentMap ( )

Destruct the persistent map.

Version
1.9.2

Member Function Documentation

LUNCHBOX_API bool lunchbox::PersistentMap::contains ( const std::string &  key) const
Returns
true if the key exists.
Version
1.9.2
LUNCHBOX_API bool lunchbox::PersistentMap::fetch ( const std::string &  key,
size_t  sizeHint = 0 
) const

Asynchronously retrieve a value which to be read later.

Might be implemented as a 'NOP' by backend implementations.

Parameters
keythe key to retrieve.
sizeHintthe size of the value, may be ignored by implementation.
Returns
false on error, true otherwise.
Version
1.11
LUNCHBOX_API bool lunchbox::PersistentMap::flush ( )

Flush outstanding operations to the backend storage.

Version
1.11
template<class V >
V lunchbox::PersistentMap::get ( const std::string &  key) const
inline

Retrieve a value for a key.

Parameters
keythe key to retrieve.
Returns
the value, or an empty string if the key is not available.
Version
1.11

Definition at line 150 of file persistentMap.h.

template<class V >
std::set< V > lunchbox::PersistentMap::getSet ( const std::string &  key) const
inline

Retrieve a value as a set for a key.

Parameters
keythe key to retrieve.
Returns
the values, or an empty set if the key is not available.
Version
1.9.2

Definition at line 270 of file persistentMap.h.

References lunchbox::byteswap().

+ Here is the call graph for this function:

template<class V >
std::vector< V > lunchbox::PersistentMap::getVector ( const std::string &  key) const
inline

Retrieve a value as a vector for a key.

Parameters
keythe key to retrieve.
Returns
the values, or an empty vector if the key is not available.
Version
1.9.2

Definition at line 256 of file persistentMap.h.

References lunchbox::byteswap().

+ Here is the call graph for this function:

static LUNCHBOX_API bool lunchbox::PersistentMap::handles ( const servus::URI &  uri)
static
Returns
true if an implementation for the given URI is available.
Version
1.9.2
template<class V >
bool lunchbox::PersistentMap::insert ( const std::string &  key,
const V &  value 
)
inline

Insert or update a value in the database.

Parameters
keythe key to store the value.
valuethe value stored at the key.
Returns
true on success, false otherwise
Exceptions
std::runtime_errorif the value is not copyable
Version
1.9.2

Definition at line 105 of file persistentMap.h.

Referenced by insert().

+ Here is the caller graph for this function:

template<class V >
bool lunchbox::PersistentMap::insert ( const std::string &  key,
const std::vector< V > &  values 
)
inline

Insert or update a vector of values in the database.

Parameters
keythe key to store the value.
valuesthe values stored at the key.
Returns
true on success, false otherwise
Exceptions
std::runtime_errorif the vector values are not copyable
Version
1.9.2

Definition at line 118 of file persistentMap.h.

template<class V >
bool lunchbox::PersistentMap::insert ( const std::string &  key,
const std::set< V > &  values 
)
inline

Insert or update a set of values in the database.

Parameters
keythe key to store the value.
valuesthe values stored at the key.
Returns
true on success, false otherwise
Exceptions
std::runtime_errorif the set values are not copyable
Version
1.9.2

Definition at line 131 of file persistentMap.h.

References insert().

+ Here is the call graph for this function:

LUNCHBOX_API std::string lunchbox::PersistentMap::operator[] ( const std::string &  key) const

Retrieve a value for a key.

Parameters
keythe key to retrieve.
Returns
the value, or an empty string if the key is not available.
Version
1.9.2
LUNCHBOX_API void lunchbox::PersistentMap::setByteswap ( const bool  swap)

Enable or disable endianness conversion on reads.

Version
1.11
LUNCHBOX_API size_t lunchbox::PersistentMap::setQueueDepth ( const size_t  depth)

Set the maximum number of asynchronous outstanding write operations.

Some backend implementations support asynchronous writes, which can be enabled by setting a non-zero queue depth. Applications then need to quarantee that the inserted values stay valid until 'depth' other elements have been inserted or flush() has been called. Implementations which do not support asynchronous writes return 0.

Returns
the queue depth chosen by the implementation, smaller or equal to the given depth.
Version
1.11

The documentation for this class was generated from the following file: