Lunchbox  1.13.0
Multi-threaded C++ toolbox library for all application developers creating high-performance multi-threaded programs.
lunchbox::PersistentMap Class Reference

Unified interface to save key-value pairs in a persistent store. More...

#include <persistentMap.h>

+ Inheritance diagram for lunchbox::PersistentMap:
+ Collaboration diagram for lunchbox::PersistentMap:

Public Member Functions

 PersistentMap (const std::string &uri=std::string())
 Construct a new persistent map. More...
 
 PersistentMap (const servus::URI &uri)
 Construct a persistent map using an URI. More...
 
 ~PersistentMap ()
 Destruct the persistent map. More...
 
size_t setQueueDepth (const size_t depth)
 Set the maximum number of asynchronous outstanding write operations. More...
 
template<class V >
bool insert (const std::string &key, const V &value)
 Insert or update a value in the database. More...
 
template<class V >
bool insert (const std::string &key, const std::vector< V > &values)
 Insert or update a vector of values in the database. More...
 
template<class V >
bool insert (const std::string &key, const std::set< V > &values)
 Insert or update a set of values in the database. More...
 
std::string operator[] (const std::string &key) const
 Retrieve a value for a key. More...
 
template<class V >
get (const std::string &key) const
 Retrieve a value for a key. More...
 
template<class V >
std::vector< V > getVector (const std::string &key) const
 Retrieve a value as a vector for a key. More...
 
template<class V >
std::set< V > getSet (const std::string &key) const
 Retrieve a value as a set for a key. More...
 
bool fetch (const std::string &key, size_t sizeHint=0) const
 Asynchronously retrieve a value which to be read later. More...
 
bool contains (const std::string &key) const
 
bool flush ()
 Flush outstanding operations to the backend storage. More...
 
void setByteswap (const bool swap)
 Enable or disable endianness conversion on reads. More...
 
template<>
bool _insert (const std::string &k, const std::string &v, const boost::false_type &)
 

Static Public Member Functions

static bool handles (const servus::URI &uri)
 

Detailed Description

Unified interface to save key-value pairs in a persistent store.

Example:

/* Copyright (c) 2014-2015, Stefan.Eilemann@epfl.ch
*
* This library is free software; you can redistribute it and/or modify it under
* the terms of the GNU Lesser General Public License version 2.1 as published
* by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
* details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this library; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#define TEST_RUNTIME 240 //seconds
#include <test.h>
#include <lunchbox/clock.h>
#include <lunchbox/os.h>
#include <lunchbox/persistentMap.h>
#include <lunchbox/rng.h>
#ifdef LUNCHBOX_USE_LEVELDB
# include <leveldb/db.h>
#endif
#ifdef LUNCHBOX_USE_SKV
# include <FxLogger/FxLogger.hpp>
#endif
#include <boost/format.hpp>
#include <stdexcept>
const int ints[] = { 17, 53, 42, 65535, 32768 };
const size_t numInts = sizeof( ints ) / sizeof( int );
const int64_t loopTime = 1000;
bool perfTest = false;
template< class T > void insertVector( PersistentMap& map )
{
std::vector< T > vector;
for( size_t i = 0; i < numInts; ++i )
vector.push_back( T( ints[ i ] ));
TEST( map.insert( typeid( vector ).name(), vector ));
}
template< class T > void readVector( const PersistentMap& map )
{
const std::vector< T >& vector =
map.getVector< T >( typeid( vector ).name( ));
TESTINFO( vector.size() == numInts, vector.size() << " != " << numInts );
for( size_t i = 0; i < numInts; ++i )
TEST( vector[ i ] == T( ints[i] ));
}
template< class T > void insertVector( PersistentMap& map, const size_t elems )
{
std::vector< T > vector;
for( size_t i = 0; i < elems; ++i )
vector.push_back( i );
TEST( map.insert( std::string( "bulk" ) + typeid( vector ).name(), vector ));
}
template< class T > void readVector( const PersistentMap& map, const size_t elems )
{
const std::vector< T >& vector =
map.getVector< T >( std::string( "bulk" ) + typeid( vector ).name());
TESTINFO( vector.size() == elems, vector.size() << " != " << elems );
for( size_t i = 0; i < numInts; ++i )
TESTINFO( vector[ i ] == T( i ), vector[ i ] << " != " << i );
}
void read( const PersistentMap& map )
{
const std::set< uint32_t >& bigSet =
map.getSet< uint32_t >( "std::set< uint32_t >" );
TEST( bigSet.size() == 1000 );
for( uint32_t i = 1; i <= 1000; ++i )
TEST( bigSet.find( i ) != bigSet.end( ));
TEST( map[ "foo" ] == "bar" );
TEST( map[ "bar" ].empty( ));
TEST( map.get< bool >( "bValue" ) == true );
TEST( map.get< int >( "iValue" ) == 42 );
readVector< int >( map );
readVector< uint16_t >( map );
const std::set< int >& set = map.getSet< int >( "std::set< int >" );
TESTINFO( set.size() == numInts, set.size() << " != " << numInts );
for( size_t i = 0; i < numInts; ++i )
TESTINFO( set.find( ints[i] ) != set.end(),
ints[i] << " not found in set" );
}
void read( const std::string& uri )
{
PersistentMap map( uri );
read( map );
}
void setup( const std::string& uri )
{
PersistentMap map( uri );
TEST( map.insert( "foo", "bar" ));
TEST( map.contains( "foo" ));
TESTINFO( map[ "foo" ] == "bar",
map[ "foo" ] << " length " << map[ "foo" ].length( ));
TEST( map[ "bar" ].empty( ));
TEST( map.insert( "the quick brown fox", "jumped over something" ));
TESTINFO( map[ "the quick brown fox" ] == "jumped over something",
map[ "the quick brown fox" ] );
TEST( map.insert( "hans", std::string( "dampf" )));
TESTINFO( map[ "hans" ] == "dampf", map[ "hans" ] );
const bool bValue = true;
TEST( map.insert( "bValue", bValue ));
TEST( map.get< bool >( "bValue" ) == bValue );
const int iValue = 42;
TEST( map.insert( "iValue", iValue ));
TEST( map.get< int >( "iValue" ) == iValue );
TEST( map.insert( "coffee", 0xC0FFEE ));
map.setByteswap( true );
TEST( map.get< unsigned >( "coffee" ) == 0xEEFFC000u );
map.setByteswap( false );
TEST( map.get< int >( "coffee" ) == 0xC0FFEE );
insertVector< int >( map );
insertVector< uint16_t >( map );
readVector< int >( map );
readVector< uint16_t >( map );
insertVector< int >( map, LB_128KB );
insertVector< uint16_t >( map, LB_128KB );
map.fetch( std::string( "bulk" ) + typeid( std::vector< int > ).name( ));
map.fetch( std::string( "bulk" ) + typeid( std::vector<uint16_t> ).name( ));
readVector< int >( map, LB_128KB );
readVector< uint16_t >( map, LB_128KB );
std::set< int > set( ints, ints + numInts );
TEST( map.insert( "std::set< int >", set ));
std::set< uint32_t > bigSet;
for( uint32_t i = 1; i <= 1000; ++i )
bigSet.insert( i );
TEST( map.insert( "std::set< uint32_t >", bigSet ));
read( map );
}
void benchmark( const std::string& uri, const uint64_t queueDepth,
const size_t valueSize )
{
static std::string lastURI;
if( uri != lastURI )
{
std::cout << uri << std::endl;
lastURI = uri;
}
PersistentMap map( uri );
map.setQueueDepth( queueDepth );
// Prepare keys and value
keys.resize( queueDepth + 1 );
for( uint64_t i = 0; i <= queueDepth; ++i )
keys[i].assign( reinterpret_cast< char* >( &i ), 8 );
std::string value( valueSize, '*' );
for( size_t i = 0; i < valueSize; ++i )
value[i] = rng.get<char>();
// write performance
uint64_t i = 0;
while( clock.getTime64() < loopTime )
{
map.insert( keys[ i % (queueDepth+1) ], value );
++i;
}
map.flush();
const float writeTime = clock.getTimef() / 1000.f;
const uint64_t wOps = i;
TEST( i > queueDepth );
// read performance
clock.reset();
if( queueDepth == 0 ) // sync read
{
for( i = 0; i < wOps && clock.getTime64() < loopTime; ++i ) // read keys
map[ keys[ i % (queueDepth+1) ]];
}
else // fetch + async read
{
for( i = 0; i < queueDepth; ++i ) // prefetch queueDepth keys
TEST( map.fetch( keys[ i % (queueDepth+1) ], valueSize ) );
for( ; i < wOps && clock.getTime64() < loopTime; ++i ) // read keys
{
map[ keys[ (i - queueDepth) % (queueDepth+1) ] ];
TEST( map.fetch( keys[ i % (queueDepth+1) ], valueSize ));
}
for( uint64_t j = i - queueDepth; j <= i; ++j ) // drain fetched keys
map[ keys[ j % (queueDepth+1) ]];
}
const float readTime = clock.getTimef() / 1000.f;
const size_t rOps = i;
std::cout << boost::format( "%6i, %6i, %9.2f, %9.2f, %9.2f, %9.2f")
// cppcheck-suppress zerodivcond
% queueDepth % valueSize % (rOps/readTime) % (wOps/writeTime)
% (rOps/1024.f/1024.f*valueSize/readTime)
% (wOps/1024.f/1024.f*valueSize/writeTime) << std::endl;
if( !perfTest )
{
// check contents of store (not all to save time on bigger tests)
for( uint64_t j = 0; j < wOps && clock.getTime64() < loopTime; ++j )
{
const std::string& val = map[ keys[ j % (queueDepth+1) ]];
TESTINFO( val.size() == valueSize,
val.size() << " != " << valueSize );
TEST( val == value );
}
}
// try to make sure there's nothing outstanding if we messed up in our test
map.flush();
}
void testGenericFailures()
{
try
{
setup( "foobar://" );
}
catch( const std::runtime_error& )
{
return;
}
TESTINFO( false, "Missing exception" );
}
void testLevelDBFailures()
{
#ifdef LUNCHBOX_USE_LEVELDB
try
{
setup( "leveldb:///doesnotexist/deadbeef/coffee" );
}
catch( const std::runtime_error& )
{
return;
}
TESTINFO( false, "Missing exception" );
#endif
}
int main( int, char* argv[] )
{
perfTest = std::string( argv[0] ).find( "perf-" ) != std::string::npos;
if( perfTest )
std::cout
<< " async, value, reads/s, writes/s, read MB/s, write MB/s"
<< std::endl;
try
{
#ifdef LUNCHBOX_USE_LEVELDB
setup( "" );
setup( "leveldb://" );
setup( "leveldb://persistentMap2.leveldb" );
read( "" );
read( "leveldb://" );
read( "leveldb://persistentMap2.leveldb" );
if( perfTest )
for( size_t i=1; i <= 65536; i = i<<2 )
benchmark( "leveldb://", 0, i );
#endif
#ifdef LUNCHBOX_USE_SKV
FxLogger_Init( argv[0] );
setup( "skv://" );
read( "skv://" );
if( perfTest )
{
benchmark( "skv://", 0, 64 );
for( size_t i=1; i <= 65536; i = i<<1 )
benchmark( "skv://", i, 64 );
for( size_t i=1; i <= 65536; i = i<<2 )
benchmark( "skv://", 65536, i );
}
#endif
}
#ifdef LUNCHBOX_USE_LEVELDB
catch( const leveldb::Status& status )
{
TESTINFO( !"exception", status.ToString( ));
}
#endif
catch( const std::runtime_error& error )
{
#ifdef LUNCHBOX_USE_SKV
if( error.what() !=
std::string( "skv init failed: SKV_ERRNO_CONN_FAILED" ))
#endif
{
TESTINFO( !"exception", error.what( ));
}
}
testGenericFailures();
testLevelDBFailures();
return EXIT_SUCCESS;
}

Definition at line 48 of file persistentMap.h.

Constructor & Destructor Documentation

lunchbox::PersistentMap::PersistentMap ( const std::string &  uri = std::string())
explicit

Construct a new persistent map.

Depending on the URI scheme an implementation backend is chosen. If no URI is given, a default one is selected. Available implementations are:

  • leveldb://path (if LUNCHBOX_USE_LEVELDB is defined)
  • skv://path_to_config#pdsname (if LUNCHBOX_USE_SKV is defined)
Parameters
urithe storage backend and destination.
Exceptions
std::runtime_errorif no suitable implementation is found.
std::runtime_errorif opening the leveldb failed.
Version
1.9.2
lunchbox::PersistentMap::PersistentMap ( const servus::URI &  uri)
explicit

Construct a persistent map using an URI.

See other ctor for details.

Version
1.9.2
lunchbox::PersistentMap::~PersistentMap ( )

Destruct the persistent map.

Version
1.9.2

Member Function Documentation

bool lunchbox::PersistentMap::contains ( const std::string &  key) const
Returns
true if the key exists.
Version
1.9.2
bool lunchbox::PersistentMap::fetch ( const std::string &  key,
size_t  sizeHint = 0 
) const

Asynchronously retrieve a value which to be read later.

Might be implemented as a 'NOP' by backend implementations.

Parameters
keythe key to retrieve.
sizeHintthe size of the value, may be ignored by implementation.
Returns
false on error, true otherwise.
Version
1.11
bool lunchbox::PersistentMap::flush ( )

Flush outstanding operations to the backend storage.

Version
1.11
template<class V >
V lunchbox::PersistentMap::get ( const std::string &  key) const
inline

Retrieve a value for a key.

Parameters
keythe key to retrieve.
Returns
the value, or an empty string if the key is not available.
Version
1.11

Definition at line 150 of file persistentMap.h.

template<class V >
std::set< V > lunchbox::PersistentMap::getSet ( const std::string &  key) const
inline

Retrieve a value as a set for a key.

Parameters
keythe key to retrieve.
Returns
the values, or an empty set if the key is not available.
Version
1.9.2

Definition at line 270 of file persistentMap.h.

References lunchbox::byteswap().

+ Here is the call graph for this function:

template<class V >
std::vector< V > lunchbox::PersistentMap::getVector ( const std::string &  key) const
inline

Retrieve a value as a vector for a key.

Parameters
keythe key to retrieve.
Returns
the values, or an empty vector if the key is not available.
Version
1.9.2

Definition at line 256 of file persistentMap.h.

References lunchbox::byteswap().

+ Here is the call graph for this function:

static bool lunchbox::PersistentMap::handles ( const servus::URI &  uri)
static
Returns
true if an implementation for the given URI is available.
Version
1.9.2
template<class V >
bool lunchbox::PersistentMap::insert ( const std::string &  key,
const V &  value 
)
inline

Insert or update a value in the database.

Parameters
keythe key to store the value.
valuethe value stored at the key.
Returns
true on success, false otherwise
Exceptions
std::runtime_errorif the value is not copyable
Version
1.9.2

Definition at line 105 of file persistentMap.h.

Referenced by insert().

+ Here is the caller graph for this function:

template<class V >
bool lunchbox::PersistentMap::insert ( const std::string &  key,
const std::vector< V > &  values 
)
inline

Insert or update a vector of values in the database.

Parameters
keythe key to store the value.
valuesthe values stored at the key.
Returns
true on success, false otherwise
Exceptions
std::runtime_errorif the vector values are not copyable
Version
1.9.2

Definition at line 118 of file persistentMap.h.

template<class V >
bool lunchbox::PersistentMap::insert ( const std::string &  key,
const std::set< V > &  values 
)
inline

Insert or update a set of values in the database.

Parameters
keythe key to store the value.
valuesthe values stored at the key.
Returns
true on success, false otherwise
Exceptions
std::runtime_errorif the set values are not copyable
Version
1.9.2

Definition at line 131 of file persistentMap.h.

References insert().

+ Here is the call graph for this function:

std::string lunchbox::PersistentMap::operator[] ( const std::string &  key) const

Retrieve a value for a key.

Parameters
keythe key to retrieve.
Returns
the value, or an empty string if the key is not available.
Version
1.9.2
void lunchbox::PersistentMap::setByteswap ( const bool  swap)

Enable or disable endianness conversion on reads.

Version
1.11
size_t lunchbox::PersistentMap::setQueueDepth ( const size_t  depth)

Set the maximum number of asynchronous outstanding write operations.

Some backend implementations support asynchronous writes, which can be enabled by setting a non-zero queue depth. Applications then need to quarantee that the inserted values stay valid until 'depth' other elements have been inserted or flush() has been called. Implementations which do not support asynchronous writes return 0.

Returns
the queue depth chosen by the implementation, smaller or equal to the given depth.
Version
1.11

The documentation for this class was generated from the following file: