NoSQL databases are pretty hot these days.
NoSQL databases are actually many different types of databases only sharing the fact that they are not relational and do not use SQL as interface language.
This article will look at the family of key value store NoSQL databases.
This article will not cover key value store databases that are primarily used to cache data in memory - not even if they have an option for persisting to disk. This includes: Oracle Coherence, Hazelcast, Infinispan, Apache Ignite, memcached and Redis.
These are covered here.
For other NoSQL database families see:
A key value store basically supports these CRUD operations:
and optionally these iteration operations (conceptually):
The DBM database was released for Unix back in 1979. It has since been cloned many times including NDBM, GDBM and SDBM.
This example will use the DBM implementation from APR (Apache Portable Runtime).
Supported platforms | most |
Supported languages | C/C++ |
Features | |
Missing features | Find key range |
Searching for a key range is not possible because it is hash based not tree based.
C example:
/* standard C headers */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
/* APR headers */
#include "apr.h"
#include "apr_pools.h"
#include "apr_dbm.h"
/* data structure */
struct data
{
int iv;
double xv;
char sv[50];
};
/* error handling functions */
void db_exit(apr_dbm_t *db, apr_status_t stat)
{
char buf[256];
printf("APR DBM error: %s\n", apr_dbm_geterror(db, &stat, buf, sizeof(buf)));
exit(1);
}
void other_exit(char *msg)
{
printf("%s\n", msg);
exit(1);
}
static void dump(apr_dbm_t *db, char *keybuf)
{
apr_status_t stat;
apr_datum_t key;
apr_datum_t value;
struct data *d;
printf("Key=%s\n", keybuf);
key.dptr = keybuf;
key.dsize = strlen(keybuf);
stat = apr_dbm_fetch(db, key, &value);
if(stat != APR_SUCCESS)
{
db_exit(db, stat);
}
if(value.dptr != NULL)
{
d = (struct data *)value.dptr;
printf("(%d,%f,%s)\n", d->iv, d->xv, d->sv);
apr_dbm_freedatum(db, value);
}
else
{
printf("Not found\n");
}
}
static const int NREC = 1000;
static const char *DBNAME = "testdb";
int main()
{
apr_status_t stat;
apr_pool_t *pool;
apr_dbm_t *db;
apr_datum_t key, value, itkey;
char keybuf[32];
struct data valuebuf, *d;
int n;
/* initialize */
stat = apr_pool_initialize();
if(stat != APR_SUCCESS)
{
other_exit("pool_initialize failed");
}
stat = apr_pool_create_core(&pool);
if(stat != APR_SUCCESS)
{
other_exit("pool_create_core failed");
}
/* get connection */
stat = apr_dbm_open(&db, DBNAME, APR_DBM_RWCREATE, 0, pool);
if(stat != APR_SUCCESS)
{
other_exit("dbm_open failed");
}
/* put data */
for(int i = 0; i < NREC; i++)
{
sprintf(keybuf, "Key#%d", 1000 + i + 1);
key.dptr = keybuf;
key.dsize = strlen(keybuf);
valuebuf.iv = i + 1;
valuebuf.xv = i + 1.0;
sprintf(valuebuf.sv, "This is value %d", i + 1);
value.dptr = (char *)&valuebuf.iv;
value.dsize = sizeof(valuebuf);
stat = apr_dbm_store(db, key, value);
if(stat != APR_SUCCESS)
{
db_exit(db, stat);
}
}
/* */
sprintf(keybuf, "Key#%d", 1077);
/* get */
dump(db, keybuf);
/* delete */
key.dptr = keybuf;
key.dsize = strlen(keybuf);
stat = apr_dbm_delete(db, key);
if(stat != APR_SUCCESS)
{
db_exit(db, stat);
}
/* get non existing */
dump(db, keybuf);
/* */
sprintf(keybuf, "Key#%d", 1088);
/* update and get */
dump(db, keybuf);
key.dptr = keybuf;
key.dsize = strlen(keybuf);
stat = apr_dbm_fetch(db, key, &value);
if(stat != APR_SUCCESS)
{
db_exit(db, stat);
}
memcpy(&valuebuf.iv, value.dptr, sizeof(struct data));
apr_dbm_freedatum(db, value);
valuebuf.iv = valuebuf.iv + 1;
valuebuf.xv = valuebuf.xv + 0.1;
strcat(valuebuf.sv, " updated");
value.dptr = (char *)&valuebuf.iv;
value.dsize = sizeof(valuebuf);
stat = apr_dbm_store(db, key, value);
if(stat != APR_SUCCESS)
{
db_exit(db, stat);
}
dump(db, keybuf);
/* list all */
n = 0;
stat = apr_dbm_firstkey(db, &itkey);
while(stat == APR_SUCCESS && itkey.dptr != NULL)
{
memcpy(keybuf, itkey.dptr, itkey.dsize);
keybuf[itkey.dsize] = '\0';
if(strstr(keybuf, "Key#") != keybuf)
{
printf("Unexpected key: %s\n", keybuf);
}
key.dptr = keybuf;
key.dsize = strlen(keybuf);
stat = apr_dbm_fetch(db, key, &value);
if(stat != APR_SUCCESS)
{
db_exit(db, stat);
}
d = (struct data *)value.dptr;
if (d->iv < 1 || NREC < d->iv)
{
printf("Unexpected value: (%d,%f,%s)\n", d->iv, d->xv, d->sv);
}
apr_dbm_freedatum(db, value);
n++;
stat = apr_dbm_nextkey(db, &itkey);
}
printf("%d\n", n);
apr_dbm_freedatum(db, itkey);
/* list keys where "Key#1075" <= key < "Key#1085" */
/* **** NOT SUPPORTED **** */
/* close */
apr_dbm_close(db);
apr_pool_clear(pool);
apr_pool_destroy(pool);
apr_pool_terminate();
return 0;
}
Python provide an interface to DBM as well. An interface thay is much more high level than the C API.
import sys
if sys.version_info[0] < 3:
import anydbm
else:
import dbm
class Data(object):
def __init__(self, _iv = 0, _xv = 0.0, _sv = ''):
self.iv = _iv
self.xv = _xv
self.sv = _sv
def __str__(self):
return '(%d, %f, %s)' % (self.iv, self.xv, self.sv)
# hackish serialize and deserialize
def serialize(o):
return str('%d;%f;%s' % (o.iv, o.xv, o.sv))
def deserialize(d):
parts = d.decode().split(';', 2)
return Data(int(parts[0]), float(parts[1]), parts[2])
def dump(db, key):
print('Key=%s' % (key))
if key in db:
val = deserialize(db[key])
print(val)
else:
print('Not found')
NREC = 1000
DBNAME = 'testdb'
# open database
if sys.version_info[0] < 3:
db = anydbm.open(DBNAME, 'c')
else:
db = dbm.open(DBNAME, 'c')
# put data
for i in range(NREC):
key = 'Key#%d' % (1000 + i + 1)
val = Data(i + 1, i + 1.0, 'This is value %d' % (i + 1))
db[key] = serialize(val)
#
key = 'Key#1077'
# get
dump(db, key)
# delete
db.pop(key)
# get non existing
dump(db, key)
#
key = 'Key#1088'
# update and get
dump(db, key)
val = deserialize(db[key])
val.iv = val.iv + 1
val.xv = val.xv + 0.1
val.sv = val.sv + ' updated'
db[key] = serialize(val)
dump(db, key)
# list all
n = 0
for key in db:
if not key.decode().startswith('Key#'):
print('Unexpected key: ' + key)
val = deserialize(db[key])
if val.iv < 1 or NREC < val.iv:
print('Unexpected value: ' + val)
n = n + 1
print(n)
# list keys where "Key#1075" <= key < "Key#1085"
# **** NOT SUPPORTED ****
# close database
db.close()
Berkeley University created Berkeley DB (BDB) in 1994. A commercial company Sleepycat took over in 1996. Oracle acquired SleepyCat and BDB in 2006.
Supported platforms | Linux, Windows, *BSD, Solaris etc. |
Supported languages | C Java and other JVM languages C# and VB.NET Perl Python |
Features | Transaction support Multiple keys |
Missing features |
License has changed over time:
Note that unless you are OK with AGPL and its implications for your code then you need to buy a commercial license from Oracle.
There are two approaches to data format:
Let us first see a language neutral serialization format. This makes it equal for different languages.
The C/C++ struct is:
struct Data
{
int Iv;
double Xv;
char Sv[50];
};
And we define serialization format as:
/* standard C headers */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
/* BDB headers */
#include "db.h"
/* data structure */
struct data
{
int iv;
double xv;
char sv[50];
};
static int length(struct data *d)
{
return 4 + 8 + 2 + strlen(d->sv);
}
static void serialize(struct data *d, char *buf)
{
int slen = strlen(d->sv);
char *p = buf;
memcpy(p, &d->iv, 4);
p += 4;
memcpy(p, &d->xv, 8);
p += 8;
memcpy(p, &slen, 2);
p += 2;
memcpy(p, &d->sv[0], slen);
}
static void deserialize(char *buf, struct data *d)
{
int slen = 0;
char *p = buf;
memcpy(&d->iv, p, 4);
p += 4;
memcpy(&d->xv, p, 8);
p += 8;
memcpy(&slen, p, 2);
p += 2;
memcpy(&d->sv[0], p, slen);
d->sv[slen] = '\0';
}
/* error handling functions */
void db_exit(char *msg, int stat)
{
char buf[256];
printf("DB error %s: %s\n", msg, db_strerror(stat));
exit(1);
}
static void dump(DB *db, char *keybuf)
{
int stat;
DBT key;
DBT value;
struct data d;
printf("Key=%s\n", keybuf);
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
key.data = keybuf;
key.size = strlen(keybuf);
value.size = sizeof(struct data);
stat = db->get(db, NULL, &key, &value, 0);
if(stat == 0)
{
deserialize(value.data, &d);
printf("(%d,%f,%s)\n", d.iv, d.xv, d.sv);
}
else if(stat == DB_NOTFOUND)
{
printf("Not found\n");
}
else
{
db_exit("db->get", stat);
}
}
static const char *DBNAME = "test.db";
static const int NREC = 1000;
static const int C_OFFSET = 1000; // 1 * NREC;
int main()
{
int stat;
DB *db;
DBC *it, *it2;
DBT key, value, itkey;
struct data d;
char keybuf[32], termkeybuf[32], valuebuf[256];
int n, n2;
/* initialize */
stat = db_create(&db, NULL, 0);
if(stat != 0)
{
db_exit("db_create", stat);
}
/* get connection */
stat = db->open(db, NULL, DBNAME, NULL, DB_BTREE, DB_CREATE, 0664); /* DB_HASH is alternative to DB_BTREE */
if(stat != 0)
{
db_exit("db->open", stat);
}
/* put data */
for(int i = 0; i < NREC; i++)
{
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
sprintf(keybuf, "Key#%d", C_OFFSET + i + 1);
key.data = keybuf;
key.size = strlen(keybuf);
d.iv = i + 1;
d.xv = i + 1.0;
sprintf(d.sv, "This is value %d", i + 1);
serialize(&d, valuebuf);
value.data = valuebuf;
value.size = length(&d);
stat = db->put(db, NULL, &key, &value, 0);
if(stat != 0)
{
db_exit("db->put", stat);
}
}
/* */
sprintf(keybuf, "Key#%d", C_OFFSET + 77);
/* get */
dump(db, keybuf);
/* delete */
memset(&key, 0, sizeof(DBT));
key.data = keybuf;
key.size = strlen(keybuf);
stat = db->del(db, NULL, &key, 0);
if(stat != 0)
{
db_exit("db->del", stat);
}
/* get non existing */
dump(db, keybuf);
/* */
sprintf(keybuf, "Key#%d", C_OFFSET + 88);
/* update and get */
dump(db, keybuf);
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
key.data = keybuf;
key.size = strlen(keybuf);
value.size = sizeof(struct data);
stat = db->get(db, NULL, &key, &value, 0);
if(stat != 0)
{
db_exit("db->get", stat);
}
deserialize(value.data, &d);
d.iv = d.iv + 1;
d.xv = d.xv + 0.1;
strcat(d.sv, " updated");
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
serialize(&d, valuebuf);
key.data = keybuf;
key.size = strlen(keybuf);
value.data = valuebuf;
value.size = length(&d);
stat = db->put(db, NULL, &key, &value, 0);
if(stat != 0)
{
db_exit("db->put", stat);
}
dump(db, keybuf);
/* list all */
n = 0;
stat = db->cursor(db, NULL, &it, 0);
if(stat != 0)
{
db_exit("db->cursor", stat);
}
for(;;)
{
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
stat = it->get(it, &key, &value, DB_NEXT);
if(stat == 0)
{
memcpy(keybuf, key.data, key.size);
keybuf[key.size] = '\0';
if(strstr(keybuf, "Key#") != keybuf)
{
printf("Unexpected key: %s\n", keybuf);
}
deserialize(value.data, &d);
if (d.iv < 1 || NREC < d.iv)
{
printf("Unexpected value: (%d,%f,%s)\n", d.iv, d.xv, d.sv);
}
n++;
}
else if(stat == DB_NOTFOUND)
{
/* done */
break;
}
else
{
db_exit("dbc->get", stat);
}
}
printf("%d\n", n);
/* list keys where "Key#n075" <= key < "Key#n085" */
n2 = 0;
stat = db->cursor(db, NULL, &it2, 0);
if(stat != 0)
{
db_exit("db->cursor", stat);
}
sprintf(keybuf, "Key#%d", C_OFFSET + 75);
sprintf(termkeybuf, "Key#%d", C_OFFSET + 85);
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
key.data = keybuf;
key.size = strlen(keybuf);
value.size = sizeof(struct data);
stat = it2->get(it2, &key, &value, DB_SET);
for(;;)
{
if(stat == 0)
{
memcpy(keybuf, key.data, key.size);
keybuf[key.size] = '\0';
if(strcmp(keybuf, termkeybuf) >= 0) break;
n2++;
}
else if(stat == DB_NOTFOUND)
{
/* done */
break;
}
else
{
db_exit("dbc->get", stat);
}
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
stat = it2->get(it2, &key, &value, DB_NEXT);
}
printf("%d\n", n2);
/* close */
db->close(db, 0);
return 0;
}
Build on Windows with MSVC++:
cl /I%BDB% test.c %BDB%\Win32\Debug\libdb181d.lib
package nosql.bdb;
import java.io.Serializable;
public class Data implements Serializable {
private static final long serialVersionUID = 1L;
private int iv;
private double xv;
private String sv;
public Data() {
this(0, 0.0, "");
}
public Data(int iv, double xv, String sv) {
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
package nosql.bdb;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import com.sleepycat.db.Cursor;
import com.sleepycat.db.Database;
import com.sleepycat.db.DatabaseConfig;
import com.sleepycat.db.DatabaseEntry;
import com.sleepycat.db.DatabaseException;
import com.sleepycat.db.OperationStatus;
public class Test {
private static DatabaseEntry prep(String key) throws UnsupportedEncodingException {
return new DatabaseEntry(key.getBytes("UTF-8"));
}
private static DatabaseEntry serialize(Data o) throws IOException {
ByteBuffer bb = ByteBuffer.allocate(1000);
bb.order(ByteOrder.LITTLE_ENDIAN);
bb.putInt(o.getIv());
bb.putDouble(o.getXv());
byte[] b = o.getSv().getBytes("UTF-8");
bb.putShort((short)b.length);
bb.put(b);
int n = bb.position();
byte[] res = new byte[n];
bb.rewind();
bb.get(res);
return new DatabaseEntry(res);
}
private static Data deserialize(DatabaseEntry de) throws IOException, ClassNotFoundException {
ByteBuffer bb = ByteBuffer.wrap(de.getData());
bb.order(ByteOrder.LITTLE_ENDIAN);
int iv = bb.getInt();
double xv = bb.getDouble();
int len = bb.getShort();
byte[] temp = new byte[len];
bb.get(temp);
String sv = new String(temp, "UTF-8");
Data res = new Data(iv, xv, sv);
return res;
}
private static void dump(Database db, String key) throws ClassNotFoundException, IOException, DatabaseException {
System.out.printf("Key=%s\n", key);
DatabaseEntry rawvalue = new DatabaseEntry();
if(db.get(null, prep(key), rawvalue, null) == OperationStatus.SUCCESS) {
Data value = deserialize(rawvalue);
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final String DBNAME = "/work/bdb/test.db";
private static final int NREC = 1000;
private static final int JAVA_OFFSET = 2 * NREC;
public static void main(String[] args) throws DatabaseException, UnsupportedEncodingException, IOException, ClassNotFoundException {
Database db = new Database(DBNAME, null, new DatabaseConfig());
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (JAVA_OFFSET + i + 1);
Data value = new Data();
value.setIv(i + 1);
value.setXv(i + 1.0);
value.setSv(String.format("This is value %d", i + 1));
db.put(null, prep(key), serialize(value));
}
//
String key;
key = "Key#" + (JAVA_OFFSET + 77);
// get
dump(db, key);
// delete
db.delete(null, prep(key));
// get non existing
dump(db, key);
//
key = "Key#" + (JAVA_OFFSET + 88);
// update and get
dump(db, key);
DatabaseEntry rawvalue = new DatabaseEntry();
db.get(null, prep(key), rawvalue, null);
Data value = (Data)deserialize(rawvalue);
value.setIv(value.getIv() + 1);
value.setXv(value.getXv() + 0.1);
value.setSv(value.getSv() + " updated");
db.put(null, prep(key), serialize(value));
dump(db, key);
// list all
Cursor it = db.openCursor(null, null);
DatabaseEntry itrawkey = new DatabaseEntry();
DatabaseEntry itrawvalue = new DatabaseEntry();
OperationStatus itstat = it.getFirst(itrawkey, itrawvalue, null);
int n = 0;
while(itstat == OperationStatus.SUCCESS) {
String itkey = new String(itrawkey.getData(), "UTF-8");
if(!itkey.startsWith("Key#")) {
System.out.println("Unexpected key: " + itkey);
}
Data itvalue = (Data)deserialize(itrawvalue);
if(itvalue.getIv() < 1 || NREC < itvalue.getIv()) {
System.out.println("Unexpected value :" + itvalue);
}
n++;
itstat = it.getNext(itrawkey, itrawvalue, null);
}
it.close();
System.out.println(n);
// list keys where "Key#1075" <= key < "Key#1085"
Cursor it2 = db.openCursor(null, null);
DatabaseEntry it2rawkey = prep("Key#" + 1075);
DatabaseEntry it2rawvalue = new DatabaseEntry();
OperationStatus it2stat = it2.getSearchKey(it2rawkey, it2rawvalue, null);
int n2 = 0;
while(it2stat == OperationStatus.SUCCESS) {
String it2key = new String(it2rawkey.getData(), "UTF-8");
if(it2key.compareTo("Key#" + 1085) >= 0) break;
n2++;
it2stat = it2.getNext(it2rawkey, it2rawvalue, null);
}
it2.close();
System.out.println(n2);
//
db.close();
}
}
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using BerkeleyDB;
namespace BDB
{
public class Data
{
public int Iv { get; set; }
public double Xv { get; set; }
public string Sv { get; set; }
public override string ToString()
{
return string.Format("(iv: {0}, xv: {1}, sv: {2})", Iv, Xv, Sv);
}
}
public class Program
{
private static DatabaseEntry Prep(string key)
{
return new DatabaseEntry(Encoding.UTF8.GetBytes(key));
}
private static DatabaseEntry Serialize(Data d)
{
MemoryStream ms = new MemoryStream();
BinaryWriter bw = new BinaryWriter(ms);
bw.Write(d.Iv);
bw.Write(d.Xv);
bw.Write((short)d.Sv.Length);
bw.Write(Encoding.UTF8.GetBytes(d.Sv));
bw.Flush();
return new DatabaseEntry(ms.ToArray());
}
private static Data Deserialize(DatabaseEntry de)
{
MemoryStream ms = new MemoryStream(de.Data);
BinaryReader br = new BinaryReader(ms);
Data d = new Data();
d.Iv = br.ReadInt32();
d.Xv = br.ReadDouble();
int len = br.ReadInt16();
byte[] temp = new byte[len];
br.Read(temp, 0, temp.Length);
d.Sv = Encoding.UTF8.GetString(temp);
return d;
}
private static void Dump(Database db, string key)
{
Console.WriteLine("Key={0}", key);
if(db.Exists(Prep(key)))
{
Data value = Deserialize(db.Get(Prep(key)).Value);
Console.WriteLine(value);
}
else
{
Console.WriteLine("Not found");
}
}
private const int NREC = 1000;
private const int DN_OFFSET = 3 * NREC;
public static void Main(string[] args)
{
using(Database db = BTreeDatabase.Open("/work/BDB/test.db", new BTreeDatabaseConfig()))
{
// put data
for (int i = 0; i < NREC; i++)
{
string putkey = "Key#" + (DN_OFFSET + i + 1);
Data value = new Data { Iv = i + 1, Xv = i + 1.0, Sv = string.Format("This is value {0}", i + 1) };
db.Put(Prep(putkey), Serialize(value));
}
string key;
key = "Key#" + (DN_OFFSET + 77);
// get
Dump(db, key);
// delete
db.Delete(Prep(key));
// get non existing
Dump(db, key);
//
key = "Key#" + (DN_OFFSET + 88);
// update and get
Dump(db, key);
Data updvalue = (Data)Deserialize(db.Get(Prep(key)).Value);
updvalue.Iv = updvalue.Iv + 1;
updvalue.Xv = updvalue.Xv + 0.1;
updvalue.Sv = updvalue.Sv + " updated";
db.Put(Prep(key), Serialize(updvalue));
Dump(db, key);
// list all
using (Cursor it = db.Cursor())
{
int n = 0;
bool more = it.MoveFirst();
while (more)
{
string itkey = Encoding.UTF8.GetString(it.Current.Key.Data);
if(!itkey.StartsWith("Key#"))
{
Console.WriteLine("Unexpected key: " + itkey);
}
Data itvalue = Deserialize(it.Current.Value);
if (itvalue.Iv < 1 || NREC < itvalue.Iv)
{
Console.WriteLine("Unexpected value :" + itvalue);
}
n++;
more = it.MoveNext();
}
Console.WriteLine(n);
}
// list all - alternative
using (Cursor it1 = db.Cursor())
{
int n1 = 0;
foreach(KeyValuePair<DatabaseEntry, DatabaseEntry> kvp in it1)
{
String it1key = Encoding.UTF8.GetString(kvp.Key.Data);
if(!it1key.StartsWith("Key#"))
{
Console.WriteLine("Unexpected key: " + it1key);
}
Data it1value = Deserialize(kvp.Value);
if (it1value.Iv < 1 || NREC < it1value.Iv)
{
Console.WriteLine("Unexpected value :" + it1value);
}
n1++;
}
Console.WriteLine(n1);
}
// list keys where "Key#n075" <= key < "Key#n085"
using (Cursor it2 = db.Cursor())
{
int n2 = 0;
bool more = it2.Move(Prep("Key#" + (DN_OFFSET + 75)), true);
while (more)
{
string it2key = Encoding.UTF8.GetString(it2.Current.Key.Data);
if(string.Compare(it2key, "Key#" + (DN_OFFSET + 85)) >= 0) break;
n2++;
more = it2.MoveNext();
}
Console.WriteLine(n2);
}
}
}
}
}
Let now see using C/C++ struct as is. This makes it very easy for C++ code, but create a problem for Java and .NET.
The C/C++ struct is:
struct Data
{
int Iv;
double Xv;
char Sv[50];
};
/* standard C headers */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
/* BDB headers */
#include "db.h"
/* data structure */
struct data
{
int iv;
double xv;
char sv[50];
};
static void serialize(struct data *d, char *buf)
{
memcpy(buf, d, sizeof(struct data));
}
static void deserialize(char *buf, struct data *d)
{
memcpy(d, buf, sizeof(struct data));
}
/* error handling functions */
void db_exit(char *msg, int stat)
{
char buf[256];
printf("DB error %s: %s\n", msg, db_strerror(stat));
exit(1);
}
static void dump(DB *db, char *keybuf)
{
int stat;
DBT key;
DBT value;
struct data d;
printf("Key=%s\n", keybuf);
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
key.data = keybuf;
key.size = strlen(keybuf);
value.size = sizeof(struct data);
stat = db->get(db, NULL, &key, &value, 0);
if(stat == 0)
{
deserialize(value.data, &d);
printf("(%d,%f,%s)\n", d.iv, d.xv, d.sv);
}
else if(stat == DB_NOTFOUND)
{
printf("Not found\n");
}
else
{
db_exit("db->get", stat);
}
}
static const char *DBNAME = "test2.db";
static const int NREC = 1000;
static const int C_OFFSET = 1000; // 1 * NREC;
int main()
{
int stat;
DB *db;
DBC *it, *it2;
DBT key, value, itkey;
struct data d;
char keybuf[32], termkeybuf[32], valuebuf[256];
int n, n2;
/* initialize */
stat = db_create(&db, NULL, 0);
if(stat != 0)
{
db_exit("db_create", stat);
}
/* get connection */
stat = db->open(db, NULL, DBNAME, NULL, DB_BTREE, DB_CREATE, 0664); /* DB_HASH is alternative to DB_BTREE */
if(stat != 0)
{
db_exit("db->open", stat);
}
/* put data */
for(int i = 0; i < NREC; i++)
{
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
sprintf(keybuf, "Key#%d", C_OFFSET + i + 1);
key.data = keybuf;
key.size = strlen(keybuf);
d.iv = i + 1;
d.xv = i + 1.0;
sprintf(d.sv, "This is value %d", i + 1);
serialize(&d, valuebuf);
value.data = valuebuf;
value.size = sizeof(struct data);
stat = db->put(db, NULL, &key, &value, 0);
if(stat != 0)
{
db_exit("db->put", stat);
}
}
/* */
sprintf(keybuf, "Key#%d", C_OFFSET + 77);
/* get */
dump(db, keybuf);
/* delete */
memset(&key, 0, sizeof(DBT));
key.data = keybuf;
key.size = strlen(keybuf);
stat = db->del(db, NULL, &key, 0);
if(stat != 0)
{
db_exit("db->del", stat);
}
/* get non existing */
dump(db, keybuf);
/* */
sprintf(keybuf, "Key#%d", C_OFFSET + 88);
/* update and get */
dump(db, keybuf);
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
key.data = keybuf;
key.size = strlen(keybuf);
value.size = sizeof(struct data);
stat = db->get(db, NULL, &key, &value, 0);
if(stat != 0)
{
db_exit("db->get", stat);
}
deserialize(value.data, &d);
d.iv = d.iv + 1;
d.xv = d.xv + 0.1;
strcat(d.sv, " updated");
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
serialize(&d, valuebuf);
key.data = keybuf;
key.size = strlen(keybuf);
value.data = valuebuf;
value.size = sizeof(struct data);
stat = db->put(db, NULL, &key, &value, 0);
if(stat != 0)
{
db_exit("db->put", stat);
}
dump(db, keybuf);
/* list all */
n = 0;
stat = db->cursor(db, NULL, &it, 0);
if(stat != 0)
{
db_exit("db->cursor", stat);
}
for(;;)
{
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
stat = it->get(it, &key, &value, DB_NEXT);
if(stat == 0)
{
memcpy(keybuf, key.data, key.size);
keybuf[key.size] = '\0';
if(strstr(keybuf, "Key#") != keybuf)
{
printf("Unexpected key: %s\n", keybuf);
}
deserialize(value.data, &d);
if (d.iv < 1 || NREC < d.iv)
{
printf("Unexpected value: (%d,%f,%s)\n", d.iv, d.xv, d.sv);
}
n++;
}
else if(stat == DB_NOTFOUND)
{
/* done */
break;
}
else
{
db_exit("dbc->get", stat);
}
}
printf("%d\n", n);
/* list keys where "Key#n075" <= key < "Key#n085" */
n2 = 0;
stat = db->cursor(db, NULL, &it2, 0);
if(stat != 0)
{
db_exit("db->cursor", stat);
}
sprintf(keybuf, "Key#%d", C_OFFSET + 75);
sprintf(termkeybuf, "Key#%d", C_OFFSET + 85);
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
key.data = keybuf;
key.size = strlen(keybuf);
value.size = sizeof(struct data);
stat = it2->get(it2, &key, &value, DB_SET);
for(;;)
{
if(stat == 0)
{
memcpy(keybuf, key.data, key.size);
keybuf[key.size] = '\0';
if(strcmp(keybuf, termkeybuf) >= 0) break;
n2++;
}
else if(stat == DB_NOTFOUND)
{
/* done */
break;
}
else
{
db_exit("dbc->get", stat);
}
memset(&key, 0, sizeof(DBT));
memset(&value, 0, sizeof(DBT));
stat = it2->get(it2, &key, &value, DB_NEXT);
}
printf("%d\n", n2);
/* close */
db->close(db, 0);
return 0;
}
Build on Windows with MSVC++:
cl /I%BDB% test2.c %BDB%\Win32\Debug\libdb181d.lib
To use the C/C++ format the code will be using my Record library as that makes it a little bit easier.
package nosql.bdb;
import java.io.Serializable;
public class Data implements Serializable {
private static final long serialVersionUID = 1L;
private int iv;
private double xv;
private String sv;
public Data() {
this(0, 0.0, "");
}
public Data(int iv, double xv, String sv) {
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
package nosql.bdb;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import com.sleepycat.db.Cursor;
import com.sleepycat.db.Database;
import com.sleepycat.db.DatabaseConfig;
import com.sleepycat.db.DatabaseEntry;
import com.sleepycat.db.DatabaseException;
import com.sleepycat.db.OperationStatus;
import dk.vajhoej.record.Alignment;
import dk.vajhoej.record.Endian;
import dk.vajhoej.record.FieldType;
import dk.vajhoej.record.RecordException;
import dk.vajhoej.record.Struct;
import dk.vajhoej.record.StructField;
import dk.vajhoej.record.StructReader;
import dk.vajhoej.record.StructWriter;
public class Test2 {
@Struct(endianess=Endian.LITTLE, alignment=Alignment.ALIGN8, endpad=true)
public static class CData {
@StructField(n=0, type=FieldType.INT4)
public int iv;
@StructField(n=1, type=FieldType.FP8)
public double xv;
@StructField(n=2, type=FieldType.FIXSTRNULTERM, length=50)
public String sv;
//
public CData() {
iv = 0;
xv = 0.0;
sv = "";
}
public CData(Data d) {
iv = d.getIv();
xv = d.getXv();
sv = d.getSv();
}
public Data getData() {
return new Data(iv, xv, sv);
}
}
private static DatabaseEntry prep(String key) throws UnsupportedEncodingException {
return new DatabaseEntry(key.getBytes("UTF-8"));
}
private static DatabaseEntry serialize(Data o) throws IOException, RecordException {
CData o2 = new CData(o);
StructWriter sw = new StructWriter();
sw.write(o2);
return new DatabaseEntry(sw.getBytes());
}
private static Data deserialize(DatabaseEntry de) throws IOException, ClassNotFoundException, RecordException {
StructReader sr = new StructReader(de.getData());
CData o = sr.read(CData.class);
return o.getData();
}
private static void dump(Database db, String key) throws ClassNotFoundException, IOException, DatabaseException, RecordException {
System.out.printf("Key=%s\n", key);
DatabaseEntry rawvalue = new DatabaseEntry();
if(db.get(null, prep(key), rawvalue, null) == OperationStatus.SUCCESS) {
Data value = deserialize(rawvalue);
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final String DBNAME = "/work/bdb/test2.db";
private static final int NREC = 1000;
private static final int JAVA_OFFSET = 2 * NREC;
public static void main(String[] args) throws DatabaseException, UnsupportedEncodingException, IOException, ClassNotFoundException, RecordException {
Database db = new Database(DBNAME, null, new DatabaseConfig());
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (JAVA_OFFSET + i + 1);
Data value = new Data();
value.setIv(i + 1);
value.setXv(i + 1.0);
value.setSv(String.format("This is value %d", i + 1));
db.put(null, prep(key), serialize(value));
}
//
String key;
key = "Key#" + (JAVA_OFFSET + 77);
// get
dump(db, key);
// delete
db.delete(null, prep(key));
// get non existing
dump(db, key);
//
key = "Key#" + (JAVA_OFFSET + 88);
// update and get
dump(db, key);
DatabaseEntry rawvalue = new DatabaseEntry();
db.get(null, prep(key), rawvalue, null);
Data value = (Data)deserialize(rawvalue);
value.setIv(value.getIv() + 1);
value.setXv(value.getXv() + 0.1);
value.setSv(value.getSv() + " updated");
db.put(null, prep(key), serialize(value));
dump(db, key);
// list all
Cursor it = db.openCursor(null, null);
DatabaseEntry itrawkey = new DatabaseEntry();
DatabaseEntry itrawvalue = new DatabaseEntry();
OperationStatus itstat = it.getFirst(itrawkey, itrawvalue, null);
int n = 0;
while(itstat == OperationStatus.SUCCESS) {
String itkey = new String(itrawkey.getData(), "UTF-8");
if(!itkey.startsWith("Key#")) {
System.out.println("Unexpected key: " + itkey);
}
Data itvalue = (Data)deserialize(itrawvalue);
if(itvalue.getIv() < 1 || NREC < itvalue.getIv()) {
System.out.println("Unexpected value :" + itvalue);
}
n++;
itstat = it.getNext(itrawkey, itrawvalue, null);
}
it.close();
System.out.println(n);
// list keys where "Key#1075" <= key < "Key#1085"
Cursor it2 = db.openCursor(null, null);
DatabaseEntry it2rawkey = prep("Key#" + 1075);
DatabaseEntry it2rawvalue = new DatabaseEntry();
OperationStatus it2stat = it2.getSearchKey(it2rawkey, it2rawvalue, null);
int n2 = 0;
while(it2stat == OperationStatus.SUCCESS) {
String it2key = new String(it2rawkey.getData(), "UTF-8");
if(it2key.compareTo("Key#" + 1085) >= 0) break;
n2++;
it2stat = it2.getNext(it2rawkey, it2rawvalue, null);
}
it2.close();
System.out.println(n2);
//
db.close();
}
}
To use the C/C++ format the code will be using my NRecord library as that makes it a little bit easier.
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using BerkeleyDB;
using Vajhoej.Record;
namespace BDB
{
public class Data
{
public int Iv { get; set; }
public double Xv { get; set; }
public string Sv { get; set; }
public override string ToString()
{
return string.Format("(iv: {0}, xv: {1}, sv: {2})", Iv, Xv, Sv);
}
}
public class Program
{
[Struct(Endianess = Endian.LITTLE, Alignment = Alignment.ALIGN8, Endpad = true)]
public class CData
{
[StructField(N = 0, Type = FieldType.INT4)]
public int Iv;
[StructField(N = 1, Type = FieldType.FP8)]
public double Xv;
[StructField(N = 2, Type = FieldType.FIXSTRNULTERM, Length = 50)]
public String Sv;
//
public CData()
{
Iv = 0;
Xv = 0.0;
Sv = "";
}
public CData(Data o)
{
Iv = o.Iv;
Xv = o.Xv;
Sv = o.Sv;
}
public Data GetData()
{
return new Data { Iv = Iv, Xv = Xv, Sv = Sv };
}
}
private static DatabaseEntry Prep(string key)
{
return new DatabaseEntry(Encoding.UTF8.GetBytes(key));
}
private static DatabaseEntry Serialize(Data d)
{
CData d2 = new CData(d);
StructWriter sw = new StructWriter();
sw.Write(d2);
return new DatabaseEntry(sw.GetBytes());
}
private static Data Deserialize(DatabaseEntry de)
{
StructReader sr = new StructReader(de.Data);
CData o = sr.Read<CData>(typeof(CData));
return o.GetData();
}
private static void Dump(Database db, string key)
{
Console.WriteLine("Key={0}", key);
if(db.Exists(Prep(key)))
{
Data value = Deserialize(db.Get(Prep(key)).Value);
Console.WriteLine(value);
}
else
{
Console.WriteLine("Not found");
}
}
private const int NREC = 1000;
private const int DN_OFFSET = 3 * NREC;
public static void Main(string[] args)
{
using(Database db = BTreeDatabase.Open("/work/BDB/test2.db", new BTreeDatabaseConfig()))
{
// put data
for (int i = 0; i < NREC; i++)
{
string putkey = "Key#" + (DN_OFFSET + i + 1);
Data value = new Data { Iv = i + 1, Xv = i + 1.0, Sv = string.Format("This is value {0}", i + 1) };
db.Put(Prep(putkey), Serialize(value));
}
string key;
key = "Key#" + (DN_OFFSET + 77);
// get
Dump(db, key);
// delete
db.Delete(Prep(key));
// get non existing
Dump(db, key);
//
key = "Key#" + (DN_OFFSET + 88);
// update and get
Dump(db, key);
Data updvalue = (Data)Deserialize(db.Get(Prep(key)).Value);
updvalue.Iv = updvalue.Iv + 1;
updvalue.Xv = updvalue.Xv + 0.1;
updvalue.Sv = updvalue.Sv + " updated";
db.Put(Prep(key), Serialize(updvalue));
Dump(db, key);
// list all
using (Cursor it = db.Cursor())
{
int n = 0;
bool more = it.MoveFirst();
while (more)
{
string itkey = Encoding.UTF8.GetString(it.Current.Key.Data);
if(!itkey.StartsWith("Key#"))
{
Console.WriteLine("Unexpected key: " + itkey);
}
Data itvalue = Deserialize(it.Current.Value);
if (itvalue.Iv < 1 || NREC < itvalue.Iv)
{
Console.WriteLine("Unexpected value :" + itvalue);
}
n++;
more = it.MoveNext();
}
Console.WriteLine(n);
}
// list all - alternative
using (Cursor it1 = db.Cursor())
{
int n1 = 0;
foreach(KeyValuePair<DatabaseEntry, DatabaseEntry> kvp in it1)
{
String it1key = Encoding.UTF8.GetString(kvp.Key.Data);
if(!it1key.StartsWith("Key#"))
{
Console.WriteLine("Unexpected key: " + it1key);
}
Data it1value = Deserialize(kvp.Value);
if (it1value.Iv < 1 || NREC < it1value.Iv)
{
Console.WriteLine("Unexpected value :" + it1value);
}
n1++;
}
Console.WriteLine(n1);
}
// list keys where "Key#n075" <= key < "Key#n085"
using (Cursor it2 = db.Cursor())
{
int n2 = 0;
bool more = it2.Move(Prep("Key#" + (DN_OFFSET + 75)), true);
while (more)
{
string it2key = Encoding.UTF8.GetString(it2.Current.Key.Data);
if(string.Compare(it2key, "Key#" + (DN_OFFSET + 85)) >= 0) break;
n2++;
more = it2.MoveNext();
}
Console.WriteLine(n2);
}
}
}
}
}
Example showing transaction usage (classic account transfer example):
package nosql.bdb;
import java.io.File;
import java.io.FileNotFoundException;
import java.math.BigDecimal;
import com.sleepycat.db.Database;
import com.sleepycat.db.DatabaseConfig;
import com.sleepycat.db.DatabaseEntry;
import com.sleepycat.db.DatabaseException;
import com.sleepycat.db.DatabaseType;
import com.sleepycat.db.Environment;
import com.sleepycat.db.EnvironmentConfig;
import com.sleepycat.db.Transaction;
import com.sleepycat.db.TransactionConfig;
public class TestTx {
public static class Account {
private BigDecimal amount;
public Account() {
this.amount = new BigDecimal("0.00");
}
public Account(BigDecimal amount) {
this.amount = amount;
}
public BigDecimal getAmount() {
return amount;
}
public void transfer(BigDecimal delta) {
amount = amount.add(delta);
}
// used for deserialization and serialization
public Account(byte[] b) {
this.amount = new BigDecimal(new String(b));
}
public byte[] toBytes() {
return amount.toPlainString().getBytes();
}
}
public static void saveAccount(Database db, String id, Account acct) throws DatabaseException {
TransactionConfig txcfg = new TransactionConfig();
txcfg.setReadCommitted(true);
Transaction tx = db.getEnvironment().beginTransaction(null, txcfg);
db.put(null, new DatabaseEntry(id.getBytes()), new DatabaseEntry(acct.toBytes()));
tx.commitSync();
}
public static Account getAccount(Database db, String id) throws DatabaseException {
DatabaseEntry acctdata = new DatabaseEntry();
db.get(null, new DatabaseEntry(id.getBytes()), acctdata, null);
return new Account(acctdata.getData());
}
public static void transfer(Database db, String fromid, String toid, BigDecimal amount) throws DatabaseException {
TransactionConfig txcfg = new TransactionConfig();
txcfg.setReadCommitted(true);
Transaction tx = db.getEnvironment().beginTransaction(null, txcfg);
DatabaseEntry acctdata = new DatabaseEntry();
db.get(tx, new DatabaseEntry(fromid.getBytes()), acctdata, null);
Account fromacct = new Account(acctdata.getData());
db.get(tx, new DatabaseEntry(toid.getBytes()), acctdata, null);
Account toacct = new Account(acctdata.getData());
fromacct.transfer(BigDecimal.ZERO.subtract(amount));
toacct.transfer(amount);
db.put(tx, new DatabaseEntry(fromid.getBytes()), new DatabaseEntry(fromacct.toBytes()));
db.put(tx, new DatabaseEntry(toid.getBytes()), new DatabaseEntry(toacct.toBytes()));
tx.commitSync();
//tx.abort();
}
public static void main(String[] args) throws FileNotFoundException, DatabaseException {
// open database
EnvironmentConfig envcfg = new EnvironmentConfig();
envcfg.setTransactional(true);
envcfg.setAllowCreate(true);
envcfg.setReplicationManagerSSLdisabled(true);
envcfg.setInitializeCache(true);
Environment env = new Environment(new File("/work/bdb"), envcfg);
DatabaseConfig cfg = new DatabaseConfig();
cfg.setAllowCreate(true);
cfg.setTransactional(true);
cfg.setType(DatabaseType.BTREE);
Database db = env.openDatabase(null, "acct.db", null, cfg);
// setup accounts
saveAccount(db, "A", new Account(new BigDecimal("100.00")));
saveAccount(db, "B", new Account(new BigDecimal("0.00")));
// display accounts
Account acct_a = getAccount(db, "A");
System.out.printf("%s : %s\n", "A", acct_a.getAmount());
Account acct_b = getAccount(db, "B");
System.out.printf("%s : %s\n", "B", acct_b.getAmount());
// transfer money
transfer(db, "A", "B", new BigDecimal("20.00"));
// display accounts
acct_a = getAccount(db, "A");
System.out.printf("%s : %s\n", "A", acct_a.getAmount());
acct_b = getAccount(db, "B");
System.out.printf("%s : %s\n", "B", acct_b.getAmount());
// close database
db.close();
env.close();
}
}
BDB-JE (Berkeley DB Java Edition) is is a pure Java product and since version 7.3.7 licensed under Apache license (before that it was same AGPL license as BDB).
Note that BDB-JE is different from BDB:
Supported platforms | Any platform with Java |
Supported languages | Java and other JVM languages |
Features | Transaction support Multiple keys |
Missing features |
BDB-JE can be used in different modes:
This is a low level API where the application needs to explicit do any required serialization and deserialization.
Example:
package nosql.bdbje;
import java.io.Serializable;
public class Data implements Serializable {
private static final long serialVersionUID = 1L;
private int iv;
private double xv;
private String sv;
public Data() {
this(0, 0.0, "");
}
public Data(int iv, double xv, String sv) {
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
package nosql.bdbje;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import com.sleepycat.je.Cursor;
import com.sleepycat.je.CursorConfig;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseConfig;
import com.sleepycat.je.DatabaseEntry;
import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.Get;
import com.sleepycat.je.OperationResult;
import com.sleepycat.je.OperationStatus;
public class Test {
private static DatabaseEntry prep(String key) throws UnsupportedEncodingException {
return new DatabaseEntry(key.getBytes("UTF-8"));
}
private static DatabaseEntry serialize(Data o) throws IOException {
ByteBuffer bb = ByteBuffer.allocate(1000);
bb.order(ByteOrder.LITTLE_ENDIAN);
bb.putInt(o.getIv());
bb.putDouble(o.getXv());
byte[] b = o.getSv().getBytes("UTF-8");
bb.putShort((short)b.length);
bb.put(b);
int n = bb.position();
byte[] res = new byte[n];
bb.rewind();
bb.get(res);
return new DatabaseEntry(res);
}
private static Data deserialize(DatabaseEntry b) throws IOException, ClassNotFoundException {
ByteBuffer bb = ByteBuffer.wrap(b.getData());
bb.order(ByteOrder.LITTLE_ENDIAN);
int iv = bb.getInt();
double xv = bb.getDouble();
int len = bb.getShort();
byte[] temp = new byte[len];
bb.get(temp);
String sv = new String(temp, "UTF-8");
Data res = new Data(iv, xv, sv);
return res;
}
private static void dump(Database db, String key) throws ClassNotFoundException, IOException {
System.out.printf("Key=%s\n", key);
DatabaseEntry rawvalue = new DatabaseEntry();
if(db.get(null, prep(key), rawvalue, null) == OperationStatus.SUCCESS) {
Data value = deserialize(rawvalue);
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
public static void main(String[] args) throws IOException, ClassNotFoundException {
EnvironmentConfig envcfg = new EnvironmentConfig();
envcfg.setAllowCreateVoid(true);
Environment env = new Environment(new File("/work/BDB"), envcfg);
DatabaseConfig dbcfg = new DatabaseConfig();
dbcfg.setAllowCreate(true);
Database db = env.openDatabase(null, "testdb", dbcfg);
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (1000 + i + 1);
Data value = new Data();
value.setIv(i + 1);
value.setXv(i + 1.0);
value.setSv(String.format("This is value %d", i + 1));
db.put(null, prep(key), serialize(value));
}
//
String key;
key = "Key#" + 1077;
// get
dump(db, key);
// delete
db.delete(null, prep(key));
// get non existing
dump(db, key);
//
key = "Key#" + 1088;
// update and get
dump(db, key);
DatabaseEntry rawvalue = new DatabaseEntry();
db.get(null, prep(key), rawvalue, null);
Data value = (Data)deserialize(rawvalue);
value.setIv(value.getIv() + 1);
value.setXv(value.getXv() + 0.1);
value.setSv(value.getSv() + " updated");
db.put(null, prep(key), serialize(value));
dump(db, key);
// list all
Cursor it = db.openCursor(null, new CursorConfig());
DatabaseEntry itrawkey = new DatabaseEntry();
DatabaseEntry itrawvalue = new DatabaseEntry();
OperationStatus itstat = it.getFirst(itrawkey, itrawvalue, null);
int n = 0;
while(itstat == OperationStatus.SUCCESS) {
String itkey = new String(itrawkey.getData(), "UTF-8");
if(!itkey.startsWith("Key#")) {
System.out.println("Unexpected key: " + itkey);
}
Data itvalue = (Data)deserialize(itrawvalue);
if(itvalue.getIv() < 1 || NREC < itvalue.getIv()) {
System.out.println("Unexpected value :" + itvalue);
}
n++;
itstat = it.getNext(itrawkey, itrawvalue, null);
}
it.close();
System.out.println(n);
// list keys where "Key#1075" <= key < "Key#1085"
Cursor it2 = db.openCursor(null, new CursorConfig());
DatabaseEntry it2rawkey = prep("Key#" + 1075);
DatabaseEntry it2rawvalue = new DatabaseEntry();
OperationResult it2res = it2.get(it2rawkey, it2rawvalue, Get.SEARCH_GTE, null);
OperationStatus it2stat = (it2res != null) ? OperationStatus.SUCCESS : OperationStatus.NOTFOUND;
int n2 = 0;
while(it2stat == OperationStatus.SUCCESS) {
String it2key = new String(it2rawkey.getData(), "UTF-8");
if(it2key.compareTo("Key#" + 1085) >= 0) break;
n2++;
it2stat = it2.getNext(it2rawkey, it2rawvalue, null);
}
it2.close();
System.out.println(n2);
// close database
db.close();
env.close();
}
}
There is an upcoming Java EE standard for NoSQL databases: Jakarta NoSQL. The reference implementation is Eclipse JNoSQL.
I have created a BDB-JE driver for it that can be downloaded here.
The example is tested with B2 release.
package nosql.bdbje.jnosql;
import java.io.Serializable;
public class Data implements Serializable {
private static final long serialVersionUID = 1L;
private int iv;
private double xv;
private String sv;
public Data() {
this(0, 0.0, "");
}
public Data(int iv, double xv, String sv) {
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
package nosql.bdbje.jnosql;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Optional;
import jakarta.nosql.Value;
import jakarta.nosql.keyvalue.BucketManager;
import jakarta.nosql.keyvalue.BucketManagerFactory;
import jakarta.nosql.keyvalue.KeyValueConfiguration;
import dk.vajhoej.jnosql.bdbje.BDBJEConfiguration;
public class Test {
private static byte[] serialize(Data o) throws IOException {
ByteBuffer bb = ByteBuffer.allocate(1000);
bb.order(ByteOrder.LITTLE_ENDIAN);
bb.putInt(o.getIv());
bb.putDouble(o.getXv());
byte[] b = o.getSv().getBytes("UTF-8");
bb.putShort((short)b.length);
bb.put(b);
int n = bb.position();
byte[] res = new byte[n];
bb.rewind();
bb.get(res);
return res;
}
private static Data deserialize(byte[] b) throws IOException, ClassNotFoundException {
ByteBuffer bb = ByteBuffer.wrap(b);
bb.order(ByteOrder.LITTLE_ENDIAN);
int iv = bb.getInt();
double xv = bb.getDouble();
int len = bb.getShort();
byte[] temp = new byte[len];
bb.get(temp);
String sv = new String(temp, "UTF-8");
Data res = new Data(iv, xv, sv);
return res;
}
private static void dump(BucketManager bm, String key) throws ClassNotFoundException, IOException {
System.out.printf("Key=%s\n", key);
Optional<Value> rawvalue = bm.get(key);
if(rawvalue.isPresent()) {
Data value = (Data)rawvalue.get().get();
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
public static void main(String[] args) throws IOException, ClassNotFoundException {
KeyValueConfiguration cfg = new BDBJEConfiguration("/work/BDB", "testdb", false, o -> serialize((Data)o), b -> deserialize(b));
BucketManagerFactory bmf = cfg.get();
BucketManager bm = bmf.getBucketManager("testdb");
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (1000 + i + 1);
Data value = new Data();
value.setIv(i + 1);
value.setXv(i + 1.0);
value.setSv(String.format("This is value %d", i + 1));
bm.put(key, value);
}
//
String key;
key = "Key#" + 1077;
// get
dump(bm, key);
// delete
bm.delete(key);
// get non existing
dump(bm, key);
//
key = "Key#" + 1088;
// update and get
dump(bm, key);
Data value = (Data)bm.get(key).get().get();
value.setIv(value.getIv() + 1);
value.setXv(value.getXv() + 0.1);
value.setSv(value.getSv() + " updated");
bm.put(key, value);
dump(bm, key);
// list all
// **** NOT SUPPORTED ****
// list keys where "Key#1075" <= key < "Key#1085"
// **** NOT SUPPORTED ****
//
bm.close();
bmf.close();
}
}
The configuration in:
KeyValueConfiguration cfg = new BDBJEConfiguration("/work/BDB", "testdb", false, o -> serialize((Data)o), b -> deserialize(b));
can be externalized in bdbje.properties:
dbdir = /work/BDB
dbnam = testdb
prefix = false
to = nosql.bdbje.jnosql.Test$MySerializer
from = nosql.bdbje.jnosql.Test$MyDeserializer
and either:
KeyValueConfiguration cfg = new BDBJEConfiguration();
or:
KeyValueConfiguration cfg = KeyValueConfiguration.getConfiguration();
following nested classes are needed:
public static class MySerializer implements Serialize {
@Override
public byte[] convert(Object obj) throws IOException {
return serialize((Data)obj);
}
}
public static class MyDeserializer implements Deserialize {
@Override
public Object convert(byte[] b) throws IOException, ClassNotFoundException {
return deserialize(b);
}
}
Example showing transaction usage (classic account transfer example):
package nosql.bdbje;
import java.io.File;
import java.math.BigDecimal;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseConfig;
import com.sleepycat.je.DatabaseEntry;
import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.Transaction;
import com.sleepycat.je.TransactionConfig;
public class TestTx {
public static class Account {
private BigDecimal amount;
public Account() {
this.amount = new BigDecimal("0.00");
}
public Account(BigDecimal amount) {
this.amount = amount;
}
public BigDecimal getAmount() {
return amount;
}
public void transfer(BigDecimal delta) {
amount = amount.add(delta);
}
// used for deserialization and serialization
public Account(byte[] b) {
this.amount = new BigDecimal(new String(b));
}
public byte[] toBytes() {
return amount.toPlainString().getBytes();
}
}
public static void saveAccount(Database db, String id, Account acct) {
TransactionConfig txcfg = new TransactionConfig();
txcfg.setReadCommitted(true);
Transaction tx = db.getEnvironment().beginTransaction(null, txcfg);
db.put(null, new DatabaseEntry(id.getBytes()), new DatabaseEntry(acct.toBytes()));
tx.commitSync();
}
public static Account getAccount(Database db, String id) {
DatabaseEntry acctdata = new DatabaseEntry();
db.get(null, new DatabaseEntry(id.getBytes()), acctdata, null);
return new Account(acctdata.getData());
}
public static void transfer(Database db, String fromid, String toid, BigDecimal amount) {
TransactionConfig txcfg = new TransactionConfig();
txcfg.setReadCommitted(true);
Transaction tx = db.getEnvironment().beginTransaction(null, txcfg);
DatabaseEntry acctdata = new DatabaseEntry();
db.get(tx, new DatabaseEntry(fromid.getBytes()), acctdata, null);
Account fromacct = new Account(acctdata.getData());
db.get(tx, new DatabaseEntry(toid.getBytes()), acctdata, null);
Account toacct = new Account(acctdata.getData());
fromacct.transfer(BigDecimal.ZERO.subtract(amount));
toacct.transfer(amount);
db.put(tx, new DatabaseEntry(fromid.getBytes()), new DatabaseEntry(fromacct.toBytes()));
db.put(tx, new DatabaseEntry(toid.getBytes()), new DatabaseEntry(toacct.toBytes()));
tx.commitSync();
//tx.abort();
}
public static void main(String[] args) {
// open database
EnvironmentConfig envcfg = new EnvironmentConfig();
envcfg.setAllowCreateVoid(true);
envcfg.setTransactional(true);
Environment env = new Environment(new File("/work/BDB"), envcfg);
DatabaseConfig dbcfg = new DatabaseConfig();
dbcfg.setAllowCreate(true);
dbcfg.setTransactional(true);
// setup accounts
Database db = env.openDatabase(null, "acctdb", dbcfg);
saveAccount(db, "A", new Account(new BigDecimal("100.00")));
saveAccount(db, "B", new Account(new BigDecimal("0.00")));
// display accounts
Account acct_a = getAccount(db, "A");
System.out.printf("%s : %s\n", "A", acct_a.getAmount());
Account acct_b = getAccount(db, "B");
System.out.printf("%s : %s\n", "B", acct_b.getAmount());
// transfer money
transfer(db, "A", "B", new BigDecimal("20.00"));
// display accounts
acct_a = getAccount(db, "A");
System.out.printf("%s : %s\n", "A", acct_a.getAmount());
acct_b = getAccount(db, "B");
System.out.printf("%s : %s\n", "B", acct_b.getAmount());
// close database
db.close();
env.close();
}
}
This is a high level API wotking on classes with persistence isntructions given in annotaions and serialization and deserialization are handled automatically.
Example:
package nosql.bdbje;
import com.sleepycat.persist.model.Entity;
import com.sleepycat.persist.model.PrimaryKey;
@Entity
public class DataO {
@PrimaryKey
private String key;
private int iv;
private double xv;
private String sv;
public DataO() {
this("", 0, 0.0, "");
}
public DataO(String key, int iv, double xv, String sv) {
this.key = key;
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public String getKey() {
return key;
}
public void setKey(String key) {
this.key = key;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
package nosql.bdbje;
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.persist.EntityCursor;
import com.sleepycat.persist.EntityStore;
import com.sleepycat.persist.PrimaryIndex;
import com.sleepycat.persist.StoreConfig;
public class TestO {
private static void dump(PrimaryIndex<String, Data0> pk, String key) throws ClassNotFoundException, IOException {
System.out.printf("Key=%s\n", key);
if(pk.contains(key)) {
DataO o = pk.get(key);
System.out.println(o);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
public static void main(String[] args) throws IOException, ClassNotFoundException {
EnvironmentConfig envcfg = new EnvironmentConfig();
envcfg.setAllowCreateVoid(true);
Environment env = new Environment(new File("/work/BDB"), envcfg);
StoreConfig stcfg = new StoreConfig();
stcfg.setAllowCreate(true);
EntityStore est = new EntityStore(env, "testdb", stcfg);
PrimaryIndex<String, Data0> pk = est.getPrimaryIndex(String.class, DataO.class);
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (1000 + i + 1);
DataO o = new DataO();
o.setKey(key);
o.setIv(i + 1);
o.setXv(i + 1.0);
o.setSv(String.format("This is value %d", i + 1));
pk.put(o);
}
//
String key;
key = "Key#" + 1077;
// get
dump(pk, key);
// delete
pk.delete(key);
// get non existing
dump(pk, key);
//
key = "Key#" + 1088;
// update and get
dump(pk, key);
DataO o = pk.get(key);
o.setIv(o.getIv() + 1);
o.setXv(o.getXv() + 0.1);
o.setSv(o.getSv() + " updated");
pk.put(o);
dump(pk, key);
// list all
EntityCursor<Data0> c = pk.entities();
Iterator<Data0> it = c.iterator();
int n = 0;
while(it.hasNext()) {
DataO ito = it.next();
if(!ito.getKey().startsWith("Key#")) {
System.out.println("Unexpected key: " + ito.getKey());
}
if(ito.getIv() < 1 || NREC < ito.getIv()) {
System.out.println("Unexpected value :" + ito);
}
n++;
}
c.close();
System.out.println(n);
// list keys where "Key#1075" <= key < "Key#1085"
EntityCursor<Data0> c2 = pk.entities("Key#1075", true, "Key#1085", false);
Iterator<Data0> it2 = c2.iterator();
int n2 = 0;
while(it2.hasNext()) {
@SuppressWarnings("unused")
DataO ito2 = it2.next();
n2++;
}
c2.close();
System.out.println(n2);
// close database
est.close();
env.close();
}
}
Example showing transaction usage (classic account transfer example):
package nosql.bdbje;
import java.io.File;
import java.math.BigDecimal;
import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.Transaction;
import com.sleepycat.je.TransactionConfig;
import com.sleepycat.persist.EntityStore;
import com.sleepycat.persist.PrimaryIndex;
import com.sleepycat.persist.StoreConfig;
import com.sleepycat.persist.model.Entity;
import com.sleepycat.persist.model.PrimaryKey;
public class TestTxO {
@Entity
public static class Account {
@PrimaryKey
private String name;
private BigDecimal amount;
public Account() {
this("", new BigDecimal("0.00"));
}
public Account(String name, BigDecimal amount) {
this.name = name;
this.amount = amount;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public BigDecimal getAmount() {
return amount;
}
public void transfer(BigDecimal delta) {
amount = amount.add(delta);
}
}
public static void saveAccount(PrimaryIndex<String, Account> pk, Account acct) {
TransactionConfig txcfg = new TransactionConfig();
txcfg.setReadCommitted(true);
Transaction tx = pk.getDatabase().getEnvironment().beginTransaction(null, txcfg);
pk.put(acct);
tx.commitSync();
}
public static void transfer(PrimaryIndex<String, Account> pk, String fromid, String toid, BigDecimal amount) {
TransactionConfig txcfg = new TransactionConfig();
txcfg.setReadCommitted(true);
Transaction tx = pk.getDatabase().getEnvironment().beginTransaction(null, txcfg);
Account fromacct = pk.get(fromid);
Account toacct = pk.get(toid);
fromacct.transfer(BigDecimal.ZERO.subtract(amount));
toacct.transfer(amount);
pk.put(fromacct);
pk.put(toacct);
tx.commitSync();
//tx.abort();
}
public static void main(String[] args) {
// open database
EnvironmentConfig envcfg = new EnvironmentConfig();
envcfg.setAllowCreateVoid(true);
envcfg.setTransactional(true);
Environment env = new Environment(new File("/work/BDB"), envcfg);
StoreConfig stcfg = new StoreConfig();
stcfg.setAllowCreate(true);
stcfg.setTransactional(true);
EntityStore est = new EntityStore(env, "acctdb", stcfg);
PrimaryIndex<String, Account> pk = est.getPrimaryIndex(String.class, Account.class);
// setup accounts
saveAccount(pk, new Account("A", new BigDecimal("100.00")));
saveAccount(pk, new Account("B", new BigDecimal("0.00")));
// display accounts
Account acct_a = pk.get("A");
System.out.printf("%s : %s\n", acct_a.getName(), acct_a.getAmount());
Account acct_b = pk.get("B");
System.out.printf("%s : %s\n", acct_b.getName(), acct_b.getAmount());
// transfer money
transfer(pk, "A", "B", new BigDecimal("20.00"));
// display accounts
acct_a = pk.get("A");
System.out.printf("%s : %s\n", acct_a.getName(), acct_a.getAmount());
acct_b = pk.get("B");
System.out.printf("%s : %s\n", acct_b.getName(), acct_b.getAmount());
// close database
est.close();
env.close();
}
}
Example:
package nosql.bdbje;
import java.io.File;
import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.persist.EntityStore;
import com.sleepycat.persist.PrimaryIndex;
import com.sleepycat.persist.SecondaryIndex;
import com.sleepycat.persist.StoreConfig;
import com.sleepycat.persist.model.Entity;
import com.sleepycat.persist.model.PrimaryKey;
import com.sleepycat.persist.model.Relationship;
import com.sleepycat.persist.model.SecondaryKey;
public class MultiKey {
@Entity
public static class Data {
@PrimaryKey
private int id;
@SecondaryKey(relate=Relationship.MANY_TO_ONE)
private String name;
@SecondaryKey(relate=Relationship.MANY_TO_ONE)
private String address;
private int value;
public Data() {
this(0, "", "", 0);
}
public Data(int id, String name, String address, int value) {
super();
this.id = id;
this.name = name;
this.address = address;
this.value = value;
}
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
public int getValue() {
return value;
}
public void setValue(int value) {
this.value = value;
}
}
public static void main(String[] args) {
// open database
EnvironmentConfig envcfg = new EnvironmentConfig();
envcfg.setAllowCreateVoid(true);
envcfg.setTransactional(true);
Environment env = new Environment(new File("/work/BDB"), envcfg);
StoreConfig stcfg = new StoreConfig();
stcfg.setAllowCreate(true);
stcfg.setTransactional(true);
EntityStore est = new EntityStore(env, "mulkeydb", stcfg);
PrimaryIndex<Integer, Data> idix = est.getPrimaryIndex(Integer.class, Data.class);
SecondaryIndex<String, Integer, Data> namix = est.getSecondaryIndex(idix, String.class, "name");
SecondaryIndex<String, Integer, Data> addrix = est.getSecondaryIndex(idix, String.class, "address");
// put 3 records
idix.put(new Data(1, "A", "A A A", 123));
idix.put(new Data(2, "B", "B B B", 456));
idix.put(new Data(3, "C", "C C C", 789));
// lookup by id
System.out.println(idix.get(2).getValue());
System.out.println(idix.get(3).getValue());
// lookup by name
System.out.println(namix.get("B").getValue());
System.out.println(namix.get("C").getValue());
// lookup by address
System.out.println(addrix.get("B B B").getValue());
System.out.println(addrix.get("C C C").getValue());
// close database
est.close();
env.close();
}
}
MapDB was first released in 2013. It is open source under Apache 2.0 license and written in Java and Kotlin.
It is really a NoSQL Key Value Store but it is special because it exposes an API based on standard Java Map<K,V> interface (and other collections).
Supported platforms | Any platform with Java |
Supported languages | Java and other JVM languages |
Features | |
Missing features |
There are two approaches to data format:
Common value class:
package nosql.mapdb;
import java.io.Serializable;
public class Data implements Serializable {
private static final long serialVersionUID = 1L;
private int iv;
private double xv;
private String sv;
public Data() {
this(0, 0.0, "");
}
public Data(int iv, double xv, String sv) {
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
package nosql.mapdb;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import org.mapdb.DB;
import org.mapdb.DBMaker;
public class Test {
private static void dump(Map<String,Data> db, String key) throws ClassNotFoundException, IOException {
System.out.printf("Key=%s\n", key);
Data value = db.get(key);
if(value != null) {
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
public static void main(String[] args) throws IOException, ClassNotFoundException {
DB dbf = DBMaker.fileDB("/work/testdb.mdb").make();
@SuppressWarnings("unchecked")
Map<String,Data> db = (Map<String,Data>) dbf.hashMap("data").createOrOpen();
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (1000 + i + 1);
Data value = new Data();
value.setIv(i + 1);
value.setXv(i + 1.0);
value.setSv(String.format("This is value %d", i + 1));
db.put(key, value);
}
//
String key;
key = "Key#" + 1077;
// get
dump(db, key);
// delete
db.remove(key);
// get non existing
dump(db, key);
//
key = "Key#" + 1088;
// update and get
dump(db, key);
Data value = db.get(key);
value.setIv(value.getIv() + 1);
value.setXv(value.getXv() + 0.1);
value.setSv(value.getSv() + " updated");
db.put(key, value);
dump(db, key);
// list all
Iterator<String> it = db.keySet().iterator();
int n = 0;
while(it.hasNext()) {
String itkey = it.next();
if(!itkey.startsWith("Key#")) {
System.out.println("Unexpected key: " + itkey);
}
Data itvalue = db.get(itkey);
if(itvalue.getIv() < 1 || NREC < itvalue.getIv()) {
System.out.println("Unexpected value :" + itvalue);
}
n++;
}
System.out.println(n);
// list keys where "Key#1075" <= key < "Key#1085"
// **** not supported by HTreeMap (BTreeMap required) ****
// close database
dbf.close();
}
}
package nosql.mapdb;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import org.mapdb.DB;
import org.mapdb.DBMaker;
import org.mapdb.BTreeMap;
public class Test2 {
private static void dump(Map<String,Data> db, String key) throws ClassNotFoundException, IOException {
System.out.printf("Key=%s\n", key);
Data value = db.get(key);
if(value != null) {
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
public static void main(String[] args) throws IOException, ClassNotFoundException {
DB dbf = DBMaker.fileDB("/work/test2db.mdb").make();
@SuppressWarnings("unchecked")
Map<String,Data> db = (Map<String,Data>) dbf.treeMap("data").createOrOpen();
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (1000 + i + 1);
Data value = new Data();
value.setIv(i + 1);
value.setXv(i + 1.0);
value.setSv(String.format("This is value %d", i + 1));
db.put(key, value);
}
//
String key;
key = "Key#" + 1077;
// get
dump(db, key);
// delete
db.remove(key);
// get non existing
dump(db, key);
//
key = "Key#" + 1088;
// update and get
dump(db, key);
Data value = db.get(key);
value.setIv(value.getIv() + 1);
value.setXv(value.getXv() + 0.1);
value.setSv(value.getSv() + " updated");
db.put(key, value);
dump(db, key);
// list all
Iterator<String> it = db.keySet().iterator();
int n = 0;
while(it.hasNext()) {
String itkey = it.next();
if(!itkey.startsWith("Key#")) {
System.out.println("Unexpected key: " + itkey);
}
Data itvalue = db.get(itkey);
if(itvalue.getIv() < 1 || NREC < itvalue.getIv()) {
System.out.println("Unexpected value :" + itvalue);
}
n++;
}
System.out.println(n);
// list keys where "Key#1075" <= key < "Key#1085"
Iterator<String> it2 = ((BTreeMap<String,Data>)db).subMap("Key#1075", "Key#1085").keySet().iterator();
int n2 = 0;
while(it2.hasNext()) {
String it2key = it2.next();
@SuppressWarnings("unused")
Data it2value = db.get(it2key);
n2++;
}
System.out.println(n2);
// close database
dbf.close();
}
}
package nosql.mapdb;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Iterator;
import java.util.Map;
import org.mapdb.DB;
import org.mapdb.DBMaker;
import org.mapdb.Serializer;
public class TestX {
private static byte[] prep(String key) throws UnsupportedEncodingException {
return key.getBytes("UTF-8");
}
private static byte[] serialize(Data o) throws IOException {
ByteBuffer bb = ByteBuffer.allocate(1000);
bb.order(ByteOrder.LITTLE_ENDIAN);
bb.putInt(o.getIv());
bb.putDouble(o.getXv());
byte[] b = o.getSv().getBytes("UTF-8");
bb.putShort((short)b.length);
bb.put(b);
int n = bb.position();
byte[] res = new byte[n];
bb.rewind();
bb.get(res);
return res;
}
private static Data deserialize(byte[] b) throws IOException, ClassNotFoundException {
ByteBuffer bb = ByteBuffer.wrap(b);
bb.order(ByteOrder.LITTLE_ENDIAN);
int iv = bb.getInt();
double xv = bb.getDouble();
int len = bb.getShort();
byte[] temp = new byte[len];
bb.get(temp);
String sv = new String(temp, "UTF-8");
Data res = new Data(iv, xv, sv);
return res;
}
private static void dump(Map<byte[],byte[]> db, String key) throws ClassNotFoundException, IOException {
System.out.printf("Key=%s\n", key);
byte[] rawvalue = db.get(prep(key));
if(rawvalue != null) {
Data value = deserialize(rawvalue);
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
public static void main(String[] args) throws IOException, ClassNotFoundException {
DB dbf = DBMaker.fileDB("/work/testxdb.mdb").make();
Map<byte[],byte[]> db = dbf.hashMap("data", Serializer.BYTE_ARRAY, Serializer.BYTE_ARRAY).createOrOpen();
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (1000 + i + 1);
Data value = new Data();
value.setIv(i + 1);
value.setXv(i + 1.0);
value.setSv(String.format("This is value %d", i + 1));
db.put(prep(key), serialize(value));
}
//
String key;
key = "Key#" + 1077;
// get
dump(db, key);
// delete
db.remove(prep(key));
// get non existing
dump(db, key);
//
key = "Key#" + 1088;
// update and get
dump(db, key);
Data value = deserialize(db.get(prep(key)));
value.setIv(value.getIv() + 1);
value.setXv(value.getXv() + 0.1);
value.setSv(value.getSv() + " updated");
db.put(prep(key), serialize(value));
dump(db, key);
// list all
Iterator<byte[]> it = db.keySet().iterator();
int n = 0;
while(it.hasNext()) {
String itkey = new String(it.next());
if(!itkey.startsWith("Key#")) {
System.out.println("Unexpected key: " + itkey);
}
Data itvalue = deserialize(db.get(prep(itkey)));
if(itvalue.getIv() < 1 || NREC < itvalue.getIv()) {
System.out.println("Unexpected value :" + itvalue);
}
n++;
}
System.out.println(n);
// list keys where "Key#1075" <= key < "Key#1085"
// **** not supported by HTreeMap (BTreeMap required) ****
// close database
dbf.close();
}
}
package nosql.mapdb;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Iterator;
import java.util.Map;
import org.mapdb.DB;
import org.mapdb.DBMaker;
import org.mapdb.Serializer;
import org.mapdb.BTreeMap;
public class Test2X {
private static byte[] prep(String key) throws UnsupportedEncodingException {
return key.getBytes("UTF-8");
}
private static byte[] serialize(Data o) throws IOException {
ByteBuffer bb = ByteBuffer.allocate(1000);
bb.order(ByteOrder.LITTLE_ENDIAN);
bb.putInt(o.getIv());
bb.putDouble(o.getXv());
byte[] b = o.getSv().getBytes("UTF-8");
bb.putShort((short)b.length);
bb.put(b);
int n = bb.position();
byte[] res = new byte[n];
bb.rewind();
bb.get(res);
return res;
}
private static Data deserialize(byte[] b) throws IOException, ClassNotFoundException {
ByteBuffer bb = ByteBuffer.wrap(b);
bb.order(ByteOrder.LITTLE_ENDIAN);
int iv = bb.getInt();
double xv = bb.getDouble();
int len = bb.getShort();
byte[] temp = new byte[len];
bb.get(temp);
String sv = new String(temp, "UTF-8");
Data res = new Data(iv, xv, sv);
return res;
}
private static void dump(Map<byte[],byte[]> db, String key) throws ClassNotFoundException, IOException {
System.out.printf("Key=%s\n", key);
byte[] rawvalue = db.get(prep(key));
if(rawvalue != null) {
Data value = deserialize(rawvalue);
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
public static void main(String[] args) throws IOException, ClassNotFoundException {
DB dbf = DBMaker.fileDB("/work/test2xdb.mdb").make();
Map<byte[],byte[]> db = dbf.treeMap("data", Serializer.BYTE_ARRAY, Serializer.BYTE_ARRAY).createOrOpen();
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (1000 + i + 1);
Data value = new Data();
value.setIv(i + 1);
value.setXv(i + 1.0);
value.setSv(String.format("This is value %d", i + 1));
db.put(prep(key), serialize(value));
}
//
String key;
key = "Key#" + 1077;
// get
dump(db, key);
// delete
db.remove(prep(key));
// get non existing
dump(db, key);
//
key = "Key#" + 1088;
// update and get
dump(db, key);
Data value = deserialize(db.get(prep(key)));
value.setIv(value.getIv() + 1);
value.setXv(value.getXv() + 0.1);
value.setSv(value.getSv() + " updated");
db.put(prep(key), serialize(value));
dump(db, key);
// list all
Iterator<byte[]> it = db.keySet().iterator();
int n = 0;
while(it.hasNext()) {
String itkey = new String(it.next());
if(!itkey.startsWith("Key#")) {
System.out.println("Unexpected key: " + itkey);
}
Data itvalue = deserialize(db.get(prep(itkey)));
if(itvalue.getIv() < 1 || NREC < itvalue.getIv()) {
System.out.println("Unexpected value :" + itvalue);
}
n++;
}
System.out.println(n);
// list keys where "Key#1075" <= key < "Key#1085"
Iterator<byte[]> it2 = ((BTreeMap<byte[],byte[]>)db).subMap(prep("Key#1075"), prep("Key#1085")).keySet().iterator();
int n2 = 0;
while(it2.hasNext()) {
String it2key = new String(it2.next());
byte[] it2rawvalue = db.get(prep(it2key));
@SuppressWarnings("unused")
Data it2value = deserialize(it2rawvalue);
n2++;
}
System.out.println(n2);
// close database
dbf.close();
}
}
Google created LevelDB in 2004. Facebook created a fork RocksDB in 2012. And since then RocksDB has gained some popularity.
Supported platforms | Linux, Windows, macOS, *BSD, Solaris, AIX |
Supported languages | C++ Java and other JVM languages C# and VB.NET |
Features | |
Missing features |
There are two approaches to data format:
Let us first see a language neutral serialization format. This makes it equal for different languages.
The C/C++ struct is:
struct Data
{
int Iv;
double Xv;
char Sv[50];
};
And we define serialization format as:
#include <iostream>
#include <sstream>
#include <string>
#include <cstdlib>
#include <cstring>
using namespace std;
#include "rocksdb/db.h"
using namespace rocksdb;
struct Data
{
int Iv;
double Xv;
char Sv[50];
};
static string Prep(string key)
{
return key;
}
static void Serialize(Data *d, string *buf)
{
int slen = strlen(d->Sv);
buf->resize(4 + 8 + 2 + slen);
char *p = (char *)buf->data();
memcpy(p, &d->Iv, 4);
p += 4;
memcpy(p, &d->Xv, 8);
p += 8;
memcpy(p, &slen, 2);
p += 2;
memcpy(p, &d->Sv[0], slen);
}
static void Deserialize(string buf, Data *d)
{
char *p = (char *)buf.data();
memcpy(&d->Iv, p, 4);
p += 4;
memcpy(&d->Xv, p, 8);
p += 8;
int slen;
memcpy(&slen, p, 2);
p += 2;
memcpy(&d->Sv[0], p, slen);
d->Sv[slen] = '\0';
}
static void check(Status status)
{
if (!status.ok())
{
cout << status.ToString() << endl;
exit(1);
}
}
static void Dump(DB *db, string key)
{
cout << "Key=" << key << endl;
string rawvalue;
Status status = db->Get(ReadOptions(), Prep(key), &rawvalue);
if(status.ok())
{
Data value;
Deserialize(rawvalue, &value);
cout << "(" << value.Iv << "," << value.Xv << "," << value.Sv << ")" << endl;
}
else
{
cout << "Not found" << endl;
}
}
static string ItoA(int v)
{
stringstream ss;
ss << v;
return ss.str();
}
static const int NREC = 1000;
static const int CPP_OFFSET = NREC;
int main()
{
Status status;
DB *db;
Options options;
options.create_if_missing = true;
status = DB::Open(options, "/work/R/DB", &db);
check(status);
// put data
for(int i = 0; i < NREC; i++)
{
string key = "Key#" + ItoA(CPP_OFFSET + i + 1);
Data value;
value.Iv = i + 1;
value.Xv = i + 1.0;
sprintf(value.Sv, "This is value %d", i + 1);
string temp;
Serialize(&value, &temp);
status = db->Put(WriteOptions(), Prep(key), temp);
check(status);
}
//
string key;
key = "Key#" + ItoA(CPP_OFFSET + 77);
// get
Dump(db, key);
// delete
db->Delete(WriteOptions(), Prep(key));
// get non existing
Dump(db, key);
//
key = "Key#" + ItoA(CPP_OFFSET + 88);
// update and get
Dump(db, key);
string rawvalue;
status = db->Get(ReadOptions(), Prep(key), &rawvalue);
Data value;
Deserialize(rawvalue, &value);
value.Iv = value.Iv + 1;
value.Xv = value.Xv + 0.1;
strcat(value.Sv, " updated");
string temp;
Serialize(&value, &temp);
status = db->Put(WriteOptions(), Prep(key), temp);
Dump(db, key);
// list all
Iterator *it = db->NewIterator(ReadOptions());
it->SeekToFirst();
int n = 0;
while(it->Valid())
{
string itkey = it->key().ToString();
if(itkey.rfind("Key#", 0) != 0)
{
cout << "Unexpected key: " << itkey << endl;
}
string itrawvalue;
status = db->Get(ReadOptions(), Prep(itkey), &itrawvalue);
Data itvalue;
Deserialize(itrawvalue, &itvalue);
if (itvalue.Iv < 1 || NREC < itvalue.Iv)
{
cout << "Unexpected value :" << "(" << itvalue.Iv << "," << itvalue.Xv << "," << itvalue.Sv << ")" << endl;
}
n++;
it->Next();
}
cout << n << endl;
delete it;
// list keys where "Key#n075" <= key < "Key#n085"
Iterator *it2 = db->NewIterator(ReadOptions());
it2->Seek(Prep("Key#" + ItoA(CPP_OFFSET + 75)));
int n2 = 0;
while(it2->Valid())
{
string it2key = it2->key().ToString();
if(it2key >= (string)"Key#" + ItoA(CPP_OFFSET + 85)) break;
n2++;
it2->Next();
}
cout << n2 << endl;
delete it2;
//
status = db->SyncWAL();
check(status);
// This is the correct way of closing but on my Windows 7 it hangs
//status = db->Close();
//check(status);
//delete db;
exit(0);
}
Build on Windows with MSVC++:
cl /EHsc /MD /I%ROCKSDB_DIR%\include Test.cpp %ROCKSDB_DIR%\lib\rocksdb.lib %ZLIB_DIR%\lib\zlib.lib rpcrt4.lib Shlwapi.lib
package nosql.rocksdb;
import java.io.Serializable;
public class Data implements Serializable {
private static final long serialVersionUID = 1L;
private int iv;
private double xv;
private String sv;
public Data() {
this(0, 0.0, "");
}
public Data(int iv, double xv, String sv) {
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
package nosql.rocksdb;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import org.rocksdb.RocksIterator;
public class Test {
private static byte[] prep(String key) throws UnsupportedEncodingException {
return key.getBytes("UTF-8");
}
private static byte[] serialize(Data o) throws IOException {
ByteBuffer bb = ByteBuffer.allocate(1000);
bb.order(ByteOrder.LITTLE_ENDIAN);
bb.putInt(o.getIv());
bb.putDouble(o.getXv());
byte[] b = o.getSv().getBytes("UTF-8");
bb.putShort((short)b.length);
bb.put(b);
int n = bb.position();
byte[] res = new byte[n];
bb.rewind();
bb.get(res);
return res;
}
private static Data deserialize(byte[] b) throws IOException, ClassNotFoundException {
ByteBuffer bb = ByteBuffer.wrap(b);
bb.order(ByteOrder.LITTLE_ENDIAN);
int iv = bb.getInt();
double xv = bb.getDouble();
int len = bb.getShort();
byte[] temp = new byte[len];
bb.get(temp);
String sv = new String(temp, "UTF-8");
Data res = new Data(iv, xv, sv);
return res;
}
private static void dump(RocksDB db, String key) throws ClassNotFoundException, IOException, RocksDBException {
System.out.printf("Key=%s\n", key);
byte[] rawvalue = db.get(prep(key));
if(rawvalue != null) {
Data value = deserialize(rawvalue);
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
private static final int JAVA_OFFSET = 2 * NREC;
public static void main(String[] args) throws RocksDBException, IOException, ClassNotFoundException {
RocksDB.loadLibrary();
try(RocksDB db = RocksDB.open("/work/R/DB")) {
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (JAVA_OFFSET + i + 1);
Data value = new Data();
value.setIv(i + 1);
value.setXv(i + 1.0);
value.setSv(String.format("This is value %d", i + 1));
db.put(prep(key), serialize(value));
}
//
String key;
key = "Key#" + (JAVA_OFFSET + 77);
// get
dump(db, key);
// delete
db.delete(prep(key));
// get non existing
dump(db, key);
//
key = "Key#" + (JAVA_OFFSET + 88);
// update and get
dump(db, key);
Data value = (Data)deserialize(db.get(prep(key)));
value.setIv(value.getIv() + 1);
value.setXv(value.getXv() + 0.1);
value.setSv(value.getSv() + " updated");
db.put(prep(key), serialize(value));
dump(db, key);
// list all
try(RocksIterator it = db.newIterator()) {
it.seekToFirst();
int n = 0;
while(it.isValid()) {
String itkey = new String(it.key(), "UTF-8");
if(!itkey.startsWith("Key#")) {
System.out.println("Unexpected key: " + itkey);
}
Data itvalue = (Data)deserialize(db.get(prep(itkey)));
if(itvalue.getIv() < 1 || NREC < itvalue.getIv()) {
System.out.println("Unexpected value :" + itvalue);
}
n++;
it.next();
}
System.out.println(n);
}
// list keys where "Key#n075" <= key < "Key#n085"
try(RocksIterator it2 = db.newIterator()) {
it2.seek(prep("Key#" + (JAVA_OFFSET + 75)));
int n2 = 0;
while(it2.isValid()) {
String it2key = new String(it2.key());
if(it2key.compareTo("Key#" + (JAVA_OFFSET + 85)) >= 0) break;
n2++;
it2.next();
}
System.out.println(n2);
}
}
}
}
There is an upcoming Java EE standard for NoSQL databases: Jakarta NoSQL. The reference implementation is Eclipse JNoSQL.
I have created a RocksDB driver for it that can be downloaded here.
The example is tested with B2 release.
package nosql.rocksdb.jnosql;
import java.io.Serializable;
public class Data implements Serializable {
private static final long serialVersionUID = 1L;
private int iv;
private double xv;
private String sv;
public Data() {
this(0, 0.0, "");
}
public Data(int iv, double xv, String sv) {
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
package nosql.rocksdb.jnosql;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Optional;
import jakarta.nosql.Value;
import jakarta.nosql.keyvalue.BucketManager;
import jakarta.nosql.keyvalue.BucketManagerFactory;
import jakarta.nosql.keyvalue.KeyValueConfiguration;
import dk.vajhoej.jnosql.rocksdb.RocksDBConfiguration;
public class Test {
private static byte[] serialize(Data o) throws IOException {
ByteBuffer bb = ByteBuffer.allocate(1000);
bb.order(ByteOrder.LITTLE_ENDIAN);
bb.putInt(o.getIv());
bb.putDouble(o.getXv());
byte[] b = o.getSv().getBytes("UTF-8");
bb.putShort((short)b.length);
bb.put(b);
int n = bb.position();
byte[] res = new byte[n];
bb.rewind();
bb.get(res);
return res;
}
private static Data deserialize(byte[] b) throws IOException, ClassNotFoundException {
ByteBuffer bb = ByteBuffer.wrap(b);
bb.order(ByteOrder.LITTLE_ENDIAN);
int iv = bb.getInt();
double xv = bb.getDouble();
int len = bb.getShort();
byte[] temp = new byte[len];
bb.get(temp);
String sv = new String(temp, "UTF-8");
Data res = new Data(iv, xv, sv);
return res;
}
private static void dump(BucketManager bm, String key) throws ClassNotFoundException, IOException {
System.out.printf("Key=%s\n", key);
Optional<Value> rawvalue = bm.get(key);
if(rawvalue.isPresent()) {
Data value = (Data)rawvalue.get().get();
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
public static void main(String[] args) throws IOException, ClassNotFoundException {
KeyValueConfiguration cfg = new RocksDBConfiguration("/work/R/DB", false, o -> serialize((Data)o), b -> deserialize(b));
BucketManagerFactory bmf = cfg.get();
BucketManager bm = bmf.getBucketManager("testdb");
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (1000 + i + 1);
Data value = new Data();
value.setIv(i + 1);
value.setXv(i + 1.0);
value.setSv(String.format("This is value %d", i + 1));
bm.put(key, value);
}
//
String key;
key = "Key#" + 1077;
// get
dump(bm, key);
// delete
bm.delete(key);
// get non existing
dump(bm, key);
//
key = "Key#" + 1088;
// update and get
dump(bm, key);
Data value = (Data)bm.get(key).get().get();
value.setIv(value.getIv() + 1);
value.setXv(value.getXv() + 0.1);
value.setSv(value.getSv() + " updated");
bm.put(key, value);
dump(bm, key);
// list all
// **** NOT SUPPORTED ****
// list keys where "Key#1075" <= key < "Key#1085"
// **** NOT SUPPORTED ****
//
bm.close();
bmf.close();
}
}
The configuration in:
KeyValueConfiguration cfg = new RocksDBConfiguration("/work/R/DB", false, o -> serialize((Data)o), b -> deserialize(b));
can be externalized in rocksdb.properties:
dbpath = /work/R/DB
prefix = false
to = nosql.rocksdb.jnosql.Test$MySerializer
from = nosql.rocksdb.jnosql.Test$MyDeserializer
and either:
KeyValueConfiguration cfg = new RocksDBConfiguration();
or:
KeyValueConfiguration cfg = KeyValueConfiguration.getConfiguration();
following nested classes are needed:
public static class MySerializer implements Serialize {
@Override
public byte[] convert(Object obj) throws IOException {
return serialize((Data)obj);
}
}
public static class MyDeserializer implements Deserialize {
@Override
public Object convert(byte[] b) throws IOException, ClassNotFoundException {
return deserialize(b);
}
}
using System;
using System.IO;
using System.Text;
using RocksDbSharp;
namespace R
{
public class Data
{
public int Iv { get; set; }
public double Xv { get; set; }
public string Sv { get; set; }
public override string ToString()
{
return string.Format("(iv: {0}, xv: {1}, sv: {2})", Iv, Xv, Sv);
}
}
public class Program
{
private static byte[] Prep(string key)
{
return Encoding.UTF8.GetBytes(key);
}
private static byte[] Serialize(Data d)
{
MemoryStream ms = new MemoryStream();
BinaryWriter bw = new BinaryWriter(ms);
bw.Write(d.Iv);
bw.Write(d.Xv);
bw.Write((short)d.Sv.Length);
bw.Write(Encoding.UTF8.GetBytes(d.Sv));
bw.Flush();
return ms.ToArray();
}
private static Data Deserialize(byte[] b)
{
MemoryStream ms = new MemoryStream(b);
BinaryReader br = new BinaryReader(ms);
Data d = new Data();
d.Iv = br.ReadInt32();
d.Xv = br.ReadDouble();
int len = br.ReadInt16();
byte[] temp = new byte[len];
br.Read(temp, 0, temp.Length);
d.Sv = Encoding.UTF8.GetString(temp);
return d;
}
private static void Dump(RocksDb db, string key)
{
Console.WriteLine("Key={0}", key);
byte[] rawvalue = db.Get(Prep(key));
if(rawvalue != null) {
Data value = Deserialize(rawvalue);
Console.WriteLine(value);
}
else
{
Console.WriteLine("Not found");
}
}
private const int NREC = 1000;
private const int DN_OFFSET = 3 * NREC;
public static void Main(string[] args)
{
using (RocksDb db = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), "/work/R/DB"))
{
// put data
for (int i = 0; i < NREC; i++)
{
string putkey = "Key#" + (DN_OFFSET + i + 1);
Data value = new Data { Iv = i + 1, Xv = i + 1.0, Sv = string.Format("This is value {0}", i + 1) };
db.Put(Prep(putkey), Serialize(value));
}
//
string key;
key = "Key#" + (DN_OFFSET + 77);
// get
Dump(db, key);
// delete
db.Remove(Prep(key));
// get non existing
Dump(db, key);
//
key = "Key#" + (DN_OFFSET + 88);
// update and get
Dump(db, key);
Data updvalue = (Data)Deserialize(db.Get(Prep(key)));
updvalue.Iv = updvalue.Iv + 1;
updvalue.Xv = updvalue.Xv + 0.1;
updvalue.Sv = updvalue.Sv + " updated";
db.Put(Prep(key), Serialize(updvalue));
Dump(db, key);
// list all
using (Iterator it = db.NewIterator())
{
it.SeekToFirst();
int n = 0;
while (it.Valid())
{
string itkey = Encoding.UTF8.GetString(it.Key());
if(!itkey.StartsWith("Key#"))
{
Console.WriteLine("Unexpected key: " + itkey);
}
Data itvalue = (Data)Deserialize(db.Get(Prep(itkey)));
if (itvalue.Iv < 1 || NREC < itvalue.Iv)
{
Console.WriteLine("Unexpected value :" + itvalue);
}
n++;
it.Next();
}
Console.WriteLine(n);
}
// list keys where "Key#75" <= key < "Key#85"
using (Iterator it2 = db.NewIterator())
{
it2.Seek(Prep("Key#" + (DN_OFFSET + 75)));
int n2 = 0;
while (it2.Valid())
{
string it2key = Encoding.UTF8.GetString(it2.Key());
if(string.Compare(it2key, "Key#" + (DN_OFFSET + 85)) >= 0) break;
n2++;
it2.Next();
}
Console.WriteLine(n2);
}
}
}
}
}
Let now see using C/C++ struct as is. This makes it very easy for C++ code, but create a problem for Java and .NET.
The C/C++ struct is:
struct Data
{
int Iv;
double Xv;
char Sv[50];
};
#include <iostream>
#include <sstream>
#include <string>
#include <cstdlib>
#include <cstring>
using namespace std;
#include "rocksdb/db.h"
using namespace rocksdb;
struct Data
{
int Iv;
double Xv;
char Sv[50];
};
static string Prep(string key)
{
return key;
}
static void Serialize(Data *d, string *buf)
{
buf->resize(sizeof(Data));
char *p = (char *)buf->data();
memcpy(p, d, sizeof(Data));;
}
static void Deserialize(string buf, Data *d)
{
char *p = (char *)buf.data();
memcpy(d, p, sizeof(Data));;
}
static void check(Status status)
{
if (!status.ok())
{
cout << status.ToString() << endl;
exit(1);
}
}
static void Dump(DB *db, string key)
{
cout << "Key=" << key << endl;
string rawvalue;
Status status = db->Get(ReadOptions(), Prep(key), &rawvalue);
if(status.ok())
{
Data value;
Deserialize(rawvalue, &value);
cout << "(" << value.Iv << "," << value.Xv << "," << value.Sv << ")" << endl;
}
else
{
cout << "Not found" << endl;
}
}
static string ItoA(int v)
{
stringstream ss;
ss << v;
return ss.str();
}
static const int NREC = 1000;
static const int CPP_OFFSET = NREC;
int main()
{
Status status;
DB *db;
Options options;
options.create_if_missing = true;
status = DB::Open(options, "/work/R/DB", &db);
check(status);
// put data
for(int i = 0; i < NREC; i++)
{
string key = "Key#" + ItoA(CPP_OFFSET + i + 1);
Data value;
value.Iv = i + 1;
value.Xv = i + 1.0;
sprintf(value.Sv, "This is value %d", i + 1);
string temp;
Serialize(&value, &temp);
status = db->Put(WriteOptions(), Prep(key), temp);
check(status);
}
//
string key;
key = "Key#" + ItoA(CPP_OFFSET + 77);
// get
Dump(db, key);
// delete
db->Delete(WriteOptions(), Prep(key));
// get non existing
Dump(db, key);
//
key = "Key#" + ItoA(CPP_OFFSET + 88);
// update and get
Dump(db, key);
string rawvalue;
status = db->Get(ReadOptions(), Prep(key), &rawvalue);
Data value;
Deserialize(rawvalue, &value);
value.Iv = value.Iv + 1;
value.Xv = value.Xv + 0.1;
strcat(value.Sv, " updated");
string temp;
Serialize(&value, &temp);
status = db->Put(WriteOptions(), Prep(key), temp);
Dump(db, key);
// list all
Iterator *it = db->NewIterator(ReadOptions());
it->SeekToFirst();
int n = 0;
while(it->Valid())
{
string itkey = it->key().ToString();
if(itkey.rfind("Key#", 0) != 0)
{
cout << "Unexpected key: " << itkey << endl;
}
string itrawvalue;
status = db->Get(ReadOptions(), Prep(itkey), &itrawvalue);
Data itvalue;
Deserialize(itrawvalue, &itvalue);
if (itvalue.Iv < 1 || NREC < itvalue.Iv)
{
cout << "Unexpected value :" << "(" << itvalue.Iv << "," << itvalue.Xv << "," << itvalue.Sv << ")" << endl;
}
n++;
it->Next();
}
cout << n << endl;
delete it;
// list keys where "Key#n075" <= key < "Key#n085"
Iterator *it2 = db->NewIterator(ReadOptions());
it2->Seek(Prep("Key#" + ItoA(CPP_OFFSET + 75)));
int n2 = 0;
while(it2->Valid())
{
string it2key = it2->key().ToString();
if(it2key >= (string)"Key#" + ItoA(CPP_OFFSET + 85)) break;
n2++;
it2->Next();
}
cout << n2 << endl;
delete it2;
//
status = db->SyncWAL();
check(status);
// This is the correct way of closing but on my Windows 7 it hangs
//status = db->Close();
//check(status);
//delete db;
exit(0);
}
Build on Windows with MSVC++:
cl /EHsc /MD /I%ROCKSDB_DIR%\include Test.cpp %ROCKSDB_DIR%\lib\rocksdb.lib %ZLIB_DIR%\lib\zlib.lib rpcrt4.lib Shlwapi.lib
To use the C/C++ format the code will be using my Record library as that makes it a little bit easier.
package nosql.rocksdb;
import java.io.Serializable;
public class Data implements Serializable {
private static final long serialVersionUID = 1L;
private int iv;
private double xv;
private String sv;
public Data() {
this(0, 0.0, "");
}
public Data(int iv, double xv, String sv) {
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
package nosql.rocksdb;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import org.rocksdb.RocksIterator;
import dk.vajhoej.record.Alignment;
import dk.vajhoej.record.Endian;
import dk.vajhoej.record.FieldType;
import dk.vajhoej.record.RecordException;
import dk.vajhoej.record.Struct;
import dk.vajhoej.record.StructField;
import dk.vajhoej.record.StructReader;
import dk.vajhoej.record.StructWriter;
public class TestAlt {
@Struct(endianess=Endian.LITTLE, alignment=Alignment.ALIGN8)
public static class CData {
@StructField(n=0, type=FieldType.INT4)
public int iv;
@StructField(n=1, type=FieldType.FP8)
public double xv;
@StructField(n=2, type=FieldType.FIXSTRNULTERM, length=50)
public String sv;
//
public CData() {
iv = 0;
xv = 0.0;
sv = "";
}
public CData(Data d) {
iv = d.getIv();
xv = d.getXv();
sv = d.getSv();
}
public Data getData() {
return new Data(iv, xv, sv);
}
}
private static byte[] prep(String key) throws UnsupportedEncodingException {
return key.getBytes("UTF-8");
}
private static byte[] serialize(Data o) throws IOException, RecordException {
CData o2 = new CData(o);
StructWriter sw = new StructWriter();
sw.write(o2);
return sw.getBytes();
}
private static Data deserialize(byte[] b) throws IOException, ClassNotFoundException, RecordException {
StructReader sr = new StructReader(b);
CData o = sr.read(CData.class);
return o.getData();
}
private static void dump(RocksDB db, String key) throws ClassNotFoundException, IOException, RocksDBException, RecordException {
System.out.printf("Key=%s\n", key);
byte[] rawvalue = db.get(prep(key));
if(rawvalue != null) {
Data value = deserialize(rawvalue);
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
private static final int JAVA_OFFSET = 2 * NREC;
public static void main(String[] args) throws RocksDBException, IOException, ClassNotFoundException, RecordException {
RocksDB.loadLibrary();
try(RocksDB db = RocksDB.open("/work/R/DB")) {
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (JAVA_OFFSET + i + 1);
Data value = new Data();
value.setIv(i + 1);
value.setXv(i + 1.0);
value.setSv(String.format("This is value %d", i + 1));
db.put(prep(key), serialize(value));
}
//
String key;
key = "Key#" + (JAVA_OFFSET + 77);
// get
dump(db, key);
// delete
db.delete(prep(key));
// get non existing
dump(db, key);
//
key = "Key#" + (JAVA_OFFSET + 88);
// update and get
dump(db, key);
Data value = (Data)deserialize(db.get(prep(key)));
value.setIv(value.getIv() + 1);
value.setXv(value.getXv() + 0.1);
value.setSv(value.getSv() + " updated");
db.put(prep(key), serialize(value));
dump(db, key);
// list all
try(RocksIterator it = db.newIterator()) {
it.seekToFirst();
int n = 0;
while(it.isValid()) {
String itkey = new String(it.key(), "UTF-8");
if(!itkey.startsWith("Key#")) {
System.out.println("Unexpected key: " + itkey);
}
Data itvalue = (Data)deserialize(db.get(prep(itkey)));
if(itvalue.getIv() < 1 || NREC < itvalue.getIv()) {
System.out.println("Unexpected value :" + itvalue);
}
n++;
it.next();
}
System.out.println(n);
}
// list keys where "Key#n075" <= key < "Key#n085"
try(RocksIterator it2 = db.newIterator()) {
it2.seek(prep("Key#" + (JAVA_OFFSET + 75)));
int n2 = 0;
while(it2.isValid()) {
String it2key = new String(it2.key());
if(it2key.compareTo("Key#" + (JAVA_OFFSET + 85)) >= 0) break;
n2++;
it2.next();
}
System.out.println(n2);
}
}
}
}
There is an upcoming Java EE standard for NoSQL databases: Jakarta NoSQL. The reference implementation is Eclipse JNoSQL.
I have created a RocksDB driver for it that can be downloaded here.
The example is tested with B2 release.
To use the C/C++ format the code will be using my Record library as that makes it a little bit easier.
package nosql.rocksdb.jnosql;
import java.io.Serializable;
public class Data implements Serializable {
private static final long serialVersionUID = 1L;
private int iv;
private double xv;
private String sv;
public Data() {
this(0, 0.0, "");
}
public Data(int iv, double xv, String sv) {
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
package nosql.rocksdb.jnosql;
import java.io.IOException;
import java.util.Optional;
import jakarta.nosql.Value;
import jakarta.nosql.keyvalue.BucketManager;
import jakarta.nosql.keyvalue.BucketManagerFactory;
import jakarta.nosql.keyvalue.KeyValueConfiguration;
import dk.vajhoej.jnosql.rocksdb.RocksDBConfiguration;
import dk.vajhoej.record.Alignment;
import dk.vajhoej.record.Endian;
import dk.vajhoej.record.FieldType;
import dk.vajhoej.record.RecordException;
import dk.vajhoej.record.Struct;
import dk.vajhoej.record.StructField;
import dk.vajhoej.record.StructReader;
import dk.vajhoej.record.StructWriter;
public class Test2 {
@Struct(endianess=Endian.LITTLE, alignment=Alignment.ALIGN8)
public static class CData {
@StructField(n=0, type=FieldType.INT4)
public int iv;
@StructField(n=1, type=FieldType.FP8)
public double xv;
@StructField(n=2, type=FieldType.FIXSTRNULTERM, length=50)
public String sv;
//
public CData() {
iv = 0;
xv = 0.0;
sv = "";
}
public CData(Data d) {
iv = d.getIv();
xv = d.getXv();
sv = d.getSv();
}
public Data getData() {
return new Data(iv, xv, sv);
}
}
private static byte[] serialize(Data o) throws IOException {
try {
CData o2 = new CData(o);
StructWriter sw = new StructWriter();
sw.write(o2);
return sw.getBytes();
} catch (RecordException e) {
throw new IOException(e);
}
}
private static Data deserialize(byte[] b) throws IOException, ClassNotFoundException {
try {
StructReader sr = new StructReader(b);
CData o = sr.read(CData.class);
return o.getData();
} catch (RecordException e) {
throw new IOException(e);
}
}
private static void dump(BucketManager bm, String key) throws ClassNotFoundException, IOException {
System.out.printf("Key=%s\n", key);
Optional<Value> rawvalue = bm.get(key);
if(rawvalue.isPresent()) {
Data value = (Data)rawvalue.get().get();
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
public static void main(String[] args) throws IOException, ClassNotFoundException {
KeyValueConfiguration cfg = new RocksDBConfiguration("/work/R/DB", false, o -> serialize((Data)o), b -> deserialize(b));
BucketManagerFactory bmf = cfg.get();
BucketManager bm = bmf.getBucketManager("testdb");
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (1000 + i + 1);
Data value = new Data();
value.setIv(i + 1);
value.setXv(i + 1.0);
value.setSv(String.format("This is value %d", i + 1));
bm.put(key, value);
}
//
String key;
key = "Key#" + 1077;
// get
dump(bm, key);
// delete
bm.delete(key);
// get non existing
dump(bm, key);
//
key = "Key#" + 1088;
// update and get
dump(bm, key);
Data value = (Data)bm.get(key).get().get();
value.setIv(value.getIv() + 1);
value.setXv(value.getXv() + 0.1);
value.setSv(value.getSv() + " updated");
bm.put(key, value);
dump(bm, key);
// list all
// **** NOT SUPPORTED ****
// list keys where "Key#1075" <= key < "Key#1085"
// **** NOT SUPPORTED ****
//
bm.close();
bmf.close();
}
}
To use the C/C++ format the code will be using my NRecord library as that makes it a little bit easier.
using System;
using System.IO;
using System.Text;
using RocksDbSharp;
using Vajhoej.Record;
namespace R
{
public class Data
{
public int Iv { get; set; }
public double Xv { get; set; }
public string Sv { get; set; }
public override string ToString()
{
return string.Format("(iv: {0}, xv: {1}, sv: {2})", Iv, Xv, Sv);
}
}
public class Program
{
[Struct(Endianess = Endian.LITTLE, Alignment = Alignment.ALIGN8)]
public class CData
{
[StructField(N = 0, Type = FieldType.INT4)]
public int Iv;
[StructField(N = 1, Type = FieldType.FP8)]
public double Xv;
[StructField(N = 2, Type = FieldType.FIXSTRNULTERM, Length = 50)]
public String Sv;
//
public CData()
{
Iv = 0;
Xv = 0.0;
Sv = "";
}
public CData(Data o)
{
Iv = o.Iv;
Xv = o.Xv;
Sv = o.Sv;
}
public Data GetData()
{
return new Data { Iv = Iv, Xv = Xv, Sv = Sv };
}
}
private static byte[] Prep(string key)
{
return Encoding.UTF8.GetBytes(key);
}
private static byte[] Serialize(Data d)
{
CData d2 = new CData(d);
StructWriter sw = new StructWriter();
sw.Write(d2);
return sw.GetBytes();
}
private static Data Deserialize(byte[] b)
{
StructReader sr = new StructReader(b);
CData o = sr.Read<CData>(typeof(CData));
return o.GetData();
}
private static void Dump(RocksDb db, string key)
{
Console.WriteLine("Key={0}", key);
byte[] rawvalue = db.Get(Prep(key));
if (rawvalue != null)
{
Data value = Deserialize(rawvalue);
Console.WriteLine(value);
}
else
{
Console.WriteLine("Not found");
}
}
private const int NREC = 1000;
private const int DN_OFFSET = 3 * NREC;
public static void Main(string[] args)
{
using (RocksDb db = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), "/work/R/DB"))
{
// put data
for (int i = 0; i < NREC; i++)
{
string putkey = "Key#" + (DN_OFFSET + i + 1);
Data value = new Data { Iv = i + 1, Xv = i + 1.0, Sv = string.Format("This is value {0}", i + 1) };
db.Put(Prep(putkey), Serialize(value));
}
//
string key;
key = "Key#" + (DN_OFFSET + 77);
// get
Dump(db, key);
// delete
db.Remove(Prep(key));
// get non existing
Dump(db, key);
//
key = "Key#" + (DN_OFFSET + 88);
// update and get
Dump(db, key);
Data updvalue = (Data)Deserialize(db.Get(Prep(key)));
updvalue.Iv = updvalue.Iv + 1;
updvalue.Xv = updvalue.Xv + 0.1;
updvalue.Sv = updvalue.Sv + " updated";
db.Put(Prep(key), Serialize(updvalue));
Dump(db, key);
// list all
using (Iterator it = db.NewIterator())
{
it.SeekToFirst();
int n = 0;
while (it.Valid())
{
string itkey = Encoding.UTF8.GetString(it.Key());
if (!itkey.StartsWith("Key#"))
{
Console.WriteLine("Unexpected key: " + itkey);
}
Data itvalue = (Data)Deserialize(db.Get(Prep(itkey)));
if (itvalue.Iv < 1 || NREC < itvalue.Iv)
{
Console.WriteLine("Unexpected value :" + itvalue);
}
n++;
it.Next();
}
Console.WriteLine(n);
}
// list keys where "Key#75" <= key < "Key#85"
using (Iterator it2 = db.NewIterator())
{
it2.Seek(Prep("Key#" + (DN_OFFSET + 75)));
int n2 = 0;
while (it2.Valid())
{
string it2key = Encoding.UTF8.GetString(it2.Key());
if (string.Compare(it2key, "Key#" + (DN_OFFSET + 85)) >= 0) break;
n2++;
it2.Next();
}
Console.WriteLine(n2);
}
}
}
}
}
Voldemort was created by LinkedIn in 2009.
Strivtly speaking Voldemort is not a key value store - Voldemort is a distributed data store on top of a simple key value store.
Voldemort is Java based.
Voldemort can use different key value stores, but a common choice and the one used here is BDB-JE (described in a previous section). It can also use MySQL relational database. And there is experimental support for RocksDB (described in a previous section).
There is little reason to use Voldemort over BDB-JE in a single node scenario. The main reason for using Voldemort is a need for multi nodes providing redundancy and high performance.
Supported platforms | Any platform with Java |
Supported languages | Java and other JVM languages |
Features | Distributed AP database with eventual consistency |
Missing features | Iteration over keys |
Voldemort handle serialization and has builtin support for several serialization methods.
The following example will show:
package nosql.voldemort;
import java.io.Serializable;
public class Data implements Serializable {
private static final long serialVersionUID = 1L;
private int iv;
private double xv;
private String sv;
public Data() {
this(0, 0.0, "");
}
public Data(int iv, double xv, String sv) {
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
package nosql.voldemort;
import voldemort.client.ClientConfig;
import voldemort.client.SocketStoreClientFactory;
import voldemort.client.StoreClient;
import voldemort.client.StoreClientFactory;
import voldemort.versioning.Versioned;
public class TestUnstructured {
private static void dump(StoreClient<Object, Object> client, String key) {
System.out.printf("Key=%s\n", key);
Versioned<Object> vervalue = client.get(key);
if(vervalue != null) {
Data value = (Data)vervalue.getValue();
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
public static void main(String[] args) {
StoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls("tcp://localhost:6666"));
StoreClient<Object, Object> client = factory.getStoreClient("TestUnstructured");
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (1000 + i + 1);
Data value = new Data();
value.setIv(i + 1);
value.setXv(i + 1.0);
value.setSv(String.format("This is value %d", i + 1));
client.put(key, value);
}
//
String key;
key = "Key#" + 1077;
// get
dump(client, key);
// delete
client.delete(key);
// get non existing
dump(client, key);
//
key = "Key#" + 1088;
// update and get
dump(client, key);
Data value = (Data)client.get(key).getValue();
value.setIv(value.getIv() + 1);
value.setXv(value.getXv() + 0.1);
value.setSv(value.getSv() + " updated");
client.put(key, value);
dump(client, key);
// list all
// **** NOT SUPPORTED ****
// list keys where "Key#1075" <= key < "Key#1085"
// **** NOT SUPPORTED ****
}
}
package nosql.voldemort;
import java.util.HashMap;
import java.util.Map;
import voldemort.client.ClientConfig;
import voldemort.client.SocketStoreClientFactory;
import voldemort.client.StoreClient;
import voldemort.client.StoreClientFactory;
import voldemort.versioning.Versioned;
public class TestStructured {
private static void dump(StoreClient<Object, Object> client, String key) {
System.out.printf("Key=%s\n", key);
Versioned<Object> vervalue = client.get(key);
if(vervalue != null) {
@SuppressWarnings("unchecked")
Map<String,Object> value = (Map<String,Object>)vervalue.getValue();
System.out.println(value);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
public static void main(String[] args) {
StoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls("tcp://localhost:6666"));
StoreClient<Object, Object> client = factory.getStoreClient("TestStructured");
// put data
for(int i = 0; i < NREC; i++) {
String key = "Key#" + (1000 + i + 1);
Map<String,Object> value = new HashMap<>();
value.put("iv", i + 1);
value.put("xv", i + 1.0);
value.put("sv", String.format("This is value %d", i + 1));
client.put(key, value);
}
//
String key;
key = "Key#" + 1077;
// get
dump(client, key);
// delete
client.delete(key);
// get non existing
dump(client, key);
//
key = "Key#" + 1088;
// update and get
dump(client, key);
@SuppressWarnings("unchecked")
Map<String,Object> value = (Map<String,Object> )client.get(key).getValue();
value.put("iv", (Integer)value.get("iv") + 1);
value.put("xv", (Double)value.get("xv") + 0.1);
value.put("sv", (String)value.get("sv") + " updated");
client.put(key, value);
dump(client, key);
// list all
// **** NOT SUPPORTED ****
// list keys where "Key#1075" <= key < "Key#1085"
// **** NOT SUPPORTED ****
}
}
Here are a simple 1 node config (which is not a particular relevant config).
Start command on Windows:
java -Xmx2G -server -Dcom.sun.management.jmxremote -cp .;%VOLDEMORT_HOME%\dist\*;%VOLDEMORT_HOME%\lib\* voldemort.server.VoldemortServer %VOLDEMORT_HOME% .
server.properties:
node.id = 0
# ports
http.enable = false
socket.enable = true
admin.enable = true
# storage options
storage.configs = voldemort.store.bdb.BdbStorageConfiguration
# BDB
bdb.data.directory = data
cluster.xml:
<cluster>
<name>TestCluster</name>
<server>
<id>0</id>
<host>localhost</host>
<http-port>8888</http-port>
<socket-port>6666</socket-port>
<admin-port>6667</admin-port>
<partitions>0,1</partitions>
</server>
</cluster>
stores.xml:
<stores>
<store>
<name>TestUnstructured</name>
<replication-factor>1</replication-factor>
<preferred-reads>1</preferred-reads>
<required-reads>1</required-reads>
<preferred-writes>1</preferred-writes>
<required-writes>1</required-writes>
<persistence>bdb</persistence>
<routing>client</routing>
<routing-strategy>consistent-routing</routing-strategy>
<key-serializer>
<type>string</type>
<schema-info>utf8</schema-info>
</key-serializer>
<value-serializer>
<type>java-serialization</type>
</value-serializer>
</store>
<store>
<name>TestStructured</name>
<replication-factor>1</replication-factor>
<preferred-reads>1</preferred-reads>
<required-reads>1</required-reads>
<preferred-writes>1</preferred-writes>
<required-writes>1</required-writes>
<persistence>bdb</persistence>
<routing>client</routing>
<routing-strategy>consistent-routing</routing-strategy>
<key-serializer>
<type>string</type>
<schema-info>utf8</schema-info>
</key-serializer>
<value-serializer>
<type>json</type>
<schema-info>{"iv": "int32", "xv": "float64", "sv": "string"}</schema-info>
</value-serializer>
</store>
</stores>
ISAM (Index Sequential Access Method) files is a type of database very similar to key value store NoSQL databases and is often considered a form of key value store NoSQL database.
The main difference between other key value stores and ISAM is that ISAM operates on records that include both key and value:
record {
key
value
}
ISAM supports these operations (conceptually):
The COBOL standard define support for indexed files. And COBOL must have had the feature since the 1960's.
Supported platforms | Most (IBM mainframe, VMS, Unix, Windows, Linux etc.) |
Supported languages | Cobol |
Features | |
Missing features |
Example:
IDENTIFICATION DIVISION.
PROGRAM-ID. TEST-PROGRAM.
ENVIRONMENT DIVISION.
INPUT-OUTPUT SECTION.
FILE-CONTROL.
SELECT OPTIONAL D-FILE ASSIGN TO "TESTDB.ISQ" ORGANIZATION IS INDEXED ACCESS MODE IS DYNAMIC RECORD KEY IS D-ID.
DATA DIVISION.
FILE SECTION.
FD D-FILE.
01 D-RECORD.
03 D-ID PIC X(8).
03 D-IV PIC 9(8) COMP.
03 D-XV COMP-2.
03 D-SLEN PIC 9(4) COMP.
03 D-SV PIC X(50).
WORKING-STORAGE SECTION.
01 NREC PIC 9(4) COMP VALUE 1000.
01 BLANK-VAL PIC X(8) VALUE " ".
01 EOF-FLAG PIC X.
01 DONE-FLAG PIC X.
01 WID PIC 9(4) DISPLAY.
01 IV PIC 9(4) DISPLAY.
01 XV PIC 9(4)V9(2) DISPLAY.
01 SLEN PIC 9(2) DISPLAY.
01 SV PIC X(50).
01 I PIC 9(4) DISPLAY.
01 N PIC 9(4) DISPLAY.
01 N2 PIC 9(4) DISPLAY.
01 STOP-ID PIC X(8).
PROCEDURE DIVISION.
MAIN-PARAGRAPH.
OPEN I-O D-FILE
MOVE BLANK-VAL TO D-ID
START D-FILE KEY GREATER THAN D-ID
INVALID KEY DISPLAY "Empty file"
END-START
PERFORM VARYING I FROM 1 BY 1 UNTIL I > NREC
COMPUTE WID = 1000 + I
STRING "Key#" DELIMITED BY SIZE WID DELIMITED BY SIZE INTO D-ID
COMPUTE D-IV = I
COMPUTE D-XV = I
COMPUTE D-SLEN = 18
STRING "This is value " DELIMITED BY SIZE I DELIMITED BY SIZE INTO D-SV
WRITE D-RECORD
INVALID KEY DISPLAY "Error writing"
NOT INVALID KEY CONTINUE
END-WRITE
END-PERFORM
COMPUTE WID = 1077
STRING "Key#" DELIMITED BY SIZE WID DELIMITED BY SIZE INTO D-ID
PERFORM DUMP-PARAGRAPH
DELETE D-FILE
INVALID KEY DISPLAY "Error deleting"
NOT INVALID KEY CONTINUE
END-DELETE
PERFORM DUMP-PARAGRAPH
COMPUTE WID = 1088
STRING "Key#" DELIMITED BY SIZE WID DELIMITED BY SIZE INTO D-ID
PERFORM DUMP-PARAGRAPH
READ D-FILE
INVALID KEY DISPLAY "Error reading"
NOT INVALID KEY CONTINUE
END-READ
COMPUTE D-IV = D-IV + 1
COMPUTE D-XV = D-XV + 0.1
MOVE D-SLEN TO SLEN
MOVE D-SV TO SV
STRING SV(1:SLEN) DELIMITED BY SIZE " updated" DELIMITED BY SIZE INTO D-SV
COMPUTE D-SLEN = D-SLEN + 8
REWRITE D-RECORD
INVALID KEY DISPLAY "Error updating"
NOT INVALID KEY CONTINUE
END-REWRITE
PERFORM DUMP-PARAGRAPH
MOVE BLANK-VAL TO D-ID
START D-FILE KEY IS GREATER THAN D-ID
INVALID KEY DISPLAY "Error rewinding"
NOT INVALID KEY CONTINUE
END-START
MOVE 'N' TO EOF-FLAG
MOVE 0 TO N
PERFORM UNTIL EOF-FLAG = 'Y'
READ D-FILE NEXT
AT END MOVE 'Y' TO EOF-FLAG
NOT AT END PERFORM CHECK-AND-COUNT-PARAGRAPH
END-READ
END-PERFORM
DISPLAY N
COMPUTE WID = 1075
STRING "Key#" DELIMITED BY SIZE WID DELIMITED BY SIZE INTO D-ID
START D-FILE KEY IS GREATER THAN OR EQUAL TO D-ID
INVALID KEY DISPLAY "Error searching"
NOT INVALID KEY CONTINUE
END-START
COMPUTE WID = 1085
STRING "Key#" DELIMITED BY SIZE WID DELIMITED BY SIZE INTO STOP-ID
MOVE 'N' TO EOF-FLAG
MOVE 'N' TO DONE-FLAG
MOVE 0 TO N2
PERFORM UNTIL EOF-FLAG = 'Y' OR DONE-FLAG = 'Y'
READ D-FILE NEXT
AT END MOVE 'Y' TO EOF-FLAG
NOT AT END PERFORM COUNT-PARAGRAPH
END-READ
END-PERFORM
DISPLAY N2
CLOSE D-FILE
STOP RUN.
DUMP-PARAGRAPH.
DISPLAY "Key=" D-ID
READ D-FILE
INVALID KEY DISPLAY "Not found"
NOT INVALID KEY PERFORM DUMP-DATA-PARAGRAPH
END-READ.
DUMP-DATA-PARAGRAPH.
MOVE D-IV TO IV
MOVE D-XV TO XV
MOVE D-SLEN TO SLEN
MOVE D-SV TO SV
DISPLAY "(" IV "," XV "," SV(1:SLEN) ")".
CHECK-AND-COUNT-PARAGRAPH.
MOVE D-IV TO IV
MOVE D-XV TO XV
MOVE D-SLEN TO SLEN
MOVE D-SV TO SV
IF IV < 1 OR NREC < IV THEN
DISPLAY "Unexpected value: (" IV "," XV "," SV(1:SLEN) ")"
END-IF
COMPUTE N = N + 1.
COUNT-PARAGRAPH.
IF D-ID GREATER THAN OR EQUAL TO STOP-ID THEN
MOVE 'Y' TO DONE-FLAG
ELSE
COMPUTE N2 = N2 + 1
END-IF.
Build with GNU Cobol:
cobc -Wall -free -x test.cob
Build with VMS Cobol:
$ cobol test
$ link test
Disclaimer: I am not experienced with COBOL, so I can not guarantee that above code is good COBOL code. It does work with GNU Cobol and VMS Cobol though.
For examples on how to read such a Cobol indexed file from other languages see Cobol files.
The VMS operating systems comes with builtin RMS (Record Management System) that among many other features provide support for index-sequential files. And VMS dates back to 1977.
Supported platforms | VMS |
Supported languages | Pascal, Basic, Cobol, Fortran, C/C++ |
Features | Active/active cluster with shared storage and access managed via distributed lock manager Multiple keys |
Missing features |
Example:
program test(input, output);
label
finished;
const
NREC = 1000;
PAS_OFFSET = NREC;
type
keystr = packed array [1..8] of char;
strdatalen = 0..65535;
strdata = packed array [1..50] of char;
data = packed record
id : [key(0),aligned(2)] keystr;
iv : integer;
xv : double;
slen : strdatalen;
sv : strdata;
end;
database = file of data;
var
db : database; (* has to be global *)
i : integer;
procedure dump(id : keystr);
var
d : data;
begin
writeln('Key=', id);
findk(db, 0, id);
if not ufb(db) then begin
d := db^;
writeln('(', d.iv:1, ',', d.xv:3:1, ',', substr(d.sv, 1, d.slen), ')');
end else begin
writeln('Not found');
end;
end;
var
d : data;
id : keystr;
n, n2 : integer;
begin
open(db, 'testdb.isq', unknown, organization := indexed, access_method := keyed);
(* put data *)
for i := 1 to NREC do begin
d.id := 'Key#' + dec(PAS_OFFSET + i, 4);
d.iv := i;
d.xv := i;
d.slen := 18;
d.sv := 'This is value ' + dec(i, 4);
db^ := d;
put(db);
end;
(* *)
id := 'Key#' + dec(PAS_OFFSET + 77, 4);
(* get *)
dump(id);
(* delete *)
findk(db, 0, id);
delete(db);
(* get non existing *)
dump(id);
(* *)
id := 'Key#' + dec(PAS_OFFSET + 88, 4);
(* update and get *)
dump(id);
findk(db, 0, id);
d := db^;
d.iv := d.iv + 1;
d.xv := d.xv + 0.1;
d.slen := d.slen + 8;
d.sv := substr(d.sv, 1, d.slen - 8) + ' updated';
db^ := d;
update(db);
dump(id);
(* list all *)
resetk(db, 0);
n := 0;
while not eof(db) do begin
d := db^;
if (d.iv < 1) or (NREC < d.iv) then begin
writeln('Unepected value: ', '(', d.iv:1, ',', d.xv:3:1, ',', substr(d.sv, 1, d.slen), ')');
end;
n := n + 1;
get(db);
end;
writeln(n:1);
(* list keys where "Key#n075" <= key < "Key#n085" *)
findk(db, 0, 'Key#' + dec(PAS_OFFSET + 75, 4));
n2 := 0;
while not eof(db) do begin
d := db^;
if d.id >= ('Key#' + dec(PAS_OFFSET + 85, 4)) then goto finished;
n2 := n2 + 1;
get(db);
end;
finished:
writeln(n2:1);
(* *)
close(db);
end.
program test
record datarec
string id = 8
long iv
gfloat xv
word slen
string sv = 50
end record
declare integer constant NREC = 1000
declare integer constant BASIC_OFFSET = 6 * NREC
declare integer i, n, n2
declare string id0, idstart, idstop
map (databuf) datarec d
open "testdb.isq" as file #1, indexed fixed, recordtype none, map databuf, primary key d::id
! put
for i = 1 to NREC
d::id = format$(BASIC_OFFSET + i, "Key_#####")
d::iv = i
d::xv = i
d::slen = 18
d::sv = format$(i, "This is value ####")
put #1
next i
!
id0 = format$(BASIC_OFFSET + 77, "Key_#####")
! get
gosub dump
! delete
find #1, key #0 eq id0
delete #1
! get non existing
gosub dump
!
id0 = format$(BASIC_OFFSET + 88, "Key_#####")
! update and get
gosub dump
get #1, key #0 eq id0
d::iv = d::iv + 1
d::xv = d::xv + 0.1
d::slen = d::slen + 8
mid(d::sv,19,8) = " updated"
update #1
gosub dump
! list all
n = 0
reset #1
handler eof_handler
end handler
when error use eof_handler
while 1 = 1
get #1
if mid(d::id,1, 4) <> "Key#" then
print using "Unexpected key: 'E", d::id
end if
if d::iv < 1 or NREC < d::iv then
print using "Unexpected value: (#### ####.## 'E)",d::iv,d::xv,mid(d::sv,1,d::slen)
end if
n = n + 1
next
end when
print n
! list keys where "Key#n075" <= key < "Key#n085"
idstart = format$(BASIC_OFFSET + 75, "Key_#####")
idstop = format$(BASIC_OFFSET + 85, "Key_#####")
find #1, key #0 ge idstart
n2 = 0
when error use eof_handler
loop:
while 1 = 1
get #1
if d::id >= idstop then
exit loop
end if
n2 = n2 + 1
next
end when
print n2
!
close #1
exit program
dump:
print id0
handler notf_handler
print "Not found"
end handler
when error use notf_handler
get #1, key #0 eq id0
print using "(####_,####.##_,'E)",d::iv,d::xv,mid(d::sv,1,d::slen)
end when
return
end program
On VMS the Cobol standard indexed file is mapped to RMS index-sequential files.
IDENTIFICATION DIVISION.
PROGRAM-ID. TEST-PROGRAM.
ENVIRONMENT DIVISION.
INPUT-OUTPUT SECTION.
FILE-CONTROL.
SELECT OPTIONAL D-FILE ASSIGN TO "TESTDB.ISQ" ORGANIZATION IS INDEXED ACCESS MODE IS DYNAMIC RECORD KEY IS D-ID.
DATA DIVISION.
FILE SECTION.
FD D-FILE.
01 D-RECORD.
03 D-ID PIC X(8).
03 D-IV PIC 9(8) COMP.
03 D-XV COMP-2.
03 D-SLEN PIC 9(4) COMP.
03 D-SV PIC X(50).
WORKING-STORAGE SECTION.
01 NREC PIC 9(4) COMP VALUE 1000.
01 COB_OFFSET PIC 9(4) COMP VALUE 2000.
01 BLANK-VAL PIC X(8) VALUE " ".
01 EOF-FLAG PIC X.
01 DONE-FLAG PIC X.
01 WID PIC 9(4) DISPLAY.
01 IV PIC 9(4) DISPLAY.
01 XV PIC 9(4)V9(2) DISPLAY.
01 SLEN PIC 9(2) DISPLAY.
01 SV PIC X(50).
01 I PIC 9(4) DISPLAY.
01 N PIC 9(4) DISPLAY.
01 N2 PIC 9(4) DISPLAY.
01 STOP-ID PIC X(8).
PROCEDURE DIVISION.
MAIN-PARAGRAPH.
OPEN I-O D-FILE
MOVE BLANK-VAL TO D-ID
START D-FILE KEY GREATER THAN D-ID
INVALID KEY DISPLAY "Empty file"
END-START
PERFORM VARYING I FROM 1 BY 1 UNTIL I > NREC
COMPUTE WID = COB_OFFSET + I
STRING "Key#" DELIMITED BY SIZE WID DELIMITED BY SIZE INTO D-ID
COMPUTE D-IV = I
COMPUTE D-XV = I
COMPUTE D-SLEN = 18
STRING "This is value " DELIMITED BY SIZE I DELIMITED BY SIZE INTO D-SV
WRITE D-RECORD
INVALID KEY DISPLAY "Error writing"
NOT INVALID KEY CONTINUE
END-WRITE
END-PERFORM
COMPUTE WID = COB_OFFSET + 77
STRING "Key#" DELIMITED BY SIZE WID DELIMITED BY SIZE INTO D-ID
PERFORM DUMP-PARAGRAPH
DELETE D-FILE
INVALID KEY DISPLAY "Error deleting"
NOT INVALID KEY CONTINUE
END-DELETE
PERFORM DUMP-PARAGRAPH
COMPUTE WID = COB_OFFSET + 88
STRING "Key#" DELIMITED BY SIZE WID DELIMITED BY SIZE INTO D-ID
PERFORM DUMP-PARAGRAPH
READ D-FILE
INVALID KEY DISPLAY "Error reading"
NOT INVALID KEY CONTINUE
END-READ
COMPUTE D-IV = D-IV + 1
COMPUTE D-XV = D-XV + 0.1
MOVE D-SLEN TO SLEN
MOVE D-SV TO SV
STRING SV(1:SLEN) DELIMITED BY SIZE " updated" DELIMITED BY SIZE INTO D-SV
COMPUTE D-SLEN = D-SLEN + 8
REWRITE D-RECORD
INVALID KEY DISPLAY "Error updating"
NOT INVALID KEY CONTINUE
END-REWRITE
PERFORM DUMP-PARAGRAPH
MOVE BLANK-VAL TO D-ID
START D-FILE KEY IS GREATER THAN D-ID
INVALID KEY DISPLAY "Error rewinding"
NOT INVALID KEY CONTINUE
END-START
MOVE 'N' TO EOF-FLAG
MOVE 0 TO N
PERFORM UNTIL EOF-FLAG = 'Y'
READ D-FILE NEXT
AT END MOVE 'Y' TO EOF-FLAG
NOT AT END PERFORM CHECK-AND-COUNT-PARAGRAPH
END-READ
END-PERFORM
DISPLAY N
COMPUTE WID = COB_OFFSET + 75
STRING "Key#" DELIMITED BY SIZE WID DELIMITED BY SIZE INTO D-ID
START D-FILE KEY IS GREATER THAN OR EQUAL TO D-ID
INVALID KEY DISPLAY "Error searching"
NOT INVALID KEY CONTINUE
END-START
COMPUTE WID = COB_OFFSET + 85
STRING "Key#" DELIMITED BY SIZE WID DELIMITED BY SIZE INTO STOP-ID
MOVE 'N' TO EOF-FLAG
MOVE 'N' TO DONE-FLAG
MOVE 0 TO N2
PERFORM UNTIL EOF-FLAG = 'Y' OR DONE-FLAG = 'Y'
READ D-FILE NEXT
AT END MOVE 'Y' TO EOF-FLAG
NOT AT END PERFORM COUNT-PARAGRAPH
END-READ
END-PERFORM
DISPLAY N2
CLOSE D-FILE
STOP RUN.
DUMP-PARAGRAPH.
DISPLAY "Key=" D-ID
READ D-FILE
INVALID KEY DISPLAY "Not found"
NOT INVALID KEY PERFORM DUMP-DATA-PARAGRAPH
END-READ.
DUMP-DATA-PARAGRAPH.
MOVE D-IV TO IV
MOVE D-XV TO XV
MOVE D-SLEN TO SLEN
MOVE D-SV TO SV
DISPLAY "(" IV "," XV "," SV(1:SLEN) ")".
CHECK-AND-COUNT-PARAGRAPH.
MOVE D-IV TO IV
MOVE D-XV TO XV
MOVE D-SLEN TO SLEN
MOVE D-SV TO SV
IF IV < 1 OR NREC < IV THEN
DISPLAY "Unexpected value: (" IV "," XV "," SV(1:SLEN) ")"
END-IF
COMPUTE N = N + 1.
COUNT-PARAGRAPH.
IF D-ID GREATER THAN OR EQUAL TO STOP-ID THEN
MOVE 'Y' TO DONE-FLAG
ELSE
COMPUTE N2 = N2 + 1
END-IF.
/* standard C headers */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
/* RMS headers */
#include <starlet.h>
#include <rms.h>
/* data structure */
#pragma member_alignment save
#pragma nomember_alignment
#define KEYSIZ 8
struct data
{
char id[KEYSIZ];
int iv;
double xv;
short slen;
char sv[50];
};
#define RECSIZ sizeof(struct data)
#pragma member_alignment restore
/* error handling functions */
void rms_exit(long stat)
{
printf("Error code: %d\n", stat);
exit(1);
}
static void dump(struct RAB *rab, char *id)
{
long stat;
struct data d;
printf("Key=%s\n", id);
rab->rab$l_kbf = id;
rab->rab$b_ksz = KEYSIZ;
rab->rab$b_krf = 0;
rab->rab$l_rop = 0;
rab->rab$l_ubf = &d.id[0];
rab->rab$w_usz = RECSIZ;
stat = sys$get(rab, 0, 0);
if((stat & 1) && (memcmp(d.id, id, KEYSIZ) == 0)) {
printf("(%d,%f,%s)\n", d.iv, d.xv, d.sv);
} else {
printf("Not found\n");
}
}
static const char *FNM = "testdb.isq";
static const int NREC = 1000;
static const int C_OFFSET = 3 * NREC;
int main()
{
long stat;
struct FAB fab;
struct RAB rab;
struct XABKEY xab;
char id[KEYSIZ + 1], startid[KEYSIZ + 1], stopid[KEYSIZ + 1];
struct data d;
int n, n2;
/* open */
fab = cc$rms_fab;
fab.fab$l_fna = (char *)FNM;
fab.fab$b_fns = strlen(FNM);
fab.fab$b_org = FAB$C_IDX;
fab.fab$b_rfm = FAB$C_FIX;
fab.fab$b_rat = FAB$M_CR;
fab.fab$l_fop = FAB$M_CIF;
fab.fab$w_mrs = RECSIZ;
fab.fab$b_fac = FAB$M_GET | FAB$M_PUT | FAB$M_UPD | FAB$M_DEL;
fab.fab$l_xab = (char *)&xab;
xab = cc$rms_xabkey;
xab.xab$b_dtp = XAB$C_STG;
xab.xab$w_pos0 = 0;
xab.xab$b_ref = 0;
xab.xab$b_siz0 = KEYSIZ;
stat = sys$create(&fab, 0, 0);
if(!(stat & 1)) {
rms_exit(stat);
}
fab.fab$l_xab = 0;
rab = cc$rms_rab;
rab.rab$l_fab = &fab;
rab.rab$b_rac = RAB$C_KEY;
stat = sys$connect(&rab, 0 ,0);
if(!(stat & 1)) {
rms_exit(stat);
}
/* put data */
for(int i = 0; i < NREC; i++)
{
sprintf(id, "Key#%d", C_OFFSET + i + 1);
memcpy(d.id, id, KEYSIZ);
d.iv = i + 1;
d.xv = i + 1.0;
sprintf(d.sv, "This is value %d", i + 1);
d.slen = strlen(d.sv);
rab.rab$l_rbf = &d.id[0];
rab.rab$w_rsz = RECSIZ;
stat = sys$put(&rab, 0, 0);
if(!(stat & 1)) {
rms_exit(stat);
}
}
/* */
sprintf(id, "Key#%d", C_OFFSET + 77);
/* get */
dump(&rab, id);
/* delete */
rab.rab$l_kbf = id;
rab.rab$b_ksz = KEYSIZ;
rab.rab$b_krf = 0;
rab.rab$l_rop = RAB$M_KGE;
stat = sys$find(&rab, 0, 0);
if(!(stat & 1)) {
rms_exit(stat);
}
stat = sys$delete(&rab, 0, 0);
if(!(stat & 1)) {
rms_exit(stat);
}
/* get non existing */
dump(&rab, id);
/* */
sprintf(id, "Key#%d", C_OFFSET + 88);
/* update and get */
dump(&rab, id);
rab.rab$l_kbf = id;
rab.rab$b_ksz = KEYSIZ;
rab.rab$b_krf = 0;
rab.rab$l_rop = 0;
rab.rab$l_ubf = &d.id[0];
rab.rab$w_usz = RECSIZ;
stat = sys$get(&rab, 0, 0);
if(!(stat & 1)) {
rms_exit(stat);
}
d.iv = d.iv + 1;
d.xv = d.xv + 0.1;
strcat(d.sv, " updated");
d.slen = strlen(d.sv);
rab.rab$l_rbf = &d.id[0];
rab.rab$w_rsz = RECSIZ;
stat = sys$update(&rab, 0, 0);
if(!(stat & 1)) {
rms_exit(stat);
}
dump(&rab, id);
/* list all */
n = 0;
stat = sys$rewind(&rab, 0, 0);
if(!(stat & 1)) {
rms_exit(stat);
}
rab.rab$b_rac = RAB$C_SEQ; /* switch to sequential access mode */
for(;;)
{
rab.rab$l_kbf = 0;
rab.rab$b_ksz = 0;
rab.rab$b_krf = 0;
rab.rab$l_rop = 0;
rab.rab$l_ubf = &d.id[0];
rab.rab$w_usz = RECSIZ;
stat = sys$get(&rab, 0, 0);
if(!(stat & 1)) break;
if(strstr(d.id, "Key#") != d.id)
{
printf("Unexpected key: %s\n", d.id);
}
if (d.iv < 1 || NREC < d.iv)
{
printf("Unexpected value: (%d,%f,%s)\n", d.iv, d.xv, d.sv);
}
n++;
}
rab.rab$b_rac = RAB$C_KEY; /* switch back to keyed access mode */
printf("%d\n", n);
/* list keys where "Key#n075" <= key < "Key#n085" */
sprintf(startid, "Key#%d", C_OFFSET + 75);
sprintf(stopid, "Key#%d", C_OFFSET + 85);
rab.rab$l_kbf = startid;
rab.rab$b_ksz = KEYSIZ;
rab.rab$b_krf = 0;
rab.rab$l_rop = RAB$M_KGE;
stat = sys$find(&rab, 0, 0);
if(!(stat & 1)) {
rms_exit(stat);
}
n2 = 0;
rab.rab$b_rac = RAB$C_SEQ; /* switch to sequential access mode */
for(;;)
{
rab.rab$l_kbf = 0;
rab.rab$b_ksz = 0;
rab.rab$b_krf = 0;
rab.rab$l_rop = 0;
rab.rab$l_ubf = &d.id[0];
rab.rab$w_usz = RECSIZ;
stat = sys$get(&rab, 0, 0);
if(!(stat & 1)) break;
if(strcmp(d.id, stopid) > 0) break;
n2++;
}
rab.rab$b_rac = RAB$C_KEY; /* switch back to keyed access mode */
printf("%d\n", n2);
/* close */
stat = sys$disconnect(&rab, 0, 0);
if(!(stat & 1)) {
rms_exit(stat);
}
stat = sys$close(&fab, 0, 0);
if(!(stat & 1)) {
rms_exit(stat);
}
return 0;
}
We note that the C code is more cumbersome than other languages, because it is a pure API usage, while other languages benefit from language builtin support.
program test
implicit none
structure /mydata/
character*8 id
integer*4 iv
real*8 xv
integer*2 slen
character*50 sv
endstructure
record /mydata/d
character*8 BLANK_ID
character*8 id,stopid
integer*4 DB,NREC,FOR_OFFSET
integer*4 i,n,n2
parameter (DB=1,NREC=1000,FOR_OFFSET=4*NREC,BLANK_ID=' ')
c open
open(unit=DB,file='testdb.isq',status='old',
+ recordtype='fixed',form='unformatted',recl=18,
+ organization='indexed',access='keyed',key=(1:8:character))
c put data
do 100 i=1,NREC
write(d.id,'(A4,I4)') 'Key#',(FOR_OFFSET+i)
d.iv=i
d.xv=i
d.slen=18
write(d.sv,'(A,I4.4)') 'This is value ',i
write(unit=DB) d
100 continue
c
write(id,'(A4,I4)') 'Key#',(FOR_OFFSET + 77)
c get
call dump(DB, id)
c delete
read(unit=DB,keyge=id,keyid=0) d
delete(unit=DB)
c get non-existing
call dump(DB, id)
c get and update
write(id,'(A4,I4)') 'Key#',(FOR_OFFSET + 88)
call dump(DB,id)
read(unit=DB,keyeq=id,keyid=0) d
d.iv=d.iv+1
d.xv=d.xv+0.1
d.sv(d.slen+1:d.slen+8)=' updated'
d.slen=d.slen+8
rewrite(unit=DB) d
call dump(DB,id)
c list all
n=0
read(unit=DB,keyge=BLANK_ID,keyid=0,err=300) d
200 if(d.id(1:4).ne.'Key#') then
write(*,*) 'Unexpected key: ',d.id
endif
if(d.iv.lt.1.or.NREC.lt.d.iv) then
write(*,*) 'Unexpected value: ',
+ '(',d.iv,',',d.xv,',',d.sv(1:d.slen),')'
endif
n=n+1
read(unit=DB,end=300) d
goto 200
300 write(*,*) n
c list keys where "Key#n075" <= key < "Key#n085"
write(id,'(A4,I4)') 'Key#',(FOR_OFFSET+75)
write(stopid,'(A4,I4)') 'Key#',(FOR_OFFSET+85)
n2=0
read(unit=DB,keyge=id,keyid=0,err=500) d
400 if(d.id.ge.stopid) goto 500
n2=n2+1
read(unit=DB,end=500) d
goto 400
500 write(*,*) n2
c close
close(unit=DB)
end
c
subroutine dump(u,id)
implicit none
integer*4 u
character*8 id
structure /mydata/
character*8 id
integer*4 iv
real*8 xv
integer*2 slen
character*50 sv
endstructure
record /mydata/d
write(*,*) 'Key=',id
read(unit=u,keyeq=id,keyid=0,err=100) d
write(*,*) '(',d.iv,',',d.xv,',',d.sv(1:d.slen),')'
return
100 write(*,*) 'Not found'
return
end
VMS Python comes with an IndexedFile module capable of accessing index-sequential files and a construct module capable of mapping between a native record format and Python data objects.
from construct import *
from vms.rms.IndexedFile import IndexedFile
from extras import *
class Data:
def __init__(self, id, iv, xv, sv):
self.id = id
self.iv = iv
self.xv = xv
self.slen = len(sv.rstrip())
self.sv = sv.ljust(50)
def update_sv(self, sv):
self.slen = len(sv.rstrip())
self.sv = sv.ljust(50)
def __str__(self):
return '(%d, %f, %s)' % (self.iv, self.xv, self.sv.rstrip())
class DataFil(IndexedFile):
def __init__(self, fnm):
IndexedFile.__init__(self, fnm, Struct('data', String('id', 8), SNInt32('iv'), VAXGFloat('xv'), UNInt16('slen'), String('sv', 50)))
def primary_keynum(self):
return 0
def pack_key(self, keynum, keyval):
return keyval
def keyval(self, rec, keynum):
return rec.id
def toData(obj):
return Data(obj.id, obj.iv, obj.xv, obj.sv)
def dump(db, id):
print('Key=%s' % (id))
d = db.fetch(0, id)
if d != None:
d = toData(d)
print(d)
else:
print('Not found')
NREC = 1000
PY_OFFSET = 7 * NREC
# open
db = DataFil('testdb.isq')
# put data
for i in range(NREC):
d = Data('Key#' + str(PY_OFFSET + i + 1), i + 1, i + 1.0, 'This is value %d' % (PY_OFFSET + i + 1))
db.put(d)
#
id = 'Key#' + str(PY_OFFSET + 77)
# get
dump(db, id)
# delete
db.delete(0, id)
# get non-existing
dump(db, id)
# get and update
id = 'Key#' + str(PY_OFFSET + 88)
dump(db, id)
d = db.fetch(0, id)
d = toData(d)
d.iv = d.iv + 1
d.xv = d.xv + 0.1
d.update_sv(d.sv[0:d.slen] + ' updated')
db.update(d)
dump(db, id)
# list all
n = 0
for d in db:
d = toData(d)
if not d.id.startswith('Key#'):
print('Unexpected key: ' + d.id)
if d.iv < 1 or NREC < d.iv:
print('Unexpected value: ' + d)
n = n + 1
print(n)
# list keys where "Key#n075" <= key < "Key#n085"
n2 = 0
for i in range(10): # hack - there should be a better way, but I can't find it
d = db.fetch(0, 'Key#' + str(PY_OFFSET + 75 + i))
if d != None:
d = toData(d)
n2 = n2 + 1
print(n2)
extras.py (with additions to construct moule for VAX floating point and packed decimals support):
from ctypes import CDLL, c_double, c_char_p, cast, byref
from construct import Adapter, Bytes
from vms.rms import BCD2Tuple, Tuple2BCD
from decimal import Decimal
CVT_K_VAX_F = 0
CVT_K_VAX_D = 1
CVT_K_VAX_G = 2
CVT_K_VAX_H = 3
CVT_K_IEEE_S = 4
CVT_K_IEEE_T = 5
CVT_K_IBM_LONG = 6
CVT_K_IBM_SHORT = 7
CVT_K_CRAY = 8
librtl = CDLL('LIBRTL.EXE')
lib_convert_float = getattr(librtl, 'CVT$CONVERT_FLOAT')
def float_to_python(val, typ):
res = c_double()
lib_convert_float(cast(val, c_char_p), typ, byref(res), CVT_K_IEEE_T, 0)
return res.value
def float_from_python(val, typ):
res = '12345678'
lib_convert_float(byref(c_double(val)), CVT_K_IEEE_T, cast(res, c_char_p), typ, 0)
return res
class VAXFFloatAdapter(Adapter):
def _decode(self, obj, context):
return float_to_python(obj, CVT_K_VAX_F)
def _encode(self, obj, context):
return float_from_python(obj, CVT_K_VAX_F)
def VAXFFloat(name):
return VAXFFloatAdapter(Bytes(name, 4))
class VAXDFloatAdapter(Adapter):
def _decode(self, obj, context):
return float_to_python(obj, CVT_K_VAX_D)
def _encode(self, obj, context):
return float_from_python(obj, CVT_K_VAX_D)
def VAXDFloat(name):
return VAXDFloatAdapter(Bytes(name, 8))
class VAXGFloatAdapter(Adapter):
def _decode(self, obj, context):
return float_to_python(obj, CVT_K_VAX_G)
def _encode(self, obj, context):
return float_from_python(obj, CVT_K_VAX_G)
def VAXGFloat(name):
return VAXGFloatAdapter(Bytes(name, 8))
class PackedDecimalAdapter(Adapter):
def __init__(self, byts, befdec, aftdec):
Adapter.__init__(self, byts)
self.befdec = befdec
self.aftdec = aftdec
def _decode(self, obj, context):
return Decimal(BCD2Tuple(obj, self.aftdec))
def _encode(self, obj, context):
return Tuple2BCD(obj.as_tuple(), self.befdec, self.aftdec)
def PackedDecimal(name, befdec, aftdec):
return PackedDecimalAdapter(Bytes(name, (befdec + aftdec + 2) / 2), befdec, aftdec)
Java cannot call a native API like RMS directly. There need to be a conversion layer utilizing JNI.
Instead of using a custom layer then the code will be using my ISAM library.
import dk.vajhoej.isam.KeyField;
import dk.vajhoej.record.Alignment;
import dk.vajhoej.record.Endian;
import dk.vajhoej.record.FieldType;
import dk.vajhoej.record.Struct;
import dk.vajhoej.record.StructField;
@Struct(endianess=Endian.LITTLE, alignment=Alignment.ALIGN1)
public class Data {
@KeyField(n=0)
@StructField(n=0, type=FieldType.FIXSTRNULTERM, length=8)
private String id;
@StructField(n=1, type=FieldType.INT4)
private int iv;
@StructField(n=2, type=FieldType.FP8)
private double xv;
@StructField(n=3, type=FieldType.VARFIXSTR, length=50)
private String sv;
public Data() {
this("", 0, 0.0, "");
}
public Data(String id, int iv, double xv, String sv) {
super();
this.id = id;
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public String getId() {
return id;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
import dk.vajhoej.isam.IsamException;
import dk.vajhoej.isam.IsamResult;
import dk.vajhoej.isam.IsamSource;
import dk.vajhoej.isam.Key0;
import dk.vajhoej.isam.local.LocalIsamSource;
import dk.vajhoej.record.RecordException;
public class Test {
private static void dump(IsamSource db, String id) throws IsamException, RecordException {
System.out.printf("Key=%s\n", id);
Data d = db.read(Data.class, new Key0<String>(id));
if(d != null) {
System.out.println(d);
} else {
System.out.println("Not found");
}
}
private static final int NREC = 1000;
private static final int JAVA_OFFSET = 6 * NREC;
public static void main(String[] args) throws IsamException, RecordException {
try {
// open
IsamSource db = new LocalIsamSource("testdb.isq", "dk.vajhoej.vms.rms.IndexSequential", false);
// put data
for(int i = 0; i < NREC; i++) {
Data d = new Data("Key#" + (JAVA_OFFSET + i + 1), i + 1, i + 1.0, String.format("This is value %d", i + 1));
db.create(d);
}
//
Data d;
String id;
id = "Key#" + (JAVA_OFFSET + 77);
// get
dump(db, id);
// delete
db.delete(Data.class, new Key0<String>(id));
// get non existing
dump(db, id);
// get and update
id = "Key#" + (JAVA_OFFSET + 88);
dump(db, id);
d = db.read(Data.class, new Key0<String>(id));
d.setIv(d.getIv() + 1);
d.setXv(d.getXv() + 0.1);
d.setSv(d.getSv() + " updated");
db.update(d);
dump(db, id);
// list all
IsamResult<Data> it = db.readGE(Data.class, new Key0<String>(" "));
int n = 0;
while(it.read()) {
d = it.current();
if(!d.getId().startsWith("Key#")) {
System.out.println("Unexpected key: " + d.getId());
}
if(d.getIv() < 1 || NREC < d.getIv()) {
System.out.println("Unexpected value :" + d);
}
n++;
}
System.out.println(n);
// list keys where "Key#n075" <= key < "Key#n085"
IsamResult<Data> it2 = db.readGE(Data.class, new Key0<String>("Key#" + (JAVA_OFFSET + 75)));
int n2 = 0;
while(it2.read()) {
d = it2.current();
if(d.getId().compareTo("Key#" + (JAVA_OFFSET + 85)) >= 0) break;
n2++;
}
System.out.println(n2);
// close
db.close();
} catch (Exception ex) {
ex.printStackTrace();
}
}
}
Jython (JVM Python) can use the same library as Java.
It is easiest to keep the data class in Java.
import dk.vajhoej.isam.KeyField;
import dk.vajhoej.record.Alignment;
import dk.vajhoej.record.Endian;
import dk.vajhoej.record.FieldType;
import dk.vajhoej.record.Struct;
import dk.vajhoej.record.StructField;
@Struct(endianess=Endian.LITTLE, alignment=Alignment.ALIGN1)
public class Data {
@KeyField(n=0)
@StructField(n=0, type=FieldType.FIXSTRNULTERM, length=8)
private String id;
@StructField(n=1, type=FieldType.INT4)
private int iv;
@StructField(n=2, type=FieldType.FP8)
private double xv;
@StructField(n=3, type=FieldType.VARFIXSTR, length=50)
private String sv;
public Data() {
this("", 0, 0.0, "");
}
public Data(String id, int iv, double xv, String sv) {
super();
this.id = id;
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public String getId() {
return id;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
from dk.vajhoej.isam import Key0
from dk.vajhoej.isam.local import LocalIsamSource
import Data
def dump(db, id):
print('Key=%s' % (id))
d = db.read(Data, Key0(id))
if d != None:
print(d)
else:
print('Not found')
NREC = 1000
PY_OFFSET = 7 * NREC
# open
db = LocalIsamSource('testdb.isq', 'dk.vajhoej.vms.rms.IndexSequential', False)
# put data
for i in range(NREC):
d = Data('Key#' + str(PY_OFFSET + i + 1), i + 1, i + 1.0, 'This is value %d' % (PY_OFFSET + i + 1))
db.create(d)
#
id = 'Key#' + str(PY_OFFSET + 77)
# get
dump(db, id)
# delete
db.delete(Data,Key0(id))
# get non-existing
dump(db, id)
# get and update
id = 'Key#' + str(PY_OFFSET + 88)
dump(db, id)
d = db.read(Data, Key0(id))
d.iv = d.iv + 1
d.xv = d.xv + 0.1
d.sv = d.sv + ' updated'
db.update(d)
dump(db, id)
# list all
it = db.readGE(Data, Key0(' '))
n = 0
while it.read():
d = it.current()
if not d.id.decode().startswith('Key#'):
print('Unexpected key: ' + d.id)
if d.iv < 1 or NREC < d.iv:
print('Unexpected value: ' + d)
n = n + 1
print(n)
# list keys where "Key#n075" <= key < "Key#n085"
it2 = db.readGE(Data, Key0('Key#' + str(PY_OFFSET + 75)))
n2 = 0
while it2.read():
d = it2.current()
if d.id >= ('Key#' + str(PY_OFFSET + 85)):
break
n2 = n2 + 1
print(n2)
# close
db.close()
Rhino (JVM JavaScript) can use the same library as Java.
It is easiest to keep the data class in Java.
import dk.vajhoej.isam.KeyField;
import dk.vajhoej.record.Alignment;
import dk.vajhoej.record.Endian;
import dk.vajhoej.record.FieldType;
import dk.vajhoej.record.Struct;
import dk.vajhoej.record.StructField;
@Struct(endianess=Endian.LITTLE, alignment=Alignment.ALIGN1)
public class Data {
@KeyField(n=0)
@StructField(n=0, type=FieldType.FIXSTRNULTERM, length=8)
private String id;
@StructField(n=1, type=FieldType.INT4)
private int iv;
@StructField(n=2, type=FieldType.FP8)
private double xv;
@StructField(n=3, type=FieldType.VARFIXSTR, length=50)
private String sv;
public Data() {
this("", 0, 0.0, "");
}
public Data(String id, int iv, double xv, String sv) {
super();
this.id = id;
this.iv = iv;
this.xv = xv;
this.sv = sv;
}
public String getId() {
return id;
}
public int getIv() {
return iv;
}
public void setIv(int iv) {
this.iv = iv;
}
public double getXv() {
return xv;
}
public void setXv(double xv) {
this.xv = xv;
}
public String getSv() {
return sv;
}
public void setSv(String sv) {
this.sv = sv;
}
@Override
public String toString() {
return String.format("{iv: %d, xv: %f, sv: %s}", iv, xv, sv);
}
}
importClass(Packages.java.lang.System);
function printf() {
System.out.printf.apply(System.out, arguments);
}
importClass(Packages.dk.vajhoej.isam.Key0);
importClass(Packages.dk.vajhoej.isam.local.LocalIsamSource);
importClass(Packages.Data);
function dump(db, id) {
printf("Key=%s\n", id);
d = db.read(Data, new Key0(id));
if(d != null) {
print(d);
} else {
print("Not found");
}
}
NREC = 1000;
JS_OFFSET = 8 * NREC;
// open
db = new LocalIsamSource("testdb.isq", "dk.vajhoej.vms.rms.IndexSequential", false);
// put data
for(i = 0; i < NREC; i++) {
d = new Data("Key#" + (JS_OFFSET + i + 1), i + 1, i + 1.0, "This is value " + (JS_OFFSET + i + 1));
db.create(d);
}
//
id = "Key#" + (JS_OFFSET + 77);
// get
dump(db, id);
// delete
db.delete(Data, new Key0(id));
// get non-existing
dump(db, id);
// get and update
id = "Key#" + (JS_OFFSET + 88);
dump(db, id);
d = db.read(Data, new Key0(id));
d.iv = d.iv + 1;
d.xv = d.xv + 0.1;
d.sv = d.sv + " updated";
db.update(d);
dump(db, id);
// list all
it = db.readGE(Data, new Key0(" "));
n = 0;
while(it.read()) {
d = it.current();
if(!d.id.startsWith("Key#")) {
print("Unexpected key: " + d.id);
}
if(d.iv < 1 || NREC < d.iv) {
print("Unexpected value: " + d);
}
n = n + 1;
}
print(n);
// list keys where "Key#n075" <= key < "Key#n085"
it2 = db.readGE(Data, Key0("Key#" + (JS_OFFSET + 75)));
n2 = 0;
while(it2.read()) {
d = it2.current();
if(d.id >= ("Key#" + (JS_OFFSET + 85))) break;
n2 = n2 + 1;
}
print(n2);
// close
db.close();
DCL (VMS shell) is capable of accessing index-sequential files.
$ NREC = 1000
$ DCL_OFFSET = 5 * NREC
$! open
$ open/read/write db testdb.isq
$! put
$ i = 0
$ loop0:
$ if i .ge. NREC then goto endloop0
$ id = "Key#" + f$string(DCL_OFFSET + i + 1)
$ iv = i + 1
$ xv = "XXXXXXXX" ! no FP support in DCL
$ sv = "This is value " + f$string(DCL_OFFSET + i + 1)
$ str_iv[0,32] = iv
$ str_slen[0,16] = f$length(sv)
$ rec = id + str_iv + xv + str_slen + sv + f$extract(0, 50 - f$length(sv), " ")
$ write db rec
$ i = i + 1
$ goto loop0
$ endloop0:
$!
$ id = "Key#" + f$string(DCL_OFFSET + 77)
$! get
$ call dump
$! delete
$ read/index=0/key="''id'"/delete db rec
$! get non-existing
$ call dump
$! update
$ id = "Key#" + f$string(DCL_OFFSET + 88)
$ call dump
$ read/index=0/key="''id'" db rec
$ xid = f$extract(0, 8, rec)
$ iv = f$cvsi(0, 32, f$extract(8, 4, rec))
$ xv = f$extract(12, 8, rec) ! no FP support in DCL
$ slen = f$cvsi(0, 16, f$extract(20, 2, rec))
$ sv = f$extraxt(22, slen, rec)
$ iv = iv + 1
$ ! can't change xv
$ sv = sv + " updated"
$ str_iv[0,32] = iv
$ str_slen[0,16] = f$length(sv)
$ rec = xid + str_iv + xv + str_slen + sv + f$extract(0, 50 - f$length(sv), " ")
$ write/update db rec
$ call dump
$! close
$ close db
$! list all
$ open/read db testdb.isq
$ n = 0
$ loop1:
$ read/error=endloop1 db rec
$ xid = f$extract(0, 8, rec)
$ if f$locate("Key#", xid) .ne. 0
$ then
$ write sys$output f$fao("Unexpected key: !AS", xid)
$ endif
$ iv = f$cvsi(0, 32, f$extract(8, 4, rec))
$ if iv .lt. 1 .or. NREC .lt. iv
$ then
$ sv = f$edit(f$extraxt(20, f$length(rec) - 20, rec), "TRIM")
$ write sys$output f$fao("Unexpected value: (!AS,!SL,!AS)", xid, iv, sv)
$ endif
$ n = n + 1
$ goto loop1
$ endloop1:
$ write sys$output n
$ close db
$! list keys where "Key#n075" <= key < "Key#n085"
$ open/read db testdb.isq
$ id = "Key#" + f$string(DCL_OFFSET + 75 - 1)
$ read/index=0/key="''id'" db rec
$ n2 = 0
$ loop2:
$ read/error=endloop2 db rec
$ xid = f$extract(0, 8, rec)
$ if xid .ges. "Key#" + f$string(DCL_OFFSET + 85) then goto endloop2
$ n2 = n2 + 1
$ goto loop2
$ endloop2:
$ write sys$output n2
$ close db
$!
$ exit
$!
$ dump: subroutine
$ write sys$output f$fao("Key=!AS", id)
$ read/index=0/key="''id'"/error=notf db rec
$ xid = f$extract(0, 8, rec)
$ iv = f$cvsi(0, 32, f$extract(8, 4, rec))
$ xv = f$extract(12, 8, rec) ! no FP support in DCL
$ slen = f$cvsi(0, 16, f$extract(20, 2, rec))
$ sv = f$extraxt(22, slen, rec)
$ write sys$output f$fao("(!AS,!SL,?,!AS)", xid, iv, sv)
$ goto finish
$ notf:
$ write sys$output "Not found"
$ goto finish
$ finish:
$ endsubroutine
Let us see an example of multi key usage:
program mulkey(input, output);
type
fixstr = packed array [1..32] of char;
data = record
id : [key(0)] integer;
nam : [key(1)] fixstr;
addr : [key(2)] fixstr;
val : integer;
end;
database = file of data;
var
db : database;
d : data;
i : integer;
begin
open(db, 'mulkeydb.isq', unknown, organization := indexed, access_method := keyed);
(* put 3 records *)
d.id := 1;
d.nam := 'A';
d.addr := 'A A A';
d.val := 123;
db^ := d;
put(db);
d.id := 2;
d.nam := 'B';
d.addr := 'B B B';
d.val := 456;
db^ := d;
put(db);
d.id := 3;
d.nam := 'C';
d.addr := 'C C C';
d.val := 789;
db^ := d;
put(db);
(* lookup by id *)
findk(db, 0, 2);
if not ufb(db) then begin
d := db^;
writeln(d.val);
end;
findk(db, 0, 3);
if not ufb(db) then begin
d := db^;
writeln(d.val);
end;
(* lookup by nam *)
findk(db, 1, 'B');
if not ufb(db) then begin
d := db^;
writeln(d.val);
end;
findk(db, 1, 'C');
if not ufb(db) then begin
d := db^;
writeln(d.val);
end;
(* lookup by addr *)
findk(db, 2, 'B B B');
if not ufb(db) then begin
d := db^;
writeln(d.val);
end;
findk(db, 2, 'C C C');
if not ufb(db) then begin
d := db^;
writeln(d.val);
end;
close(db);
end.
IDENTIFICATION DIVISION.
PROGRAM-ID.MULKEY.
ENVIRONMENT DIVISION.
INPUT-OUTPUT SECTION.
FILE-CONTROL.
SELECT OPTIONAL DATA-FILE ASSIGN TO "mulkeydb.isq"
ORGANIZATION IS INDEXED
ACCESS MODE IS DYNAMIC
RECORD KEY IS DATA-ID
ALTERNATE RECORD KEY IS DATA-NAME WITH DUPLICATES
ALTERNATE RECORD KEY IS DATA-ADDR WITH DUPLICATES.
DATA DIVISION.
FILE SECTION.
FD DATA-FILE.
01 DATA-RECORD.
03 DATA-ID PIC S9(8) COMP.
03 DATA-NAME PIC X(32).
03 DATA-ADDR PIC X(32).
03 DATA-VAL PIC S9(8) COMP.
WORKING-STORAGE SECTION.
01 TEMP PIC S9(8) DISPLAY.
PROCEDURE DIVISION.
MAIN-PARAGRAPH.
OPEN I-O DATA-FILE
* lookup by id
MOVE 2 TO DATA-ID
READ DATA-FILE
INVALID KEY DISPLAY "Error reading"
NOT INVALID KEY CONTINUE
END-READ
MOVE DATA-VAL TO TEMP
DISPLAY TEMP
MOVE 3 TO DATA-ID
READ DATA-FILE
INVALID KEY DISPLAY "Error reading"
NOT INVALID KEY CONTINUE
END-READ
MOVE DATA-VAL TO TEMP
DISPLAY TEMP
* lookup by name
MOVE "B" TO DATA-NAME
READ DATA-FILE KEY IS DATA-NAME
INVALID KEY DISPLAY "Error reading"
NOT INVALID KEY CONTINUE
END-READ
MOVE DATA-VAL TO TEMP
DISPLAY TEMP
MOVE "C" TO DATA-NAME
READ DATA-FILE KEY IS DATA-NAME
INVALID KEY DISPLAY "Error reading"
NOT INVALID KEY CONTINUE
END-READ
MOVE DATA-VAL TO TEMP
DISPLAY TEMP
* lookup by addr
MOVE "B B B" TO DATA-ADDR
READ DATA-FILE KEY IS DATA-ADDR
INVALID KEY DISPLAY "Error reading"
NOT INVALID KEY CONTINUE
END-READ
MOVE DATA-VAL TO TEMP
DISPLAY TEMP
MOVE "C C C" TO DATA-ADDR
READ DATA-FILE KEY IS DATA-ADDR
INVALID KEY DISPLAY "Error reading"
NOT INVALID KEY CONTINUE
END-READ
MOVE DATA-VAL TO TEMP
DISPLAY TEMP
*
CLOSE DATA-FILE
STOP RUN.
import dk.vajhoej.isam.KeyField;
import dk.vajhoej.isam.IsamSource;
import dk.vajhoej.isam.Key0;
import dk.vajhoej.isam.Key1;
import dk.vajhoej.isam.Key2;
import dk.vajhoej.isam.local.LocalIsamSource;
import dk.vajhoej.record.Alignment;
import dk.vajhoej.record.Endian;
import dk.vajhoej.record.FieldType;
import dk.vajhoej.record.Struct;
import dk.vajhoej.record.StructField;
public class MulKey {
@Struct(endianess=Endian.LITTLE, alignment=Alignment.ALIGN1)
public static class BaseData {
@KeyField(n=0)
@StructField(n=0, type=FieldType.INT4)
private int id;
@KeyField(n=1)
@StructField(n=1, type=FieldType.FIXSTR, length=32)
private String name;
@KeyField(n=2)
@StructField(n=2, type=FieldType.FIXSTR, length=32)
private String address;
@StructField(n=3, type=FieldType.INT4)
private int value;
public BaseData() {
this(0, "", "", 0);
}
public BaseData(int id, String name, String address, int value) {
super();
this.id = id;
this.name = name;
this.address = address;
this.value = value;
}
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
public int getValue() {
return value;
}
public void setValue(int value) {
this.value = value;
}
}
public static void main(String[] args) throws Exception {
// open
IsamSource db = new LocalIsamSource("mulkeydb.isq", "dk.vajhoej.vms.rms.IndexSequential", false);
// lookup by id
System.out.println(db.read(BaseData.class, new Key0(2)).getValue());
System.out.println(db.read(BaseData.class, new Key0(3)).getValue());
// lookup by name
System.out.println(db.read(BaseData.class, new Key1("B")).getValue());
System.out.println(db.read(BaseData.class, new Key1("C")).getValue());
// lookup by address
System.out.println(db.read(BaseData.class, new Key2("B B B")).getValue());
System.out.println(db.read(BaseData.class, new Key2("C C C")).getValue());
// close
db.close();
}
}
$ javac -cp /isamdir/isam.jar:/isamdir/record.jar MulKey.java
$ java -cp .:/isamdir/isam.jar:/isamdir/isam-vms.jar:/isamdir/record.jar "MulKey"
One obvious question is: is it better to use a key value store NoSQL database than a traditional relational database?
In general I think the answer is no. Relational databases comes with so many benefits: rich query language (SQL), standardized database API's, higher level data access technologies (ORM), admin tools, reporting tools etc..
And there are plenty of relational databases to choose from. Native embedded databases include Sqlite and FireBird. Java embedded databases include HSQLDB and H2.
There are cases where a key value store NoSQL database may make sense:
Version | Date | Description |
---|---|---|
1.0 | March 12th 2020 | Initial version |
1.1 | March 18th 2020 | Add Fortran and Java for VMS index-sequential |
1.2 | April 15th 2020 | Add Python DBM section |
1.3 | October 13th 2020 | Add Jython and Rhino for VMS index-sequential |
1.4 | January 6th 2021 | Add JNoSQL for BDB-JE and RocksDB |
1.5 | January 19th 2021 | Add DCL for VMS index-sequential |
1.6 | January 28th 2021 | Add BDB section |
1.7 | December 12th 2021 | Add Basic for VMS index-sequential |
1.8 | February 6th 2022 | Add MapDB |
1.9 | September 3rd 2022 | Add Python for VMS index-sequential |
1.10 | December 31st 2022 | Add BDB-JE object oriented (Entity) interface |
1.11 | January 2nd 2023 | Add Cobol and Java examples for VMS multi key |
See list of all articles here
Please send comments to Arne Vajhøj
Very simple concept. And as a result then key value stores can produce very high performance on modest hardware.
But for anything but very simple CRUD operations then key value stores can result in a lot of code compared to a relational database.
An interesting aspect is that while the term NoSQL is relative new (first used in 1998 and widely used since 2009), then the key value stores are much older and many of them date back to the 1960's and 1970's. They just got put into the NoSQL class later.