Miss any of our Open RFC calls?Watch the recordings here! »

lem

1.0.0 • Public • Published

lem

lem logo

Build status

database for time-series data using LevelDB and node.js

installation

$ npm install lem

usage

var lem = require('lem');
var level = require('level');
 
// create a new leveldb - this can also be a sub-level
var leveldb = level('/tmp/lemtest');
 
// create a new lem store using the leveldb
var lemdb = lem(leveldb);
 
// when nodes are indexed
lemdb.on('index', function(key, meta){
 
})
 
// a live stream from the database
lemdb.on('data', function(data){
 
})
 
// nodes are represented by keys
var key = 'myhouse.kitchen.fridge.temperature';
 
// index a node with some meta data
lemdb.index(key, 'My Fridge Temp');
 
// create a recorder which will write data to the node
var temp = lemdb.recorder(key);
 
// write a value every second
setInterval(function(){
    temp(Math.random()*100);
}, 1000)
 

timestamps

When values are written to recorders - they are timestamped. Sometimes - more acurate timestamping (like a GPS source) is used - you can provide the timestamp to the recorder:

var temp = lemdb.recorder('timestamp.test');
setInterval(function(){
    // get a custom timestamp from somewhere - the current time is the default
    var timestamp = new Date().getTime();
    temp(Math.random()*100, timestamp);
}, 1000)

index

You can read the index from any point in the tree - it returns a ReadStream of the keys that have been indexed:

...
var through = require('through');
 
// index a key into the tree
lemdb.index('cars.red5.speed', 'The speed of the car', function(){
    var keysfound = {};
 
    // keys returns a readstream of objects each with a 'key' and 'data' property
    lemdb.keys('cars.red5').pipe(through(function(data){
        keysfound[data.key] = data.value;
    }, function(){
        console.log('Meta: ' + keysfound.speed);
    })
})

This will log:

Meta: The speed of the car

valuestream

Create a ReadStream of telemetry values for a node - you can specify start and end keys to view windows in time:

 
// create a range - this can be a 'session' to make meaningful groups within lem
var sessionstart = new Date('04/05/2013 12:34:43');
var sessionend = new Date('04/05/2013 12:48:10');
var counter = 0;
var total = 0;
 
var secs = (sessionend.getTime() - sessionstart.getTime()) / 1000;
 
lemdb.valuestream('cars.red5.speed', {          
    start:sessionstart.getTime(),
    end:sessionend.getTime()
}).pipe(through(function(data){
 
    // this is the timestamp of the value
    var key = data.key;
 
    // this is the actual value
    var value = data.value;
 
    // map-reduce beginnings
    total += value;
    counter++;
}, function(){
 
    var avg = 0;
 
    if(counter>0){
        avg = total / counter;
    }
 
    console.log('average speed of: ' + avg);
    console.log('data points: ' + total);
    console.log('time period: ' + secs + ' secs');
    
}))

api

var lemdb = lem(leveldb);

Create a new lem database from the provided leveldb. This can be a level-sublevel so you can partition lem into an existing database.

var lem = require('lem');
var level = require('level');
 
var leveldb = level('/tmp/mylem');
var lemdb = lem(leveldb);

lemdb.index(path, meta, [done])

Write a node and some meta data to the index.

The index is used to build a tree of key-values that exist without having to traverse the time-stamped keys.

The stream returned can be used to build any kind of data structure you want (list, tree, etc).

The meta data for each node is saved as a string - you can use your own encoding (e.g. JSON).

Create some indexes:

lemdb.index('myhouse.kitchen.fridge.temperature', '{"title":"Fridge Temp","owner":344}');
lemdb.index('myhouse.kitchen.thermostat.temperature', '{"title":"Stat Temp","owner":344}');

lemdb.keys(path)

keys returns a ReadStream of all keys in the index beneath the key you provide.

For example - convert the stream into a tree representing all nodes in the kitchen:

...
var through = require('through');
var tree = {};
lemdb.keys('myhouse.kitchen').pipe(through(function(data){
    tree[data.key] = data.value;
}, function(){
    console.dir(tree);
}))

This outputs:

{
    "fridge.temperature":'{"title":"Fridge Temp","owner":344}',
    "thermostat.temperature":'{"title":"Stat Temp","owner":344}'
}

lemdb.recorder(path)

A recorder is used to write time-series data to a node.

You create it with the path of the node:

var recorder = lemdb.recorder('myhouse.kitchen.fridge.temperature');

recorder(value, [timestamp], [done])

The recorder itself is a function that you run with a value and optional timestamp and callback.

If no timestamp is provided a default is created:

var timestamp = new Date().getTime();

The callback is run once the value has been committed to disk:

 
// a function to get an accurate time-stamp from somewhere
function getProperTime(){
    return ...;
}
 
// a function to return the current value of an external sensor
function getSensorValue(){
    return ...;
}
var recorder = lemdb.recorder('myhouse.kitchen.fridge.temperature');
 
// sample the value every second
setInterval(function(){
    var value = getSensorValue();
    var timestamp = getProperTime();
    recorder(value, timestamp, function(){
        console.log(timestamp + ':' + value);
    })
}, 1000)
 

events

lemdb.on('index', function(key, meta){})

the 'index' event is emitted when a node is added to the index:

lemdb.on('index', function(key, meta){
    console.log('the key is: ' + key);
 
    // the meta is a string
    var obj = JSON.parse(meta);
    console.dir(obj);
})

lemdb.on('data', function(key, value){})

This is a livestream from leveldb and so contains a full description of the operation:

lemdb.on('index', function(data){
    console.dir(data);
})

This would log:

{ type: 'put',
  key: 'values~cars~red5~speed~1394886656496',
  value: '85'
}

license

MIT

Install

npm i lem

DownloadsWeekly Downloads

5

Version

1.0.0

License

MIT

Last publish

Collaborators

  • avatar