crudlet

Crudlet is a universal, streamable interface for data stores that works on any platform. Basically you can use just about any database (or even your API) without it being coupled to your application.

Why?

• Decoupled. Crudlet allows you to decouple any store (even your own API) from your front-end / backend application.
• Interoperable. Use Crudlet with just about any library, or framework.
• Tiny. Crudlet is only 11kb minified.
• Isomorphic. Easily use your application code on multiple platforms (client & server-side). Just swap out the database adapter.
• Testable. Crudlet makes it super easy to stub-out any data store for testing purposes. Super useful especially for server-side apps (e.g: stubbing-out mongodb).
• Extensible. Easily add offline-mode & peer-to-peer (realtime) with just a few lines of code.

Installation

npm install crudlet

Adapters

• pubnub - pubnub sync adapter
• socket.io - socket.io sync adapter
• webrtc - webrtc sync adapter
• loki - loki in-memory database
• memory - another in-memory database
• local-storage - local storage database
• http - HTTP adapter

Examples

• collaborative todos

Example

Below is an example of a realtime DB that uses pubnub, and local storage.

var crud          = require("crudlet");
var pubnub        = require("crudlet-pubnub");
var localStorage  = require("crudlet-local-storage");
 
// store data locally on the users machine
var localdb = localStorage();
 
// pubnub adapter for sending operations to other connected clients
var pubdb   = pubnub({
  publishKey   : "publish key",
  subscribeKey : "subscribe key",
  channel      : "chatroom"
});
 
// the actual DB we're going to use. Pass
// all operations to localstorage, and pubnub
var db = crud.parallel(localdb, pubdb);
 
// tail all operations send to pubnub back into the database. Note
// that remote calls won't get re-published to pubnub
pubdb("tail").pipe(crud.open(db));
 
// create a child database - collection will get passed to each operation
var peopleDb = crud.child(db, { collection: "people" });
 
// insert some people
peopleDb(crudlet.operation("insert", {
  data: [
    { name: "Gordon Ramsay" },
    { name: "Ben Stiller"   }
  ]
})).on("data", function() {
  // handle data here
});

stream.Readable db(operationName, options)

Runs a new operation.

Note that the supported operations & required options may change depending on the data store you're using.

var localStorage = require("crudlet-local-storage");
 
var localdb = localStorage();
localdb(crudlet.operation("insert", {
  collection: "people",
  data: { name: "Arnold Schwarzenegger" }
})).on("data", function() {
  // handle data here
});

stream.Stream crud.open(db)

Creates a new operation stream.

var operationStream = crud.open(db);
 
// emitted when the operation is performed
operationStream.on("data", function() {
 
});
 
operationStream.write(crud.operation("insert", {
  collection: "people",
  data: { name: "Sandra Bullock" }
}));
 
operationStream.write(crud.operation("remove", {
  collection: "people",
  query: { name: "Jeff Goldbloom" }
}));

operation db.operation(name, option)

creates a new operation which can be written to a database stream. See crud.open(db).

crud.open(db).write(crud.operation("insert", {
  collection: "friends",
  data: { name: "Blakers" }
}));

operation crud.op(name, options)

shorthand for crud.operation(name options).

db crud.top(db)

to operation - Makes it so that you can simply call db(operationName, options) instead of passing in the operation each time.

var db = crud.top(localStorage());
 
// enables this
db("insert", {
  collection: { name: "Jorge" }
});
 
// also accepts this
db(crud.operation("insert"));

db crud.child(db, options)

Creates a new child database. options is essentially just added to each operation performed.

var peopleDb = crud.top(crud.child(db, { collection: "people" }));
 
// insert a new person into the people collection
peopleDb("insert", {
  data: { name: "Shrek" }
});

db crud.tailable(db, reject)

Makes the db tailable. This simply allows you to listen for any operations invoked on a db such as create, update, remove, and load.

reject is an array of operations to ignore. Default is [load].

var db = crud.tailable(localdb);
db("tail", function() {
 
});
 
var peopleDb = crud.top(crud.child(db, { collection: "people" }));
 
peopleDb("insert", { data: { name: "Donkey" }}); // trigger tail
peopleDb("remove", { query: { name: "Donkey" }}); // trigger tail
peopleDb("update", { query: { name: "Donkey" }, data: { name: "Donkay" }}); // trigger tail
peopleDb("load", { query: { name: "Donkey" }}); // ignored by tail

db crud.parallel(...dbs)

Combines databases and executes operations in parallel.

var db = crud.parallel(localdb, httpdb);
 
// execute "load" on localdb at the same time
db(crud.op("load")).on("data", function() {
  // Note that his will get called TWICE
}).on("end", function() {
  // called when operation is executed on all dbs
});

db crud.sequence(...dbs)

Combines databases and executes operations in sequence.

var db = crud.top(crud.parallel(localdb, httpdb));
 
// load data from localdb first, then move to httpdb
db("load").on("data", function() {
  // Note that his will get called TWICE
});

db crud.first(...dbs)

Runs dbs in sequence, but stops when a result is emitted from a database.

var db = crud.top(crud.first(localStorage(), http()));
 
// load data from local storage if it exists, or continue
// to http storage
db("load", { collection: "people" }).on("data", function() {
 
});

db crud.accept([...operationNames, ]db)

Accepts only the provided operations.

// main DB - api server
var httpdb  = crud.tailable(http());
 
// temporary cache
var localdb = localStorage();
 
// main DB - get cached data from local storage before
// checking the server
var db      = crud.first(crud.accept("load", localdb), httpdb);
 
// pipe all persistence operations back to local storage
httpdb(crud.op("tail")).pipe(crud.open(localdb));

db crud.reject([...operationNames, ]db)

Runs all operations except the ones provided.

Building a custom database

Building a custom database is pretty easy. All you need to do is return a stream when db(opName, options) is called.

Here's some scaffolding for a custom db:

// slimmed down version of node streams.
var stream = require("obj-stream");
 
function createDatabase(options) {
 
  // create database here
 
  // return fn that executes operations
  return function (operation) {
    var writable = stream.writable();
 
    // this is important so that data can be piped to other things
    process.nextTick(function() {
 
      // collection MUST exist
      if (!operation.collection) return writable.reader.emit("error", new Error("missing collection"));
 
      // perform task here
 
      // write data from insert/load
      writable.write(data);
 
      // must call end operation when complete
      writable.end();
    });
 
    return writable.reader;
  };
}

Keep in mind that there are a few conventions you should follow when writing custom database adapters. These conventions are here to ensure that databases are interoperable with each other.

db(insert, options)

Insert a new item in the database. Note that data is emitted for each item inserted in the database.

• options - db options
  • data - data to insert. Accepts 1 or many items
  • collection - collection to insert (optional for dbs that don't have it)

var _ = require("highland");
var peopleDb = crud.top(crud.child(db, { collection: "people" }));
 
// insert one item
peopleDb("insert", {
  data: { name: "jeff" }
});
 
// insert many items & collect the results in 1
// array
peopleDb("insert", {
  data: [
    { name: "Joe" },
    { name: "Rogan" }
  ]
}).pipe(_.pipeline(_.collect)).on("data", function(people) {
 
});

db(update, options)

Updates an item in the database. Doesn't return any values.

• options
  • query - search query for items to update
  • data - data to merge with
  • collection - db collection
  • multi - true to update multiple items. false is default.

var peopleDb = crud.top(crud.child(db, { collection: "people" }));
 
peopleDb("update", {
  query: { name: "jake" },
  data : { age: 17 }
});
 
// update multiple items
peopleDb("update", {
  multi: true,
  query: { name: "joe" },
  data : { age: 17 }
});

db(upsert, options)

Updates an item if it exists. Inserts an item if it doesn't.

• options
  • query - search query for items to update
  • data - data to merge or insert
  • collection - db collection

var peopleDb = crud.top(crud.child(db, { collection: "people" }));
 
// insert
peopleDb("upsert", {
  query: { name: "jake" },
  data : { name: "jake", age: 17 }
}).on("end", function() {
 
  // update
  peopleDb("upsert", {
    query: { name: "jake" },
    data : { name: "jake", age: 18 }
  })
});

db(load, options)

Loads one or many items from the database.

• options
  • query - search query for items
  • collection - db collection
  • multi - true if loading multiple. false is default.

 
var peopleDb = crud.top(crud.child(db, { collection: "people" }));
 
// load one item
peopleDb("load", {
  query: { name: "tina" }
}).on("data", function() {
  // handle
});
 
 
// load many items
peopleDb("load", {
  multi: true,
  query: { name: "tina" }
}).pipe(_.pipeline(_.collect)).on("data", function(people) {
  // handle
});

db(remove, options)

Removes an item from the database.

• options
  • query - query to search
  • collection - collection to search
  • multi - true to remove multiple items

 
// remove one item
db("remove", {
  collection: "people",
  query: { name: "batman" }
});
 
 
// remove all instances where age = 54
db("remove", {
  collection: "people",
  query: { age: 54 },
  multi: true
});