ts-datastore-orm (Typescript Orm wrapper for Google Datastore)
ts-datastore-orm targets to provide a strong typed and structural ORM feature for Datastore (Firestore in Datastore mode).
Please check the examples below to check out all the amazing features!
This package has 0 dependencies. You have to install the @google-cloud/datastore manually.
npm install -s @google-cloud/datastore@latest
This package is compatible with @google-cloud/datastore v5.0.0 or above. I have tested the library specifically with the following versions: v5.0.6, v5.1.0, v6.1.1, v6.2.0, v6.3.0 v7.0.0
You can also compare the native performance of @google-cloud/datastore with ts-datastorem-orm. Basically there is no significant overhead compared with native nodejs-datastore package.
Project Setup
- npm install -s @google-cloud/datastore@latest
- npm install -s ts-datastore-orm@latest
- In tsconfig.json
- set "experimentalDecorators" to true.
- set "emitDecoratorMetadata" to true.
- set "strictNullChecks" to true.
- Generate service-account.json from Goolge APIs. (Details won't be covered here)
Example: connection
async function connectionExamples() {
const connection1 = await createConnection({keyFilename: "./datastoreServiceAccount.json"});
const connection2 = await createConnection({clientEmail: "", privateKey: ""});
const datastore1 = connection1.datastore;
const datastore2 = connection2.datastore;
}Example: define Entity
import { BaseEntity, CompositeIndex, CompositeIndexExporter, createConnection, Entity, Field,
tsDatastoreOrm, TsDatastoreOrmError, BeforeDelete, BeforeInsert, BeforeUpdate, BeforeUpsert, AfterLoad} from "ts-datastore-orm";
@CompositeIndex({_id: "desc"})
@Entity({namespace: "testing", kind: "User", enumerable: true})
export class User extends BaseEntity {
@Field({generateId: true})
public _id: number = 0;
@Field()
public date: Date = new Date();
@Field({index: true})
public string: string = "";
@Field()
public number: number = 10;
@Field()
public buffer: Buffer = Buffer.alloc(1);
@Field()
public array: number[] = [];
@Field({index: true, excludeFromIndexes: ["object.name"]})
public object: any = {};
@Field()
public undefined: undefined = undefined;
@Field()
public null: null = null;
}
@CompositeIndex({number: "desc", name: "desc"})
@CompositeIndex({_id: "desc"})
@Entity() // namespace: default, kind: same as class name
export class TaskGroup extends BaseEntity {
@Field({generateId: true})
public _id: number = 0;
@Field()
public name: string = "";
@Field()
public number: number = 0;
@AfterLoad()
@BeforeInsert()
@BeforeUpsert()
@BeforeUpdate()
@BeforeDelete()
public hook(type: string) {
// you can update the entity after certain events happened
}
}Example: general
async function keyExamples() {
const connection = await createConnection({keyFilename: "./datastoreServiceAccount.json"});
const userRepository = connection.getRepository(User);
const user1 = new User();
const userKey1 = user1.getKey();
const userKey2 = userRepository.create().getKey();
const user3 = userRepository.create({_id: 3});
user3._ancestorKey = userKey1;
const userKey3 = user3.getKey();
const result1 = isSameKey(userKey1, userKey2); // true
const result2 = isSameNamespace(userKey1, userKey2); // true
const result3 = isSameKind(userKey1, userKey2); // true
const result4 = isSamePath(userKey1, userKey2); // true
const result5 = isSameKey(userKey1, userKey3); // false
const [encoded1] = await userRepository.datastore.keyToLegacyUrlSafe(userKey1);
const restoredKey = userRepository.datastore.keyFromLegacyUrlsafe(encoded1);
const result6 = isSameKey(userKey1, restoredKey); // true
}Example: general
async function generalExamples() {
const connection = await createConnection({keyFilename: "./datastoreServiceAccount.json"});
const repository = connection.getRepository(User, {namespace: "mynamespace", kind: "NewUser"});
const taskGroupRepository = connection.getRepository(TaskGroup);
const datastore = connection.datastore; // access to native datastore
const user1 = repository.create();
await repository.insert(user1);
const key = user1.getKey(); // the native datastore key
// the kind and namespace is attached to entity, but they are not enumerable by default
// use @Entity({enumerable: true}) such that if you console.log(entity), _kind and _namespace will be displayed as well
const {_kind, _namespace, _ancestorKey} = user1;
// simple query
const findUser1 = repository.findOne(user1._id);
// find users
const users = await repository
.query()
.filter("_id", x => x.gt(5))
.limit(100)
.findMany();
// get id
const ids = await repository.allocateIds(10);
// remove all data
await repository.truncate();
}Example: multiple entities
async function multipleEntities() {
const connection = await createConnection({keyFilename: "./datastoreServiceAccount.json"});
const repository = connection.getRepository(User, {namespace: "mynamespace", kind: "NewUser"});
const users = Array(10).map((_, i) => repository.create({number: i}));
await repository.insert(users);
await repository.update(users);
await repository.upsert(users);
await repository.delete(users);
}Example: ancestors
async function ancestorExamples() {
const connection = await createConnection({clientEmail: "", privateKey: ""});
const userRepository = connection.getRepository(User);
const taskGroupRepository = connection.getRepository(TaskGroup);
const user1 = await userRepository.insert(userRepository.create({_id: 1}));
const taskGroup = taskGroupRepository.create({_id: 1, name: "group 1", _ancestorKey: user1.getKey()});
await taskGroupRepository.insert(taskGroup);
// ignore the strong type on method call
const findTaskGroup1 = await taskGroupRepository.query()
.filterKey(taskGroup.getKey())
.findOne();
// get back the user
if (findTaskGroup1?._ancestorKey) {
const findUser1 = await userRepository.findOne(findTaskGroup1._ancestorKey);
}
// another way to query the entities
const findTaskGroup2 = await taskGroupRepository.query()
.setAncestorKey(user1.getKey())
.filter("_id", 1)
.findOne();
}Example: admin
async function adminExamples() {
const connection = await createConnection({clientEmail: "", privateKey: ""});
const myAdmin = connection.getAdmin();
const namespaces = await myAdmin.getNamespaces();
const kinds = await myAdmin.getKinds();
}Example: query
async function queryExamples() {
const connection = await createConnection({clientEmail: "", privateKey: ""});
const userRepository = connection.getRepository(User);
const user = userRepository.create({_id: 1});
const findUser1 = await userRepository.query().findOne();
const findUsers2 = await userRepository.findMany([1, 2, 3, 4, 5]);
const findUsers3 = await userRepository.query().filter("_id", x => x.ge(1).lt(6)).findMany();
const findUsers4 = await userRepository.query().limit(10).offset(3).order("number", {descending: true}).findMany();
// complex query with strong type
const query1 = userRepository.query()
.filter("number", 10)
.setAncestorKey(user.getKey())
.groupBy("number")
.order("number", {descending: true})
.offset(5)
.limit(10);
// complex query with strong type
const query = userRepository.query({weakType: true})
.filter("object.name", 10)
.setAncestorKey(user.getKey())
.groupBy("object.name")
.order("object.name", {descending: true})
.offset(5)
.limit(10);
// iterator
const batch = 500;
const iterator = userRepository.query().limit(batch).getAsyncIterator();
for await (const entities of iterator) {
if (entities.length === batch) {
// true
}
}
// select key query
// this can save some query cost and also return faster
const keys = await userRepository.selectKeyQuery().findMany();
}Example: transaction manager
async function transactionManagerExamples() {
const connection = await createConnection({clientEmail: "", privateKey: ""});
const userRepository = connection.getRepository(User);
const transactionManager1 = connection.getTransactionManager();
// customize behavior of the transaction
// for readonly transaction, please refer to datastore documentation
const transactionManager2 = connection.getTransactionManager({maxRetry: 3, retryDelay: 200, readOnly: true});
const result = await transactionManager1.start(async (session) => {
const findEntity1 = await userRepository.findOneWithSession(1, session);
const findEntity2 = await userRepository.queryWithSession( session).findOne();
const ids = await userRepository.allocateIdsWithSession(10, session);
if (findEntity2) {
// only the last operation of the same entity will applies only
userRepository.insertWithSession(findEntity2, session);
userRepository.updateWithSession(findEntity2, session);
userRepository.upsertWithSession(findEntity2, session);
userRepository.deleteWithSession(findEntity2, session);
} else {
await session.rollback();
}
return 5;
});
// value === 5 in above case
const {value, hasCommitted, totalRetry} = result;
}Example: error
async function errorExamples() {
// this help you to provide a better stack upon error
tsDatastoreOrm.useFriendlyErrorStack = true;
const connection = await createConnection({clientEmail: "", privateKey: ""});
const userRepository = connection.getRepository(User);
const user = userRepository.create({_id: 1});
try {
await userRepository.delete(user);
} catch (err) {
if (err instanceof TsDatastoreOrmError) {
// error from this library
}
}
}Example: increment helper
async function incrementHelperExamples() {
const connection = await createConnection({clientEmail: "", privateKey: ""});
const userRepository = connection.getRepository(User);
const user = userRepository.create({_id: 1});
const incrementHelper = userRepository.getIncrementHelper();
// take all kind of parameters
const latestValue1 = await incrementHelper.increment(user._id, "number", 10);
const latestValue2 = await incrementHelper.increment(user, "number", 10);
const latestValue3 = await incrementHelper.increment(user.getKey(), "number", 10);
}Example: index resave helper
async function indexResaveHelperExamples() {
// sometimes u added new index and need to resave the entities
const connection = await createConnection({clientEmail: "", privateKey: ""});
const userRepository = connection.getRepository(User);
const helper = userRepository.getIndexResaveHelper();
await helper.resave("number");
await helper.resave(["number", "string"]);
}Example: lock manager
async function lockManagerExamples() {
// this is a tool for atomic lock
const connection = await createConnection({clientEmail: "", privateKey: ""});
const lockManager1 = connection.getLockManager({expiresIn: 1000});
// this look will try to acquire the lock 2 more times if it failed, waiting 100ms for each retry
// you can also customize which namespace and kind to save temporary data of the lock
const lockManager2 = connection.getLockManager({expiresIn: 1000, maxRetry: 2, retryDelay: 100, namespace: "custom", kind: "Lock"});
try {
const lockKey = "anyKey";
const result = await lockManager1.start(lockKey, async () => {
return 5;
});
console.log(result.value);
} catch (err) {
// your own error or lock acquire error
}
}Example: composite index exporter
async function compositeIndexExamples() {
const filename = "./index.yaml";
const exporter = new CompositeIndexExporter();
exporter.addEntity(User, {kind: "NewUser"});
exporter.addEntity(TaskGroup, {kind: "NewTaskGroup"});
// you can add multiple class, but you can't customize the kind name
exporter.addEntity([User, TaskGroup]);
const yaml = exporter.getYaml();
exporter.exportTo(filename);
}More Examples
Examples are in the tests/ directory.
| Sample | Source Code |
|---|---|
| General | source code |
| Query | source code |
| Subclass | source code |
| Errors | source code |
| CompositeIndex | source code |
| Admin | source code |
| TransactionManager | source code |
| LockManager | source code |
| Hook | source code |
| IndexResaveHelper | source code |
| IncrementHelper | source code |