npm

Ready to take your JavaScript development to the next level? Meet npm Enterprise - the ultimate in enterprise JavaScript.Learn more »

@subtree/peasy-db

1.1.12 • Public • Published

peasy-db

Shared database code for Peasy.

Publish new version using: npm publish --access public

storage

TABLE_USERS
  Keep track of all user meta-data. 
  Index: userId

TABLE_STATS
  Store pre-calculated statistics and insights for users. 
  Index: userId_PERIOD + periodEnd

TABLE_SOURCES
  Raw input data to use for calculating data. Currently contains stats from GA, but can contain more in the future. One document per source and period. 
  Index: userId_sourceId_PERIOD + periodEnd

TABLE_ORDERS
  Raw order data to use for calculating data. Stripped from personal data to comply with GDPR. Typically contains orders; one order per file. 
  Index: userId_sourceId + orderDate_orderId

To use locally

# in root folder:
npm link

# in using repo:
npm link @subtree/peasy-db

To publish new version: npm publish --access public

docs

Located in ./docs/index.html if you first run npm doc

How do we access data?

Keeping track of the access patterns for data is key to having a good model in NoSQL systems. Here are some access patterns we are using:

  • Get customer meta data, such as email, company name etc (for use in emails)
  • Update customer meta data
  • Get all customers as list in order to kick off jobs for all users
  • Store final results from analysis
  • Read and update intermediary analysis data (for larger jobs, caches or performance)
  • Retrieve analysis conclusions for a period and customers data needed for sending out an email
  • Get customer technical details for use during ingestion of order and statistics data

Data model

https://www.slideshare.net/AmazonWebServices/amazon-dynamodb-deep-dive-advanced-design-patterns-for-dynamodb-dat401-aws-reinvent-2018pdf

 
// Customer data
{
  "userId": "testuserIndexTestJS",
  "email": "peasy@subtree.se",
  "name": "Pål Testing Userson",
  "companyName": "Peasy Stats",
  "preferredCurrency": "SEK",
  "locale": "sv-SE",
  "distribution": [
    { "periodicity": "DAY", "destinationEmail": "pal@subtree.se" },
    { "periodicity": "DAY", "destinationEmail": "ida@subtree.se" },
    { "periodicity": "MONTH", "destinationEmail": "ida+bigboss@subtree.se" }
  ],
  "sources": [
    {
      "sourceId": "centra_peasy",
      "sourceType": "CENTRA", // SOURCE.CENTRA|SOURCE.JETSHOP|SOURCE.GOOGLE_ANALYTICS
      "credentials": {
        "apikey": "asecretkey",
        "apiurl": "https://mysite.centra.com/api/peasy/1"
      },
      "lastProcessedDate": "2018-01-01",
      "currency": "SEK"
    },
    {
      "sourceId": "ga_peasy",
      "sourceType": "GOOGLE_ANALYTICS", 
      "credentials": {
        "type": "service_account",
        "project_id": "myProjectID",
        "private_key_id": "myPrivateKeyId",
        "private_key": "-----BEGIN PRIVATE KEY-----\n(MY PRIVATE KEY)\n-----END PRIVATE KEY-----\n",
        "client_email": "peasy-205@icebug-peasy-exporter-1337.iam.gserviceaccount.com",
        "client_id": "myClientId",
        "auth_uri": "https://accounts.google.com/o/oauth2/auth",
        "token_uri": "https://oauth2.googleapis.com/token",
        "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
        "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/myProject.iam.gserviceaccount.com"
      },
      "lastProcessedDate": "2018-01-01",
      "viewId": "1234567890"
    }
  ],
  "updatedAt": "2019-05-31T12:37:13.203Z",
  "createdAt": "2019-05-09T21:47:44.678Z"
}
 
// Data: Webshop Order (Centra)
{
  "userId_sourceId": "570e60dc-5373-4350-9083-5bd40ed5bd08_347e1802-1558-4c42-9ae1-6f744c66e6c0",
  "orderDate_orderId": "2019-05-02_23017",
  "orderId": "23017",
  "orderStatus": "completed",
  "selectionId": "10caca9f09c35ba87abace15c0a6b2fd",
  "orderDate": "2019-05-02 22:54:19",
  "cancelDate": "",
  "preferredDeliveryDate": "",
  "estimatedDeliveryDate": "",
  "orderStore": "retail",
  "orderCurrency": "EUR",
  "customerId": "6094",
  "accountId": "0",
  "billingVAT": "",
  "poNumber": "",
  "shippingList": "eu",
  "suspect": 0,
  "internalOrder": 0,
  "hold": 0,
  "shippingValue": 0,
  "voucherValue": 0,
  "grandTotalValue": 129.95,
  "grandTotalTaxValue": 25.99,
  "defaultCarrier": null,
  "deliveryCity": "Luxembourg",
  "deliveryState": "",
  "deliveryZipcode": "1321",
  "deliveryCountry": "LU",
  "billingCity": "Luxembourg",
  "billingState": "",
  "billingZipcode": "1321",
  "billingCountry": "LU",
  "shipwallet_converted_method_id": "UPSSTDP",
  "shipwallet_courier_instructions": "",
  "shipwallet_deliverytime": "2019-05-08",
  "shipwallet_doorcode": "",
  "shipwallet_id": "11ff88c8-6f1c-42ce-97ad-75062ff4b734",
  "shipwallet_method": "UPSSTDP",
  "shipwallet_pickup": "",
  "shipwallet_reference_id": "",
  "shipwallet_pickuppoint_address": "",
  "shipwallet_expected_delivery_time": "2019-05-08",
  "products": [
    {
      "lineId": "18915",
      "sku": "D5441-0B",
      "variantSku": "",
      "sizeSku": "D5441-0B/11.0",
      "ean": "7310710185311",
      "name": "DTS5 M RB9X",
      "variant": "Flame/Black",
      "brand": "Icebug",
      "size": "US 11",
      "qty": 1,
      "originalPrice": 129.95,
      "price": 129.95,
      "weight": 0.29,
      "weightUnit": "kg",
      "countryOfOrigin": "VN",
      "harmCode": "6404110000",
      "comment": ""
    }
  ]
}
 
// Data: Analytics
{
  "userId": "570e60dc-5373-4350-9083-5bd40ed5bd08",
  "sourceId": "347e1802-1558-4c42-9ae1-6f744c66e6c0",
  "objectType": "SOURCESTATS",
  "sourceType": "GOOGLE_ANALYTICS", 
  "periodicity": "DAY",
  "periodStart": "2019-05-10",
  "periodEnd": "2019-05-10",
  "statistics": {
    "SessionsCount": 581212,
    "TopVisitedPages": [
      {"url": "/", "fullUrl": "https://icebug.com/", "count": 412200 }, 
      {"url": "/all-shoes", "fullUrl": "https://icebug.com/all-shoes", "count": 232911 }, 
      {"url": "/m/five-tips-for-the-trail-round", "fullUrl": "https://icebug.com/m/five-tips-for-the-trail-round", "count": 110330 }, 
      {"url": "/about-us", "fullUrl": "https://icebug.com/about-us", "count": 80302 }, 
      {"url": "/p/metro2-w-bugrip-r-chili", "fullUrl": "https://icebug.com/p/metro2-w-bugrip-r-chili", "count": 1238 }
    ],
    "TopChannels": [
      {"name": "Search", "count": 65 },
      {"name": "Newsletter", "count": 34 },
      {"name": "Paid Search", "count": 12 },
      {"name": "Organic", "count": 5 },
      {"name": "Direct", "count": 1 }
    ],
  },
  "createdAt": "2019-05-10T10:09:44.678Z"
}
 
// Merged data store
{
  "userId_period": "testuser_DAY",
  "objectType": "STATS",
  "periodicity": "DAY",
  "periodStart": "2019-05-10",
  "periodEnd": "2019-05-10",
  "statistics": {
    "currency": "SEK",
    "TotalCancelledOrdersCount": 0,
    "TotalOrderCount": 2634,
    "TotalProductCount": 3077,
    "TotalValueOfOrders": 2366055.02,
    "MostSoldProducts": [
      {"sku": "F13006-9B", "name": "Ivalo2 W BUGrip®", "count": 78, "orderValue": 21331.43 }, 
      {"sku": "C6799", "name": "Overshoe", "count": 76, "orderValue": 2882.12 }, 
      {"sku": "F6202-9A", "name": "Pytho4 W BUGrip®", "count": 74, "orderValue": 18721.01 }
    ],
    "TopCountries": [
      {"code": "SE", "orders": 65 },
      {"code": "NO", "orders": 34 },
      {"code": "DE", "orders": 12 }
    ],
    "SessionsCount": 581212,
    "ConversionRatio": 0.021,
    "TopVisitedPages": [
      { "url": "icebug.com/", "urlPath": "/", "count": 172 },
      { "url": "icebug.se/", "urlPath": "/", "count": 128 },
      { "url": "icebug.se/p/eli-w-rb9x-black", "urlPath": "/p/eli-w-rb9x-black", "count": 84 },
      { "url": "icebug.se/acceleritas7", "urlPath": "/acceleritas7", "count": 34 },
      { "url": "icebug.de/", "urlPath": "/", "count": 28 }
    ],
    "TopChannels": [
      { "name": "Organic Search", "count": 294 },
      { "name": "Social", "count": 207 },
      { "name": "Direct", "count": 149 },
      { "name": "Paid Search", "count": 77 },
      { "name": "Referral", "count": 32 }
    ],
  },
  "createdAt": "2019-05-10T10:09:44.678Z"
}
 

npm version Standard - JavaScript Style Guide

Keywords

install

npm i @subtree/peasy-db

Downloadsweekly downloads

143

version

1.1.12

license

UNLICENSED

homepage

github.com

repository

Gitgithub

last publish

collaborators

  • avatar
Report a vulnerability