Newsworthy Presidential Mistakes
Ready to take your JavaScript development to the next level? Meet npm Enterprise - the ultimate in enterprise JavaScript. Learn more »

lex-bnf

0.2.0 • Public • Published

Lex-BNF

The lexical token analyzer and BNF parser for JavaScript.

Here is a sample that parses a SQL-like-sentence implemented to operate the AWS DynamoDb in the npm aws-node-util:

"use strict";
const BNF = require("lex-bnf");
 
//
// Basic Terms
//
const TABLE = BNF.literal("TABLE");
const SELECT = BNF.literal("SELECT");
const UPDATE = BNF.literal("UPDATE");
const FROM = BNF.literal("FROM");
const WHERE = BNF.literal("WHERE");
const FILTER = BNF.literal("FILTER");
const LIMIT = BNF.literal("LIMIT");
const BETWEEN = BNF.literal("BETWEEN");
const AND = BNF.literal("AND");
const OR = BNF.literal("OR");
const NOT = BNF.literal("NOT");
const IN = BNF.literal("IN");
 
let SQLISH_LEX_TOKEN_DEF = new BNF("words", {
    "words": [
        [ "word", "words", ],
        [ "word", ],
    ],
    "word": [
        [ "num-literal" ],
        [ "comparison-operator" ],
        [ "attribute-value-placeholder" ],
        [ "attribute-path-name" ],
        [ "comment-mark" ],
        [ "escaped-char" ],
        [ "string-literal" ],
        [ "block-comment" ],
        [ "line-comment" ],
        [ BNF.ident ],
        [ BNF.lex("PUNCT") ],
        [ BNF.strlitDQ ],
        [ BNF.strlitSQ ],
        [ BNF.lex("WS") ],
        [ BNF.lex("WS-LINCMT") ],
        [ BNF.lex("WS-BLKCMT") ],
    ],
    "num-literal": [
        [ "fractional-literal" ],
        [ "int-literal" ],
    ],
    "int-literal": [
        [ "dec-num-literal" ],
    ],
    "dec-num-literal": [
        [ "[sign]", BNF.lex("NUMLIT"), ],
    ],
    "sign": [
        [ BNF.literal("+") ],
        [ BNF.literal("-") ],
    ],
    "fractional-literal": [
        [ "[sign]", BNF.lex("NUMLIT"), "fractional-part" ],
    ],
    "fractional-part": [
        [ BNF.literal("."), BNF.lex("NUMLIT"), "[exp-part]", ],
    ],
    "exp-part": [
        [ BNF.literal("E"), "[sign]", BNF.lex("NUMLIT"), ],
        [ BNF.literal("e"), "[sign]", BNF.lex("NUMLIT"), ],
    ],
    "comparison-operator": [
        [ BNF.literal("<"), BNF.literal("=") ],
        [ BNF.literal("<"), BNF.literal(">") ],
        [ BNF.literal(">"), BNF.literal("=") ],
        [ BNF.literal(">"), ],
        [ BNF.literal("="), ],
        [ BNF.literal("<"), ],
    ],
    "attribute-value-placeholder": [
        [ BNF.literal(":"), BNF.ident ],
    ],
    "attribute-path-name": [
        [ BNF.ident, BNF.literal("."), "attribute-path-name" ],
        [ BNF.ident, BNF.literal("."), BNF.ident ],
    ],
    "block-comment":[
        [ BNF.literal("/*"), BNF.literalUntil("*/") ],
    ],
    "line-comment":[
        [ BNF.literal("//"), BNF.literalUntil("\n") ],
        [ BNF.literal("--"), BNF.literalUntil("\n") ],
    ],
    "comment-mark": [
        [ BNF.literal("/"), BNF.literal("*") ],
        [ BNF.literal("*"), BNF.literal("/") ],
        [ BNF.literal("/"), BNF.literal("/") ],
        [ BNF.literal("-"), BNF.literal("-") ],
    ],
    "escaped-char": [
        [ BNF.literal("\\"), BNF.literal("\\"), ],
        [ BNF.literal("\\"), BNF.literal("\""), ],
        [ BNF.literal("\\"), BNF.literal("'"), ],
    ],
    "string-literal": [
        [ "string-literal-dq" ],
        [ "string-literal-sq" ],
    ],
    "string-literal-dq": [
        [ BNF.literal('"'), "string-literal-dq-end" ],
    ],
    "string-literal-dq-end": [
        [ BNF.literalUntil('"') ],
        [ BNF.literal('"') ],
    ],
    "string-literal-sq": [
        [ BNF.literal("'"), "string-literal-sq-end" ],
    ],
    "string-literal-sq-end": [
        [ BNF.literalUntil("'") ],
        [ BNF.literal("'") ],
    ],
}, {
    "num-literal": "NUMLIT",
    "comparison-operator": "PUNCT",
    "attribute-value-placeholder": "IDENT",
    "attribute-path-name": "IDENT",
    "block-comment": "WS-BLKCMT",
    "line-comment": "WS-LINCMT",
    "comment-mark": "WS",
    "escaped-char": "PUNCT",
    "string-literal-dq": "STRLIT-DQ",
    "string-literal-sq": "STRLIT-SQ",
});
 
let SQLISH_BNF_DEF = {
    "sqlish-query": [
        [ "[select-clause]", "from-clause", "where-key-clause",
            "[filter-clause]", "[limit-clause]" ],
    ],
    "select-clause": [
        [ SELECT, "key-list" ],
    ],
    "key-list": [
        [ "column-name", BNF.comma, "key-list" ],
        [ "column-name" ],
    ],
    "column-name": [
        [ BNF.ident ],
    ],
    "from-clause": [
        [ FROM, "table-name" ],
    ],
    "table-name": [
        [ BNF.ident ],
    ],
    "where-key-clause": [
        [ WHERE, "condition-expression" ],
    ],
    "filter-clause": [
        [ FILTER, "condition-expression" ],
    ],
    "condition-expression" : [
        [ "or-expression" ],
    ],
    "or-expression" : [
        [ "and-expression", OR, "condition-expression" ],
        [ "and-expression" ],
    ],
    "and-expression" : [
        [ "compare-expression", AND, "condition-expression" ],
        [ "compare-expression" ],
    ],
    "compare-expression": [
        [ BNF.literal("("), "condition-expression", BNF.literal(")") ],
        [ BNF.ident, "comparator", "value" ],
        [ BNF.ident, BETWEEN, "between-range" ],
        [ BNF.ident, IN, BNF.literal("("), "value-list", BNF.literal(")") ],
        [ "function" ],
        [ NOT, "condition-expression" ],
    ],
    "comparator": [
        [BNF.literal("=") ],
        [BNF.literal("<") ],
        [BNF.literal("<=") ],
        [BNF.literal(">") ],
        [BNF.literal(">=") ],
        [BNF.literal("<>") ],
    ],
    "function": [
        [ BNF.literal("attribute_exists"), BNF.literal("("), "path", BNF.literal(")") ],
        [ BNF.literal("attribute_not_exists"), BNF.literal("("), "path", BNF.literal(")") ],
        [ BNF.literal("attribute_type"), BNF.literal("("),
                "path", BNF.comma, "attribute-type",
            BNF.literal(")") ],
        [ BNF.literal("begins_with"), BNF.literal("("),
                "path", BNF.comma, "value",
            BNF.literal(")") ],
        [ BNF.literal("contains"), BNF.literal("("),
                "path", BNF.comma, "value",
            BNF.literal(")") ],
        [ BNF.literal("size"), BNF.literal("("), "path", BNF.literal(")") ],
    ],
    "path": [
        [ BNF.ident, BNF.literal("."), "path" ],
        [ BNF.ident ],
    ],
    "between-range": [
        [ "value", AND, "value" ],
    ],
    "value-list": [
        ["value", BNF.comma, "value-list" ],
        ["value" ],
    ],
    "value": [
        [ BNF.numlit ],
        [ BNF.strlitDQ ],
        [ BNF.strlitSQ ],
        [ BNF.ident ],
    ],
    "limit-clause": [
        [ LIMIT, "limit-count" ],
    ],
    "limit-count": [
        [ BNF.numlit ],
    ],
    "key-type": [
        [ BNF.literal("HASH") ],
        [ BNF.literal("RANGE") ],
    ],
    "key-attribute-type": [
        [ BNF.literal("S") ],
        [ BNF.literal("N") ],
        [ BNF.literal("BOOL") ],
    ],
    "attribute-type": [
        [ BNF.literal("S") ],
        [ BNF.literal("SS") ],
        [ BNF.literal("N") ],
        [ BNF.literal("NS") ],
        [ BNF.literal("B") ],
        [ BNF.literal("BS") ],
        [ BNF.literal("BOOL") ],
        [ BNF.literal("NULL") ],
        [ BNF.literal("L") ],
        [ BNF.literal("M") ],
    ],
};
 
/**
 * Interpret a SQLish sentence to a DynamoDb QUERY API parameter.
 * @param {string} source A sentence to convert
 * @returns {object} as a parameter for the QUERY API of DynamoDb
 */
function parseSqlishQuery(source) {
 
    let lexTokens = LexAnalyzer.parse(source);
    let bindedTokens = SQLISH_LEX_TOKEN_DEF.buildWords(lexTokens);
    let tokens = bindedTokens.filter(
        lex => lex != null && !lex.isWhiteSpace());
 
    if(tokens != null && !Array.isArray(tokens) && !tokens.match) {
        return tokens; // tokens is BNF.Result object.
    }
    tokens.forEach( token => {
        let term = token.getTerm();
        switch(token.getType()) {
        case "STRLIT-DQ":
            token.setTerm("\"" + unescapeDQ(term.replace(/^"(.*)"$/, "$1")) + "\"");
            break;
        case "STRLIT-SQ":
            token.setTerm("'" + unescapeSQ(term.replace(/^'(.*)'$/, "$1")) + "'");
            break;
        default:
            break;
        }
    });
 
    let bnf = new BNF("sqlish-query", SQLISH_BNF_DEF);
    let st = bnf.parseTokens(tokens);
    let opt = {};
 
    let fromClause = st.getTerm("from-clause");
    if(!fromClause.match) {
        throw new Error("the from-clause not found");
    } else {
        opt.TableName = fromClause.getWordsList("table-name")[0].join("");
    }
 
    let whereClause = st.getTerm("where-key-clause");
    if(!whereClause.match) {
        throw new Error("the where clause not found");
    } else {
        opt.KeyConditionExpression =
            whereClause.getWordsList("condition-expression")[0].join(" ");
    }
 
    let selectClause = st.getTerm("select-clause");
    if(selectClause.match) {
        opt.ProjectionExpression =
            selectClause.getWordsList("key-list")[0].join("");
    }
    let filterClause = st.getTerm("filter-clause");
    if(filterClause.match) {
        opt.FilterExpression =
            filterClause.getWordsList("condition-expression")[0].join(" ");
    }
    let limitClause = st.getTerm("limit-clause");
    if(limitClause.match) {
        opt.Limit = limitClause.getWordsList("limit-count")[0].join(" ");
    }
    return opt;
};
let source = `
    SELECT mainStar, orbitOrder, name
    FROM stars
    WHERE mainStar=:mainStar`;
let result = parseSqlishQuery(source);
 
console.log(`SOURCE:${source}
RESULT:${JSON.stringify(result, null, "    ")}`);

LICENSE

This software is released under the MIT License, see LICENSE

Keywords

install

npm i lex-bnf

Downloadsweekly downloads

5

version

0.2.0

license

MIT

homepage

github.com

repository

Gitgithub

last publish

collaborators

  • avatar
Report a vulnerability