w-json-stream

    1.0.14 • Public • Published

    w-json-stream

    A tool for JSON parse and stringify by stream and web worker.

    language npm version license gzip file size npm download jsdelivr download

    Documentation

    To view documentation or get support, visit docs.

    Installation

    Using npm(ES6 module):

    Note: w-json-stream is mainly dependent on from2 and jsonparse.

    npm i w-json-stream
    

    Example for samll data:

    Link: [dev source code]

    
    async function testSmall() {
    
        let obj = {
            a: 123,
            b: 145.67,
            c: 'test中文1',
            d: true,
            f: [11, 'xyz', false, [166, 197, 215]],
            g: {
                ga: 223,
                gb: 245.67,
                gc: 'test中文2',
                gd: [66, 97, 115],
            },
        }
        let arr = [
            {
                a: 123,
                b: 145.67,
                c: 'test中文1',
                d: true,
            },
            {
                f: [11, 'xyz', false, [166, 197, 215]],
                g: {
                    ga: 223,
                    gb: 245.67,
                    gc: 'test中文2',
                    gd: [66, 97, 115],
                },
            },
        ]
    
        let jobj = await json.stringify(obj)
        console.log(jobj)
        // => {"a":123,"b":145.67,"c":"test中文1","d":true,"f":[11,"xyz",false,[166,197,215]],"g":{"ga":223,"gb":245.67,"gc":"test中文2","gd":[66,97,115]}}
    
        let jarr = await json.stringify(arr)
        console.log(jarr)
        // => [{"a":123,"b":145.67,"c":"test中文1","d":true},{"f":[11,"xyz",false,[166,197,215]],"g":{"ga":223,"gb":245.67,"gc":"test中文2","gd":[66,97,115]}}]
    
        let robj = await json.parse(jobj)
        console.log(robj)
        // => {
        //     a: 123,
        //     b: 145.67,
        //     c: 'test中文1',
        //     d: true,
        //     f: [ 11, 'xyz', false, [ 166, 197, 215 ] ],
        //     g: { ga: 223, gb: 245.67, gc: 'test中文2', gd: [ 66, 97, 115 ] }
        // }
    
        let rarr = await json.parse(jarr)
        console.log(rarr)
        // => [
        //     { a: 123, b: 145.67, c: 'test中文1', d: true },
        //     {
        //       f: [ 11, 'xyz', false, [Array] ],
        //       g: { ga: 223, gb: 245.67, gc: 'test中文2', gd: [Array] }
        //     }
        // ]
    
    }
    testSmall()
        .catch((err) => {
            console.log(err)
        })
    
    

    Example for large data:

    Link: [dev source code]

    
    async function testLarge() {
    
        let n = 700 //7000, 9000
        let lgRows = []
        for (let i = 1; i <= 1000; i++) {
            lgRows.push([i, 23.4, '567', 'abc', true, false, [], {}, [8, 9.01], { a: 12.34, b: 'xyz', i }])
        }
        let lgArr = []
        for (let i = 1; i <= n; i++) {
            lgArr.push({
                key: 'k' + i,
                value: {
                    a: 123,
                    b: 145.67,
                    c: 'test中文1',
                    d: true,
                    f: [11, 'xyz', false, [166, 197, 215]],
                    g: {
                        ga: 223,
                        gb: 245.67,
                        gc: 'test中文2',
                        gd: [66, 97, 115],
                        ge: JSON.parse(JSON.stringify(lgRows)),
                    },
                }
            })
        }
    
        if (true) {
    
            try {
                let res = JSON.stringify(lgArr)
                console.log('JSON.stringify(lgArr)', res.length, res.substr(0, 200) + '...')
            }
            catch (err) {
                console.log('JSON.stringify(lgArr) catch', err)
                // n=7000 => catch RangeError: Invalid string length => Out Of Memory
                // n=9000 => FATAL ERROR: MarkCompactCollector: young object promotion failed Allocation failed - JavaScript heap out of memory
            }
    
        }
    
        if (true) {
    
            await json.stringify(lgArr)
                .then((res) => {
                    console.log('json.stringify(lgArr) then', res.length, res.substr(0, 200) + '...')
                })
                .catch((err) => {
                    console.log('json.stringify(lgArr) catch', err)
                    // n=7000 => FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory
                })
    
        }
    
    }
    testLarge()
        .catch((err) => {
            console.log(err)
        })
    
    

    Example for stream:

    Link: [dev source code]

    
    async function testStream() {
    
        async function testStringify() {
            return new Promise((resolve, reject) => {
    
                //fp
                let fp = './temp-stringify.json'
    
                //arr
                let n = 4000 //4000000
                let arr = []
                for (let i = 1; i <= n; i++) {
                    arr.push({
                        key: 'k' + i,
                        value: {
                            a: 123,
                            b: 145.67,
                            c: 'test中文1',
                            d: true,
                            f: [11, 'xyz', false, [166, 197, 215]],
                            g: {
                                ga: 223,
                                gb: 245.67,
                                gc: 'test中文2',
                                gd: [66, 97, 115],
                            },
                        }
                    })
                }
    
                //createStringifyStream
                let stringifyStream = json.createStringifyStream(arr)
    
                //createWriteStream
                let writerStream = fs.createWriteStream(fp)
    
                // //pipe
                // stringifyStream.pipe(writerStream)
    
                //stringifyStream onData
                stringifyStream.on('data', function(chunk) {
                    writerStream.write(chunk, 'UTF8')
                })
    
                //stringifyStream onEnd
                stringifyStream.on('end', function() {
                    writerStream.end()
                })
    
                //writerStream onFinish
                writerStream.on('finish', function() {
                    console.log('writerStream finish')
                    let res = fs.readFileSync(fp, 'utf8')
                    fs.unlinkSync(fp)
                    console.log('res.length', res.length)
                    console.log('res', res.substr(0, 200) + '...')
                    resolve()
                })
    
                //writerStream onError
                writerStream.on('error', function(err) {
                    console.log('writerStream error', err)
                })
    
            })
        }
    
        async function testStringifys() {
            await testStringify()
            // writerStream finish
            // res.length 646894
            // res [{"key":"k1","value":{"a":123,"b":145.67,"c":"test中文1","d":true,"f":[11,"xyz",false,[166,197,215]],"g":{"ga":223,"gb":245.67,"gc":"test中文2","gd":[66,97,115]}}},{"key":"k2","value":{"a":123,"b":145.67,...
        }
    
        console.log('testStringify...')
        await testStringifys()
            .catch((err) => {
                console.log(err)
            })
    
        async function testParse(pre, filter) {
            return new Promise((resolve, reject) => {
    
                //fp
                let fp = './temp-write.json'
    
                //arr
                let arr = [
                    {
                        key: 'k1',
                        value: {
                            a: 123,
                            b: 145.67,
                            c: 'test中文1',
                            d: true,
                            f: [11, 'xyz', false, [166, 197, 215]],
                            g: {
                                ga: 223,
                                gb: 245.67,
                                gc: 'test中文2',
                                gd: [66, 97, 115],
                            },
                        },
                    },
                    {
                        child: [
                            { name: 'child:abc', labels: ['child:label-abc'] },
                            { name: 'child:def', labels: ['child:label-def'] },
                            true,
                            false,
                            2020,
                            {
                                name: 'child:xyz',
                                query: {
                                    labels: ['child:query:label-xyz']
                                }
                            },
                        ],
                    },
                    { name: 'abc', labels: ['label-abc'] },
                    { name: 'def', labels: ['label-def'] },
                    true,
                    false,
                    -1,
                    {
                        name: 'xyz',
                        query: {
                            labels: ['query:label-xyz']
                        }
                    },
                ]
    
                //jarr
                let jarr = JSON.stringify(arr)
    
                //writeFileSync
                fs.writeFileSync(fp, jarr, 'utf8')
    
                //createReadStream
                let readerStream = fs.createReadStream(fp)
    
                //createParseStream
                let parseStream = json.createParseStream(filter)
    
                //onData
                let res = []
                parseStream.on('data', function(chunk) {
                    console.log(pre, 'parseStream data chunk=', chunk)
                    res.push(chunk)
                })
    
                //onEnd
                parseStream.on('end', function() {
                    console.log(pre, 'parseStream end res', res)
                    fs.unlinkSync(fp)
                    resolve()
                })
    
                //onError
                parseStream.on('error', function(err) {
                    console.log(pre, 'parseStream error', err)
                })
    
                //pipe
                readerStream.pipe(parseStream)
    
            })
        }
    
        async function testParses() {
    
            let filter
    
            filter = '' //default
            console.log('test0', filter)
            await testParse('test0', filter)
            // test0
            // test0 parseStream data chunk= {
            //   value: {
            //     key: 'k1',
            //     value: {
            //       a: 123,
            //       b: 145.67,
            //       c: 'test中文1',
            //       d: true,
            //       f: [Array],
            //       g: [Object]
            //     }
            //   },
            //   key: 0
            // }
            // test0 parseStream data chunk= {
            //   value: { child: [ [Object], [Object], true, false, 2020, [Object] ] },
            //   key: 1
            // }
            // test0 parseStream data chunk= { value: { name: 'abc', labels: [ 'label-abc' ] }, key: 2 }
            // test0 parseStream data chunk= { value: { name: 'def', labels: [ 'label-def' ] }, key: 3 }
            // test0 parseStream data chunk= { value: true, key: 4 }
            // test0 parseStream data chunk= { value: false, key: 5 }
            // test0 parseStream data chunk= { value: -1, key: 6 }
            // test0 parseStream data chunk= { value: { name: 'xyz', query: { labels: [Array] } }, key: 7 }
            // test0 parseStream end res [
            //   { value: { key: 'k1', value: [Object] }, key: 0 },
            //   { value: { child: [Array] }, key: 1 },
            //   { value: { name: 'abc', labels: [Array] }, key: 2 },
            //   { value: { name: 'def', labels: [Array] }, key: 3 },
            //   { value: true, key: 4 },
            //   { value: false, key: 5 },
            //   { value: -1, key: 6 },
            //   { value: { name: 'xyz', query: [Object] }, key: 7 }
            // ]
    
            filter = '*.labels' //無emit, 只能監聽onData個別處理
            console.log('test1', filter)
            await testParse('test1', filter)
            // test1 *.labels
            // test1 parseStream data chunk= [ 'label-abc' ]
            // test1 parseStream data chunk= [ 'label-def' ]
            // test1 parseStream end res [ [ 'label-abc' ], [ 'label-def' ] ]
    
            filter = [true, 'labels', { emitPath: true }]
            console.log('test2', filter, 'true=*, 開啟emitPath才能塞回物件取得過濾後結果')
            await testParse('test2', filter)
            // test2 [ true, 'labels', { emitPath: true } ] true=*, 開啟emitPath才能塞回物件取得過濾後結果
            // test2 parseStream data chunk= { value: 'label-abc', path: [ 2, 'labels', 0 ] }
            // test2 parseStream data chunk= { value: 'label-def', path: [ 3, 'labels', 0 ] }
            // test2 parseStream end res [
            //   { value: 'label-abc', path: [ 2, 'labels', 0 ] },
            //   { value: 'label-def', path: [ 3, 'labels', 0 ] }
            // ]
    
            filter = [{ recurse: true }, 'labels', { emitPath: true }]
            console.log('test3', filter, '開啟recurse代表為任意子節點, 開啟emitPath才能塞回物件取得過濾後結果')
            await testParse('test3', filter)
            // test3 [ { recurse: true }, 'labels', { emitPath: true } ] 開啟recurse代表為任意子節點, 開啟emitPath才能塞回物件取得過濾後結果
            // test3 parseStream data chunk= { value: 'child:label-abc', path: [ 1, 'child', 0, 'labels', 0 ] }
            // test3 parseStream data chunk= { value: 'child:label-def', path: [ 1, 'child', 1, 'labels', 0 ] }
            // test3 parseStream data chunk= {
            // value: 'child:query:label-xyz',
            // path: [ 1, 'child', 5, 'query', 'labels', 0 ]
            // }
            // test3 parseStream data chunk= { value: 'label-abc', path: [ 2, 'labels', 0 ] }
            // test3 parseStream data chunk= { value: 'label-def', path: [ 3, 'labels', 0 ] }
            // test3 parseStream data chunk= { value: 'query:label-xyz', path: [ 7, 'query', 'labels', 0 ] }
            // test3 parseStream end res [
            // { value: 'child:label-abc', path: [ 1, 'child', 0, 'labels', 0 ] },
            // { value: 'child:label-def', path: [ 1, 'child', 1, 'labels', 0 ] },
            // {
            //     value: 'child:query:label-xyz',
            //     path: [ 1, 'child', 5, 'query', 'labels', 0 ]
            // },
            // { value: 'label-abc', path: [ 2, 'labels', 0 ] },
            // { value: 'label-def', path: [ 3, 'labels', 0 ] },
            // { value: 'query:label-xyz', path: [ 7, 'query', 'labels', 0 ] }
            // ]
    
        }
    
        console.log('testParse...')
        await testParses()
            .catch((err) => {
                console.log(err)
            })
    
    }
    testStream()
        .catch((err) => {
            console.log(err)
        })
    
    

    In a browser(UMD module):

    Note: w-json-stream does not dependent on any package.

    [Necessary] Add script for w-json-stream.

    
    <!-- for basic -->
    <script src="https://cdn.jsdelivr.net/npm/w-json-stream@1.0.14/dist/w-json-stream.umd.js"></script>
    
    <!-- for web workers -->
    <script src="https://cdn.jsdelivr.net/npm/w-json-stream@1.0.14/dist/w-json-stream.wk.umd.js"></script>
    
    

    stringify and parse for small data: ex-small.html [source code]

    stringifyByStream and parseByStream for small data: ex-small-stream.html [source code]

    stringify and parse for large data: ex-large.html [source code]

    createParseStream: ex-stream-createParseStream.html [source code] * ReadableStream does not support IE11.

    createParseStream with filter: ex-stream-createParseStreamWithFilter.html [source code]

    createStringifyStream: ex-stream-createStringifyStream.html [source code] * WritableStream does not support IE11 and Firefox.

    stringify and parse for large data in web worker: ex-large-webworker.html [source code] * WebWorkers(from blob) does not support IE11.

    
    console.log('WebWorkers(from blob) does not support IE11')
    
    let json = window['w-json-stream']
    console.log(json)
    
    console.log('build data...')
    
    let n = 700 //7000, 9000
    let lgRows = []
    for (let i = 1; i <= 1000; i++) {
        lgRows.push([i, 23.4, '567', 'abc', true, false, [], {}, [8, 9.01], { a: 12.34, b: 'xyz', i }])
    }
    let carr = ''
    let arr = []
    for (let i = 1; i <= n; i++) {
        arr.push({
            key: 'k' + i,
            value: {
                a: 123,
                b: 145.67,
                c: 'test中文1',
                d: true,
                f: [11, 'xyz', false, [166, 197, 215]],
                g: {
                    ga: 223,
                    gb: 245.67,
                    gc: 'test中文2',
                    gd: [66, 97, 115],
                    ge: JSON.parse(JSON.stringify(lgRows)),
                },
            }
        })
    }
    
    console.log('build data finish')
    
    let t=0
    let timer=setInterval(function(){
        console.log('t',t/10+'(s)')
        t++
        if(t>50){
            clearInterval(timer)
        }
    },100)
    
    let pmArr = json.stringify(arr) 
        .then(function(res){
            console.log('json.stringify(arr) then',res)
            carr = res
        })
        .catch(function(err){
            console.log('json.stringify(arr) catch',err)
        })
    
    Promise.all([pmArr])
        .then(function(){
    
            json.parse(carr) 
                .then(function(r){
                    console.log('json.parse(carr) then',r)
                })
                .catch(function(err){
                    console.log('json.parse(carr) catch',err)
                })
            
        })
        .catch(function(err){
            console.log(err)
        })
    
    

    Install

    npm i w-json-stream

    DownloadsWeekly Downloads

    12

    Version

    1.0.14

    License

    MIT

    Unpacked Size

    2.49 MB

    Total Files

    84

    Last publish

    Collaborators

    • semisphere