Have ideas to improve npm?Join in the discussion! »

generate-robotstxt

4.0.1 • Public • Published

generate-robotstxt

NPM version Travis Build Status dependencies Status devDependencies Status

Awesome generator robots.txt.

Installation

npm install generate-robotstxt

Usage

const robotstxt = require('generate-robotstxt').default;
 
robotstxt({
    policy: [
        {
            userAgent: 'Googlebot',
            allow: '/',
            disallow: '/search',
            crawlDelay: 2
        },
        {
            userAgent: '*',
            allow: '/',
            disallow: '/search',
            crawlDelay: 10,
            cleanParam: 'ref /articles/'
        }
    ],
    sitemap: 'sitemap.xml',
    host: 'http://example.com'
})
  .then((content) => {
    console.log(content);
  });

Or

import robotstxt from 'generate-robotstxt';
 
robotstxt({
    policy: [
        {
            userAgent: 'Googlebot',
            allow: '/',
            disallow: '/search',
            crawlDelay: 2
        },
        {
            userAgent: '*',
            allow: '/',
            disallow: '/search',
            crawlDelay: 10,
            cleanParam: 'ref /articles/'
        }
    ],
    sitemap: 'sitemap.xml',
    host: 'http://example.com'
})
  .then((content) => {
    console.log(content);
  });

Contribution

Feel free to push your code if you agree with publishing under the MIT license.

Changelog

License

Install

npm i [email protected]

Version

4.0.1

License

MIT

Last publish

Collaborators

  • avatar
  • avatar
  • avatar