Unverified Commit 16d4c5e6 authored by Victor Macias Castillo's avatar Victor Macias Castillo Committed by GitHub

Merge pull request #3 from Woorank/add-support-noindex

Add support noindex
parents ddfeec07 25d8e389
version: 2
jobs:
build:
docker:
- image: circleci/node:8
steps:
- checkout
- run: npm install --quiet
- run: npm test
- deploy:
command: |
export PACKAGE_EXISTS="$(npm info robots-txt-parse@$(npm run -s version) version)"
echo "Existing version: ${PACKAGE_EXISTS}"
if [[ $CIRCLE_BRANCH == "master" && ! $PACKAGE_EXISTS ]]; then
echo "Publishing..."
echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
npm publish
fi
language: "node_js"
node_js:
- "0.10"
cache:
directories:
- node_modules
sudo: false
......@@ -29,6 +29,7 @@ function parseLine(line) {
};
case 'allow':
case 'disallow':
case 'noindex':
return {
type: GROUP_MEMBER,
rule: field,
......
{
"name": "robots-txt-parse",
"version": "0.0.4",
"version": "1.0.0",
"description": "Streaming parser for robots.txt files",
"main": "lib/parse.js",
"repository": {
"type": "git",
"url": "https://github.com/Woorank/robots-txt-parse"
},
"scripts": {
"test": "mocha -R spec ./test",
"test-watch": "mocha -w -R spec ./test"
"test-watch": "mocha -w -R spec ./test",
"version": "echo $npm_package_version"
},
"author": "Jan Potoms",
"license": "MIT",
......
User-agent: *
Allow: /path1
Disallow: /*/path2/
Noindex: /*/path2/
Noindex: /*/path3/
......@@ -104,4 +104,25 @@ describe('parser', function () {
.catch(done);
});
it('should parse noindex', function (done) {
parse(getFixture('noindex'))
.then(function (parsed) {
assert.deepPropertyVal(parsed, 'groups[0].agents[0]', '*');
assert.deepPropertyVal(parsed, 'groups[0].rules[0].rule', 'allow');
assert.deepPropertyVal(parsed, 'groups[0].rules[0].path', '/path1');
assert.deepPropertyVal(parsed, 'groups[0].rules[1].rule', 'disallow');
assert.deepPropertyVal(parsed, 'groups[0].rules[1].path', '/*/path2/');
assert.deepPropertyVal(parsed, 'groups[0].rules[2].rule', 'noindex');
assert.deepPropertyVal(parsed, 'groups[0].rules[2].path', '/*/path2/');
assert.deepPropertyVal(parsed, 'groups[0].rules[3].rule', 'noindex');
assert.deepPropertyVal(parsed, 'groups[0].rules[3].path', '/*/path3/');
done();
})
.catch(done);
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment