BOT_NAME = 'linkchecker'
SPIDER_MODULES = ['linkchecker.spiders']
NEWSPIDER_MODULE = 'linkchecker.spiders'
ROBOTSTXT_OBEY = False

BOT_NAME = 'linkchecker'
SPIDER_MODULES = ['linkchecker.spiders']
NEWSPIDER_MODULE = 'linkchecker.spiders'
ROBOTSTXT_OBEY = False
