{ "log_path": "/etc/scrapyd/logs/linkschecker/crawl_one/2371481c4e1b11f1872b4ec0d7e9e9ab.log", "json_path": "/etc/scrapyd/logs/linkschecker/crawl_one/2371481c4e1b11f1872b4ec0d7e9e9ab.json", "json_url": "http://127.0.0.1:8080/logs/linkschecker/crawl_one/2371481c4e1b11f1872b4ec0d7e9e9ab.json", "size": 4785, "position": 4785, "status": "ok", "_head": "2026-05-12 15:56:29 [scrapy.utils.log] INFO: Scrapy 2.11.1 started (bot: linkschecker)\n2026-05-12 15:56:29 [scrapy.utils.log] INFO: Versions: lxml 5.1.0.0, libxml2 2.12.3, cssselect 1.2.0, parsel 1.8.1, w3lib 2.1.2, Twisted 22.10.0, Python 3.8.18 (default, Feb 13 2024, 11:03:53) - [GCC 12.2.0], pyOpenSSL 24.0.0 (OpenSSL 3.2.1 30 Jan 2024), cryptography 42.0.4, Platform Linux-5.15.0-152-generic-x86_64-with-glibc2.34\n2026-05-12 15:56:29 [scrapy.addons] INFO: Enabled addons:\n[]\n2026-05-12 15:56:29 [py.warnings] WARNING: /usr/local/lib/python3.8/site-packages/scrapy/utils/request.py:254: ScrapyDeprecationWarning: '2.6' is a deprecated value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n\nIt is also the default value. In other words, it is normal to get this warning if you have not defined a value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so for backward compatibility reasons, but it will change in a future version of Scrapy.\n\nSee the documentation of the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for information on how to handle this deprecation.\n return cls(crawler)\n\n2026-05-12 15:56:29 [scrapy.extensions.telnet] INFO: Telnet Password: 0d6296d981364c8e\n2026-05-12 15:56:29 [scrapy.middleware] INFO: Enabled extensions:\n['scrapy.extensions.corestats.CoreStats',\n 'scrapy.extensions.telnet.TelnetConsole',\n 'scrapy.extensions.memusage.MemoryUsage',\n 'scrapy.extensions.logstats.LogStats']\n2026-05-12 15:56:29 [scrapy.crawler] INFO: Overridden settings:\n{'BOT_NAME': 'linkschecker',\n 'CONCURRENT_REQUESTS': 8,\n 'DEPTH_LIMIT': '1',\n 'DOWNLOAD_TIMEOUT': 60,\n 'LOG_FILE': 'logs/linkschecker/crawl_one/2371481c4e1b11f1872b4ec0d7e9e9ab.log',\n 'LOG_LEVEL': 'INFO',\n 'MEMUSAGE_CHECK_INTERVAL_SECONDS': 5,\n 'MEMUSAGE_LIMIT_MB': 800,\n 'NEWSPIDER_MODULE': 'linkschecker.spiders',\n 'RETRY_HTTP_CODES': [500, 502, 503, 504, 522, 524, 408, 429, 403],\n 'RETRY_TIMES': 1,\n 'SPIDER_MODULES': ['linkschecker.spiders'],\n 'USER_AGENT': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 '\n '(KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}\n2026-05-12 15:56:29 [scrapy.middleware] INFO: Enabled downloader middlewares:\n['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n2026-05-12 15:56:29 [scrapy.middleware] INFO: Enabled spider middlewares:\n['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n2026-05-12 15:56:29 [scrapy.middleware] INFO: Enabled item pipelines:\n['linkschecker.pipelines.LinkscheckerPipelineCrawlOne']\n2026-05-12 15:56:29 [scrapy.core.engine] INFO: Spider opened\n2026-05-12 15:56:29 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n2026-05-12 15:56:29 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023\n2026-05-12 15:56:29 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-12 15:56:30 [scrapy.core.engine] INFO: Closing spider (finished)\n2026-05-12 15:56:30 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n{'downloader/request_bytes': 384,\n 'downloader/request_count': 1,\n 'downloader/request_method_count/GET': 1,\n 'downloader/response_bytes': 31602,\n 'downloader/response_count': 1,\n 'downloader/response_status_count/200': 1,\n 'elapsed_time_seconds': 0.960146,\n 'finish_reason': 'finished',\n 'finish_time': datetime.datetime(2026, 5, 12, 15, 56, 30, 788385, tzinfo=datetime.timezone.utc),\n 'httpcompression/response_bytes': 112983,\n 'httpcompression/response_count': 1,\n 'item_scraped_count': 1,\n 'log_count/INFO': 11,\n 'log_count/WARNING': 1,\n 'memusage/max': 74850304,\n 'memusage/startup': 74850304,\n 'response_received_count': 1,\n 'scheduler/dequeued': 1,\n 'scheduler/dequeued/memory': 1,\n 'scheduler/enqueued': 1,\n 'scheduler/enqueued/memory': 1,\n 'start_time': datetime.datetime(2026, 5, 12, 15, 56, 29, 828239, tzinfo=datetime.timezone.utc)}\n2026-05-12 15:56:30 [scrapy.core.engine] INFO: Spider closed (finished)\n", "pages": 1, "items": 1, "first_log_time": "2026-05-12 15:56:29", "latest_log_time": "2026-05-12 15:56:30", "runtime": "0:00:01", "shutdown_reason": "N/A", "finish_reason": "finished", "last_update_time": "2026-05-12 15:56:40", "head": "2026-05-12 15:56:29 [scrapy.utils.log] INFO: Scrapy 2.11.1 started (bot: linkschecker)\n2026-05-12 15:56:29 [scrapy.utils.log] INFO: Versions: lxml 5.1.0.0, libxml2 2.12.3, cssselect 1.2.0, parsel 1.8.1, w3lib 2.1.2, Twisted 22.10.0, Python 3.8.18 (default, Feb 13 2024, 11:03:53) - [GCC 12.2.0], pyOpenSSL 24.0.0 (OpenSSL 3.2.1 30 Jan 2024), cryptography 42.0.4, Platform Linux-5.15.0-152-generic-x86_64-with-glibc2.34\n2026-05-12 15:56:29 [scrapy.addons] INFO: Enabled addons:\n[]\n2026-05-12 15:56:29 [py.warnings] WARNING: /usr/local/lib/python3.8/site-packages/scrapy/utils/request.py:254: ScrapyDeprecationWarning: '2.6' is a deprecated value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n\nIt is also the default value. In other words, it is normal to get this warning if you have not defined a value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so for backward compatibility reasons, but it will change in a future version of Scrapy.\n\nSee the documentation of the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for information on how to handle this deprecation.\n return cls(crawler)\n\n2026-05-12 15:56:29 [scrapy.extensions.telnet] INFO: Telnet Password: 0d6296d981364c8e\n2026-05-12 15:56:29 [scrapy.middleware] INFO: Enabled extensions:\n['scrapy.extensions.corestats.CoreStats',\n 'scrapy.extensions.telnet.TelnetConsole',\n 'scrapy.extensions.memusage.MemoryUsage',\n 'scrapy.extensions.logstats.LogStats']\n2026-05-12 15:56:29 [scrapy.crawler] INFO: Overridden settings:\n{'BOT_NAME': 'linkschecker',\n 'CONCURRENT_REQUESTS': 8,\n 'DEPTH_LIMIT': '1',\n 'DOWNLOAD_TIMEOUT': 60,\n 'LOG_FILE': 'logs/linkschecker/crawl_one/2371481c4e1b11f1872b4ec0d7e9e9ab.log',\n 'LOG_LEVEL': 'INFO',\n 'MEMUSAGE_CHECK_INTERVAL_SECONDS': 5,\n 'MEMUSAGE_LIMIT_MB': 800,\n 'NEWSPIDER_MODULE': 'linkschecker.spiders',\n 'RETRY_HTTP_CODES': [500, 502, 503, 504, 522, 524, 408, 429, 403],\n 'RETRY_TIMES': 1,\n 'SPIDER_MODULES': ['linkschecker.spiders'],\n 'USER_AGENT': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 '\n '(KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}\n2026-05-12 15:56:29 [scrapy.middleware] INFO: Enabled downloader middlewares:\n['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n2026-05-12 15:56:29 [scrapy.middleware] INFO: Enabled spider middlewares:\n['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n2026-05-12 15:56:29 [scrapy.middleware] INFO: Enabled item pipelines:\n['linkschecker.pipelines.LinkscheckerPipelineCrawlOne']\n2026-05-12 15:56:29 [scrapy.core.engine] INFO: Spider opened\n2026-05-12 15:56:29 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n2026-05-12 15:56:29 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023\n2026-05-12 15:56:29 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-12 15:56:30 [scrapy.core.engine] INFO: Closing spider (finished)\n2026-05-12 15:56:30 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n{'downloader/request_bytes': 384,\n 'downloader/request_count': 1,\n 'downloader/request_method_count/GET': 1,\n 'downloader/response_bytes': 31602,\n 'downloader/response_count': 1,\n 'downloader/response_status_count/200': 1,\n 'elapsed_time_seconds': 0.960146,\n 'finish_reason': 'finished',\n 'finish_time': datetime.datetime(2026, 5, 12, 15, 56, 30, 788385, tzinfo=datetime.timezone.utc),\n 'httpcompression/response_bytes': 112983,\n 'httpcompression/response_count': 1,\n 'item_scraped_count': 1,\n 'log_count/INFO': 11,\n 'log_count/WARNING': 1,\n 'memusage/max': 74850304,\n 'memusage/startup': 74850304,\n 'response_received_count': 1,\n 'scheduler/dequeued': 1,\n 'scheduler/dequeued/memory': 1,\n 'scheduler/enqueued': 1,\n 'scheduler/enqueued/memory': 1,\n 'start_time': datetime.datetime(2026, 5, 12, 15, 56, 29, 828239, tzinfo=datetime.timezone.utc)}\n2026-05-12 15:56:30 [scrapy.core.engine] INFO: Spider closed (finished)\n", "tail": "2026-05-12 15:56:29 [scrapy.utils.log] INFO: Scrapy 2.11.1 started (bot: linkschecker)\n2026-05-12 15:56:29 [scrapy.utils.log] INFO: Versions: lxml 5.1.0.0, libxml2 2.12.3, cssselect 1.2.0, parsel 1.8.1, w3lib 2.1.2, Twisted 22.10.0, Python 3.8.18 (default, Feb 13 2024, 11:03:53) - [GCC 12.2.0], pyOpenSSL 24.0.0 (OpenSSL 3.2.1 30 Jan 2024), cryptography 42.0.4, Platform Linux-5.15.0-152-generic-x86_64-with-glibc2.34\n2026-05-12 15:56:29 [scrapy.addons] INFO: Enabled addons:\n[]\n2026-05-12 15:56:29 [py.warnings] WARNING: /usr/local/lib/python3.8/site-packages/scrapy/utils/request.py:254: ScrapyDeprecationWarning: '2.6' is a deprecated value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n\nIt is also the default value. In other words, it is normal to get this warning if you have not defined a value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so for backward compatibility reasons, but it will change in a future version of Scrapy.\n\nSee the documentation of the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for information on how to handle this deprecation.\n return cls(crawler)\n\n2026-05-12 15:56:29 [scrapy.extensions.telnet] INFO: Telnet Password: 0d6296d981364c8e\n2026-05-12 15:56:29 [scrapy.middleware] INFO: Enabled extensions:\n['scrapy.extensions.corestats.CoreStats',\n 'scrapy.extensions.telnet.TelnetConsole',\n 'scrapy.extensions.memusage.MemoryUsage',\n 'scrapy.extensions.logstats.LogStats']\n2026-05-12 15:56:29 [scrapy.crawler] INFO: Overridden settings:\n{'BOT_NAME': 'linkschecker',\n 'CONCURRENT_REQUESTS': 8,\n 'DEPTH_LIMIT': '1',\n 'DOWNLOAD_TIMEOUT': 60,\n 'LOG_FILE': 'logs/linkschecker/crawl_one/2371481c4e1b11f1872b4ec0d7e9e9ab.log',\n 'LOG_LEVEL': 'INFO',\n 'MEMUSAGE_CHECK_INTERVAL_SECONDS': 5,\n 'MEMUSAGE_LIMIT_MB': 800,\n 'NEWSPIDER_MODULE': 'linkschecker.spiders',\n 'RETRY_HTTP_CODES': [500, 502, 503, 504, 522, 524, 408, 429, 403],\n 'RETRY_TIMES': 1,\n 'SPIDER_MODULES': ['linkschecker.spiders'],\n 'USER_AGENT': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 '\n '(KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}\n2026-05-12 15:56:29 [scrapy.middleware] INFO: Enabled downloader middlewares:\n['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n2026-05-12 15:56:29 [scrapy.middleware] INFO: Enabled spider middlewares:\n['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n2026-05-12 15:56:29 [scrapy.middleware] INFO: Enabled item pipelines:\n['linkschecker.pipelines.LinkscheckerPipelineCrawlOne']\n2026-05-12 15:56:29 [scrapy.core.engine] INFO: Spider opened\n2026-05-12 15:56:29 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n2026-05-12 15:56:29 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023\n2026-05-12 15:56:29 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-12 15:56:30 [scrapy.core.engine] INFO: Closing spider (finished)\n2026-05-12 15:56:30 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n{'downloader/request_bytes': 384,\n 'downloader/request_count': 1,\n 'downloader/request_method_count/GET': 1,\n 'downloader/response_bytes': 31602,\n 'downloader/response_count': 1,\n 'downloader/response_status_count/200': 1,\n 'elapsed_time_seconds': 0.960146,\n 'finish_reason': 'finished',\n 'finish_time': datetime.datetime(2026, 5, 12, 15, 56, 30, 788385, tzinfo=datetime.timezone.utc),\n 'httpcompression/response_bytes': 112983,\n 'httpcompression/response_count': 1,\n 'item_scraped_count': 1,\n 'log_count/INFO': 11,\n 'log_count/WARNING': 1,\n 'memusage/max': 74850304,\n 'memusage/startup': 74850304,\n 'response_received_count': 1,\n 'scheduler/dequeued': 1,\n 'scheduler/dequeued/memory': 1,\n 'scheduler/enqueued': 1,\n 'scheduler/enqueued/memory': 1,\n 'start_time': datetime.datetime(2026, 5, 12, 15, 56, 29, 828239, tzinfo=datetime.timezone.utc)}\n2026-05-12 15:56:30 [scrapy.core.engine] INFO: Spider closed (finished)\n", "first_log_timestamp": 1778601389, "latest_log_timestamp": 1778601390, "datas": [ [ "2026-05-12 15:56:29", 0, 0, 0, 0 ] ], "latest_matches": { "scrapy_version": "2.11.1", "telnet_console": "127.0.0.1:6023", "telnet_username": "", "telnet_password": "0d6296d981364c8e", "resuming_crawl": "", "latest_offsite": "", "latest_duplicate": "", "latest_crawl": "", "latest_scrape": "", "latest_item": "", "latest_stat": "2026-05-12 15:56:29 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)" }, "latest_crawl_timestamp": 0, "latest_scrape_timestamp": 0, "log_categories": { "critical_logs": { "count": 0, "details": [] }, "error_logs": { "count": 0, "details": [] }, "warning_logs": { "count": 1, "details": [ "2026-05-12 15:56:29 [py.warnings] WARNING: /usr/local/lib/python3.8/site-packages/scrapy/utils/request.py:254: ScrapyDeprecationWarning: '2.6' is a deprecated value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n\nIt is also the default value. In other words, it is normal to get this warning if you have not defined a value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so for backward compatibility reasons, but it will change in a future version of Scrapy.\n\nSee the documentation of the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for information on how to handle this deprecation.\n return cls(crawler)\n" ] }, "redirect_logs": { "count": 0, "details": [] }, "retry_logs": { "count": 0, "details": [] }, "ignore_logs": { "count": 0, "details": [] } }, "crawler_stats": { "source": "log", "last_update_time": "2026-05-12 15:56:30", "last_update_timestamp": 1778601390, "downloader/request_bytes": 384, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/response_bytes": 31602, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "elapsed_time_seconds": 0.960146, "finish_reason": "finished", "finish_time": "datetime.datetime(2026, 5, 12, 15, 56, 30, 788385, tzinfo=datetime.timezone.utc)", "httpcompression/response_bytes": 112983, "httpcompression/response_count": 1, "item_scraped_count": 1, "log_count/INFO": 11, "log_count/WARNING": 1, "memusage/max": 74850304, "memusage/startup": 74850304, "response_received_count": 1, "scheduler/dequeued": 1, "scheduler/dequeued/memory": 1, "scheduler/enqueued": 1, "scheduler/enqueued/memory": 1, "start_time": "datetime.datetime(2026, 5, 12, 15, 56, 29, 828239, tzinfo=datetime.timezone.utc)" }, "last_update_timestamp": 1778601400, "logparser_version": "0.8.2", "crawler_engine": {} }