{
"log_path": "/etc/scrapyd/logs/linkschecker/crawl_one/9ff4865aa7ae11f18a56f2ee302452b8.log",
"json_path": "/etc/scrapyd/logs/linkschecker/crawl_one/9ff4865aa7ae11f18a56f2ee302452b8.json",
"json_url": "http://127.0.0.1:8080/logs/linkschecker/crawl_one/9ff4865aa7ae11f18a56f2ee302452b8.json",
"size": 7569,
"position": 7569,
"status": "ok",
"_head": 100,
"pages": 2,
"items": 1,
"first_log_time": "2026-09-03 15:46:28",
"latest_log_time": "2026-09-03 15:48:20",
"runtime": "0:01:52",
"shutdown_reason": "N/A",
"finish_reason": "finished",
"last_update_time": "2026-09-03 15:48:22",
"head": "2026-09-03 15:46:28 [scrapy.utils.log] INFO: Scrapy 2.11.1 started (bot: linkschecker)\n2026-09-03 15:46:28 [scrapy.utils.log] INFO: Versions: lxml 5.1.0.0, libxml2 2.12.3, cssselect 1.2.0, parsel 1.8.1, w3lib 2.1.2, Twisted 22.10.0, Python 3.8.18 (default, Feb 13 2024, 11:03:53) - [GCC 12.2.0], pyOpenSSL 24.0.0 (OpenSSL 3.2.1 30 Jan 2024), cryptography 42.0.4, Platform Linux-5.15.0-185-generic-x86_64-with-glibc2.34\n2026-09-03 15:46:28 [scrapy.addons] INFO: Enabled addons:\n[]\n2026-09-03 15:46:28 [py.warnings] WARNING: /usr/local/lib/python3.8/site-packages/scrapy/utils/request.py:254: ScrapyDeprecationWarning: '2.6' is a deprecated value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n\nIt is also the default value. In other words, it is normal to get this warning if you have not defined a value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so for backward compatibility reasons, but it will change in a future version of Scrapy.\n\nSee the documentation of the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for information on how to handle this deprecation.\n return cls(crawler)\n\n2026-09-03 15:46:28 [scrapy.extensions.telnet] INFO: Telnet Password: b43453f693bd948c\n2026-09-03 15:46:29 [scrapy.middleware] INFO: Enabled extensions:\n['scrapy.extensions.corestats.CoreStats',\n 'scrapy.extensions.telnet.TelnetConsole',\n 'scrapy.extensions.memusage.MemoryUsage',\n 'scrapy.extensions.logstats.LogStats']\n2026-09-03 15:46:29 [scrapy.crawler] INFO: Overridden settings:\n{'BOT_NAME': 'linkschecker',\n 'CONCURRENT_REQUESTS': 8,\n 'DEPTH_LIMIT': '1',\n 'DOWNLOAD_TIMEOUT': 60,\n 'LOG_FILE': 'logs/linkschecker/crawl_one/9ff4865aa7ae11f18a56f2ee302452b8.log',\n 'LOG_LEVEL': 'INFO',\n 'MEMUSAGE_CHECK_INTERVAL_SECONDS': 5,\n 'MEMUSAGE_LIMIT_MB': 800,\n 'NEWSPIDER_MODULE': 'linkschecker.spiders',\n 'RETRY_HTTP_CODES': [500, 502, 503, 504, 522, 524, 408, 429, 403],\n 'RETRY_TIMES': 1,\n 'SPIDER_MODULES': ['linkschecker.spiders'],\n 'USER_AGENT': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 '\n '(KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}\n2026-09-03 15:46:29 [scrapy.middleware] INFO: Enabled downloader middlewares:\n['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n2026-09-03 15:46:29 [scrapy.middleware] INFO: Enabled spider middlewares:\n['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n2026-09-03 15:46:29 [scrapy.middleware] INFO: Enabled item pipelines:\n['linkschecker.pipelines.LinkscheckerPipelineCrawlOne']\n2026-09-03 15:46:29 [scrapy.core.engine] INFO: Spider opened\n2026-09-03 15:46:29 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n2026-09-03 15:46:29 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6036\n2026-09-03 15:46:29 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:46:29 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 2 times): 403 Forbidden\n2026-09-03 15:46:34 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:46:39 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:46:44 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:46:49 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:46:54 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:46:59 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:04 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:47:09 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:14 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:47:19 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:24 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:29 [scrapy.extensions.logstats] INFO: Crawled 1 pages (at 1 pages/min), scraped 0 items (at 0 items/min)\n\n2026-09-03 15:47:29 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:34 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:47:39 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:44 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:47:49 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:54 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:47:59 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:48:04 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:48:09 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:48:14 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:48:19 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:48:19 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 2 times): 500 Internal Server Error\n2026-09-03 15:48:19 [scrapy.core.engine] INFO: Closing spider (finished)\n2026-09-03 15:48:20 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n{'downloader/request_bytes': 1836,\n 'downloader/request_count': 4,\n 'downloader/request_method_count/GET': 4,\n 'downloader/response_bytes': 12683,\n 'downloader/response_count': 4,\n 'downloader/response_status_count/403': 2,\n 'downloader/response_status_count/500': 2,\n 'elapsed_time_seconds': 111.178906,\n 'finish_reason': 'finished',\n 'finish_time': datetime.datetime(2026, 9, 3, 15, 48, 20, 280674, tzinfo=datetime.timezone.utc),\n 'httpcompression/response_bytes': 12432,\n 'httpcompression/response_count': 4,\n 'item_scraped_count': 1,\n 'log_count/ERROR': 2,\n 'log_count/INFO': 34,\n 'log_count/WARNING': 1,\n 'memusage/max': 74895360,\n 'memusage/startup': 74895360,\n 'request_depth_max': 1,\n 'response_received_count': 2,",
"tail": "2026-09-03 15:46:28 [scrapy.utils.log] INFO: Scrapy 2.11.1 started (bot: linkschecker)\n2026-09-03 15:46:28 [scrapy.utils.log] INFO: Versions: lxml 5.1.0.0, libxml2 2.12.3, cssselect 1.2.0, parsel 1.8.1, w3lib 2.1.2, Twisted 22.10.0, Python 3.8.18 (default, Feb 13 2024, 11:03:53) - [GCC 12.2.0], pyOpenSSL 24.0.0 (OpenSSL 3.2.1 30 Jan 2024), cryptography 42.0.4, Platform Linux-5.15.0-185-generic-x86_64-with-glibc2.34\n2026-09-03 15:46:28 [scrapy.addons] INFO: Enabled addons:\n[]\n2026-09-03 15:46:28 [py.warnings] WARNING: /usr/local/lib/python3.8/site-packages/scrapy/utils/request.py:254: ScrapyDeprecationWarning: '2.6' is a deprecated value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n\nIt is also the default value. In other words, it is normal to get this warning if you have not defined a value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so for backward compatibility reasons, but it will change in a future version of Scrapy.\n\nSee the documentation of the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for information on how to handle this deprecation.\n return cls(crawler)\n\n2026-09-03 15:46:28 [scrapy.extensions.telnet] INFO: Telnet Password: b43453f693bd948c\n2026-09-03 15:46:29 [scrapy.middleware] INFO: Enabled extensions:\n['scrapy.extensions.corestats.CoreStats',\n 'scrapy.extensions.telnet.TelnetConsole',\n 'scrapy.extensions.memusage.MemoryUsage',\n 'scrapy.extensions.logstats.LogStats']\n2026-09-03 15:46:29 [scrapy.crawler] INFO: Overridden settings:\n{'BOT_NAME': 'linkschecker',\n 'CONCURRENT_REQUESTS': 8,\n 'DEPTH_LIMIT': '1',\n 'DOWNLOAD_TIMEOUT': 60,\n 'LOG_FILE': 'logs/linkschecker/crawl_one/9ff4865aa7ae11f18a56f2ee302452b8.log',\n 'LOG_LEVEL': 'INFO',\n 'MEMUSAGE_CHECK_INTERVAL_SECONDS': 5,\n 'MEMUSAGE_LIMIT_MB': 800,\n 'NEWSPIDER_MODULE': 'linkschecker.spiders',\n 'RETRY_HTTP_CODES': [500, 502, 503, 504, 522, 524, 408, 429, 403],\n 'RETRY_TIMES': 1,\n 'SPIDER_MODULES': ['linkschecker.spiders'],\n 'USER_AGENT': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 '\n '(KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}\n2026-09-03 15:46:29 [scrapy.middleware] INFO: Enabled downloader middlewares:\n['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n2026-09-03 15:46:29 [scrapy.middleware] INFO: Enabled spider middlewares:\n['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n2026-09-03 15:46:29 [scrapy.middleware] INFO: Enabled item pipelines:\n['linkschecker.pipelines.LinkscheckerPipelineCrawlOne']\n2026-09-03 15:46:29 [scrapy.core.engine] INFO: Spider opened\n2026-09-03 15:46:29 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n2026-09-03 15:46:29 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6036\n2026-09-03 15:46:29 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:46:29 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 2 times): 403 Forbidden\n2026-09-03 15:46:34 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:46:39 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:46:44 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:46:49 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:46:54 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:46:59 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:04 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:47:09 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:14 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:47:19 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:24 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:29 [scrapy.extensions.logstats] INFO: Crawled 1 pages (at 1 pages/min), scraped 0 items (at 0 items/min)\n\n2026-09-03 15:47:29 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:34 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:47:39 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:44 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:47:49 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:47:54 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:47:59 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:48:04 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-09-03 15:48:09 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:48:14 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:48:19 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-09-03 15:48:19 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 2 times): 500 Internal Server Error\n2026-09-03 15:48:19 [scrapy.core.engine] INFO: Closing spider (finished)\n2026-09-03 15:48:20 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n{'downloader/request_bytes': 1836,\n 'downloader/request_count': 4,\n 'downloader/request_method_count/GET': 4,\n 'downloader/response_bytes': 12683,\n 'downloader/response_count': 4,\n 'downloader/response_status_count/403': 2,\n 'downloader/response_status_count/500': 2,\n 'elapsed_time_seconds': 111.178906,\n 'finish_reason': 'finished',\n 'finish_time': datetime.datetime(2026, 9, 3, 15, 48, 20, 280674, tzinfo=datetime.timezone.utc),\n 'httpcompression/response_bytes': 12432,\n 'httpcompression/response_count': 4,\n 'item_scraped_count': 1,\n 'log_count/ERROR': 2,\n 'log_count/INFO': 34,\n 'log_count/WARNING': 1,\n 'memusage/max': 74895360,\n 'memusage/startup': 74895360,\n 'request_depth_max': 1,\n 'response_received_count': 2,\n 'retry/count': 2,\n 'retry/max_reached': 2,\n 'retry/reason_count/403 Forbidden': 1,\n 'retry/reason_count/500 Internal Server Error': 1,\n 'scheduler/dequeued': 4,\n 'scheduler/dequeued/memory': 4,\n 'scheduler/enqueued': 4,\n 'scheduler/enqueued/memory': 4,\n 'start_time': datetime.datetime(2026, 9, 3, 15, 46, 29, 101768, tzinfo=datetime.timezone.utc)}\n2026-09-03 15:48:20 [scrapy.core.engine] INFO: Spider closed (finished)\n",
"first_log_timestamp": 1788450388,
"latest_log_timestamp": 1788450500,
"datas": [
[
"2026-09-03 15:46:29",
0,
0,
0,
0
],
[
"2026-09-03 15:47:29",
1,
1,
0,
0
]
],
"latest_matches": {
"scrapy_version": "2.11.1",
"telnet_console": "127.0.0.1:6036",
"telnet_username": "",
"telnet_password": "b43453f693bd948c",
"resuming_crawl": "",
"latest_offsite": "",
"latest_duplicate": "",
"latest_crawl": "",
"latest_scrape": "",
"latest_item": "",
"latest_stat": "2026-09-03 15:47:29 [scrapy.extensions.logstats] INFO: Crawled 1 pages (at 1 pages/min), scraped 0 items (at 0 items/min)"
},
"latest_crawl_timestamp": 0,
"latest_scrape_timestamp": 0,
"log_categories": {
"critical_logs": {
"count": 0,
"details": []
},
"error_logs": {
"count": 2,
"details": [
"2026-09-03 15:46:29 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 2 times): 403 Forbidden",
"2026-09-03 15:48:19 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 2 times): 500 Internal Server Error"
]
},
"warning_logs": {
"count": 1,
"details": [
"2026-09-03 15:46:28 [py.warnings] WARNING: /usr/local/lib/python3.8/site-packages/scrapy/utils/request.py:254: ScrapyDeprecationWarning: '2.6' is a deprecated value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n\nIt is also the default value. In other words, it is normal to get this warning if you have not defined a value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so for backward compatibility reasons, but it will change in a future version of Scrapy.\n\nSee the documentation of the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for information on how to handle this deprecation.\n return cls(crawler)\n"
]
},
"redirect_logs": {
"count": 0,
"details": []
},
"retry_logs": {
"count": 2,
"details": [
"2026-09-03 15:46:29 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 2 times): 403 Forbidden",
"2026-09-03 15:48:19 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 2 times): 500 Internal Server Error"
]
},
"ignore_logs": {
"count": 0,
"details": []
}
},
"crawler_stats": {
"source": "log",
"last_update_time": "2026-09-03 15:48:20",
"last_update_timestamp": 1788450500,
"downloader/request_bytes": 1836,
"downloader/request_count": 4,
"downloader/request_method_count/GET": 4,
"downloader/response_bytes": 12683,
"downloader/response_count": 4,
"downloader/response_status_count/403": 2,
"downloader/response_status_count/500": 2,
"elapsed_time_seconds": 111.178906,
"finish_reason": "finished",
"finish_time": "datetime.datetime(2026, 9, 3, 15, 48, 20, 280674, tzinfo=datetime.timezone.utc)",
"httpcompression/response_bytes": 12432,
"httpcompression/response_count": 4,
"item_scraped_count": 1,
"log_count/ERROR": 2,
"log_count/INFO": 34,
"log_count/WARNING": 1,
"memusage/max": 74895360,
"memusage/startup": 74895360,
"request_depth_max": 1,
"response_received_count": 2,
"retry/count": 2,
"retry/max_reached": 2,
"retry/reason_count/403 Forbidden": 1,
"retry/reason_count/500 Internal Server Error": 1,
"scheduler/dequeued": 4,
"scheduler/dequeued/memory": 4,
"scheduler/enqueued": 4,
"scheduler/enqueued/memory": 4,
"start_time": "datetime.datetime(2026, 9, 3, 15, 46, 29, 101768, tzinfo=datetime.timezone.utc)"
},
"last_update_timestamp": 1788450502,
"logparser_version": "0.8.2",
"crawler_engine": {}
}