{
"log_path": "/etc/scrapyd/logs/linkschecker/crawl_one/04279d6c4d1211f1872b4ec0d7e9e9ab.log",
"json_path": "/etc/scrapyd/logs/linkschecker/crawl_one/04279d6c4d1211f1872b4ec0d7e9e9ab.json",
"json_url": "http://127.0.0.1:8080/logs/linkschecker/crawl_one/04279d6c4d1211f1872b4ec0d7e9e9ab.json",
"size": 5878,
"position": 5878,
"status": "ok",
"_head": "2026-05-11 08:18:40 [scrapy.utils.log] INFO: Scrapy 2.11.1 started (bot: linkschecker)\n2026-05-11 08:18:40 [scrapy.utils.log] INFO: Versions: lxml 5.1.0.0, libxml2 2.12.3, cssselect 1.2.0, parsel 1.8.1, w3lib 2.1.2, Twisted 22.10.0, Python 3.8.18 (default, Feb 13 2024, 11:03:53) - [GCC 12.2.0], pyOpenSSL 24.0.0 (OpenSSL 3.2.1 30 Jan 2024), cryptography 42.0.4, Platform Linux-5.15.0-152-generic-x86_64-with-glibc2.34\n2026-05-11 08:18:40 [scrapy.addons] INFO: Enabled addons:\n[]\n2026-05-11 08:18:40 [py.warnings] WARNING: /usr/local/lib/python3.8/site-packages/scrapy/utils/request.py:254: ScrapyDeprecationWarning: '2.6' is a deprecated value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n\nIt is also the default value. In other words, it is normal to get this warning if you have not defined a value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so for backward compatibility reasons, but it will change in a future version of Scrapy.\n\nSee the documentation of the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for information on how to handle this deprecation.\n return cls(crawler)\n\n2026-05-11 08:18:40 [scrapy.extensions.telnet] INFO: Telnet Password: 0c2868b2a9afe17b\n2026-05-11 08:18:40 [scrapy.middleware] INFO: Enabled extensions:\n['scrapy.extensions.corestats.CoreStats',\n 'scrapy.extensions.telnet.TelnetConsole',\n 'scrapy.extensions.memusage.MemoryUsage',\n 'scrapy.extensions.logstats.LogStats']\n2026-05-11 08:18:40 [scrapy.crawler] INFO: Overridden settings:\n{'BOT_NAME': 'linkschecker',\n 'CONCURRENT_REQUESTS': 8,\n 'DEPTH_LIMIT': '1',\n 'DOWNLOAD_TIMEOUT': 60,\n 'LOG_FILE': 'logs/linkschecker/crawl_one/04279d6c4d1211f1872b4ec0d7e9e9ab.log',\n 'LOG_LEVEL': 'INFO',\n 'MEMUSAGE_CHECK_INTERVAL_SECONDS': 5,\n 'MEMUSAGE_LIMIT_MB': 800,\n 'NEWSPIDER_MODULE': 'linkschecker.spiders',\n 'RETRY_HTTP_CODES': [500, 502, 503, 504, 522, 524, 408, 429, 403],\n 'RETRY_TIMES': 1,\n 'SPIDER_MODULES': ['linkschecker.spiders'],\n 'USER_AGENT': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 '\n '(KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}\n2026-05-11 08:18:40 [scrapy.middleware] INFO: Enabled downloader middlewares:\n['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n2026-05-11 08:18:40 [scrapy.middleware] INFO: Enabled spider middlewares:\n['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n2026-05-11 08:18:40 [scrapy.middleware] INFO: Enabled item pipelines:\n['linkschecker.pipelines.LinkscheckerPipelineCrawlOne']\n2026-05-11 08:18:40 [scrapy.core.engine] INFO: Spider opened\n2026-05-11 08:18:40 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n2026-05-11 08:18:40 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023\n\n2026-05-11 08:18:40 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:18:40 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 2 times): 403 Forbidden\n2026-05-11 08:18:45 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-11 08:18:50 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:18:55 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-11 08:19:00 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:19:05 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-11 08:19:10 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:19:15 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-11 08:19:20 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:19:25 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:19:26 [scrapy.core.engine] INFO: Closing spider (finished)\n2026-05-11 08:19:27 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n{'downloader/request_bytes': 1098,\n 'downloader/request_count': 3,\n 'downloader/request_method_count/GET': 3,\n 'downloader/response_bytes': 261824,\n 'downloader/response_count': 3,\n 'downloader/response_status_count/200': 1,\n 'downloader/response_status_count/403': 2,\n 'elapsed_time_seconds': 46.650655,\n 'finish_reason': 'finished',\n 'finish_time': datetime.datetime(2026, 5, 11, 8, 19, 27, 255784, tzinfo=datetime.timezone.utc),\n 'httpcompression/response_bytes': 683484,\n 'httpcompression/response_count': 3,\n 'item_scraped_count': 1,\n 'log_count/ERROR': 1,\n 'log_count/INFO': 20,\n 'log_count/WARNING': 1,\n 'memusage/max': 75333632,\n 'memusage/startup': 75333632,\n 'request_depth_max': 1,\n 'response_received_count': 2,\n 'retry/count': 1,\n 'retry/max_reached': 1,\n 'retry/reason_count/403 Forbidden': 1,\n 'scheduler/dequeued': 3,\n 'scheduler/dequeued/memory': 3,\n 'scheduler/enqueued': 3,\n 'scheduler/enqueued/memory': 3,\n 'start_time': datetime.datetime(2026, 5, 11, 8, 18, 40, 605129, tzinfo=datetime.timezone.utc)}\n2026-05-11 08:19:27 [scrapy.core.engine] INFO: Spider closed (finished)\n",
"pages": 2,
"items": 1,
"first_log_time": "2026-05-11 08:18:40",
"latest_log_time": "2026-05-11 08:19:27",
"runtime": "0:00:47",
"shutdown_reason": "N/A",
"finish_reason": "finished",
"last_update_time": "2026-05-11 08:19:30",
"head": "2026-05-11 08:18:40 [scrapy.utils.log] INFO: Scrapy 2.11.1 started (bot: linkschecker)\n2026-05-11 08:18:40 [scrapy.utils.log] INFO: Versions: lxml 5.1.0.0, libxml2 2.12.3, cssselect 1.2.0, parsel 1.8.1, w3lib 2.1.2, Twisted 22.10.0, Python 3.8.18 (default, Feb 13 2024, 11:03:53) - [GCC 12.2.0], pyOpenSSL 24.0.0 (OpenSSL 3.2.1 30 Jan 2024), cryptography 42.0.4, Platform Linux-5.15.0-152-generic-x86_64-with-glibc2.34\n2026-05-11 08:18:40 [scrapy.addons] INFO: Enabled addons:\n[]\n2026-05-11 08:18:40 [py.warnings] WARNING: /usr/local/lib/python3.8/site-packages/scrapy/utils/request.py:254: ScrapyDeprecationWarning: '2.6' is a deprecated value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n\nIt is also the default value. In other words, it is normal to get this warning if you have not defined a value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so for backward compatibility reasons, but it will change in a future version of Scrapy.\n\nSee the documentation of the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for information on how to handle this deprecation.\n return cls(crawler)\n\n2026-05-11 08:18:40 [scrapy.extensions.telnet] INFO: Telnet Password: 0c2868b2a9afe17b\n2026-05-11 08:18:40 [scrapy.middleware] INFO: Enabled extensions:\n['scrapy.extensions.corestats.CoreStats',\n 'scrapy.extensions.telnet.TelnetConsole',\n 'scrapy.extensions.memusage.MemoryUsage',\n 'scrapy.extensions.logstats.LogStats']\n2026-05-11 08:18:40 [scrapy.crawler] INFO: Overridden settings:\n{'BOT_NAME': 'linkschecker',\n 'CONCURRENT_REQUESTS': 8,\n 'DEPTH_LIMIT': '1',\n 'DOWNLOAD_TIMEOUT': 60,\n 'LOG_FILE': 'logs/linkschecker/crawl_one/04279d6c4d1211f1872b4ec0d7e9e9ab.log',\n 'LOG_LEVEL': 'INFO',\n 'MEMUSAGE_CHECK_INTERVAL_SECONDS': 5,\n 'MEMUSAGE_LIMIT_MB': 800,\n 'NEWSPIDER_MODULE': 'linkschecker.spiders',\n 'RETRY_HTTP_CODES': [500, 502, 503, 504, 522, 524, 408, 429, 403],\n 'RETRY_TIMES': 1,\n 'SPIDER_MODULES': ['linkschecker.spiders'],\n 'USER_AGENT': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 '\n '(KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}\n2026-05-11 08:18:40 [scrapy.middleware] INFO: Enabled downloader middlewares:\n['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n2026-05-11 08:18:40 [scrapy.middleware] INFO: Enabled spider middlewares:\n['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n2026-05-11 08:18:40 [scrapy.middleware] INFO: Enabled item pipelines:\n['linkschecker.pipelines.LinkscheckerPipelineCrawlOne']\n2026-05-11 08:18:40 [scrapy.core.engine] INFO: Spider opened\n2026-05-11 08:18:40 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n2026-05-11 08:18:40 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023\n\n2026-05-11 08:18:40 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:18:40 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 2 times): 403 Forbidden\n2026-05-11 08:18:45 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-11 08:18:50 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:18:55 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-11 08:19:00 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:19:05 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-11 08:19:10 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:19:15 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-11 08:19:20 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:19:25 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:19:26 [scrapy.core.engine] INFO: Closing spider (finished)\n2026-05-11 08:19:27 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n{'downloader/request_bytes': 1098,\n 'downloader/request_count': 3,\n 'downloader/request_method_count/GET': 3,\n 'downloader/response_bytes': 261824,\n 'downloader/response_count': 3,\n 'downloader/response_status_count/200': 1,\n 'downloader/response_status_count/403': 2,\n 'elapsed_time_seconds': 46.650655,\n 'finish_reason': 'finished',\n 'finish_time': datetime.datetime(2026, 5, 11, 8, 19, 27, 255784, tzinfo=datetime.timezone.utc),\n 'httpcompression/response_bytes': 683484,\n 'httpcompression/response_count': 3,\n 'item_scraped_count': 1,\n 'log_count/ERROR': 1,\n 'log_count/INFO': 20,\n 'log_count/WARNING': 1,\n 'memusage/max': 75333632,\n 'memusage/startup': 75333632,\n 'request_depth_max': 1,\n 'response_received_count': 2,\n 'retry/count': 1,\n 'retry/max_reached': 1,\n 'retry/reason_count/403 Forbidden': 1,\n 'scheduler/dequeued': 3,\n 'scheduler/dequeued/memory': 3,\n 'scheduler/enqueued': 3,\n 'scheduler/enqueued/memory': 3,\n 'start_time': datetime.datetime(2026, 5, 11, 8, 18, 40, 605129, tzinfo=datetime.timezone.utc)}\n2026-05-11 08:19:27 [scrapy.core.engine] INFO: Spider closed (finished)\n",
"tail": "2026-05-11 08:18:40 [scrapy.utils.log] INFO: Scrapy 2.11.1 started (bot: linkschecker)\n2026-05-11 08:18:40 [scrapy.utils.log] INFO: Versions: lxml 5.1.0.0, libxml2 2.12.3, cssselect 1.2.0, parsel 1.8.1, w3lib 2.1.2, Twisted 22.10.0, Python 3.8.18 (default, Feb 13 2024, 11:03:53) - [GCC 12.2.0], pyOpenSSL 24.0.0 (OpenSSL 3.2.1 30 Jan 2024), cryptography 42.0.4, Platform Linux-5.15.0-152-generic-x86_64-with-glibc2.34\n2026-05-11 08:18:40 [scrapy.addons] INFO: Enabled addons:\n[]\n2026-05-11 08:18:40 [py.warnings] WARNING: /usr/local/lib/python3.8/site-packages/scrapy/utils/request.py:254: ScrapyDeprecationWarning: '2.6' is a deprecated value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n\nIt is also the default value. In other words, it is normal to get this warning if you have not defined a value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so for backward compatibility reasons, but it will change in a future version of Scrapy.\n\nSee the documentation of the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for information on how to handle this deprecation.\n return cls(crawler)\n\n2026-05-11 08:18:40 [scrapy.extensions.telnet] INFO: Telnet Password: 0c2868b2a9afe17b\n2026-05-11 08:18:40 [scrapy.middleware] INFO: Enabled extensions:\n['scrapy.extensions.corestats.CoreStats',\n 'scrapy.extensions.telnet.TelnetConsole',\n 'scrapy.extensions.memusage.MemoryUsage',\n 'scrapy.extensions.logstats.LogStats']\n2026-05-11 08:18:40 [scrapy.crawler] INFO: Overridden settings:\n{'BOT_NAME': 'linkschecker',\n 'CONCURRENT_REQUESTS': 8,\n 'DEPTH_LIMIT': '1',\n 'DOWNLOAD_TIMEOUT': 60,\n 'LOG_FILE': 'logs/linkschecker/crawl_one/04279d6c4d1211f1872b4ec0d7e9e9ab.log',\n 'LOG_LEVEL': 'INFO',\n 'MEMUSAGE_CHECK_INTERVAL_SECONDS': 5,\n 'MEMUSAGE_LIMIT_MB': 800,\n 'NEWSPIDER_MODULE': 'linkschecker.spiders',\n 'RETRY_HTTP_CODES': [500, 502, 503, 504, 522, 524, 408, 429, 403],\n 'RETRY_TIMES': 1,\n 'SPIDER_MODULES': ['linkschecker.spiders'],\n 'USER_AGENT': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 '\n '(KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}\n2026-05-11 08:18:40 [scrapy.middleware] INFO: Enabled downloader middlewares:\n['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n2026-05-11 08:18:40 [scrapy.middleware] INFO: Enabled spider middlewares:\n['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n2026-05-11 08:18:40 [scrapy.middleware] INFO: Enabled item pipelines:\n['linkschecker.pipelines.LinkscheckerPipelineCrawlOne']\n2026-05-11 08:18:40 [scrapy.core.engine] INFO: Spider opened\n2026-05-11 08:18:40 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n2026-05-11 08:18:40 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023\n\n2026-05-11 08:18:40 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:18:40 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 2 times): 403 Forbidden\n2026-05-11 08:18:45 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-11 08:18:50 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:18:55 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-11 08:19:00 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:19:05 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-11 08:19:10 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:19:15 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n\n2026-05-11 08:19:20 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:19:25 [scrapy.extensions.memusage] INFO: Peak memory usage is 71MiB\n2026-05-11 08:19:26 [scrapy.core.engine] INFO: Closing spider (finished)\n2026-05-11 08:19:27 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n{'downloader/request_bytes': 1098,\n 'downloader/request_count': 3,\n 'downloader/request_method_count/GET': 3,\n 'downloader/response_bytes': 261824,\n 'downloader/response_count': 3,\n 'downloader/response_status_count/200': 1,\n 'downloader/response_status_count/403': 2,\n 'elapsed_time_seconds': 46.650655,\n 'finish_reason': 'finished',\n 'finish_time': datetime.datetime(2026, 5, 11, 8, 19, 27, 255784, tzinfo=datetime.timezone.utc),\n 'httpcompression/response_bytes': 683484,\n 'httpcompression/response_count': 3,\n 'item_scraped_count': 1,\n 'log_count/ERROR': 1,\n 'log_count/INFO': 20,\n 'log_count/WARNING': 1,\n 'memusage/max': 75333632,\n 'memusage/startup': 75333632,\n 'request_depth_max': 1,\n 'response_received_count': 2,\n 'retry/count': 1,\n 'retry/max_reached': 1,\n 'retry/reason_count/403 Forbidden': 1,\n 'scheduler/dequeued': 3,\n 'scheduler/dequeued/memory': 3,\n 'scheduler/enqueued': 3,\n 'scheduler/enqueued/memory': 3,\n 'start_time': datetime.datetime(2026, 5, 11, 8, 18, 40, 605129, tzinfo=datetime.timezone.utc)}\n2026-05-11 08:19:27 [scrapy.core.engine] INFO: Spider closed (finished)\n",
"first_log_timestamp": 1778487520,
"latest_log_timestamp": 1778487567,
"datas": [
[
"2026-05-11 08:18:40",
0,
0,
0,
0
]
],
"latest_matches": {
"scrapy_version": "2.11.1",
"telnet_console": "127.0.0.1:6023",
"telnet_username": "",
"telnet_password": "0c2868b2a9afe17b",
"resuming_crawl": "",
"latest_offsite": "",
"latest_duplicate": "",
"latest_crawl": "",
"latest_scrape": "",
"latest_item": "",
"latest_stat": "2026-05-11 08:18:40 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)"
},
"latest_crawl_timestamp": 0,
"latest_scrape_timestamp": 0,
"log_categories": {
"critical_logs": {
"count": 0,
"details": []
},
"error_logs": {
"count": 1,
"details": [
"2026-05-11 08:18:40 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 2 times): 403 Forbidden"
]
},
"warning_logs": {
"count": 1,
"details": [
"2026-05-11 08:18:40 [py.warnings] WARNING: /usr/local/lib/python3.8/site-packages/scrapy/utils/request.py:254: ScrapyDeprecationWarning: '2.6' is a deprecated value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n\nIt is also the default value. In other words, it is normal to get this warning if you have not defined a value for the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so for backward compatibility reasons, but it will change in a future version of Scrapy.\n\nSee the documentation of the 'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for information on how to handle this deprecation.\n return cls(crawler)\n"
]
},
"redirect_logs": {
"count": 0,
"details": []
},
"retry_logs": {
"count": 1,
"details": [
"2026-05-11 08:18:40 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 2 times): 403 Forbidden"
]
},
"ignore_logs": {
"count": 0,
"details": []
}
},
"crawler_stats": {
"source": "log",
"last_update_time": "2026-05-11 08:19:27",
"last_update_timestamp": 1778487567,
"downloader/request_bytes": 1098,
"downloader/request_count": 3,
"downloader/request_method_count/GET": 3,
"downloader/response_bytes": 261824,
"downloader/response_count": 3,
"downloader/response_status_count/200": 1,
"downloader/response_status_count/403": 2,
"elapsed_time_seconds": 46.650655,
"finish_reason": "finished",
"finish_time": "datetime.datetime(2026, 5, 11, 8, 19, 27, 255784, tzinfo=datetime.timezone.utc)",
"httpcompression/response_bytes": 683484,
"httpcompression/response_count": 3,
"item_scraped_count": 1,
"log_count/ERROR": 1,
"log_count/INFO": 20,
"log_count/WARNING": 1,
"memusage/max": 75333632,
"memusage/startup": 75333632,
"request_depth_max": 1,
"response_received_count": 2,
"retry/count": 1,
"retry/max_reached": 1,
"retry/reason_count/403 Forbidden": 1,
"scheduler/dequeued": 3,
"scheduler/dequeued/memory": 3,
"scheduler/enqueued": 3,
"scheduler/enqueued/memory": 3,
"start_time": "datetime.datetime(2026, 5, 11, 8, 18, 40, 605129, tzinfo=datetime.timezone.utc)"
},
"last_update_timestamp": 1778487570,
"logparser_version": "0.8.2",
"crawler_engine": {}
}