You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
使用gerapy管理scrapy-redis,当在一个scrapyd中运行多个scrapy-redis爬虫会报数据库锁住的错误
15/Apr/2022 11:20:07] "GET /api/task/5/status HTTP/1.1" 200 904
[15/Apr/2022 11:20:10] "GET /api/task/5/status HTTP/1.1" 200 904
[15/Apr/2022 11:20:13] "GET /api/task/5/status HTTP/1.1" 200 904
INFO - 2022-04-15 11:20:16,064 - process: 590282 - scheduler.py - gerapy.server.core.scheduler - 34 - scheduler - execute job of client k2, project env_crawlers, spider coastal_sea_area
DB error executing 'update_job' (database is locked). Retrying with a new DB connection...
Exception in thread APScheduler:
Traceback (most recent call last):
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/sqlite3/base.py", line 383, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.OperationalError: database is locked
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django_apscheduler/util.py", line 99, in func_wrapper
result = func(*args, **kwargs)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django_apscheduler/jobstores.py", line 258, in update_job
db_job.save()
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 743, in save
self.save_base(using=using, force_insert=force_insert,
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 780, in save_base
updated = self._save_table(
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 853, in _save_table
updated = self._do_update(base_qs, using, pk_val, values, update_fields,
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 903, in _do_update
return filtered._update(values) > 0
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/query.py", line 760, in _update
return query.get_compiler(self.db).execute_sql(CURSOR)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1471, in execute_sql
cursor = super().execute_sql(result_type)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1142, in execute_sql
cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 99, in execute
return super().execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 67, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers
return executor(sql, params, many, context)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/utils.py", line 89, in exit
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/sqlite3/base.py", line 383, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.OperationalError: database is locked
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/sqlite3/base.py", line 383, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.OperationalError: database is locked
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/threading.py", line 932, in _bootstrap_inner
self.run()
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/apscheduler/schedulers/blocking.py", line 32, in _main_loop
wait_seconds = self._process_jobs()
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/apscheduler/schedulers/base.py", line 1000, in _process_jobs
jobstore.update_job(job)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django_apscheduler/util.py", line 105, in func_wrapper
result = func(*args, **kwargs)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django_apscheduler/jobstores.py", line 258, in update_job
db_job.save()
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 743, in save
self.save_base(using=using, force_insert=force_insert,
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 780, in save_base
updated = self._save_table(
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 853, in _save_table
updated = self._do_update(base_qs, using, pk_val, values, update_fields,
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 903, in _do_update
return filtered._update(values) > 0
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/query.py", line 760, in _update
return query.get_compiler(self.db).execute_sql(CURSOR)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1471, in execute_sql
cursor = super().execute_sql(result_type)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1142, in execute_sql
cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 99, in execute
return super().execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 67, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers
return executor(sql, params, many, context)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/utils.py", line 89, in exit
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/sqlite3/base.py", line 383, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.OperationalError: database is locked
The text was updated successfully, but these errors were encountered:
使用gerapy管理scrapy-redis,当在一个scrapyd中运行多个scrapy-redis爬虫会报数据库锁住的错误
15/Apr/2022 11:20:07] "GET /api/task/5/status HTTP/1.1" 200 904
[15/Apr/2022 11:20:10] "GET /api/task/5/status HTTP/1.1" 200 904
[15/Apr/2022 11:20:13] "GET /api/task/5/status HTTP/1.1" 200 904
INFO - 2022-04-15 11:20:16,064 - process: 590282 - scheduler.py - gerapy.server.core.scheduler - 34 - scheduler - execute job of client k2, project env_crawlers, spider coastal_sea_area
DB error executing 'update_job' (database is locked). Retrying with a new DB connection...
Exception in thread APScheduler:
Traceback (most recent call last):
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/sqlite3/base.py", line 383, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.OperationalError: database is locked
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django_apscheduler/util.py", line 99, in func_wrapper
result = func(*args, **kwargs)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django_apscheduler/jobstores.py", line 258, in update_job
db_job.save()
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 743, in save
self.save_base(using=using, force_insert=force_insert,
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 780, in save_base
updated = self._save_table(
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 853, in _save_table
updated = self._do_update(base_qs, using, pk_val, values, update_fields,
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 903, in _do_update
return filtered._update(values) > 0
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/query.py", line 760, in _update
return query.get_compiler(self.db).execute_sql(CURSOR)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1471, in execute_sql
cursor = super().execute_sql(result_type)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1142, in execute_sql
cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 99, in execute
return super().execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 67, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers
return executor(sql, params, many, context)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/utils.py", line 89, in exit
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/sqlite3/base.py", line 383, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.OperationalError: database is locked
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/sqlite3/base.py", line 383, in execute
return Database.Cursor.execute(self, query, params)
sqlite3.OperationalError: database is locked
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/threading.py", line 932, in _bootstrap_inner
self.run()
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/apscheduler/schedulers/blocking.py", line 32, in _main_loop
wait_seconds = self._process_jobs()
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/apscheduler/schedulers/base.py", line 1000, in _process_jobs
jobstore.update_job(job)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django_apscheduler/util.py", line 105, in func_wrapper
result = func(*args, **kwargs)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django_apscheduler/jobstores.py", line 258, in update_job
db_job.save()
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 743, in save
self.save_base(using=using, force_insert=force_insert,
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 780, in save_base
updated = self._save_table(
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 853, in _save_table
updated = self._do_update(base_qs, using, pk_val, values, update_fields,
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/base.py", line 903, in _do_update
return filtered._update(values) > 0
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/query.py", line 760, in _update
return query.get_compiler(self.db).execute_sql(CURSOR)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1471, in execute_sql
cursor = super().execute_sql(result_type)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1142, in execute_sql
cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 99, in execute
return super().execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 67, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers
return executor(sql, params, many, context)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/utils.py", line 89, in exit
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/anaconda3/envs/env_crawlers/lib/python3.8/site-packages/django/db/backends/sqlite3/base.py", line 383, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.OperationalError: database is locked
The text was updated successfully, but these errors were encountered: