My web application posts multiple files (FileField) that is associated with
the same item. For each item I make a directory for storing all the related
files. However, when frontend sends a sequence of file posts to this model,
I run into race condition in the filesystemstorage and have an error there.
When the first file is uploaded, the item directory is not existed yet and
is going to make a directory. Then the second file also checks the item
directory existence and trying to do the same directory creation. Then the
thread switched to the first file upload and creates the directory. Then
the second file upload it also creates the same directory and gets the
creation error since the directory has been created by the first upload
already. Here is my model definition.
class Item(models.Model):
name = models.CharField(max_length=256)
def item_id_path(instance, filename):
return '{}/{}'.format(instance.item.id, filename)
# Create your models here.
class File(models.Model):
file = models.FileField(upload_to=item_id_path)
timestamp = models.DateTimeField(auto_now_add=True)
item = models.ForeignKey(Item, on_delete=models.CASCADE)
def __str__(self):
return self.file.name
what approach can I take to avoid this kind of race condition? maybe a fix
on the django source code in core/files/storage.py line 236 in
FileSystemStorage _save method could work?
from
os.makedirs(directory)
to
os.makedirs(directory, exist_ok=True)
Here is is the call traceback information.
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/rest_framework/serializers.py",
line 214, in save
self.instance = self.create(validated_data)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/rest_framework/serializers.py",
line 940, in create
instance = ModelClass.objects.create(**validated_data)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/manager.py",
line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/query.py",
line 413, in create
obj.save(force_insert=True, using=self.db)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/base.py",
line 717, in save
force_update=force_update, update_fields=update_fields)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/base.py",
line 747, in save_base
updated = self._save_table(raw, cls, force_insert, force_update, using,
update_fields)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/base.py",
line 830, in _save_table
result = self._do_insert(cls._base_manager, using, fields, update_pk,
raw)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/base.py",
line 868, in _do_insert
using=using, raw=raw)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/manager.py",
line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/query.py",
line 1133, in _insert
return query.get_compiler(using=using).execute_sql(return_id)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/sql/compiler.py",
line 1284, in execute_sql
for sql, params in self.as_sql():
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/sql/compiler.py",
line 1237, in as_sql
for obj in self.query.objs
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/sql/compiler.py",
line 1237, in <listcomp>
for obj in self.query.objs
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/sql/compiler.py",
line 1236, in <listcomp>
[self.prepare_value(field, self.pre_save_val(field, obj)) for field in
fields]
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/sql/compiler.py",
line 1188, in pre_save_val
return field.pre_save(obj, add=True)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/fields/files.py",
line 288, in pre_save
file.save(file.name, file.file, save=False)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/fields/files.py",
line 87, in save
self.name = self.storage.save(name, content,
max_length=self.field.max_length)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/core/files/storage.py",
line 49, in save
return self._save(name, content)
File
"/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/core/files/storage.py",
line 236, in _save
os.makedirs(directory)
File "/Users/han/anaconda3/envs/django/lib/python3.6/os.py", line 220, in
makedirs
mkdir(name, mode)
FileExistsError: [Errno 17] File exists:
--
You received this message because you are subscribed to the Google Groups
"Django developers (Contributions to Django itself)" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
Visit this group at https://groups.google.com/group/django-developers.
To view this discussion on the web visit
https://groups.google.com/d/msgid/django-developers/e066c935-c2a5-4ffc-a6f6-543cd8fcd0be%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
