Hi,
The multi_delete in swift3, perform sequential DELETE. In my 3
storage-node configuration to delete a 1000 objects, it took 30 second.
Following code change to create 100 thread pool to delete 1000 object took
only 12 second. (This may even reduce if more storage nodes in picture).
If the following code change look fine, How can we formally
(propose/review/commit) take this to the swift3 github code base ?
*Code diff:*
diff --git a/swift3/swift-plugin-s3/swift3/controllers/multi_delete.py
b/swift3/swift-plugin-s3/swift3/controllers/multi_delete.py
index 1bfde1d..5140529 100644
--- a/swift3/swift-plugin-s3/swift3/controllers/multi_delete.py
+++ b/swift3/swift-plugin-s3/swift3/controllers/multi_delete.py
@@ -21,9 +21,9 @@ from swift3.response import HTTPOk, S3NotImplemented,
NoSuchKey, \
from swift3.cfg import CONF
from swift3.utils import LOGGER
-# Zadara-Begin
+from eventlet import GreenPool
+import copy
MAX_MULTI_DELETE_BODY_SIZE = 262144
-# Zadara-End
class MultiObjectDeleteController(Controller):
@@ -44,6 +44,24 @@ class MultiObjectDeleteController(Controller):
return tostring(elem)
+ def async_delete(self, reqs, key, elem):
+ req = copy.copy(reqs)
+ req.object_name = key
+ try:
+ req.get_response(self.app, method='DELETE')
+ except NoSuchKey:
+ pass
+ except ErrorResponse as e:
+ error = SubElement(elem, 'Error')
+ SubElement(error, 'Key').text = key
+ SubElement(error, 'Code').text = e.__class__.__name__
+ SubElement(error, 'Message').text = e._msg
+ return
+
+ if not self.quiet:
+ deleted = SubElement(elem, 'Deleted')
+ SubElement(deleted, 'Key').text = key
+
@bucket_operation
def POST(self, req):
"""
@@ -90,27 +108,17 @@ class MultiObjectDeleteController(Controller):
body = self._gen_error_body(error, elem, delete_list)
return HTTPOk(body=body)
+ parallel_delete = 100
+ run_pool = GreenPool(size=parallel_delete)
for key, version in delete_list:
if version is not None:
# TODO: delete the specific version of the object
raise S3NotImplemented()
- req.object_name = key
-
- try:
- req.get_response(self.app, method='DELETE')
- except NoSuchKey:
- pass
- except ErrorResponse as e:
- error = SubElement(elem, 'Error')
- SubElement(error, 'Key').text = key
- SubElement(error, 'Code').text = e.__class__.__name__
- SubElement(error, 'Message').text = e._msg
- continue
-
- if not self.quiet:
- deleted = SubElement(elem, 'Deleted')
- SubElement(deleted, 'Key').text = key
+ run_pool.spawn(self.async_delete, req, key, elem)
+
+ # Wait for all the process to complete
+ run_pool.waitall()
body = tostring(elem)
__________________________________________________________________________
OpenStack Development Mailing List (not for usage questions)
Unsubscribe: openstack-dev-requ...@lists.openstack.org?subject:unsubscribe
http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack-dev
