xgiovio/openstack-swift-backup-restore-tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

changed sleep to 2 sec. fixed the setting of swift_conn during deletion process. Timeout of ssl increased to 10. Added checks to segment files: they have to be power of 2.

Browse Source
This commit is contained in:
Giovanni Di Grezia
2016-05-08 22:46:08 +02:00
parent facda776b1
commit 77f3a9fa05
10 changed files with 23 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
authentication.py
View File

@@ -11,7 +11,7 @@ swift_auth = "https://www.########"
url = "https://########/v1/AUTH_###############"
tok ="########"
#######################################end authentication
timeout = 3 #sec
timeout = 10 #sec
insecure = False

2
local_to_swift.py
View File

@@ -299,7 +299,7 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
except Exception as e:
print("Exception during upload")
print(e)
time.sleep(1)
time.sleep(2)
errors_upload = errors_upload + 1
swift_conn = authentication.set_authentication ()
if fail_tries_counter == fail_tries - 1 :

2
settings_generate_temp_url.py
View File

@@ -7,7 +7,7 @@ set_secretkey = False
create_temp_url = True
duration_in_seconds = 60*60*2
objectpath = "#####" # container/object
fail_tries = 100
fail_tries = 30
swift_generate_temp_url.launch(secretkey,set_secretkey,create_temp_url,duration_in_seconds,objectpath,fail_tries)

2
settings_local_to_swift.py
View File

@@ -28,7 +28,7 @@ size_limit_reading_os = 134217728 #must be a power of 2 and smaller/equal than s
# 4GB 4294967296
upload = True
enableLarge = True
fail_tries = 200
fail_tries = 30
temp_path = "\\\\?\\" + "c:\\temp\\"
excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir"]
delete_excluded_patterns = ["!CB_"]

2
settings_swift_delete_0_byte_folders.py
View File

@@ -7,7 +7,7 @@ swift_containers = [
"default_segments"
]
delete = True
fail_tries = 10
fail_tries = 30
for i in range(len(swift_containers)):
swift_delete_0_byte_folders.launch(swift_containers[i],delete,fail_tries)

2
settings_swift_delete_orphan_segments.py
View File

@@ -4,6 +4,6 @@ import swift_delete_orphan_segments
#swift
swift_container = "default" # container
delete = True
fail_tries = 10
fail_tries = 30
swift_delete_orphan_segments.launch(swift_container,delete,fail_tries)

2
settings_swift_to_local.py
View File

@@ -17,7 +17,7 @@ size_limit_reading_os = 134217728 #must be a power of 2 and smaller/equal than s
# 2GB 2147483648
# 4GB 4294967296
download = True
fail_tries = 200
fail_tries = 30
excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir","!CB_"]
batch = [
#source, swift container, swift prefix, md5 comparison enabled?, encrypted?, encryption_key, additional_excluded_patterns

2
swift_generate_temp_url.py
View File

@@ -20,7 +20,7 @@ def launch(secretkey,set_secretkey,create_temp_url,duration_in_seconds,objectpat
except Exception as e:
print("Exception during setting / getting the secret key.")
print(e)
time.sleep(1)
time.sleep(2)
if fail_tries_counter == fail_tries - 1 :
print("Maximum tries reached. Exiting.")
sys.exit(-1)

2
swift_to_local.py
View File

@@ -142,7 +142,7 @@ def launch(localpath,swift_container,prefix,size_limit_reading_os,download,fail_
except Exception as e:
print("Exception during download")
print(e)
time.sleep(1)
time.sleep(2)
errors_download = errors_download + 1
swift_conn = authentication.set_authentication ()
if fail_tries_counter == fail_tries - 1 :

24
utility.py
View File

@@ -3,6 +3,7 @@ __author__ = 'xgiovio'
import authentication
import sys, platform,os
import time
import math
def get_local_files (localpath):
@@ -28,7 +29,7 @@ def get_list (fail_tries,swift_conn,swift_container,prefix) :
except Exception as e:
print("Exception during the download of remote list")
print(e)
time.sleep(1)
time.sleep(2)
if fail_tries_counter == fail_tries - 1 :
print("Maximum tries reached. Can't download remote list for container " + swift_container + " with prefix " +prefix +". Exiting.")
sys.exit(-1)
@@ -73,7 +74,7 @@ def list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,pre
except Exception as e:
print("Exception during the request of metadata")
print(e)
time.sleep(1)
time.sleep(2)
if fail_tries_counter == fail_tries - 1 :
print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +". Exiting.")
sys.exit(-1)
@@ -144,7 +145,7 @@ def list_compute_manifest (fail_tries,objects,swift_conn,swift_container,prefix)
except Exception as e:
print("Exception during the request of metadata")
print(e)
time.sleep(1)
time.sleep(2)
if fail_tries_counter == fail_tries - 1 :
print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +". Exiting.")
sys.exit(-1)
@@ -175,10 +176,15 @@ def listall (objects,prefix):
return remotefiles
def check_segments_size (size_limit_reading_os,size_limit_to_segment):
if size_limit_reading_os%2 != 0 or size_limit_to_segment%2 != 0 or size_limit_reading_os > size_limit_to_segment or size_limit_reading_os < 16:
if size_limit_reading_os%2 != 0 or size_limit_to_segment%2 != 0 or size_limit_reading_os > size_limit_to_segment or size_limit_reading_os < 16 or not is_power_of_2(size_limit_reading_os) or not is_power_of_2(size_limit_to_segment):
print ("Error size segments")
sys.exit(-2)
def is_power_of_2 (number):
result = math.log(number,2)
return (result == int(result))
def check_segments_size_single (size_limit_reading_os):
if size_limit_reading_os%2 != 0 or size_limit_reading_os < 16:
print ("Error size segments")
@@ -259,24 +265,22 @@ def delete_object (swift_conn,swift_container,object,manifest,fail_tries):
print("Exception during deletion of manifest files")
print(e)
errors_deleting_files_of_large_obj +=1
time.sleep(1)
time.sleep(2)
swift_conn = authentication.set_authentication()
if fail_tries_counter1 == fail_tries - 1 :
print("Maximum tries reached. Can't delete " + swift_container + "/" + object +" manifest files.Skipping")
skipped_files_of_large_obj += 1
else:
swift_conn = authentication.set_authentication ()
else :
break
except Exception as e:
print("Exception during deletion of file")
print(e)
errors_deleting_file += 1
time.sleep(1)
time.sleep(2)
swift_conn = authentication.set_authentication()
if fail_tries_counter == fail_tries - 1 :
print("Maximum tries reached. Can't delete " + swift_container + "/" + object +".Skipping")
skipped_file +=1
else:
swift_conn = authentication.set_authentication ()
else :
break
return [swift_conn,errors_deleting_file,errors_deleting_files_of_large_obj,skipped_file,skipped_files_of_large_obj]