xgiovio/openstack-swift-backup-restore-tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added delete from remote if not present on source, fixed a bug on comparison of already uploaded files, changed md5 pathdir of large files, added print messages, fixed remote list of encrypted files if there are multiple encrypted versions of the same file

Browse Source
This commit is contained in:
Giovanni Di Grezia
2015-12-06 16:42:49 +01:00
parent 91859f7e56
commit 07972c55c5
4 changed files with 104 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
utility.py
View File

@@ -59,6 +59,7 @@ def list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,pre
remotefiles = {}
remotefiles_md5 = {}
remotefiles_xobj = {}
byte0real = 0
byte0manifest = 0
@@ -91,20 +92,25 @@ def list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,pre
print("Impossible to get remote large file md5. Cause: Not uploaded with xgiovio method (md5 in x-object-manifest)")
o["hash"] = "0"
byte0manifest = byte0manifest + 1
remotefiles_xobj[remote_dash_replace(o["name"].replace(prefix,""))] = oheaders["x-object-manifest"]
else:
remotefiles_xobj[remote_dash_replace(o["name"].replace(prefix,""))] = None
print ("0byte file " + o["name"] + " e' un file normale" )
byte0real = byte0real + 1
else:
remotefiles_xobj[remote_dash_replace(o["name"].replace(prefix,""))] = None
remotefiles[remote_dash_replace(o["name"].replace(prefix,""))]=int(o["bytes"])
remotefiles_md5[remote_dash_replace(o["name"].replace(prefix,""))]=o["hash"]
return [byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5]
return [byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5,remotefiles_xobj]
def list_compute_correct_names_for_enctyption (objects,prefix):
def list_compute_correct_names_for_encryption (objects,prefix):
remotefiles = {}
list_enc_old = []
for o in objects :
if o["content_type"] != "application/directory":
@@ -116,9 +122,11 @@ def list_compute_correct_names_for_enctyption (objects,prefix):
if encrypted_name_only in remotefiles.keys():
if int(remotefiles[encrypted_name_only].split("_xg10v10_")[3]) < int(full.split("_xg10v10_")[3]) :
remotefiles[encrypted_name_only] = full
else:
list_enc_old.append(full)
else:
remotefiles[encrypted_name_only] = full
return remotefiles
return [remotefiles,list_enc_old]
def list_compute_manifest (fail_tries,objects,swift_conn,swift_container,prefix):
@@ -223,3 +231,45 @@ def set_dash():
if platform.system() == "Windows":
return "\\"
return "/"
def delete_object (swift_conn,swift_container,object,manifest,fail_tries):
for fail_tries_counter in range (fail_tries) :
try:
print("Deleting " + swift_container + "/" + object)
swift_conn.delete_object(swift_container,object)
if manifest != None:
segment_container = manifest.split("/")[0]
prefix = manifest.replace(segment_container + "/","")
manifest_downloaded = False
for fail_tries_counter1 in range (fail_tries) :
try:
if not manifest_downloaded:
print ("Downloading remote list for " + segment_container + " with prefix " +prefix + " ... ")
headers,objects = swift_conn.get_container(segment_container, prefix =prefix,full_listing=True )
manifest_downloaded = True
for o in objects :
print("Deleting " + segment_container + "/" + o["name"])
swift_conn.delete_object(segment_container,o["name"])
except Exception as e:
print("Exception during deletion of manifest files")
print(e)
time.sleep(1)
if fail_tries_counter1 == fail_tries - 1 :
print("Maximum tries reached. Can't delete " + swift_container + "/" + object +" manifest files.Skipping")
else:
swift_conn = authentication.set_authentication ()
else :
break
except Exception as e:
print("Exception during deletion of file")
print(e)
time.sleep(1)
if fail_tries_counter == fail_tries - 1 :
print("Maximum tries reached. Can't delete " + swift_container + "/" + object +".Skipping")
else:
swift_conn = authentication.set_authentication ()
else :
break
return swift_conn