xgiovio/openstack-swift-backup-restore-tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added sync mode for local to swift, added delete function in utility, added script to remove all 0bytes directories, added script to delete large files without manifest

Browse Source
This commit is contained in:
Giovanni Di Grezia
2015-12-09 15:39:18 +01:00
parent 07972c55c5
commit cc67ad7649
11 changed files with 191 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
local_to_swift.py
View File

@@ -7,7 +7,8 @@ from utility import read_in_chunks
import time
import shutil
def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_limit_reading_os,upload,uploadlarge,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns,copy_to_dir,delete):
def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_limit_reading_os,upload,uploadlarge,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns,copy_to_dir,delete,delete_excluded_patterns):
print ("Starting Job")
print ("Localpath " + localpath)
print ("Temppath " + temp_dir)
print ("Swift container " + swift_container)
@@ -30,10 +31,12 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
print ("Encrypted key " + "hidden")
if copy_to_dir != None:
print ("Copy to dir " + copy_to_dir)
print ("Delete files on dest if not present on source " + str(delete))
print("___________")
#checks
utility.check_segments_size (size_limit_reading_os,size_limit_to_segment)
if utility.check_start_slash(swift_container) or utility.check_start_slash(prefix) or not utility.check_end_slash(localpath) or not utility.check_end_slash(temp_dir) or utility.check_end_slash(swift_container) or not utility.check_end_slash(prefix):
@@ -318,6 +321,7 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
dellist = []
delete_skipped_filters = 0
print("")
print("Computing deletion list...")
if encrypted:
@@ -328,28 +332,67 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
for o in list_enc_old:
dellist.append(o)
for rname in remotefiles_encr.keys():
if rname.endswith("_xg10v10_encrypted"):
rname_pure = rname.split("_xg10v10_encrypted")[0]
if rname_pure in remotefiles_encr.keys():
dellist.append(remotefiles_encr[rname_pure])
if rname_pure not in localfiles.keys():
dellist.append(remotefiles_encr[rname])
else:
if rname not in localfiles.keys():
dellist.append(remotefiles_encr[rname])
delete_file = True
for pattern in delete_excluded_patterns:
if pattern in rname:
delete_file = False
print("Skipped " + rname + " due to filters : " + pattern )
delete_skipped_filters +=1
break
if delete_file :
if rname.endswith("_xg10v10_encrypted"):
rname_pure = rname.split("_xg10v10_encrypted")[0]
if rname_pure in remotefiles_encr.keys():
dellist.append(remotefiles_encr[rname_pure])
if rname_pure not in localfiles.keys():
dellist.append(remotefiles_encr[rname])
else:
if rname not in localfiles.keys():
dellist.append(remotefiles_encr[rname])
else:
for rname in remotefiles.keys():
if rname not in localfiles.keys():
dellist.append(rname)
delete_file = True
for pattern in delete_excluded_patterns:
if pattern in rname:
delete_file = False
print("Skipped " + rname + " due to filters : " + pattern )
delete_skipped_filters +=1
break
if delete_file:
if rname not in localfiles.keys():
dellist.append(rname)
print("___________Files to delete______")
for files in dellist:
print(files)
print("___________")
if delete :
print("Files to delete " + str(len(dellist)))
print("Skipped files due to filters " + str(delete_skipped_filters))
if delete and len(dellist) > 0:
errors_deleting_files = 0
errors_deleting_files_of_large_objs = 0
skipped_files = 0
skipped_files_of_large_objs = 0
for object in dellist:
swift_conn = utility.delete_object(swift_conn,swift_container,prefix + object,remotefiles_xobj[object],fail_tries)
swift_conn,errors_deleting_file,errors_deleting_files_of_large_obj,skipped_file,skipped_files_of_large_obj = utility.delete_object(swift_conn,swift_container,prefix + object,remotefiles_xobj[object],fail_tries)
errors_deleting_files += errors_deleting_file
errors_deleting_files_of_large_objs += errors_deleting_files_of_large_obj
skipped_files += skipped_file
skipped_files_of_large_objs += skipped_files_of_large_obj
print("___________")
print("Errors deleting files " + str(errors_deleting_files))
print("Errors deleting files of large objs " + str(errors_deleting_files_of_large_objs))
print("Skipped files to delete " + str(skipped_files))
print("Skipped files of large objs to delete " + str(skipped_files_of_large_objs))
else:
print("Delete disabled")
if not delete:
print("Delete disabled")
swift_conn.close()
print("")
print("End Job")
print("___________")
print("___________")
print("___________")