xgiovio/openstack-swift-backup-restore-tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added sync mode for local to swift, added delete function in utility, added script to remove all 0bytes directories, added script to delete large files without manifest

Browse Source
This commit is contained in:
Giovanni Di Grezia
2015-12-09 15:39:18 +01:00
parent 07972c55c5
commit cc67ad7649
11 changed files with 191 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
local_to_swift.py
View File

@@ -7,7 +7,8 @@ from utility import read_in_chunks
import time
import shutil
def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_limit_reading_os,upload,uploadlarge,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns,copy_to_dir,delete):
def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_limit_reading_os,upload,uploadlarge,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns,copy_to_dir,delete,delete_excluded_patterns):
print ("Starting Job")
print ("Localpath " + localpath)
print ("Temppath " + temp_dir)
print ("Swift container " + swift_container)
@@ -30,10 +31,12 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
print ("Encrypted key " + "hidden")
if copy_to_dir != None:
print ("Copy to dir " + copy_to_dir)
print ("Delete files on dest if not present on source " + str(delete))
print("___________")
#checks
utility.check_segments_size (size_limit_reading_os,size_limit_to_segment)
if utility.check_start_slash(swift_container) or utility.check_start_slash(prefix) or not utility.check_end_slash(localpath) or not utility.check_end_slash(temp_dir) or utility.check_end_slash(swift_container) or not utility.check_end_slash(prefix):
@@ -318,6 +321,7 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
dellist = []
delete_skipped_filters = 0
print("")
print("Computing deletion list...")
if encrypted:
@@ -328,28 +332,67 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
for o in list_enc_old:
dellist.append(o)
for rname in remotefiles_encr.keys():
if rname.endswith("_xg10v10_encrypted"):
rname_pure = rname.split("_xg10v10_encrypted")[0]
if rname_pure in remotefiles_encr.keys():
dellist.append(remotefiles_encr[rname_pure])
if rname_pure not in localfiles.keys():
dellist.append(remotefiles_encr[rname])
else:
if rname not in localfiles.keys():
dellist.append(remotefiles_encr[rname])
delete_file = True
for pattern in delete_excluded_patterns:
if pattern in rname:
delete_file = False
print("Skipped " + rname + " due to filters : " + pattern )
delete_skipped_filters +=1
break
if delete_file :
if rname.endswith("_xg10v10_encrypted"):
rname_pure = rname.split("_xg10v10_encrypted")[0]
if rname_pure in remotefiles_encr.keys():
dellist.append(remotefiles_encr[rname_pure])
if rname_pure not in localfiles.keys():
dellist.append(remotefiles_encr[rname])
else:
if rname not in localfiles.keys():
dellist.append(remotefiles_encr[rname])
else:
for rname in remotefiles.keys():
if rname not in localfiles.keys():
dellist.append(rname)
delete_file = True
for pattern in delete_excluded_patterns:
if pattern in rname:
delete_file = False
print("Skipped " + rname + " due to filters : " + pattern )
delete_skipped_filters +=1
break
if delete_file:
if rname not in localfiles.keys():
dellist.append(rname)
print("___________Files to delete______")
for files in dellist:
print(files)
print("___________")
if delete :
print("Files to delete " + str(len(dellist)))
print("Skipped files due to filters " + str(delete_skipped_filters))
if delete and len(dellist) > 0:
errors_deleting_files = 0
errors_deleting_files_of_large_objs = 0
skipped_files = 0
skipped_files_of_large_objs = 0
for object in dellist:
swift_conn = utility.delete_object(swift_conn,swift_container,prefix + object,remotefiles_xobj[object],fail_tries)
swift_conn,errors_deleting_file,errors_deleting_files_of_large_obj,skipped_file,skipped_files_of_large_obj = utility.delete_object(swift_conn,swift_container,prefix + object,remotefiles_xobj[object],fail_tries)
errors_deleting_files += errors_deleting_file
errors_deleting_files_of_large_objs += errors_deleting_files_of_large_obj
skipped_files += skipped_file
skipped_files_of_large_objs += skipped_files_of_large_obj
print("___________")
print("Errors deleting files " + str(errors_deleting_files))
print("Errors deleting files of large objs " + str(errors_deleting_files_of_large_objs))
print("Skipped files to delete " + str(skipped_files))
print("Skipped files of large objs to delete " + str(skipped_files_of_large_objs))
else:
print("Delete disabled")
if not delete:
print("Delete disabled")
swift_conn.close()
print("")
print("End Job")
print("___________")
print("___________")
print("___________")

8
settings_generate_temp_url.py
View File

@@ -1,13 +1,13 @@
__author__ = 'xgiovio'
import generate_temp_url
import swift_generate_temp_url
secretkey =""
set_secretkey = False
create_temp_url = True
duration_in_seconds = 60*60*24*7
objectpath = "/########" # container/object
duration_in_seconds = 60*60*2
objectpath = "#####" # container/object
fail_tries = 100
generate_temp_url.launch(secretkey,set_secretkey,create_temp_url,duration_in_seconds,objectpath,fail_tries)
swift_generate_temp_url.launch(secretkey,set_secretkey,create_temp_url,duration_in_seconds,objectpath,fail_tries)

17
settings_local_to_swift.py
View File

@@ -26,21 +26,20 @@ size_limit_reading_os = 134217728 #must be a power of 2 and smaller/equal than s
# 1GB 1073741824
# 2GB 2147483648
# 4GB 4294967296
upload = False
upload = True
enableLarge = True
delete_remote_files_inexistent_on_source = True
fail_tries = 99999999999
fail_tries = 200
temp_path = "\\\\?\\" + "c:\\temp\\"
excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir"]
delete_excluded_patterns = ["!CB_"]
batch = [
#source, swift container, swift prefix, md5 comparison enabled?, encrypted?, encryption_key, additional_excluded_patterns,copy_to_dir
["\\\\?\\" + "c:\\orig\\","default","prefix/",False,False,None,[],None] ,
["\\\\?\\" + "c:\\orig2\\","default","prefix/",False,False,None,[],None] ,
#["\\\\?\\" + "c:\\orig3\\","default","prefix/",False,False,None,[],None] ,
#source, swift container, swift prefix, md5 comparison enabled?, encrypted?, encryption_key, additional_excluded_patterns,copy_to_dir,delete_inexistent_remote_files,additional_delete_excluded_patterns
["\\\\?\\" + "c:\\test\\","default","test/",False,True,"ciao",[],None,True,[]] ,
["\\\\?\\" + "c:\\test\\","default","test/",False,True,"ciao",[],None,True,[]] ,
]
###############################################################################################
for job in batch:
#local folder,temp path, swift container, swift prefix, size to segment, size reading limit os, upload enabled?. upload large enabled? , fail tries, md5 comparison enabled?, encrypted?, encryption_key,additional_excluded_patterns,copy_to_dir, delete_inexistent_remote_files
local_to_swift.launch(job[0],temp_path,job[1],job[2],size_limit_to_segment,size_limit_reading_os,upload,enableLarge,fail_tries, job[3],job[4],job[5], job[6] + excluded_patterns,job[7],delete_remote_files_inexistent_on_source)
#local folder,temp path, swift container, swift prefix, size to segment, size reading limit os, upload enabled?. upload large enabled? , fail tries, md5 comparison enabled?, encrypted?, encryption_key,additional_excluded_patterns,copy_to_dir, delete_inexistent_remote_files,additional_delete_excluded_patterns
local_to_swift.launch(job[0],temp_path,job[1],job[2],size_limit_to_segment,size_limit_reading_os,upload,enableLarge,fail_tries, job[3],job[4],job[5], job[6] + excluded_patterns,job[7],job[8],job[9] + delete_excluded_patterns)

13
settings_swift_delete_0_byte_folders.py Normal file
View File

@@ -0,0 +1,13 @@
__author__ = 'xgiovio'
import swift_delete_0_byte_folders
#swift
swift_containers = [
"default",
"default_segments"
]
delete = True
fail_tries = 10
for i in range(len(swift_containers)):
swift_delete_0_byte_folders.launch(swift_containers[i],delete,fail_tries)

6
settings_swift_delete_orphan_segments.py
View File

@@ -2,8 +2,8 @@ __author__ = 'xgiovio'
import swift_delete_orphan_segments
#swift
swift_container = "####" # container
delete = False
fail_tries = 100
swift_container = "default" # container
delete = True
fail_tries = 10
swift_delete_orphan_segments.launch(swift_container,delete,fail_tries)

6
settings_swift_to_local.py
View File

@@ -17,11 +17,11 @@ size_limit_reading_os = 134217728 #must be a power of 2 and smaller/equal than s
# 2GB 2147483648
# 4GB 4294967296
download = True
fail_tries = 1000000
excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir"]
fail_tries = 200
excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir","!CB_"]
batch = [
#source, swift container, swift prefix, md5 comparison enabled?, encrypted?, encryption_key, additional_excluded_patterns
["\\\\?\\" + "C:\\test\\","default","test/",False,True,"pass",[]] ,
["\\\\?\\" + "C:\\test\\","default","test/",False,True,"ciao",[]] ,
]

34
swift_delete_0_byte_folders.py Normal file
View File

@@ -0,0 +1,34 @@
import authentication, utility
def launch(swift_container,delete,fail_tries):
print ("Swift container " + swift_container)
print ("Delete " + str(delete))
print ("Fail tries " + str(fail_tries))
print("___________")
swift_conn = authentication.set_authentication()
swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,"")
remotefiles = utility.list_compute_0_byte_folders(objects)
for file in remotefiles:
print("0 byte directory to delete: " + file )
print("___________")
print("Directories to delete " + str(len(remotefiles)) )
if delete and len(remotefiles) > 0:
errors_deleting_files = 0
skipped_files = 0
for file in remotefiles:
swift_conn,errors_deleting_file,_,skipped_file,_ = utility.delete_object(swift_conn,swift_container,file,None,fail_tries)
errors_deleting_files += errors_deleting_file
skipped_files += skipped_file
print("___________")
print("Errors deleting files " + str(errors_deleting_files))
print("Skipped files to delete " + str(skipped_files))
else:
if not delete and len(remotefiles) > 0:
print("Delete Disabled")
print("___________")
print("___________")

49
swift_delete_orphan_segments.py
View File

@@ -6,6 +6,8 @@ import authentication, utility
def launch(swift_container,delete,fail_tries):
print ("Swift container " + swift_container)
print ("Delete " + str(delete))
print ("Fail tries " + str(fail_tries))
print("___________")
swift_conn = authentication.set_authentication ()
@@ -24,8 +26,9 @@ def launch(swift_container,delete,fail_tries):
#_____________________________________________________________________________________________________________________
#______________________________Folders with segments
listfoldersegmentsprefix = []
listfoldersegmentsprefixfull = {}
#______________________________Folders with segments : container: swift_container +"_segments"
#______________________________Folders with segments container: swift_container +"_segments"
#all files on container_segments -> they are only segments
swift_conn,container_segments_objects_raw = utility.get_list(fail_tries,swift_conn,swift_container +"_segments","")
@@ -33,52 +36,87 @@ def launch(swift_container,delete,fail_tries):
container_segments_objects = utility.list (container_segments_objects_raw,"")
#get list only with folders path name from segments
listfolder = []
for name in container_segments_objects.keys():
if (swift_container +"_segments/" + utility.folder_from_path(name,"/")) not in listfoldersegmentsprefixfull.keys():
listfoldersegmentsprefixfull[swift_container +"_segments/" + utility.folder_from_path(name,"/")]= []
listfoldersegmentsprefixfull[swift_container +"_segments/" + utility.folder_from_path(name,"/")].append(swift_container +"_segments/" + name)
if utility.folder_from_path(name,"/") not in listfolder :
listfolder.append(utility.folder_from_path(name,"/"))
for o in listfolder:
listfoldersegmentsprefix.append(swift_container +"_segments/" + o)
#______________________________Folders with segments : container: swift_container path: @SynologyCloudSync/
#______________________________Folders with segments container: swift_container path: @SynologyCloudSync/
#all files @SynologyCloudSync/ on container -> they are only segments
#get dict name:size
container_segments_objects = utility.filter_list_begin(container_objects_raw,"@SynologyCloudSync/","")
#get list only with folders path name from segments
listfolder = []
for name in container_segments_objects.keys():
if (swift_container +"/" + utility.folder_from_path(name,"/")) not in listfoldersegmentsprefixfull.keys():
listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")]=[]
listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")].append(swift_container +"/" + name)
if utility.folder_from_path(name,"/") not in listfolder :
listfolder.append(utility.folder_from_path(name,"/"))
for o in listfolder:
listfoldersegmentsprefix.append(swift_container +"/" + o)
#______________________________Folders with segments "!CB_"
#______________________________Folders with segments container: swift_container path: files with !CB_
#all files !CB_ of CloudBerry on container -> they are only segments
#get dict name:size
container_segments_objects = utility.search_list(container_objects_raw,"!CB_","")
#get list only with folders path name from segments
listfolder = []
for name in container_segments_objects.keys():
if (swift_container +"/" + utility.folder_from_path(name,"/")) not in listfoldersegmentsprefixfull.keys():
listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")]=[]
listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")].append(swift_container +"/" + name)
if utility.folder_from_path(name,"_") not in listfolder :
listfolder.append(utility.folder_from_path(name,"_"))
for o in listfolder:
listfoldersegmentsprefix.append(swift_container +"/" + o)
#_____________________________________________________________________________________________________________________
'''
for o in listfoldersegmentsprefix:
print(o)
'''
#_____________________________________________________________________________________________________________________
print("___________")
segments_to_delete = []
segments_not_listed = []
for manifesturlsegments in listfoldersegmentsprefix :
if manifesturlsegments not in listfoldermanifest:
segments_to_delete.append(manifesturlsegments)
print("Segments folders to delete: " + str(len(segments_to_delete)))
for manifest in segments_to_delete:
print("Segments folder to delete: " + manifest)
print("___________")
if delete:
errors_deleting_files = 0
skipped_files = 0
for short,longlist in listfoldersegmentsprefixfull.items():
if short in segments_to_delete:
for long in longlist:
del_container = long.split("/")[0]
del_object = long.replace(del_container + "/","")
swift_conn,errors_deleting_file,_,skipped_file,_ = utility.delete_object(swift_conn,del_container,del_object,None,fail_tries)
errors_deleting_files += errors_deleting_file
skipped_files += skipped_file
print("___________")
print("Errors deleting files " + str(errors_deleting_files))
print("Skipped files to delete " + str(skipped_files))
else:
print("Delete Disabled")
print("___________")
#_____
for manifesturl in listfoldermanifest :
if manifesturl not in listfoldersegmentsprefix:
segments_not_listed.append(manifesturl)
@@ -86,4 +124,9 @@ def launch(swift_container,delete,fail_tries):
for manifest in segments_not_listed:
print("Segments not present in given segments list : " + manifest)
#_____
swift_conn.close()

0
generate_temp_url.py → swift_generate_temp_url.py
View File

22
utility.py
View File

@@ -234,6 +234,10 @@ def set_dash():
def delete_object (swift_conn,swift_container,object,manifest,fail_tries):
errors_deleting_file = 0
errors_deleting_files_of_large_obj = 0
skipped_file = 0
skipped_files_of_large_obj = 0
for fail_tries_counter in range (fail_tries) :
try:
print("Deleting " + swift_container + "/" + object)
@@ -254,9 +258,11 @@ def delete_object (swift_conn,swift_container,object,manifest,fail_tries):
except Exception as e:
print("Exception during deletion of manifest files")
print(e)
errors_deleting_files_of_large_obj +=1
time.sleep(1)
if fail_tries_counter1 == fail_tries - 1 :
print("Maximum tries reached. Can't delete " + swift_container + "/" + object +" manifest files.Skipping")
skipped_files_of_large_obj += 1
else:
swift_conn = authentication.set_authentication ()
else :
@@ -264,12 +270,26 @@ def delete_object (swift_conn,swift_container,object,manifest,fail_tries):
except Exception as e:
print("Exception during deletion of file")
print(e)
errors_deleting_file += 1
time.sleep(1)
if fail_tries_counter == fail_tries - 1 :
print("Maximum tries reached. Can't delete " + swift_container + "/" + object +".Skipping")
skipped_file +=1
else:
swift_conn = authentication.set_authentication ()
else :
break
return swift_conn
return [swift_conn,errors_deleting_file,errors_deleting_files_of_large_obj,skipped_file,skipped_files_of_large_obj]
def list_compute_0_byte_folders (objects):
remotefiles = []
for o in objects :
if o["content_type"] == "application/directory" and int(o["bytes"]) == 0:
remotefiles.append(o["name"])
return remotefiles

1
utility_aes.py
View File

@@ -1,6 +1,7 @@
import hashlib,os
from utility import read_in_chunks
try:
pycrypto = True
from Crypto.Cipher import AES