226 lines
8.4 KiB
Python
226 lines
8.4 KiB
Python
__author__ = 'xgiovio'
|
|
|
|
import authentication
|
|
import sys, platform,os
|
|
import time
|
|
|
|
|
|
def get_local_files (localpath):
|
|
print("Building local files list")
|
|
localfiles = {}
|
|
for root, dirs, files in os.walk(localpath, topdown=True):
|
|
for name in files:
|
|
localfiles[os.path.join(root.replace(localpath,""), name)]=int(os.stat(os.path.join(root, name)).st_size)
|
|
|
|
#skip dirs
|
|
'''
|
|
for name in dirs:
|
|
print(os.path.join(root, name))
|
|
'''
|
|
#end skip
|
|
return localfiles
|
|
|
|
def get_list (fail_tries,swift_conn,swift_container,prefix) :
|
|
for fail_tries_counter in range (fail_tries) :
|
|
try:
|
|
print ("Downloading remote list for " + swift_container + " with prefix " +prefix + " ... ")
|
|
headers,objects = swift_conn.get_container(swift_container, prefix =prefix,full_listing=True )
|
|
except Exception as e:
|
|
print("Exception during the download of remote list")
|
|
print(e)
|
|
time.sleep(1)
|
|
if fail_tries_counter == fail_tries - 1 :
|
|
print("Maximum tries reached. Can't download remote list for container " + swift_container + " with prefix " +prefix +". Exiting.")
|
|
sys.exit(-1)
|
|
else:
|
|
swift_conn = authentication.set_authentication ()
|
|
else :
|
|
break
|
|
return [swift_conn,objects]
|
|
|
|
def filter_list_begin(objects,beginpattern,prefix):
|
|
filtered = {}
|
|
for o in objects :
|
|
if o["content_type"] != "application/directory":
|
|
if (o["name"]).find(beginpattern)== 0:
|
|
filtered[(o["name"].replace(prefix,""))]=int(o["bytes"])
|
|
return filtered
|
|
|
|
def search_list (objects,pattern,prefix):
|
|
filtered = {}
|
|
for o in objects :
|
|
if o["content_type"] != "application/directory":
|
|
if pattern in o["name"]:
|
|
filtered[(o["name"].replace(prefix,""))]=int(o["bytes"])
|
|
return filtered
|
|
|
|
|
|
def list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix):
|
|
|
|
remotefiles = {}
|
|
remotefiles_md5 = {}
|
|
byte0real = 0
|
|
byte0manifest = 0
|
|
|
|
for o in objects :
|
|
if o["content_type"] != "application/directory":
|
|
if int(o["bytes"]) == 0 :
|
|
print ("Requesting metadata for 0byte file " + o["name"] )
|
|
for fail_tries_counter in range (fail_tries) :
|
|
try:
|
|
oheaders = swift_conn.head_object(swift_container,o["name"])
|
|
except Exception as e:
|
|
print("Exception during the request of metadata")
|
|
print(e)
|
|
time.sleep(1)
|
|
if fail_tries_counter == fail_tries - 1 :
|
|
print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +". Exiting.")
|
|
sys.exit(-1)
|
|
else:
|
|
swift_conn = authentication.set_authentication ()
|
|
else :
|
|
break
|
|
if "x-object-manifest" in oheaders.keys():
|
|
print ("0byte file " + o["name"] + " e' un large file" )
|
|
o["bytes"] = oheaders["content-length"]
|
|
if "_xg10v10_" in oheaders["x-object-manifest"]:
|
|
split1 = oheaders["x-object-manifest"].split("_xg10v10_")[0]
|
|
split2 = split1.split("/")[1]
|
|
o["hash"] = split2
|
|
else:
|
|
print("Impossible to get remote large file md5. Cause: Not uploaded with xgiovio method (md5 in x-object-manifest)")
|
|
o["hash"] = "0"
|
|
byte0manifest = byte0manifest + 1
|
|
else:
|
|
print ("0byte file " + o["name"] + " e' un file normale" )
|
|
byte0real = byte0real + 1
|
|
remotefiles[remote_dash_replace(o["name"].replace(prefix,""))]=int(o["bytes"])
|
|
remotefiles_md5[remote_dash_replace(o["name"].replace(prefix,""))]=o["hash"]
|
|
|
|
|
|
return [byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5]
|
|
|
|
|
|
|
|
def list_compute_correct_names_for_enctyption (objects,prefix):
|
|
|
|
remotefiles = {}
|
|
|
|
for o in objects :
|
|
if o["content_type"] != "application/directory":
|
|
full = remote_dash_replace(o["name"].replace(prefix,""))
|
|
if (full.endswith("_xg10v10_encrypted")):
|
|
encrypted_name_only = full.split("_xg10v10_")[0] + "_xg10v10_encrypted"
|
|
else:
|
|
encrypted_name_only = full
|
|
if encrypted_name_only in remotefiles.keys():
|
|
if int(remotefiles[encrypted_name_only].split("_xg10v10_")[3]) < int(full.split("_xg10v10_")[3]) :
|
|
remotefiles[encrypted_name_only] = full
|
|
else:
|
|
remotefiles[encrypted_name_only] = full
|
|
return remotefiles
|
|
|
|
|
|
def list_compute_manifest (fail_tries,objects,swift_conn,swift_container,prefix):
|
|
|
|
remotefiles = {}
|
|
byte0manifest = 0
|
|
|
|
for o in objects :
|
|
if o["content_type"] != "application/directory":
|
|
if int(o["bytes"]) == 0 :
|
|
print ("Requesting metadata for 0byte file " + o["name"] )
|
|
for fail_tries_counter in range (fail_tries) :
|
|
try:
|
|
oheaders = swift_conn.head_object(swift_container,o["name"])
|
|
except Exception as e:
|
|
print("Exception during the request of metadata")
|
|
print(e)
|
|
time.sleep(1)
|
|
if fail_tries_counter == fail_tries - 1 :
|
|
print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +". Exiting.")
|
|
sys.exit(-1)
|
|
else:
|
|
swift_conn = authentication.set_authentication ()
|
|
else :
|
|
break
|
|
if "x-object-manifest" in oheaders.keys():
|
|
byte0manifest = byte0manifest + 1
|
|
remotefiles[(o["name"].replace(prefix,""))]=oheaders["x-object-manifest"]
|
|
|
|
return [byte0manifest,swift_conn,remotefiles]
|
|
|
|
def list (objects,prefix):
|
|
|
|
remotefiles = {}
|
|
for o in objects :
|
|
if o["content_type"] != "application/directory":
|
|
remotefiles[(o["name"].replace(prefix,""))]=int(o["bytes"])
|
|
|
|
return remotefiles
|
|
|
|
def listall (objects,prefix):
|
|
|
|
remotefiles = {}
|
|
for o in objects :
|
|
remotefiles[(o["name"].replace(prefix,""))]=int(o["bytes"])
|
|
return remotefiles
|
|
|
|
def check_segments_size (size_limit_reading_os,size_limit_to_segment):
|
|
if size_limit_reading_os%2 != 0 or size_limit_to_segment%2 != 0 or size_limit_reading_os > size_limit_to_segment or size_limit_reading_os < 16:
|
|
print ("Error size segments")
|
|
sys.exit(-2)
|
|
|
|
def check_segments_size_single (size_limit_reading_os):
|
|
if size_limit_reading_os%2 != 0 or size_limit_reading_os < 16:
|
|
print ("Error size segments")
|
|
sys.exit(-2)
|
|
|
|
def file_only_name (stringa,delimiter):
|
|
return stringa.split(delimiter)[len(stringa.split(delimiter))-1]
|
|
|
|
def folder_from_path (stringa,delimiter):
|
|
string_to_ret =""
|
|
for i in range(len(stringa.split(delimiter)) - 1):
|
|
string_to_ret = string_to_ret + stringa.split(delimiter)[i] + delimiter
|
|
return string_to_ret
|
|
|
|
def check_end_slash (stringa) :
|
|
lastchar = stringa[len(stringa) - 1]
|
|
if lastchar == "\\" or lastchar == "/" :
|
|
return True
|
|
return False
|
|
def check_start_slash (stringa) :
|
|
firstchar = stringa[0]
|
|
if firstchar == "\\" or firstchar == "/" :
|
|
return True
|
|
return False
|
|
|
|
def read_in_chunks(file_object, chunk_size):
|
|
while True:
|
|
data = file_object.read(chunk_size)
|
|
if not data:
|
|
break
|
|
yield data
|
|
|
|
def total_size_encrypted (original_size) :
|
|
if original_size % 16 == 0:
|
|
return original_size + 17
|
|
return original_size + ( 16 - original_size % 16) + 17
|
|
|
|
|
|
def dash_replace (string):
|
|
if platform.system() == "Windows":
|
|
return string.replace("\\","/")
|
|
return string
|
|
|
|
def remote_dash_replace (string):
|
|
if platform.system() == "Windows":
|
|
return string.replace("/","\\")
|
|
return string
|
|
|
|
def set_dash():
|
|
if platform.system() == "Windows":
|
|
return "\\"
|
|
return "/"
|