import os
from tqdm import tqdm
import botocore
import boto3
from boto3.session import Session
import threading
import sys
import warnings
warnings.filterwarnings("ignore")
class ProgressPercentage:
def __init__(self,filename):
self.filename = filename
self.size = float(os.path.getsize(filename))
self._seen_so_far = 0
self._lock = threading.Lock()
def __call__(self, bytes_amount):
with self._lock:
self._seen_so_far += bytes_amount
percentage = (self._seen_so_far / self.size) * 100
print(
"\r%s %s / %s (%.2f%%)" % (
self.filename, self._seen_so_far, self.size,
percentage),end="")
class R2:
def __init__(self,access_key=None,secret_key=None,endpoint_url=None,bucket=None):
self.access_key = access_key
self.secret_key = secret_key
self.endpoint_url = endpoint_url
self.bucket = bucket
self.client = None
def initialize(self):
if self.access_key is None or self.secret_key is None or self.endpoint_url is None:
print("Please provide access_key, secret_key and endpoint_url")
return
self.client = boto3.client('s3',
verify=False,
endpoint_url=self.endpoint_url,
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key)
return self
def upload_file(self,local_file,cloud_path,bucket=None,progress_callback=None):
if bucket is None and self.bucket is None:
print("Please provide bucket name")
return
if bucket is None:
bucket = self.bucket
if progress_callback is None:
try:
self.client.upload_file(local_file,bucket,cloud_path+'/'+os.path.basename(local_file))
except botocore.exceptions.ClientError as e:
print(e)
else:
try:
self.client.upload_file(local_file,bucket,cloud_path+'/'+os.path.basename(local_file),Callback=progress_callback)
except botocore.exceptions.ClientError as e:
print(e)
def upload_folder(self,local_folder,cloud_path,bucket=None):
if bucket is None and self.bucket is None:
print("Please provide bucket name")
return
if bucket is None:
bucket = self.bucket
stream = tqdm(os.walk(local_folder))
for root, dirs, files in stream:
for file in files:
localFilePath = os.path.join(root, file)
relativePath = os.path.relpath(localFilePath, local_folder)
cloudPath = os.path.join(cloud_path, relativePath)
cloudPath = cloudPath.replace('\\', '/')
try:
self.client.upload_file(localFilePath, bucket, cloudPath, Callback=ProgressPercentage(localFilePath))
print()
except botocore.exceptions.ClientError as e:
print(e)
使用boto3上传大文件至cloudflare R2
发布于 2022-12-30 1500 次阅读
Comments NOTHING