import logging
import mimetypes
import os
from urllib.parse import urlparse
from boto import connect_s3
import boto3
import botocore.exceptions
try:
from biothings import config
except ImportError:
# assuming key, secret and bucket will be passed
# to all functions
pass
def key_exists(bucket, s3key, aws_key=None, aws_secret=None):
client = boto3.client("s3", aws_access_key_id=aws_key, aws_secret_access_key=aws_secret)
try:
client.head_object(Bucket=bucket, Key=s3key)
return True
except Exception as e:
if e.response["Error"]["Code"] == "404":
return False
else:
raise
[docs]def send_s3_file(localfile, s3key, overwrite=False, permissions=None, metadata=None,
content=None, content_type=None, aws_key=None, aws_secret=None, s3_bucket=None):
'''save a localfile to s3 bucket with the given key.
bucket is set via S3_BUCKET
it also save localfile's lastmodified time in s3 file's metadata
'''
metadata = metadata or {}
try:
aws_key = aws_key or getattr(config, "AWS_SECRET")
aws_secret = aws_secret or getattr(config, "AWS_SECRET")
s3_bucket = s3_bucket or getattr(config, "S3_BUCKET")
except AttributeError:
logging.info("Skip sending file to S3, missing information in config file: AWS_KEY, AWS_SECRET or S3_BUCKET")
return
s3 = connect_s3(aws_key, aws_secret)
bucket = s3.get_bucket(s3_bucket)
# bucket_location = bucket.get_location() # TODO: delete this unused variable
k = bucket.new_key(s3key)
if not overwrite:
assert not k.exists(), 's3key "{}" already exists.'.format(s3key)
for header in metadata:
k.set_metadata(header, metadata[header])
if content_type:
k.content_type = content_type
if content:
k.set_contents_from_string(content)
else:
assert os.path.exists(localfile), 'localfile "{}" does not exist.'.format(localfile)
lastmodified = os.stat(localfile)[-2]
k.set_metadata('lastmodified', lastmodified)
k.set_contents_from_filename(localfile)
if permissions:
k.set_acl(permissions)
[docs]def send_s3_big_file(localfile, s3key, overwrite=False, acl=None,
aws_key=None, aws_secret=None, s3_bucket=None,
storage_class=None):
"""
Multiparts upload for file bigger than 5GiB
"""
# TODO: maybe merge with send_s3_file() based in file size ? It would need boto3 migration
try:
aws_key = aws_key or getattr(config, "AWS_SECRET")
aws_secret = aws_secret or getattr(config, "AWS_SECRET")
s3_bucket = s3_bucket or getattr(config, "S3_BUCKET")
except AttributeError:
logging.info("Skip sending file to S3, missing information in config file: AWS_KEY, AWS_SECRET or S3_BUCKET")
return
client = boto3.client("s3", aws_access_key_id=aws_key, aws_secret_access_key=aws_secret)
if not overwrite and key_exists(s3_bucket, s3key, aws_key, aws_secret):
raise Exception("Key '%s' already exist" % s3key)
tfr_config = boto3.s3.transfer.TransferConfig(
multipart_threshold=1024 * 25,
max_concurrency=10,
multipart_chunksize=1024 * 25,
use_threads=True
)
extra = {
"ACL": acl or "private",
"ContentType": mimetypes.MimeTypes().guess_type(localfile)[0] or "binary/octet-stream",
"StorageClass": storage_class or "REDUCED_REDUNDANCY"
}
client.upload_file(Filename=localfile, Bucket=s3_bucket, Key=s3key, ExtraArgs=extra, Config=tfr_config)
def get_s3_file(s3key, localfile=None, return_what=False,
aws_key=None, aws_secret=None, s3_bucket=None):
aws_key = aws_key or getattr(config, "AWS_SECRET")
aws_secret = aws_secret or getattr(config, "AWS_SECRET")
s3_bucket = s3_bucket or getattr(config, "S3_BUCKET")
localfile = localfile or os.path.basename(s3key)
s3 = connect_s3(aws_key, aws_secret)
bucket = s3.get_bucket(s3_bucket)
k = bucket.get_key(s3key)
if not k:
raise FileNotFoundError(s3key)
if return_what == "content":
return k.get_contents_as_string()
elif return_what == "key":
return k
else:
k.get_contents_to_filename(localfile)
def get_s3_folder(s3folder, basedir=None, aws_key=None, aws_secret=None, s3_bucket=None):
aws_key = aws_key or getattr(config, "AWS_SECRET")
aws_secret = aws_secret or getattr(config, "AWS_SECRET")
s3_bucket = s3_bucket or getattr(config, "S3_BUCKET")
s3 = connect_s3(aws_key, aws_secret)
bucket = s3.get_bucket(s3_bucket)
cwd = os.getcwd()
try:
if basedir:
os.chdir(basedir)
if not os.path.exists(s3folder):
os.makedirs(s3folder)
for k in bucket.list(s3folder):
get_s3_file(k.key, localfile=k.key, aws_key=aws_key, aws_secret=aws_secret, s3_bucket=s3_bucket)
finally:
os.chdir(cwd)
def send_s3_folder(folder, s3basedir=None, acl=None, overwrite=False,
aws_key=None, aws_secret=None, s3_bucket=None):
aws_key = aws_key or getattr(config, "AWS_SECRET")
aws_secret = aws_secret or getattr(config, "AWS_SECRET")
s3_bucket = s3_bucket or getattr(config, "S3_BUCKET")
s3 = connect_s3(aws_key, aws_secret)
s3.get_bucket(s3_bucket) # check if s3_bucket exists
cwd = os.getcwd()
if not s3basedir:
s3basedir = os.path.basename(cwd)
for localf in [f for f in os.listdir(folder) if not f.startswith(".")]:
fullpath = os.path.join(folder, localf)
if os.path.isdir(fullpath):
send_s3_folder(fullpath, os.path.join(s3basedir, localf),
overwrite=overwrite, acl=acl,
aws_key=aws_key, aws_secret=aws_secret, s3_bucket=s3_bucket)
else:
send_s3_big_file(fullpath, os.path.join(s3basedir, localf),
overwrite=overwrite, acl=acl,
aws_key=aws_key, aws_secret=aws_secret, s3_bucket=s3_bucket)
def get_s3_url(s3key, aws_key=None, aws_secret=None, s3_bucket=None):
try:
k = get_s3_file(s3key, return_what="key",
aws_key=aws_key, aws_secret=aws_secret, s3_bucket=s3_bucket)
except FileNotFoundError:
return None
# generate_url will include some acdesskey, signature, etc... we want to remove this
# as the bucket is public anyway and want "clean" url
url = k.generate_url(expires_in=0) # never (and whatever, we
return urlparse(url)._replace(query="").geturl()
[docs]def create_bucket(name, region=None, aws_key=None, aws_secret=None, acl=None,
ignore_already_exists=False):
"""Create a S3 bucket "name" in optional "region". If aws_key and aws_secret
are set, S3 client will these, otherwise it'll use default system-wide setting.
"acl" defines permissions on the bucket: "private" (default), "public-read",
"public-read-write" and "authenticated-read"
"""
client = boto3.client("s3", aws_access_key_id=aws_key, aws_secret_access_key=aws_secret)
acl = acl or "private"
kwargs = {"ACL": acl, "Bucket": name}
if region:
kwargs["CreateBucketConfiguration"] = {"LocationConstraint": region}
try:
client.create_bucket(**kwargs)
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == "BucketAlreadyOwnedByYou" and not ignore_already_exists:
raise
def set_static_website(name, aws_key=None, aws_secret=None, index="index.html", error="error.html"):
client = boto3.client("s3", aws_access_key_id=aws_key, aws_secret_access_key=aws_secret)
conf = {'IndexDocument': {'Suffix': index},
'ErrorDocument': {'Key': error}}
client.put_bucket_website(Bucket=name, WebsiteConfiguration=conf)
location = client.get_bucket_location(Bucket=name)
region = location["LocationConstraint"]
# generate website URL
return "http://%(name)s.s3-website-%(region)s.amazonaws.com" % {"name": name, "region": region}