Skip to content

Enable use of a subfolder in S3 #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
austinweisgrau wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Enable use of a subfolder in S3 #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,15 @@ To use an Amazon S3 bucket as the backend, you should first install the AWS CLI

[s3]
bucket = s3://your-s3-bucket

Optionally you can use a subfolder within the S3 bucket, which would
look like this:

[s3]
bucket = s3://your-s3-bucket
folder = your-subfolder

Which would resolve to the S3 folder `s3://your-s3-bucket/your-subfolder/`

# A worked example

Expand Down
38 changes: 16 additions & 22 deletions git-fat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,11 +203,13 @@ class GitFat(object):
# Cheat here
return path

def get_all_remote_files(self, bucket: str):
def get_all_remote_files(self, bucket: str, folder: str = ""):
if bucket.startswith("s3://"):
bucket = bucket.replace("s3://", "")

cmd = ["aws", "s3api", "list-objects-v2", "--bucket", bucket]
if folder:
cmd.extend(["--prefix", folder + "/"])
output = []
try:
initial_output = subprocess.check_output(cmd)
Expand Down Expand Up @@ -242,29 +244,27 @@ class GitFat(object):

return output

def get_aws_cmd(self, push, s3_bucket, files):
if not which('aws'):
sys.stderr.write('Could not find aws cli install.\n')
def get_aws_cmd(self, push, s3_bucket, files, folder=""):
if not which("aws"):
sys.stderr.write("Could not find aws cli install.\n")
sys.exit(1)

if not s3_bucket.startswith('s3://'):
if not s3_bucket.startswith("s3://"):
s3_bucket = "s3://{}".format(s3_bucket)
cmds = []
remote_files = self.get_all_remote_files(s3_bucket)
remote_files = self.get_all_remote_files(s3_bucket, folder=folder)

if folder and (not folder.endswith("/")):
folder = folder + "/"

# self.verbose(remote_files)
if push:
self.verbose('Pushing to %s' % (s3_bucket))
for file in files:
self.verbose('Processing {}'.format(file))
local_path = os.path.join(self.objdir, file)
bucket_path = s3_bucket + "/" + file
cmd = [
"aws",
"s3",
"cp",
local_path,
bucket_path
]
bucket_path = s3_bucket + "/" + folder + file
cmd = ["aws", "s3", "cp", local_path, bucket_path]

remote_item = self.get_remote_item(file, remote_files)
if remote_item == {}:
Expand All @@ -280,14 +280,8 @@ class GitFat(object):
for file in files:
self.verbose('Processing {}'.format(file))
local_path = os.path.join(self.objdir, file)
bucket_path = s3_bucket + "/" + file
cmd = [
"aws",
"s3",
"cp",
bucket_path,
local_path
]
bucket_path = s3_bucket + "/" + folder + file
cmd = ["aws", "s3", "cp", bucket_path, local_path]
if not os.path.exists(local_path):
cmds.append(cmd)
continue
Expand Down