guitarrapc/s3sync

Amazon S3 Content Synchronization with .NET

s3sync's Issues

DryRun Support

No File Change but notify what change will be shown.

Array dimensions exceeded supported range

Hi, bug found Array dimensions exceeded supported range., System.OverflowException, at S3Manager.Core.LocalFiles.FileHashHelper.GetFileBinary(String fileName) at S3Manager.Core.S3Client.<>c__DisplayClass44_0.<GetSysncStatus>b__4(KeyValuePair2 x)
at System.Linq.Enumerable.SelectEnumerableIterator2.ToArray() at System.Linq.Enumerable.ToArray[TSource](IEnumerable1 source)
at S3Manager.Core.S3Client.GetSysncStatus(IEnumerable1 localFiles, IEnumerable1 s3Objects, String prefix)
at S3Manager.Core.S3Client.SyncWithLocal(SlimFileInfo[] localFileInfos, String bucketName, String prefix, String ignorePrefix, Action1 uploadCallback) at MTBackup.Program.MainCoreAsync(String[] args) at MTBackup.Program.Main(String[] args)

Once you try to sync files larger than 2GB

Docker Support

https://hub.docker.com/r/guitarrapc/s3sync/

.NET Core 3.1 support

ContentType Support

Use latest AWS.SDK 3.7.x

S3Sync/source/S3Sync/S3Sync.csproj

Line 10 in 21e0333

<PackageReference Include="AWSSDK.S3" Version="3.3.111.33" />

What do these ServicePointManager changes accomplish?

S3Sync/source/S3Sync/Program.cs

Lines 129 to 135 in 21e0333

	static void ApplyInitialConfiguration()
	{
	// Web周りの設定
	ServicePointManager.UseNagleAlgorithm = false;
	ServicePointManager.Expect100Continue = false;
	ServicePointManager.DefaultConnectionLimit = 96;
	}

Use a command line parsing library

S3Sync/source/S3Sync/Program.cs

Lines 137 to 251 in 21e0333

	static void EvaluateArguments(string[] args)
	{
	// BucketName=nantokakantokabucket
	BucketName = args.Where(x => x.StartsWith(ArgumentType.BucketName.ToString(), StringComparison.InvariantCultureIgnoreCase))
	.SelectMany(x => x.SplitEx("="))
	.LastOrDefault()
	?? GetEnvValueString(ArgumentType.BucketName, EnvType.S3Sync_BucketName);
	// LocalRoot=c:\hogemoge
	LocalRoot = args.Where(x => x.StartsWith(ArgumentType.LocalRoot.ToString(), StringComparison.InvariantCultureIgnoreCase))
	.SelectMany(x => x.SplitEx("="))
	.LastOrDefault()
	?? GetEnvValueString(ArgumentType.LocalRoot, EnvType.S3Sync_LocalRoot);
	// KeyPrefix=image
	KeyPrefix = args.Where(x => x.StartsWith(ArgumentType.KeyPrefix.ToString(), StringComparison.InvariantCultureIgnoreCase))
	.SelectMany(x => x.SplitEx("="))
	.LastOrDefault()
	?.TrimEnd('/')
	?? GetEnvValueString(ArgumentType.KeyPrefix, EnvType.S3Sync_KeyPrefix);
	// IgnoreKeyPrefix=image
	IgnoreKeyPrefix = args.Where(x => x.StartsWith(ArgumentType.IgnoreKeyPrefix.ToString(), StringComparison.InvariantCultureIgnoreCase))
	.SelectMany(x => x.SplitEx("="))
	.LastOrDefault()
	?.TrimEnd('/')
	?? GetEnvValueString(ArgumentType.IgnoreKeyPrefix, EnvType.S3Sync_IgnoreKeyPrefix);
	// ExcludeFiles=hogemoge,fugafuga
	ExcludeFiles = args.Where(x => x.StartsWith(ArgumentType.ExcludeFiles.ToString(), StringComparison.InvariantCultureIgnoreCase))
	.SelectMany(x => x.SplitEx("="))
	.LastOrDefault()
	?.SplitEx(",")
	.Select(x => x.Trim())
	.ToArray()
	?? GetEnvValueString(ArgumentType.ExcludeFiles, EnvType.S3Sync_ExcludeFiles)
	?.SplitEx(",");
	// ExcludeDirectories=hogemoge,fugafuga
	ExcludeDirectories = args.Where(x => x.StartsWith(ArgumentType.ExcludeDirectories.ToString(), StringComparison.InvariantCultureIgnoreCase))
	.SelectMany(x => x.SplitEx("="))
	.LastOrDefault()
	?.SplitEx(",")
	?.Select(x => x.Trim())
	.ToArray()
	?? GetEnvValueString(ArgumentType.ExcludeDirectories, EnvType.S3Sync_ExcludeDirectories)
	?.SplitEx(",");
	// Silent=false
	Silent = bool.Parse(args.Where(x => x.StartsWith(ArgumentType.Silent.ToString(), StringComparison.InvariantCultureIgnoreCase))
	.SelectMany(x => x.SplitEx("="))
	.Where(x => string.Equals(x, "true", StringComparison.InvariantCultureIgnoreCase) || string.Equals(x, "false", StringComparison.InvariantCultureIgnoreCase))
	.LastOrDefault()
	?.Trim()
	?? GetEnvValueString(ArgumentType.Silent, EnvType.S3Sync_Silent)
	?? "false");
	// CredentialProfile=ProfileName
	CredentialProfile = args.Where(x => x.StartsWith(ArgumentType.CredentialProfile.ToString(), StringComparison.InvariantCultureIgnoreCase))
	.SelectMany(x => x.SplitEx("="))
	.LastOrDefault()
	?.Trim()
	?? GetEnvValueString(ArgumentType.CredentialProfile, EnvType.S3Sync_CredentialProfile);
	// DryRun=true
	Option.DryRun = bool.Parse(args.Where(x => x.StartsWith(ArgumentType.DryRun.ToString(), StringComparison.InvariantCultureIgnoreCase))
	.SelectMany(x => x.SplitEx("="))
	.Where(x => string.Equals(x, "true", StringComparison.InvariantCultureIgnoreCase) || string.Equals(x, "false", StringComparison.InvariantCultureIgnoreCase))
	.LastOrDefault()
	?.Trim()
	?? GetEnvValueString(ArgumentType.DryRun, EnvType.S3Sync_DryRun)
	?? "true");
	// ContentType=application/json
	Option.ContentType = args.Where(x => x.StartsWith(ArgumentType.ContentType.ToString(), StringComparison.InvariantCultureIgnoreCase))
	.SelectMany(x => x.SplitEx("="))
	.LastOrDefault()
	?.TrimEnd('/')
	?? GetEnvValueString(ArgumentType.ContentType, EnvType.S3Sync_ContentType);
	// Region=us-east-1
	Option.Region = args.Where(x => x.StartsWith(ArgumentType.Region.ToString(), StringComparison.InvariantCultureIgnoreCase))
	.SelectMany(x => x.SplitEx("="))
	.LastOrDefault()
	?.TrimEnd('/')
	?? GetEnvValueString(ArgumentType.Region, EnvType.S3Sync_Region);
	// Show Arguments
	Log($"{nameof(BucketName)} : {BucketName}");
	Log($"{nameof(LocalRoot)} : {LocalRoot}");
	Log($"{nameof(KeyPrefix)} : {KeyPrefix}");
	Log($"{nameof(IgnoreKeyPrefix)} : {IgnoreKeyPrefix}");
	Log($"{nameof(ExcludeFiles)} : {ExcludeFiles?.ToJoinedString(",")}");
	Log($"{nameof(ExcludeDirectories)} : {ExcludeDirectories?.ToJoinedString(",")}");
	Log($"{nameof(Silent)} : {Silent}");
	Log($"{nameof(CredentialProfile)} : {CredentialProfile}");
	Log($"{nameof(Option.DryRun)} : {Option.DryRun}");
	Log($"{nameof(Option.ContentType)} : {Option.ContentType}");
	Log($"{nameof(Option.Region)} : {Option.Region}");
	// Validate Required arguments
	if (string.IsNullOrWhiteSpace(BucketName))
	{
	Error("Please pass arguments. See detail followings.");
	PrintHelp();
	throw new NullReferenceException(nameof(BucketName));
	}
	if (string.IsNullOrWhiteSpace(LocalRoot))
	{
	Error("Please pass arguments. See detail followings.");
	PrintHelp();
	throw new NullReferenceException(nameof(LocalRoot));
	}
	}