EntityTooSmall in CompleteMultipartUploadResponse - c#

using .NET SDK v.1.5.21.0
I'm trying to upload a large file (63Mb) and I'm following the example at:
http://docs.aws.amazon.com/AmazonS3/latest/dev/LLuploadFileDotNet.html
But using a helper instead the hole code and using jQuery File Upload
https://github.com/blueimp/jQuery-File-Upload/blob/master/basic-plus.html
what I have is:
string bucket = "mybucket";
long totalSize = long.Parse(context.Request.Headers["X-File-Size"]),
maxChunkSize = long.Parse(context.Request.Headers["X-File-MaxChunkSize"]),
uploadedBytes = long.Parse(context.Request.Headers["X-File-UloadedBytes"]),
partNumber = uploadedBytes / maxChunkSize + 1,
fileSize = partNumber * inputStream.Length;
bool lastPart = inputStream.Length < maxChunkSize;
// http://docs.aws.amazon.com/AmazonS3/latest/dev/LLuploadFileDotNet.html
if (partNumber == 1) // initialize upload
{
iView.Utilities.Amazon_S3.S3MultipartUpload.InitializePartToCloud(fileName, bucket);
}
try
{
// upload part
iView.Utilities.Amazon_S3.S3MultipartUpload.UploadPartToCloud(fs, fileName, bucket, (int)partNumber, uploadedBytes, maxChunkSize);
if (lastPart)
// wrap it up and go home
iView.Utilities.Amazon_S3.S3MultipartUpload.CompletePartToCloud(fileName, bucket);
}
catch (System.Exception ex)
{
// Huston, we have a problem!
//Console.WriteLine("Exception occurred: {0}", exception.Message);
iView.Utilities.Amazon_S3.S3MultipartUpload.AbortPartToCloud(fileName, bucket);
}
and
public static class S3MultipartUpload
{
private static string accessKey = System.Configuration.ConfigurationManager.AppSettings["AWSAccessKey"];
private static string secretAccessKey = System.Configuration.ConfigurationManager.AppSettings["AWSSecretKey"];
private static AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(accessKey, secretAccessKey);
public static InitiateMultipartUploadResponse initResponse;
public static List<UploadPartResponse> uploadResponses;
public static void InitializePartToCloud(string destinationFilename, string destinationBucket)
{
// 1. Initialize.
uploadResponses = new List<UploadPartResponse>();
InitiateMultipartUploadRequest initRequest =
new InitiateMultipartUploadRequest()
.WithBucketName(destinationBucket)
.WithKey(destinationFilename.TrimStart('/'));
initResponse = client.InitiateMultipartUpload(initRequest);
}
public static void UploadPartToCloud(Stream fileStream, string destinationFilename, string destinationBucket, int partNumber, long uploadedBytes, long maxChunkedBytes)
{
// 2. Upload Parts.
UploadPartRequest request = new UploadPartRequest()
.WithBucketName(destinationBucket)
.WithKey(destinationFilename.TrimStart('/'))
.WithUploadId(initResponse.UploadId)
.WithPartNumber(partNumber)
.WithPartSize(maxChunkedBytes)
.WithFilePosition(uploadedBytes)
.WithInputStream(fileStream) as UploadPartRequest;
uploadResponses.Add(client.UploadPart(request));
}
public static void CompletePartToCloud(string destinationFilename, string destinationBucket)
{
// Step 3: complete.
CompleteMultipartUploadRequest compRequest =
new CompleteMultipartUploadRequest()
.WithBucketName(destinationBucket)
.WithKey(destinationFilename.TrimStart('/'))
.WithUploadId(initResponse.UploadId)
.WithPartETags(uploadResponses);
CompleteMultipartUploadResponse completeUploadResponse =
client.CompleteMultipartUpload(compRequest);
}
public static void AbortPartToCloud(string destinationFilename, string destinationBucket)
{
// abort.
client.AbortMultipartUpload(new AbortMultipartUploadRequest()
.WithBucketName(destinationBucket)
.WithKey(destinationFilename.TrimStart('/'))
.WithUploadId(initResponse.UploadId));
}
}
my maxChunckedSize is 6Mb (6 * (1024*1024)) as I have read that the minimum is 5Mb...
why am I getting "Your proposed upload is smaller than the minimum allowed size" exception? What am I doing wrong?
The error is:
<Error>
<Code>EntityTooSmall</Code>
<Message>Your proposed upload is smaller than the minimum allowed size</Message>
<ETag>d41d8cd98f00b204e9800998ecf8427e</ETag>
<MinSizeAllowed>5242880</MinSizeAllowed>
<ProposedSize>0</ProposedSize>
<RequestId>C70E7A23C87CE5FC</RequestId>
<HostId>pmhuMXdRBSaCDxsQTHzucV5eUNcDORvKY0L4ZLMRBz7Ch1DeMh7BtQ6mmfBCLPM2</HostId>
<PartNumber>1</PartNumber>
</Error>
How can I get ProposedSize if I'm passing the stream and stream length?

Here is a working solution for the latest Amazon SDK (as today: v.1.5.37.0)
Amazon S3 Multipart Upload works like:
Initialize the request using client.InitiateMultipartUpload(initRequest)
Send chunks of the file (loop until the end) using client.UploadPart(request)
Complete the request using client.CompleteMultipartUpload(compRequest)
If anything goes wrong, remember to dispose the client and request, as well fire the abort command using client.AbortMultipartUpload(abortMultipartUploadRequest)
I keep the client in Session as we need this for each chunk upload as well, keep an hold of the ETags that are now used to complete the process.
You can see an example and simple way of doing this in Amazon Docs itself, I ended up having a class to do everything, plus, I have integrated with the lovely jQuery File Upload plugin (Handler code below as well).
The S3MultipartUpload is as follow
public class S3MultipartUpload : IDisposable
{
string accessKey = System.Configuration.ConfigurationManager.AppSettings.Get("AWSAccessKey");
string secretAccessKey = System.Configuration.ConfigurationManager.AppSettings.Get("AWSSecretKey");
AmazonS3 client;
public string OriginalFilename { get; set; }
public string DestinationFilename { get; set; }
public string DestinationBucket { get; set; }
public InitiateMultipartUploadResponse initResponse;
public List<PartETag> uploadPartETags;
public string UploadId { get; private set; }
public S3MultipartUpload(string destinationFilename, string destinationBucket)
{
if (client == null)
{
System.Net.WebRequest.DefaultWebProxy = null; // disable proxy to make upload quicker
client = Amazon.AWSClientFactory.CreateAmazonS3Client(accessKey, secretAccessKey, new AmazonS3Config()
{
RegionEndpoint = Amazon.RegionEndpoint.EUWest1,
CommunicationProtocol = Protocol.HTTP
});
this.OriginalFilename = destinationFilename.TrimStart('/');
this.DestinationFilename = string.Format("{0:yyyy}{0:MM}{0:dd}{0:HH}{0:mm}{0:ss}{0:fffff}_{1}", DateTime.UtcNow, this.OriginalFilename);
this.DestinationBucket = destinationBucket;
this.InitializePartToCloud();
}
}
private void InitializePartToCloud()
{
// 1. Initialize.
uploadPartETags = new List<PartETag>();
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest();
initRequest.BucketName = this.DestinationBucket;
initRequest.Key = this.DestinationFilename;
// make it public
initRequest.AddHeader("x-amz-acl", "public-read");
initResponse = client.InitiateMultipartUpload(initRequest);
}
public void UploadPartToCloud(Stream fileStream, long uploadedBytes, long maxChunkedBytes)
{
int partNumber = uploadPartETags.Count() + 1; // current part
// 2. Upload Parts.
UploadPartRequest request = new UploadPartRequest();
request.BucketName = this.DestinationBucket;
request.Key = this.DestinationFilename;
request.UploadId = initResponse.UploadId;
request.PartNumber = partNumber;
request.PartSize = fileStream.Length;
//request.FilePosition = uploadedBytes // remove this line?
request.InputStream = fileStream; // as UploadPartRequest;
var up = client.UploadPart(request);
uploadPartETags.Add(new PartETag() { ETag = up.ETag, PartNumber = partNumber });
}
public string CompletePartToCloud()
{
// Step 3: complete.
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest();
compRequest.BucketName = this.DestinationBucket;
compRequest.Key = this.DestinationFilename;
compRequest.UploadId = initResponse.UploadId;
compRequest.PartETags = uploadPartETags;
string r = "Something went badly wrong";
using (CompleteMultipartUploadResponse completeUploadResponse = client.CompleteMultipartUpload(compRequest))
r = completeUploadResponse.ResponseXml;
return r;
}
public void AbortPartToCloud()
{
// abort.
client.AbortMultipartUpload(new AbortMultipartUploadRequest()
{
BucketName = this.DestinationBucket,
Key = this.DestinationFilename,
UploadId = initResponse.UploadId
});
}
public void Dispose()
{
if (client != null) client.Dispose();
if (initResponse != null) initResponse.Dispose();
}
}
I use DestinationFilename as the destination file so I can avoid the same name, but I keep the OriginalFilename as I needed later.
Using jQuery File Upload Plugin, all works inside a Generic Handler, and the process is something like this:
// Upload partial file
private void UploadPartialFile(string fileName, HttpContext context, List<FilesStatus> statuses)
{
if (context.Request.Files.Count != 1)
throw new HttpRequestValidationException("Attempt to upload chunked file containing more than one fragment per request");
var inputStream = context.Request.Files[0].InputStream;
string contentRange = context.Request.Headers["Content-Range"]; // "bytes 0-6291455/14130271"
int fileSize = int.Parse(contentRange.Split('/')[1]);,
maxChunkSize = int.Parse(context.Request.Headers["X-Max-Chunk-Size"]),
uploadedBytes = int.Parse(contentRange.Replace("bytes ", "").Split('-')[0]);
iView.Utilities.AWS.S3MultipartUpload s3Upload = null;
try
{
// ######################################################################################
// 1. Initialize Amazon S3 Client
if (uploadedBytes == 0)
{
HttpContext.Current.Session["s3-upload"] = new iView.Utilities.AWS.S3MultipartUpload(fileName, awsBucket);
s3Upload = (iView.Utilities.AWS.S3MultipartUpload)HttpContext.Current.Session["s3-upload"];
string msg = System.String.Format("Upload started: {0} ({1:N0}Mb)", s3Upload.DestinationFilename, (fileSize / 1024));
this.Log(msg);
}
// cast current session object
if (s3Upload == null)
s3Upload = (iView.Utilities.AWS.S3MultipartUpload)HttpContext.Current.Session["s3-upload"];
// ######################################################################################
// 2. Send Chunks
s3Upload.UploadPartToCloud(inputStream, uploadedBytes, maxChunkSize);
// ######################################################################################
// 3. Complete Upload
if (uploadedBytes + maxChunkSize > fileSize)
{
string completeRequest = s3Upload.CompletePartToCloud();
this.Log(completeRequest); // log S3 response
s3Upload.Dispose(); // dispose all objects
HttpContext.Current.Session["s3-upload"] = null; // we don't need this anymore
}
}
catch (System.Exception ex)
{
if (ex.InnerException != null)
while (ex.InnerException != null)
ex = ex.InnerException;
this.Log(string.Format("{0}\n\n{1}", ex.Message, ex.StackTrace)); // log error
s3Upload.AbortPartToCloud(); // abort current upload
s3Upload.Dispose(); // dispose all objects
statuses.Add(new FilesStatus(ex.Message));
return;
}
statuses.Add(new FilesStatus(s3Upload.DestinationFilename, fileSize, ""));
}
Keep in mind that to have a Session object inside a Generic Handler, you need to implement IRequiresSessionState so your handler will look like:
public class UploadHandlerSimple : IHttpHandler, IRequiresSessionState
Inside fileupload.js (under _initXHRData) I have added an extra header called X-Max-Chunk-Size so I can pass this to Amazon and calculate if it's the last part of the uploaded file.
Fell free to comment and make smart edits for everyone to use.

I guess you didn't set the content-length of the part inside the UploadPartToCloud() function.

Related

How can I add a filesystem to my HTTP-listener/ add frameworks in C# [duplicate]

I'm making a simple webserver to serve html, css, js & images (done in c#). I am using HttpListener and I can get the html, javascript and css files to work properly. I am just having trouble with the images. This is what I'm using currently:
if (request.RawUrl.ToLower().Contains(".png") || request.RawUrl.Contains(".ico") || request.RawUrl.ToLower().Contains(".jpg") || request.RawUrl.ToLower().Contains(".jpeg"))
{
string dir = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
string[] img = request.RawUrl.Split('/');
string path = dir + #"\public\imgs\" + img[img.Length - 1];
FileInfo fileInfo = new FileInfo(path);
long numBytes = fileInfo.Length;
FileStream fileStream = new FileStream(path, FileMode.Open, FileAccess.Read);
BinaryReader binaryReader = new BinaryReader(fileStream);
byte[] output = binaryReader.ReadBytes((int)numBytes);
binaryReader.Close();
fileStream.Close();
var temp = System.Text.Encoding.UTF8.GetString(output);
return temp;
}
I am converting the image into a string to return them (it's the way my boss suggested). This is the method where I am handling these requests.
private static string SendResponse(HttpListenerRequest request)
This is my WebServer classes Run() method. The call to SetContentType just goes through the request.RawUrl and determines the content type.
public void Run()
{
ThreadPool.QueueUserWorkItem((o) =>
{
Console.WriteLine("StackLight Web Server is running...");
try
{
while (_listener.IsListening)
{
ThreadPool.QueueUserWorkItem((c) =>
{
var ctx = c as HttpListenerContext;
try
{
// store html content in a byte array
string responderString = _responderMethod(ctx.Request);
// set the content type
ctx.Response.Headers[HttpResponseHeader.ContentType] = SetContentType(ctx.Request.RawUrl);
byte[] buffer = buffer = Encoding.UTF8.GetBytes(responderString);
// this writes the html out from the byte array
ctx.Response.ContentLength64 = buffer.Length;
using(Stream stream = ctx.Response.OutputStream)
{
stream.Write(buffer, 0, buffer.Length);
}
}
catch (Exception ex)
{
ConfigLogger.Instance.LogCritical(LogCategory, ex);
}
}, _listener.GetContext());
}
}
catch (Exception ex)
{
ConfigLogger.Instance.LogCritical(LogCategory, ex);
}
});
}
My html page needs to display an image to the screen, it displays a broken image so far. I know the images directory is correct, I tested that.
This is where I got my code for the webserver: here
I was thinking that maybe I have to change the SendResponse method to not return a string
I figured it out. I created a class to hold the data, content type and the request.RawUrl. Then, where I was passing a string, I changed it to pass the object I created.
So, for my WebServer class, my Run method looks like this:
public void Run()
{
ThreadPool.QueueUserWorkItem((o) =>
{
Console.WriteLine("StackLight Web Server is running...");
try
{
while (_listener.IsListening)
{
ThreadPool.QueueUserWorkItem((c) =>
{
var ctx = c as HttpListenerContext;
try
{
// set the content type
ctx.Response.Headers[HttpResponseHeader.ContentType] = SetContentType(ctx.Request.RawUrl);
WebServerRequestData data = new WebServerRequestData();
// store html content in a byte array
data = _responderMethod(ctx.Request);
string res = "";
if(data.ContentType.Contains("text"))
{
char[] chars = new char[data.Content.Length/sizeof(char)];
System.Buffer.BlockCopy(data.Content, 0, chars, 0, data.Content.Length);
res = new string(chars);
data.Content = Encoding.UTF8.GetBytes(res);
}
// this writes the html out from the byte array
ctx.Response.ContentLength64 = data.Content.Length;
ctx.Response.OutputStream.Write(data.Content, 0, data.Content.Length);
}
catch (Exception ex)
{
ConfigLogger.Instance.LogCritical(LogCategory, ex);
}
finally
{
ctx.Response.OutputStream.Close();
}
}, _listener.GetContext());
}
}
catch (Exception ex)
{
ConfigLogger.Instance.LogCritical(LogCategory, ex);
}
});
}
And my SendResponse method looks like this:
private static WebServerRequestData SendResponse(HttpListenerRequest request)
{
string dir = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
string[] fileUrl = request.RawUrl.Split('/');
// routes
if (request.RawUrl.Contains("/"))
{
// this is the main page ('/'), all other routes can be accessed from here (including css, js, & images)
if (request.RawUrl.ToLower().Contains(".png") || request.RawUrl.ToLower().Contains(".ico") || request.RawUrl.ToLower().Contains(".jpg") || request.RawUrl.ToLower().Contains(".jpeg"))
{
try
{
string path = dir + Properties.Settings.Default.ImagesPath + fileUrl[fileUrl.Length - 1];
FileInfo fileInfo = new FileInfo(path);
path = dir + #"\public\imgs\" + fileInfo.Name;
byte[] output = File.ReadAllBytes(path);
_data = new WebServerRequestData() {Content = output, ContentType = "image/png", RawUrl = request.RawUrl};
//var temp = System.Text.Encoding.UTF8.GetString(output);
//return Convert.ToBase64String(output);
return _data;
}
catch(Exception ex)
{
ConfigLogger.Instance.LogError(LogCategory, "File could not be read.");
ConfigLogger.Instance.LogCritical(LogCategory, ex);
_errorString = string.Format("<html><head><title>Test</title></head><body>There was an error processing your request:<br />{0}</body></html>", ex.Message);
_byteData = new byte[_errorString.Length * sizeof(char)];
System.Buffer.BlockCopy(_errorString.ToCharArray(), 0, _byteData, 0, _byteData.Length);
_data = new WebServerRequestData() { Content = _byteData, ContentType = "text/html", RawUrl = request.RawUrl };
return _data;
}
}
I'm still cleaning up the code a bit but it now serves the images!
Oh... And here is the object I'm using:
public class WebServerRequestData
{
public string RawUrl { get; set; }
public string ContentType { get; set; }
public byte[] Content { get; set; }
public string RawData { get; set; }
}
Some really bad stuff here:
Empty catch. You'll never find out about many bugs.
Stuffing binary data into a string. Why? There's no encoding that is able to roundtrip binary data.
You're not disposing of ctx. I don't see why you need a manual finally block. Use using.
Untrusted callers can inject arbitrary paths into path. I could request your web.config file by navigating to /img/..\..\web.config (something like that).
Consider factoring out some common expressions into variables. You've got a Copy&Paste error with ToLower. Don't do dirty stuff and you'll have less bugs.

AmazonSQSClient not refreshing AWSCredentials when Credentials File is updated

When my AWS Credentials File (see docs) is updated by an external process the AmazonSQSClient doesn't re-read it, SendMessageAsync fails with a security/token error.
We use a custom powershell script to refresh the local AWS cred's file periodically. The script works fine, the file is refreshed prior to the credentials expiring on AWS. However, if my app is running when the file is refreshed the new credentials are not re-read from the file, the "client" will show that the previous credentials are still in use.
The AWS docs list several AWSCredential providers but none of them seem to be the correct choice...I think..
Restarting the app works, the new credentials are read correctly and messages are sent until the next time the cred's file is updated.
using (var client = new AmazonSQSClient(Amazon.RegionEndpoint.EUWest1))
{
return client.SendMessageAsync(request);
}
I don't think there is a way for a running app to pick up the default credentials being refreshed in credentials file. There is a solution for Node.js loading credentials from a JSON file. You can create a similar solution in C#. You can also run a local DB to store credentials so whenever credentials file is updated DB table or JSON file is also updated. You will need to use access key and secret key in your SQS client constructor as opposed to using default credentials.
// Load these from JSON file or DB.
var accessKey = "";
var secretKey = "";
using (var client = new AmazonSQSClient(accessKey, secretKey, Amazon.RegionEndpoint.EUWest1))
{
return client.SendMessageAsync(request);
}
The following works "ok" but I've only tested it with one profile and the file watcher is not as timely as you'd like so I'd recommend you wrap your usage inside a Retry mechanism.
// Usage..
var credentials = new AwsCredentialsFile();
using (var client = new AmazonSQSClient(credentials, Amazon.RegionEndpoint.EUWest1))
{
return client.SendMessageAsync(request);
}
public class AwsCredentialsFile : AWSCredentials
{
// https://docs.aws.amazon.com/sdk-for-net/v2/developer-guide/net-dg-config-creds.html#creds-file
private const string DefaultProfileName = "default";
private static ConcurrentDictionary<string, ImmutableCredentials> _credentials = new ConcurrentDictionary<string, ImmutableCredentials>(StringComparer.OrdinalIgnoreCase);
private static FileSystemWatcher _watcher = BuildFileSystemWatcher();
private readonly System.Text.Encoding _encoding;
private readonly string _profileName;
public AwsCredentialsFile()
: this(AwsCredentialsFile.DefaultProfileName, System.Text.Encoding.UTF8)
{
}
public AwsCredentialsFile(string profileName)
: this(profileName, System.Text.Encoding.UTF8)
{
}
public AwsCredentialsFile(string profileName, System.Text.Encoding encoding)
{
_profileName = profileName;
_encoding = encoding;
}
private static FileSystemWatcher BuildFileSystemWatcher()
{
var watcher = new FileSystemWatcher
{
Path = Path.GetDirectoryName(GetDefaultCredentialsFilePath()),
NotifyFilter = NotifyFilters.LastWrite,
Filter = "credentials"
};
watcher.Changed += (object source, FileSystemEventArgs e) => { _credentials?.Clear(); };
watcher.EnableRaisingEvents = true;
return watcher;
}
public static string GetDefaultCredentialsFilePath()
{
return System.Environment.ExpandEnvironmentVariables(#"C:\Users\%USERNAME%\.aws\credentials");
}
public static (string AccessKey, string SecretAccessKey, string Token) ReadCredentialsFromFile(string profileName, System.Text.Encoding encoding)
{
var profile = $"[{profileName}]";
string awsAccessKeyId = null;
string awsSecretAccessKey = null;
string token = null;
var lines = File.ReadAllLines(GetDefaultCredentialsFilePath(), encoding);
for (int i = 0; i < lines.Length; i++)
{
var text = lines[i];
if (text.Equals(profile, StringComparison.OrdinalIgnoreCase))
{
awsAccessKeyId = lines[i + 1].Replace("aws_access_key_id = ", string.Empty);
awsSecretAccessKey = lines[i + 2].Replace("aws_secret_access_key = ", string.Empty);
if (lines.Length >= i + 3)
{
token = lines[i + 3].Replace("aws_session_token = ", string.Empty);
}
break;
}
}
var result = (AccessKey: awsAccessKeyId, SecretAccessKey: awsSecretAccessKey, Token: token);
return result;
}
public override ImmutableCredentials GetCredentials()
{
if (_credentials.TryGetValue(_profileName, out ImmutableCredentials value))
{
return value;
}
else
{
var (AccessKey, SecretAccessKey, Token) = ReadCredentialsFromFile(_profileName, _encoding);
var credentials = new ImmutableCredentials(AccessKey, SecretAccessKey, Token);
_credentials.TryAdd(_profileName, credentials);
return credentials;
}
}
}

How to write unit test for this method

I am writing unit test for this method. I have tried lot of times but still can not write any code for it. Please suggest me how to unit test it. I am using C# , nunit framework and rhino mock.
Thanks in advance.
public FileUploadJsonResult AjaxUploadProfile(int id, string branchName, string filepath, HttpPostedFileBase file)
{
// TODO: Add your business logic here and/or save the file
string statusCode = "1";
string profilePicture = string.Empty;
string fileExtension = System.IO.Path.GetExtension(file.FileName.ToLower());
string fileName = id + "_" + branchName;
string fileNameWithOriginalExtension = fileName + fileExtension;
string fileNameWithJPGExtension = fileName + ".jpg";
string fileServerPath = this.Server.MapPath("~/LO_ProfilePicture/" + fileNameWithJPGExtension);
string statusMessage = string.Empty;
if (string.IsNullOrEmpty(fileExtension) || !Utility.isCorrectExtension(fileExtension))
{
statusMessage = "Profile picture should be of JPG, BMP, PNG, GIF or JPEG format.";
return new FileUploadJsonResult { Data = new { message = string.Format(statusMessage, fileNameWithOriginalExtension), filename = string.Empty, profilepic = profilePicture, statusCode = "0" } };
}
if (file.ContentLength > PageConstants.PROFILE_PICTURE_FILE_SIZE)
{
statusMessage = "Profile picture size should be less than 2MB";
return new FileUploadJsonResult { Data = new { message = string.Format(statusMessage, fileNameWithOriginalExtension), filename = string.Empty, profilepic = profilePicture, statusCode = "0" } };
}
Utility.SaveThumbnailImage(fileServerPath, file.InputStream, PageConstants.BRANCH_PROFILE_PICTURE_FILE_HEIGTH, PageConstants.BRANCH_PROFILE_PICTURE_FILE_WIDTH);
profilePicture = PageConstants.IMAGE_PATH + "LO_ProfilePicture/" + fileNameWithJPGExtension;
// Return JSON
return new FileUploadJsonResult { Data = new { message = string.Format("Profile Picture is successfully uploaded.", fileNameWithOriginalExtension), filename = fileNameWithJPGExtension, profilepic = profilePicture, statusCode } };
}
Make it do just the essential part. Split anything that has nothing to do with the operation you're trying to handle to other classes. Put those behind interfaces, so you can mock these in your unittests. This way you'll notice you don't have to test anything with file i/o in this class. In the class below I split up the function in the essential part, some file i/o and retrieving of settings. Even these settings have nothing to do with the current method you're trying to test. The method just needs verification on, for example, the extension, but it doesn't matter on how it does this.
Tip: try to avoid static utility classes. Give them their own class. Also avoid external components such as network communication or file i/o.
As I don't have a lot of context and it may not compile. But I would go with something like:
class Controller {
public FileUploadJsonResult AjaxUploadProfile(int id, string branchName, string filepath, HttpPostedFileBase file) {
string fileName = id + "_" + branchName;
string fileExtension = _fileIO.GetExtensionForFile(file);
if (!_extensionManager.IsValidExtension(fileExtension)) {
return CreateAjaxUploadProfileError("Profile picture should be of JPG, BMP, PNG, GIF or JPEG format.");
}
if (file.ContentLength > _settingsManager.GetMaximumFileSize()) {
return CreateAjaxUploadProfileError("Profile picture size should be less than 2MB");
}
string fileNameWithJPGExtension = fileName + ".jpg";
string fileServerPath = _fileIO.GetServerProfilePicture(Server, fileNameWithJPGExtension);
string fileClientPath = _fileIO.GetClientProfilePicture(fileNameWithJPGExtension);
var dimensions = _settingsManager.GetThumbnailDimensions();
_fileIO.SaveThumbnailImage(fileServerPath, file, dimensions.Item1, dimensions.Item2);
// Return JSON
var data = new {
message = "Profile Picture is successfully uploaded.",
filename = fileClientPath,
profilepic = profilePicture,
statusCode = "1"
};
return new FileUploadJsonResult { Data = data };
}
private static CreateAjaxUploadProfileError(string message) {
var data = new {
message = message,
filename = string.Empty,
profilepic = string.Empty,
statusCode = "0"
};
return new FileUploadJsonResult { Data = data };
}
}
class FileIO : IFileIO {
public string GetExtensionForFile(HttpPostedFileBase file) {
return System.IO.Path.GetExtension(filePath.FileName.ToLower());
}
public string GetServerProfilePicture(T server, string file) {
return server.MapPath( "~/LO_ProfilePicture/" + file);
}
public void SaveThumbnailImage(string path, HttpPostedFileBase file, int height, int width) {
Utility.SaveThumbnailImage(path, file.InputStream, height, width); // or even inline
}
public string GetClientProfilePicture(string fileName) {
return _settingsManager.GetClientImagePath() + "LO_ProfilePicture/" + fileNameWithJPGExtension;
}
}
class ExtensionManager : IExtensionManager {
public bool IsValidExtension(string extension) {
return Utility.isCorrectExtension(fileExtension); // or even inline
}
}
class SettingsManager : ISettingsManager {
public Tuple<int, int> GetThumbnailDimensions() {
return Tuple.Create<int, int>(PageConstants.BRANCH_PROFILE_PICTURE_FILE_HEIGTH, PageConstants.BRANCH_PROFILE_PICTURE_FILE_WIDTH);
}
public int GetMaximumFileSize() {
return PageConstants.PROFILE_PICTURE_FILE_SIZE;
}
}
You can look at this function as a combination of multiple functions doing specific work. One function is getting target file path, another is validating extension, another is validating size, another creates thumbnail, etc.
The goal is to breakdown the complex code into small testable functions (units) which you can test independently. So when you put them together you have better confidence that your big function works as expected.

Encoder SDK 4 - Push to Publishing Point

I'm coding an application in c# using EC4 SP2 SDK.
I want to publish my file to a media server publishing point. I've searched and found 2 examples regarding seting up and auth on publishing points, but either are from older sdk's or do not work (and are for console). basicly my application doesn't encode nothing, as if it had nothing to encode.
When in degub mode checkpont i can see the correct properties for the source file and for the server.
The encoding process takes 0secs to process. I checked the logs on the server events and i get a warning "the security system has received and auth request that could not be decoded". I just havo no knowledge to break up further than this. Any help would be appreciated.
this is the piece of code:
private void broadcastSourceFileToMediaServer2()
{
using (LiveJob job = new LiveJob())
{
String filetoencode = #"c:\temp\niceday.wmv";
LiveFileSource filesource = job.AddFileSource(filetoencode);
filesource.PlaybackMode = FileSourcePlaybackMode.Loop;
job.ActivateSource(filesource);
job.ApplyPreset(LivePresets.VC1Broadband4x3);
//don't know which one is good to use
job.AcquireCredentials += new EventHandler<AcquireCredentialsEventArgs>(job_AcquireCredentials);
_myUserName = "indes";
_pw = PullPW("indes");
Uri url = new Uri("http://192.168.1.74:8080/live");
PushBroadcastPublishFormat pubpoint = new PushBroadcastPublishFormat();
pubpoint.PublishingPoint = url;
pubpoint.UserName = _myUserName;
pubpoint.Password = _pw;
job.PublishFormats.Add(pubpoint);
job.PreConnectPublishingPoint();
job.StartEncoding();
statusBox.Text = job.NumberOfEncodedSamples.ToString();
job.StopEncoding();
job.Dispose();
}
}
public static string _myUserName { get; set; }
public static SecureString _pw { get; set; }
//codificação de Password a enviar
private static SecureString PullPW(string pw)
{
SecureString s = new SecureString();
foreach (char c in pw) s.AppendChar(c);
return s;
}
static void job_AcquireCredentials(object sender, AcquireCredentialsEventArgs e)
{
e.UserName = _myUserName;
e.Password = _pw;
e.Modes = AcquireCredentialModes.None;
}
Progresses:
I managed to authenticate (at least get a positive audit event) on the server.
I changed from this:
//don't know which one is good to use
job.AcquireCredentials += new EventHandler<AcquireCredentialsEventArgs>(job_AcquireCredentials);
_myUserName = "indes";
_pw = PullPW("indes");
Uri url = new Uri("http://192.168.1.74:8080/live");
PushBroadcastPublishFormat pubpoint = new PushBroadcastPublishFormat();
pubpoint.PublishingPoint = url;
pubpoint.UserName = _myUserName;
pubpoint.Password = _pw;
To this:
job.AcquireCredentials += new EventHandler<AcquireCredentialsEventArgs>(job_AcquireCredentials);
_myUserName = #"mediaservername\user";
_pw = PullPW("user_password");
Uri url = new Uri("http://192.168.1.74:8080/live");
PushBroadcastPublishFormat pubpoint = new PushBroadcastPublishFormat();
pubpoint.PublishingPoint = url;
If you see on one side if had to include the domain (either domain or computername) before username. this changed the failed audit events on the server, so i could eliminate the manual credentials pubpoint.username and pubpoint.Password.
Now I'm just dealing with a lack of output format exception. On to it.
How about using SMOOTH Streaming, I managed to get my project going but I didn't get much more beyond Look below, to the part that has the PUBLISH switch type. ignore the file portion
internal bool StartStream()
{
Busy = true;
// Instantiates a new job for encoding
//
//***************************************Live Stream Archive******************************
if (blnRecordFromFile)
{
// Sets up publishing format for file archival type
FileArchivePublishFormat fileOut = new FileArchivePublishFormat();
// job.ApplyPreset(LivePresets.VC1512kDSL16x9);
// Gets timestamp and edits it for filename
string timeStamp = DateTime.Now.ToString();
timeStamp = timeStamp.Replace("/", "-");
timeStamp = timeStamp.Replace(":", ".");
// Sets file path and name
string path = "C:\\output\\";
string filename = "Capture" + timeStamp + ".ismv";
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
fileOut.OutputFileName = Path.Combine(path, filename);
// Adds the format to the job. You can add additional formats as well such as
// Publishing streams or broadcasting from a port
job.PublishFormats.Add(fileOut);
}
//******************************END OF Stream PORTION****************************************
////////////////////////////////////////////////////////////////////////////////////////////////////
//*************************************** Process Files or Live Stream******************************
if (blnRecordFromFile)
{
job.ApplyPreset(LivePresets.VC1IISSmoothStreaming720pWidescreen);
job = new LiveJob();
// Verifies all information is entered
if (string.IsNullOrWhiteSpace(sourcePath) || string.IsNullOrWhiteSpace(destinationPath))
return false;
job.Status += new EventHandler<EncodeStatusEventArgs>(StreamStatus);
LiveFileSource fileSource;
try
{
// Sets file to active source and checks if it is valid
fileSource = job.AddFileSource(sourcePath);
}
catch (InvalidMediaFileException)
{
return false;
}
// Sets to loop media for streaming
// fileSource.PlaybackMode = FileSourcePlaybackMode.Loop;
// Makes this file the active source. Multiple files can be added
// and cued to move to each other at their ends
job.ActivateSource(fileSource);
}
//******************************END OF FILE PORTION****************************************
// Sets up variable for fomat data
switch (publishType)
{
case Output.Archive:
// Verifies destination path exists and if not creates it
try
{
if (!Directory.Exists(destinationPath))
Directory.CreateDirectory(destinationPath);
}
catch (IOException)
{
return false;
}
FileArchivePublishFormat archiveFormat = new FileArchivePublishFormat();
// Gets the location of the old extention and removes it
string filename = Path.GetFileNameWithoutExtension(sourcePath);
// Sets the archive path and file name
archiveFormat.OutputFileName = Path.Combine(destinationPath, filename + ".ismv");
job.PublishFormats.Add(archiveFormat);
break;
case Output.Publish:
// Setups streaming of media to publishing point
job = new LiveJob();
// Aquires audio and video devices
Collection<EncoderDevice> devices = EncoderDevices.FindDevices(EncoderDeviceType.Video);
EncoderDevice video = devices.Count > 0 ? devices[0] : null;
for (int i = 0; i < devices.Count; ++i)
// devices[i].Dispose();
devices.Clear();
devices = EncoderDevices.FindDevices(EncoderDeviceType.Audio);
EncoderDevice audio = devices.Count > 0 ? devices[0] : null;
for (int i = 1; i < devices.Count; ++i)
devices[i].Dispose();
devices.Clear();
// Checks for a/v devices
if (video != null && audio != null)
{
//job.ApplyPreset(Preset.FromFile(#"C:\Tempura\LivePreset3.xml"));
job.ApplyPreset(LivePresets.H264IISSmoothStreamingLowBandwidthStandard);
job.OutputFormat.VideoProfile.SmoothStreaming = true;
deviceSource = job.AddDeviceSource(video, audio);
// Make this source the active one
job.ActivateSource(deviceSource);
}
else
{
error = true;
}
PushBroadcastPublishFormat publishFormat = new PushBroadcastPublishFormat();
try
{
// checks the path for a valid publishing point
publishFormat.PublishingPoint = new Uri(destinationPath);
}
catch (UriFormatException)
{
return false;
}
// Adds the publishing format to the job
try
{
// job.ApplyPreset(LivePresets.VC1IISSmoothStreaming480pWidescreen);
job.PublishFormats.Add(publishFormat);
job.PreConnectPublishingPoint();
}
catch (Exception e)
{
MessageBox.Show(e.StackTrace.ToString());
}
break;
default:
return false;
}
job.StartEncoding();
return true;
}
Sadly I dont have enough rep to comment, so I have to write it as an answer.
Due to you are starting a live job, in order to stream you should not call job.StopEncoding() right after StartEncoding. I think usually you would use an event to stop the encoding. If you start encoding and immediately stop it, it is only logical you have no, or only a very small output.
I changed your code to the following and it seems work well. I guess your problem is that you disposed the instance of LiveJob class. You have to keep the instance alive before it finished encoding the whole stream. So change the using part and remove the StopEncoding and Dispose will be OK.
private void broadcastSourceFileToMediaServer2()
{
LiveJob job = new LiveJob();
String filetoencode = #"c:\temp\niceday.wmv";
LiveFileSource filesource = job.AddFileSource(filetoencode);
filesource.PlaybackMode = FileSourcePlaybackMode.Loop;
job.ActivateSource(filesource);
job.ApplyPreset(LivePresets.VC1Broadband4x3);
//don't know which one is good to use
job.AcquireCredentials += new EventHandler<AcquireCredentialsEventArgs>(job_AcquireCredentials);
_myUserName = "indes";
_pw = PullPW("indes");
Uri url = new Uri("http://192.168.1.74:8080/live");
PushBroadcastPublishFormat pubpoint = new PushBroadcastPublishFormat();
pubpoint.PublishingPoint = url;
pubpoint.UserName = _myUserName;
pubpoint.Password = _pw;
job.PublishFormats.Add(pubpoint);
job.PreConnectPublishingPoint();
job.StartEncoding();
statusBox.Text = job.NumberOfEncodedSamples.ToString();
}
public static string _myUserName { get; set; }
public static SecureString _pw { get; set; }
//codificação de Password a enviar
private static SecureString PullPW(string pw)
{
SecureString s = new SecureString();
foreach (char c in pw) s.AppendChar(c);
return s;
}
static void job_AcquireCredentials(object sender, AcquireCredentialsEventArgs e)
{
e.UserName = _myUserName;
e.Password = _pw;
e.Modes = AcquireCredentialModes.None;
}

WCF Streaming File Transfer ON .NET 4

I need a good example on WCF Streaming File Transfer.
I have found several and tried them but the posts are old and I am wokding on .net 4 and IIS 7 so there are some problems.
Can you gives me a good and up-to-date example on that.
The following answers detail using a few techniques for a posting binary data to a restful service.
Post binary data to a RESTful application
What is a good way to transfer binary data to a HTTP REST API service?
Bad idea to transfer large payload using web services?
The following code is a sample of how you could write a RESTful WCF service and is by no means complete but does give you an indication on where you could start.
Sample Service, note that this is NOT production ready code.
[ServiceContract]
[AspNetCompatibilityRequirements(RequirementsMode = AspNetCompatibilityRequirementsMode.Allowed)]
[ServiceBehavior(InstanceContextMode = InstanceContextMode.PerCall)]
public class FileService
{
private IncomingWebRequestContext m_Request;
private OutgoingWebResponseContext m_Response;
[WebGet(UriTemplate = "{appName}/{id}?action={action}")]
public Stream GetFile(string appName, string id, string action)
{
var repository = new FileRepository();
var response = WebOperationContext.Current.OutgoingResponse;
var result = repository.GetById(int.Parse(id));
if (action != null && action.Equals("download", StringComparison.InvariantCultureIgnoreCase))
{
response.Headers.Add("Content-Disposition", string.Format("attachment; filename={0}", result.Name));
}
response.Headers.Add(HttpResponseHeader.ContentType, result.ContentType);
response.Headers.Add("X-Filename", result.Name);
return result.Content;
}
[WebInvoke(UriTemplate = "{appName}", Method = "POST")]
public void Save(string appName, Stream fileContent)
{
try
{
if (WebOperationContext.Current == null) throw new InvalidOperationException("WebOperationContext is null.");
m_Request = WebOperationContext.Current.IncomingRequest;
m_Response = WebOperationContext.Current.OutgoingResponse;
var file = CreateFileResource(fileContent, appName);
if (!FileIsValid(file)) throw new WebFaultException(HttpStatusCode.BadRequest);
SaveFile(file);
SetStatusAsCreated(file);
}
catch (Exception ex)
{
if (ex.GetType() == typeof(WebFaultException)) throw;
if (ex.GetType().IsGenericType && ex.GetType().GetGenericTypeDefinition() == typeof(WebFaultException<>)) throw;
throw new WebFaultException<string>("An unexpected error occurred.", HttpStatusCode.InternalServerError);
}
}
private FileResource CreateFileResource(Stream fileContent, string appName)
{
var result = new FileResource();
fileContent.CopyTo(result.Content);
result.ApplicationName = appName;
result.Name = m_Request.Headers["X-Filename"];
result.Location = #"C:\SomeFolder\" + result.Name;
result.ContentType = m_Request.Headers[HttpRequestHeader.ContentType] ?? this.GetContentType(result.Name);
result.DateUploaded = DateTime.Now;
return result;
}
private string GetContentType(string filename)
{
// this should be replaced with some form of logic to determine the correct file content type (I.E., use registry, extension, xml file, etc.,)
return "application/octet-stream";
}
private bool FileIsValid(FileResource file)
{
var validator = new FileResourceValidator();
var clientHash = m_Request.Headers[HttpRequestHeader.ContentMd5];
return validator.IsValid(file, clientHash);
}
private void SaveFile(FileResource file)
{
// This will persist the meta data about the file to a database (I.E., size, filename, file location, etc)
new FileRepository().AddFile(file);
}
private void SetStatusAsCreated(FileResource file)
{
var location = new Uri(m_Request.UriTemplateMatch.RequestUri.AbsoluteUri + "/" + file.Id);
m_Response.SetStatusAsCreated(location);
}
}
Sample Client, note that this is NOT production ready code.
// *********************************
// Sample Client
// *********************************
private void UploadButton_Click(object sender, EventArgs e)
{
var uri = "http://dev-fileservice/SampleApplication"
var fullFilename = #"C:\somefile.txt";
var fileContent = File.ReadAllBytes(fullFilename);
using (var webClient = new WebClient())
{
try
{
webClient.Proxy = null;
webClient.Headers.Add(HttpRequestHeader.ContentMd5, this.CalculateFileHash());
webClient.Headers.Add("X-DaysToKeep", DurationNumericUpDown.Value.ToString());
webClient.Headers.Add("X-Filename", Path.GetFileName(fullFilename));
webClient.UploadData(uri, "POST", fileContent);
var fileUri = webClient.ResponseHeaders[HttpResponseHeader.Location];
Console.WriteLine("File can be downloaded at" + fileUri);
}
catch (Exception ex)
{
var exception = ex.Message;
}
}
}
private string CalculateFileHash()
{
var hash = MD5.Create().ComputeHash(File.ReadAllBytes(#"C:\somefile.txt"));
var sb = new StringBuilder();
for (int i = 0; i < hash.Length; i++)
{
sb.Append(hash[i].ToString("x2"));
}
return sb.ToString();
}
private void DownloadFile()
{
var uri = "http://dev-fileservice/SampleApplication/1" // this is the URL returned by the Restful file service
using (var webClient = new WebClient())
{
try
{
webClient.Proxy = null;
var fileContent = webClient.DownloadData(uri);
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
}
}

Categories