I need a good example on WCF Streaming File Transfer.
I have found several and tried them but the posts are old and I am wokding on .net 4 and IIS 7 so there are some problems.
Can you gives me a good and up-to-date example on that.
The following answers detail using a few techniques for a posting binary data to a restful service.
Post binary data to a RESTful application
What is a good way to transfer binary data to a HTTP REST API service?
Bad idea to transfer large payload using web services?
The following code is a sample of how you could write a RESTful WCF service and is by no means complete but does give you an indication on where you could start.
Sample Service, note that this is NOT production ready code.
[ServiceContract]
[AspNetCompatibilityRequirements(RequirementsMode = AspNetCompatibilityRequirementsMode.Allowed)]
[ServiceBehavior(InstanceContextMode = InstanceContextMode.PerCall)]
public class FileService
{
private IncomingWebRequestContext m_Request;
private OutgoingWebResponseContext m_Response;
[WebGet(UriTemplate = "{appName}/{id}?action={action}")]
public Stream GetFile(string appName, string id, string action)
{
var repository = new FileRepository();
var response = WebOperationContext.Current.OutgoingResponse;
var result = repository.GetById(int.Parse(id));
if (action != null && action.Equals("download", StringComparison.InvariantCultureIgnoreCase))
{
response.Headers.Add("Content-Disposition", string.Format("attachment; filename={0}", result.Name));
}
response.Headers.Add(HttpResponseHeader.ContentType, result.ContentType);
response.Headers.Add("X-Filename", result.Name);
return result.Content;
}
[WebInvoke(UriTemplate = "{appName}", Method = "POST")]
public void Save(string appName, Stream fileContent)
{
try
{
if (WebOperationContext.Current == null) throw new InvalidOperationException("WebOperationContext is null.");
m_Request = WebOperationContext.Current.IncomingRequest;
m_Response = WebOperationContext.Current.OutgoingResponse;
var file = CreateFileResource(fileContent, appName);
if (!FileIsValid(file)) throw new WebFaultException(HttpStatusCode.BadRequest);
SaveFile(file);
SetStatusAsCreated(file);
}
catch (Exception ex)
{
if (ex.GetType() == typeof(WebFaultException)) throw;
if (ex.GetType().IsGenericType && ex.GetType().GetGenericTypeDefinition() == typeof(WebFaultException<>)) throw;
throw new WebFaultException<string>("An unexpected error occurred.", HttpStatusCode.InternalServerError);
}
}
private FileResource CreateFileResource(Stream fileContent, string appName)
{
var result = new FileResource();
fileContent.CopyTo(result.Content);
result.ApplicationName = appName;
result.Name = m_Request.Headers["X-Filename"];
result.Location = #"C:\SomeFolder\" + result.Name;
result.ContentType = m_Request.Headers[HttpRequestHeader.ContentType] ?? this.GetContentType(result.Name);
result.DateUploaded = DateTime.Now;
return result;
}
private string GetContentType(string filename)
{
// this should be replaced with some form of logic to determine the correct file content type (I.E., use registry, extension, xml file, etc.,)
return "application/octet-stream";
}
private bool FileIsValid(FileResource file)
{
var validator = new FileResourceValidator();
var clientHash = m_Request.Headers[HttpRequestHeader.ContentMd5];
return validator.IsValid(file, clientHash);
}
private void SaveFile(FileResource file)
{
// This will persist the meta data about the file to a database (I.E., size, filename, file location, etc)
new FileRepository().AddFile(file);
}
private void SetStatusAsCreated(FileResource file)
{
var location = new Uri(m_Request.UriTemplateMatch.RequestUri.AbsoluteUri + "/" + file.Id);
m_Response.SetStatusAsCreated(location);
}
}
Sample Client, note that this is NOT production ready code.
// *********************************
// Sample Client
// *********************************
private void UploadButton_Click(object sender, EventArgs e)
{
var uri = "http://dev-fileservice/SampleApplication"
var fullFilename = #"C:\somefile.txt";
var fileContent = File.ReadAllBytes(fullFilename);
using (var webClient = new WebClient())
{
try
{
webClient.Proxy = null;
webClient.Headers.Add(HttpRequestHeader.ContentMd5, this.CalculateFileHash());
webClient.Headers.Add("X-DaysToKeep", DurationNumericUpDown.Value.ToString());
webClient.Headers.Add("X-Filename", Path.GetFileName(fullFilename));
webClient.UploadData(uri, "POST", fileContent);
var fileUri = webClient.ResponseHeaders[HttpResponseHeader.Location];
Console.WriteLine("File can be downloaded at" + fileUri);
}
catch (Exception ex)
{
var exception = ex.Message;
}
}
}
private string CalculateFileHash()
{
var hash = MD5.Create().ComputeHash(File.ReadAllBytes(#"C:\somefile.txt"));
var sb = new StringBuilder();
for (int i = 0; i < hash.Length; i++)
{
sb.Append(hash[i].ToString("x2"));
}
return sb.ToString();
}
private void DownloadFile()
{
var uri = "http://dev-fileservice/SampleApplication/1" // this is the URL returned by the Restful file service
using (var webClient = new WebClient())
{
try
{
webClient.Proxy = null;
var fileContent = webClient.DownloadData(uri);
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
}
}
Related
I have a API Post method that takes is a string which represents a Bae64 string of bytes from a word document that the API converts to PDF. My test client sends multiple documents, each on its own task, to the API to be converted. The problem is with concurrency and writing the files. I end up with a file in use since the calls are parallel. I have tried a lot of different way to block the conversion process until a document is converted but none of it has worked. Everything works fine if it's jsut a single file being converted but as soon as it's 2 or more, the problem happens. Can anyone guide me in the correct direction to solve this issue?
API:
[HttpPost]
public async Task<SimpleResponse> Post([FromBody]string request)
{
var response = new SimpleResponse();
Task t = Task.Factory.StartNew(async () =>
{
try
{
Converter convert = new Converter();
var result = await convert.CovertDocToPDF(request, WebConfigurationManager.AppSettings["tempDocPath"], WebConfigurationManager.AppSettings["tempPdfPath"]);
response.Result = result;
response.Success = true;
}
catch (Exception ex)
{
response.Exception = ex;
response.Success = false;
response.Errors = new List<string>();
response.Errors.Add(string.Format("{0}, {1}", ex.Message, ex.InnerException?.Message ?? ""));
}
});
t.Wait();
return response;
}
Conversion code
public Task<string> CovertDocToPDF(string blob, string tempDocPath, string tempPdfPath)
{
try
{
// Convert blob back to bytes
byte[] bte = Convert.FromBase64String(blob);
// Process and return blob
return Process(bte, tempDocPath, tempPdfPath);
}
catch (Exception Ex)
{
throw Ex;
}
}
private async Task<string> Process(byte[] bytes, string tempDocPath, string tempPdfPath)
{
try
{
string rs = RandomString(16, true);
tempDocPath = tempDocPath + rs + ".docx";
tempPdfPath = tempPdfPath + rs + ".pdf";
// This is where the problem happens with concurrent calls. I added
// the try catch when the file is in use to generate a new
// filename but the error still happens.
try
{
// Create a temp file
File.WriteAllBytes(tempDocPath, bytes);
}
catch (Exception Ex)
{
rs = RandomString(16, true);
tempDocPath = tempDocPath + rs + ".docx";
tempPdfPath = tempPdfPath + rs + ".pdf";
File.WriteAllBytes(tempDocPath, bytes);
}
word.Application app = new word.Application();
word.Document doc = app.Documents.Open(tempDocPath);
doc.SaveAs2(tempPdfPath, word.WdSaveFormat.wdFormatPDF);
doc.Close();
app.Quit(); // Clean up the word instance.
// Need the bytes to return the blob
byte[] pdfFileBytes = File.ReadAllBytes(tempPdfPath);
// Delete temp files
File.Delete(tempDocPath);
File.Delete(tempPdfPath);
// return blob
return Convert.ToBase64String(pdfFileBytes);
}
catch (Exception Ex)
{
throw Ex;
}
}
Client:
public async void btnConvert_Click(object sender, EventArgs e)
{
var response = await StartConvert();
foreach (SimpleResponse sr in response)
{
if (sr.Success)
{
byte[] bte = Convert.FromBase64String(sr.Result.ToString());
string rs = RandomString(16, true);
string pdfFileName = tempPdfPath + rs + ".pdf";
if (File.Exists(pdfFileName))
{
File.Delete(pdfFileName);
}
System.IO.File.WriteAllBytes(pdfFileName, bte);
}
else
{
}
}
}
private async Task<IEnumerable<SimpleResponse>> StartConvert()
{
var tasks = new List<Task<SimpleResponse>>();
foreach (string s in docPaths)
{
byte[] bte = File.ReadAllBytes(s);
tasks.Add(ConvertDocuments(Convert.ToBase64String(bte)));
}
return (await Task.WhenAll(tasks));
}
private async Task<SimpleResponse> ConvertDocuments(string requests)
{
using (var client = new HttpClient(new HttpClientHandler() { UseDefaultCredentials = true }))
{
client.BaseAddress = new Uri(BaseApiUrl);
client.DefaultRequestHeaders.Add("Accept", "application/json");
// Add an Accept header for JSON format.
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));//application/json
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, BaseApiUrl + ApiUrl);
var data = JsonConvert.SerializeObject(requests);
request.Content = new StringContent(data, Encoding.UTF8, "application/json");
HttpResponseMessage response1 = await client.PostAsync(BaseApiUrl + ApiUrl, request.Content).ConfigureAwait(false);
var response = JsonConvert.DeserializeObject<SimpleResponse>(await response1.Content.ReadAsStringAsync());
return response;
}
}
Random String Generator
public string RandomString(int size, bool lowerCase = false)
{
var builder = new StringBuilder(size);
// Unicode/ASCII Letters are divided into two blocks
// (Letters 65–90 / 97–122):
// The first group containing the uppercase letters and
// the second group containing the lowercase.
// char is a single Unicode character
char offset = lowerCase ? 'a' : 'A';
const int lettersOffset = 26; // A...Z or a..z: length = 26
for (var i = 0; i < size; i++)
{
var #char = (char)_random.Next(offset, offset + lettersOffset);
builder.Append(#char);
}
return lowerCase ? builder.ToString().ToLower() : builder.ToString();
}
First, get rid of Task.Factory.StartNew ... t.Wait() - you don't need an additional task, the root level method is async and your blocking Wait just spoils the benefits of async by blocking synchronously.
Second, like a comment suggested above, the file name random string generator is most likely to be not really random. Either do not supply anything to the seed value of your pseudo-random gen, or use something like Environment.TickCount which should be sufficient for this. Guid.NewGuid() will work too.
Another good option for temp files is Path.GetTempFileName (also generates an empty file for you): https://learn.microsoft.com/en-us/dotnet/api/system.io.path.gettempfilename?view=netstandard-2.0
[HttpPost]
public async Task<SimpleResponse> Post([FromBody]string request)
{
var response = new SimpleResponse();
try
{
...
var result = await convert.CovertDocToPDF(...);
...
}
catch (Exception ex)
{
...
}
return response;
}
Based on your code it seems that you have a "faulty" random string generator for file name (I would say _random.Next is a suspect, possibly some locking and/or "app wide" instance could fix the issue). You can use Guid.NewGuid to create random part of file name (which in theory can have collisions also but in most practical cases should be fine) or Path.GetTempFileName:
var rs = Guid.NewGuid().ToString("N");
tempDocPath = tempDocPath + rs + ".docx";
tempPdfPath = tempPdfPath + rs + ".pdf";
I'm making a simple webserver to serve html, css, js & images (done in c#). I am using HttpListener and I can get the html, javascript and css files to work properly. I am just having trouble with the images. This is what I'm using currently:
if (request.RawUrl.ToLower().Contains(".png") || request.RawUrl.Contains(".ico") || request.RawUrl.ToLower().Contains(".jpg") || request.RawUrl.ToLower().Contains(".jpeg"))
{
string dir = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
string[] img = request.RawUrl.Split('/');
string path = dir + #"\public\imgs\" + img[img.Length - 1];
FileInfo fileInfo = new FileInfo(path);
long numBytes = fileInfo.Length;
FileStream fileStream = new FileStream(path, FileMode.Open, FileAccess.Read);
BinaryReader binaryReader = new BinaryReader(fileStream);
byte[] output = binaryReader.ReadBytes((int)numBytes);
binaryReader.Close();
fileStream.Close();
var temp = System.Text.Encoding.UTF8.GetString(output);
return temp;
}
I am converting the image into a string to return them (it's the way my boss suggested). This is the method where I am handling these requests.
private static string SendResponse(HttpListenerRequest request)
This is my WebServer classes Run() method. The call to SetContentType just goes through the request.RawUrl and determines the content type.
public void Run()
{
ThreadPool.QueueUserWorkItem((o) =>
{
Console.WriteLine("StackLight Web Server is running...");
try
{
while (_listener.IsListening)
{
ThreadPool.QueueUserWorkItem((c) =>
{
var ctx = c as HttpListenerContext;
try
{
// store html content in a byte array
string responderString = _responderMethod(ctx.Request);
// set the content type
ctx.Response.Headers[HttpResponseHeader.ContentType] = SetContentType(ctx.Request.RawUrl);
byte[] buffer = buffer = Encoding.UTF8.GetBytes(responderString);
// this writes the html out from the byte array
ctx.Response.ContentLength64 = buffer.Length;
using(Stream stream = ctx.Response.OutputStream)
{
stream.Write(buffer, 0, buffer.Length);
}
}
catch (Exception ex)
{
ConfigLogger.Instance.LogCritical(LogCategory, ex);
}
}, _listener.GetContext());
}
}
catch (Exception ex)
{
ConfigLogger.Instance.LogCritical(LogCategory, ex);
}
});
}
My html page needs to display an image to the screen, it displays a broken image so far. I know the images directory is correct, I tested that.
This is where I got my code for the webserver: here
I was thinking that maybe I have to change the SendResponse method to not return a string
I figured it out. I created a class to hold the data, content type and the request.RawUrl. Then, where I was passing a string, I changed it to pass the object I created.
So, for my WebServer class, my Run method looks like this:
public void Run()
{
ThreadPool.QueueUserWorkItem((o) =>
{
Console.WriteLine("StackLight Web Server is running...");
try
{
while (_listener.IsListening)
{
ThreadPool.QueueUserWorkItem((c) =>
{
var ctx = c as HttpListenerContext;
try
{
// set the content type
ctx.Response.Headers[HttpResponseHeader.ContentType] = SetContentType(ctx.Request.RawUrl);
WebServerRequestData data = new WebServerRequestData();
// store html content in a byte array
data = _responderMethod(ctx.Request);
string res = "";
if(data.ContentType.Contains("text"))
{
char[] chars = new char[data.Content.Length/sizeof(char)];
System.Buffer.BlockCopy(data.Content, 0, chars, 0, data.Content.Length);
res = new string(chars);
data.Content = Encoding.UTF8.GetBytes(res);
}
// this writes the html out from the byte array
ctx.Response.ContentLength64 = data.Content.Length;
ctx.Response.OutputStream.Write(data.Content, 0, data.Content.Length);
}
catch (Exception ex)
{
ConfigLogger.Instance.LogCritical(LogCategory, ex);
}
finally
{
ctx.Response.OutputStream.Close();
}
}, _listener.GetContext());
}
}
catch (Exception ex)
{
ConfigLogger.Instance.LogCritical(LogCategory, ex);
}
});
}
And my SendResponse method looks like this:
private static WebServerRequestData SendResponse(HttpListenerRequest request)
{
string dir = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
string[] fileUrl = request.RawUrl.Split('/');
// routes
if (request.RawUrl.Contains("/"))
{
// this is the main page ('/'), all other routes can be accessed from here (including css, js, & images)
if (request.RawUrl.ToLower().Contains(".png") || request.RawUrl.ToLower().Contains(".ico") || request.RawUrl.ToLower().Contains(".jpg") || request.RawUrl.ToLower().Contains(".jpeg"))
{
try
{
string path = dir + Properties.Settings.Default.ImagesPath + fileUrl[fileUrl.Length - 1];
FileInfo fileInfo = new FileInfo(path);
path = dir + #"\public\imgs\" + fileInfo.Name;
byte[] output = File.ReadAllBytes(path);
_data = new WebServerRequestData() {Content = output, ContentType = "image/png", RawUrl = request.RawUrl};
//var temp = System.Text.Encoding.UTF8.GetString(output);
//return Convert.ToBase64String(output);
return _data;
}
catch(Exception ex)
{
ConfigLogger.Instance.LogError(LogCategory, "File could not be read.");
ConfigLogger.Instance.LogCritical(LogCategory, ex);
_errorString = string.Format("<html><head><title>Test</title></head><body>There was an error processing your request:<br />{0}</body></html>", ex.Message);
_byteData = new byte[_errorString.Length * sizeof(char)];
System.Buffer.BlockCopy(_errorString.ToCharArray(), 0, _byteData, 0, _byteData.Length);
_data = new WebServerRequestData() { Content = _byteData, ContentType = "text/html", RawUrl = request.RawUrl };
return _data;
}
}
I'm still cleaning up the code a bit but it now serves the images!
Oh... And here is the object I'm using:
public class WebServerRequestData
{
public string RawUrl { get; set; }
public string ContentType { get; set; }
public byte[] Content { get; set; }
public string RawData { get; set; }
}
Some really bad stuff here:
Empty catch. You'll never find out about many bugs.
Stuffing binary data into a string. Why? There's no encoding that is able to roundtrip binary data.
You're not disposing of ctx. I don't see why you need a manual finally block. Use using.
Untrusted callers can inject arbitrary paths into path. I could request your web.config file by navigating to /img/..\..\web.config (something like that).
Consider factoring out some common expressions into variables. You've got a Copy&Paste error with ToLower. Don't do dirty stuff and you'll have less bugs.
I'am posting an image from a console application to a asp.net web api. I'am getting a file in the folder but the image is black (no image). Do I have something wrong in my code?
public class UploadController : ApiController
{
[System.Web.Mvc.HttpPost]
public string Upload()
{
var request = HttpContext.Current.Request;
var filePath = Path.Combine(HttpContext.Current.Server.MapPath("~/Uploads/"), request.Headers["filename"]);
try
{
using (var fs = new System.IO.FileStream(filePath, System.IO.FileMode.Create))
{
request.InputStream.CopyTo(fs);
}
}
catch (Exception e)
{
return e.Message;
}
return "uploaded";
}
}
Edit
My console app
http://pastebin.com/VsnDMYpb
try this. This works for me. I used this for multiple file upload
var httpRequest = HttpContext.Current.Request;
foreach (string file in httpRequest.Files)
{
var postedFile = httpRequest.Files[file];
var filePath = HttpContext.Current.Server.MapPath("~/Uploads/" + postedFile.FileName);
postedFile.SaveAs(filePath);
}
Use Request.Content.ReadAsMultipartAsync
public Task<IQueryable<HDFile>> Post()
{
try
{
var uploadFolderPath = HostingEnvironment.MapPath("~/App_Data/" + UploadFolder);
log.Debug(uploadFolderPath);
if (Request.Content.IsMimeMultipartContent())
{
var streamProvider = new WithExtensionMultipartFormDataStreamProvider(uploadFolderPath);
var task = Request.Content.ReadAsMultipartAsync(streamProvider).ContinueWith<IQueryable<HDFile>>(t =>
{
if (t.IsFaulted || t.IsCanceled)
{
throw new HttpResponseException(HttpStatusCode.InternalServerError);
}
var fileInfo = streamProvider.FileData.Select(i =>
{
var info = new FileInfo(i.LocalFileName);
return new HDFile(info.Name, Request.RequestUri.AbsoluteUri + "?filename=" + info.Name, (info.Length / 1024).ToString());
});
return fileInfo.AsQueryable();
});
return task;
}
else
{
throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.NotAcceptable, "This request is not properly formatted"));
}
}
catch (Exception ex)
{
log.Error(ex);
throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.BadRequest, ex.Message));
}
}
The code I get from this post
I'm trying to set up a WCF service protected by ADFS. I'm currently able to request a token and send it with the request using WIF and Thinktecture IdentityModel 4.5 with the following code:
static SecurityToken GetToken()
{
var factory = new WSTrustChannelFactory(
new UserNameWSTrustBinding(SecurityMode.TransportWithMessageCredential),
"https://fs2.server2012.local/adfs/services/trust/13/usernamemixed")
{
TrustVersion = TrustVersion.WSTrust13
};
if (factory.Credentials != null)
{
factory.Credentials.UserName.UserName = #"username";
factory.Credentials.UserName.Password = "password";
}
var rst = new RequestSecurityToken
{
RequestType = RequestTypes.Issue,
KeyType = KeyTypes.Symmetric,
AppliesTo = new EndpointReference(
"https://wcfservicecertificate/wcfservice/Service.svc/wstrust"),
};
var channel = factory.CreateChannel();
RequestSecurityTokenResponse rstr;
return channel.Issue(rst, out rstr);
}
With this I can call the WCF service by using ChannelFactory.CreateChannelWithIssuedToken:
var factory = new ChannelFactory<IService>(binding,
new EndpointAddress("https://wcfservicecertificate/wcfservice/Service.svc/wstrust"));
if (factory.Credentials != null)
{
factory.Credentials.SupportInteractive = false;
factory.Credentials.UseIdentityConfiguration = true;
}
var proxy = factory.CreateChannelWithIssuedToken(GetToken());
var result= proxy.GetData(2);
This works as expected but can only be used on (mobile) windows platforms. I would also like to be able to use the same principle on iOS and Android. Using this article I was able to request a security token from ADFS using the following code:
const string soapMessage =
#"<s:Envelope xmlns:s=""http://www.w3.org/2003/05/soap-envelope""
xmlns:a=""http://www.w3.org/2005/08/addressing""
xmlns:u=""http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd"">
<s:Header>
<a:Action s:mustUnderstand=""1"">http://docs.oasis-open.org/ws-sx/ws-trust/200512/RST/Issue</a:Action>
<a:To s:mustUnderstand=""1"">https://fs2.server2012.local/adfs/services/trust/13/UsernameMixed</a:To>
<o:Security s:mustUnderstand=""1"" xmlns:o=""http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd"">
<o:UsernameToken u:Id=""uuid-6a13a244-dac6-42c1-84c5-cbb345b0c4c4-1"">
<o:Username>username</o:Username>
<o:Password Type=""http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-username-token-profile-1.0#PasswordText"">password</o:Password>
</o:UsernameToken>
</o:Security>
</s:Header>
<s:Body>
<trust:RequestSecurityToken xmlns:trust=""http://docs.oasis-open.org/ws-sx/ws-trust/200512"">
<wsp:AppliesTo xmlns:wsp=""http://schemas.xmlsoap.org/ws/2004/09/policy"">
<a:EndpointReference>
<a:Address>https://wcfservicecertificate/wcfservice/Service.svc/wstrust</a:Address>
</a:EndpointReference>
</wsp:AppliesTo>
<trust:KeyType>http://docs.oasis-open.org/ws-sx/ws-trust/200512/SymmetricKey</trust:KeyType>
<trust:RequestType>http://docs.oasis-open.org/ws-sx/ws-trust/200512/Issue</trust:RequestType>
<trust:TokenType>urn:oasis:names:tc:SAML:2.0:assertion</trust:TokenType>
</trust:RequestSecurityToken>
</s:Body>
</s:Envelope>";
var webClient = new WebClient();
webClient.Headers.Add("Content-Type", "application/soap+xml; charset=utf-8");
var result = webClient.UploadString(
address: "https://fs2.server2012.local/adfs/services/trust/13/UsernameMixed",
method: "POST",
data: soapMessage);
This results in a SAML2.0 token which I would like to send in a request to our WCF service in order to authenticate. There are various sources (including the article mentioned earlier) which state that this should be possible but I've yet to find a solution.
Any help would be appreciated.
You can use one of hybrid solutions which use SAML with OAuth or other authorization technologies. This is more secure against phising techniques. For SAML only approach, you can refer to following link: How to pass security tokenfrom one wcf service to another wcf service. It is said that you need to enable saveBootstrapTokens property on webconfig.
This link can be useful too: Availability of Bootstrap Tokens
This can easily be done without using WIF. Lets completely avoid WIF and the .Net framework and do it in Java for illustration purposes. First make a call to the Security Token Service using the template approach like you have done. You then need to extract the SAML from the response, Base64 encode it and stuff it in the Autorization header of the subsequent request to your protected WCF service. You may also need to do the same with a ProofKey if you are coding for Non-Repudiation. Also I'm only showing authentication using username/password for brevity as Certificate Authentication involves much more work - you have to hash (SHA1 )part of the message then encrypt the hash with the private key of the cert and then add this as a xml element to the original message etc...
Here is the java helper code:
import java.io.*;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.time.Instant;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Base64;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import javax.net.ssl.HttpsURLConnection;
public class SecurityService {
private String _username;
private String _password;
private String _stsUrl;
private String _samlAssertion;
private String _samlEncoded;
private String _binarySecret;
private String _workingDirectory;
private String _platformUrl;
private String _soapBody;
private Integer _responseCode;
private Integer _plaformResponseCode;
private String _response;
private String _platformResponse;
private String _xproofSignature;
private Map<String, String> _headerDictionary;
public void setUsername(String username) {
this._username = username;
}
public void setPassword(String password) {
this._password = password;
}
public void setStsUrl(String stsUrl) {
this._stsUrl = stsUrl;
}
public String getStsUrl() {
return _stsUrl;
}
public void setplatformUrl(String platformUrl) {
this._platformUrl = platformUrl;
}
public String getSamlAssertion() {
return _samlAssertion;
}
public String getSamlEncoded() {
return _samlEncoded;
}
public String getSoapBody() {
return _soapBody;
}
public Integer getResponseCode() {
return _responseCode;
}
public Integer getPlatformResponseCode() {
return _plaformResponseCode;
}
public String getResponse() {
return _response;
}
public String getPlatformResponse() {
return _platformResponse;
}
public String getXProofSignature() {
return _xproofSignature;
}
public String getBinarySecret() {
return _binarySecret;
}
public String gePlatFormUrl() {
return _platformUrl;
}
public void setHeaderDictionary(Map<String, String> headerDictionary){
this._headerDictionary = headerDictionary;
}
public Map<String, String> getHeaderDictionary(){
return _headerDictionary;
}
public SecurityService() throws Exception {
}
public SecurityService(Boolean useConfig) throws Exception {
if (useConfig) {
this._workingDirectory = System.getProperty("user.dir") + "\\app.config";
this.getProperties();
}
}
public void sendAuthenticatedGet() throws Exception {
URL obj = new URL(_platformUrl);
HttpURLConnection con = (HttpURLConnection) obj.openConnection();
// optional default is GET
con.setRequestMethod("GET");
// Add request header
con.setRequestProperty("Authorization", "Saml " + _samlEncoded);
con.setRequestProperty("X-ProofSignature", _xproofSignature);
_plaformResponseCode = con.getResponseCode();
BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
_platformResponse = response.toString();
}
public void sendAuthenticatedPost(String body) throws Exception {
URL obj = new URL(_platformUrl);
HttpsURLConnection con = (HttpsURLConnection) obj.openConnection();
//add request header
con.setRequestMethod("POST");
con.setRequestProperty("Content-Type", "application/json");
// Add request header
con.setRequestProperty("Authorization", "Saml " + _samlEncoded);
con.setRequestProperty("X-ProofSignature", _xproofSignature);
// Add Azure Subscription Key using generic Add Headers method
if (_headerDictionary != null) {
for (String key : _headerDictionary.keySet()) {
con.setRequestProperty(key, _headerDictionary.get(key));
}
}
_soapBody = body;
// Send post request
con.setDoOutput(true);
DataOutputStream wr = new DataOutputStream(con.getOutputStream());
//wr.writeBytes(urlParameters);
wr.writeBytes(_soapBody);
wr.flush();
wr.close();
_responseCode = con.getResponseCode();
BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
_response = response.toString();
}
// HTTP POST request
public void sendPostToSts() throws Exception {
URL obj = new URL(_stsUrl);
HttpsURLConnection con = (HttpsURLConnection) obj.openConnection();
//add request header
con.setRequestMethod("POST");
con.setRequestProperty("Content-Type", "application/soap+xml");
String body = getTemplateCertificate();
_soapBody = (((body.replace("[Created]", Instant.now().toString())).replace("[Expires]", Instant.now()
.plusSeconds(300).toString())).replace("[username]", _username)).replace("[password]", _password).replace("[stsUrl]", _stsUrl);
// Send post request
con.setDoOutput(true);
DataOutputStream wr = new DataOutputStream(con.getOutputStream());
//wr.writeBytes(urlParameters);
wr.writeBytes(_soapBody);
wr.flush();
wr.close();
_responseCode = con.getResponseCode();
BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
_response = response.toString();
// Get Binary Secret
// <trust:BinarySecret></trust:BinarySecret>
final Pattern patternBinarySecret = Pattern.compile("<trust:BinarySecret>(.+?)</trust:BinarySecret>");
final Matcher matcherBinarySecret = patternBinarySecret.matcher(response.toString());
matcherBinarySecret.find();
_binarySecret = matcherBinarySecret.group(1);
// Get the SAML Assertion
final Pattern patternEncryptedAssertion = Pattern.compile("<trust:RequestedSecurityToken>(.+?)</trust:RequestedSecurityToken>");
final Matcher matcherEncryptedAssertion = patternEncryptedAssertion.matcher(response.toString());
matcherEncryptedAssertion.find();
_samlAssertion = matcherEncryptedAssertion.group(1);
byte[] proofKeyBytes = _binarySecret.getBytes("UTF-8");
String encoded = Base64.getEncoder().encodeToString(proofKeyBytes);
byte[] decoded = Base64.getDecoder().decode(encoded);
// SAML Stuff - Works beautifully
byte[] samlBytes = _samlAssertion.getBytes("UTF-8");
_samlEncoded = Base64.getEncoder().encodeToString(samlBytes);
_xproofSignature = this.encode(_samlAssertion, _binarySecret);
}
private static String readFile( String file ) throws IOException {
BufferedReader reader = new BufferedReader( new FileReader(file));
String line = null;
StringBuilder stringBuilder = new StringBuilder();
String ls = System.getProperty("line.separator");
try {
while( ( line = reader.readLine() ) != null ) {
stringBuilder.append( line );
stringBuilder.append( ls );
}
return stringBuilder.toString();
} finally {
reader.close();
}
}
// Embedded WS-Trust template for username/password RST
private static String getTemplate () {
return "<s:Envelope xmlns:s=\"http://www.w3.org/2003/05/soap-envelope\" xmlns:a=\"http://www.w3.org/2005/08/addressing\" xmlns:u= \"http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd\"><s:Header><a:Action s:mustUnderstand= \"1\">http://docs.oasis-open.org/ws-sx/ws-trust/200512/RST/Issue</a:Action><a:MessageID>urn:uuid:cfea5555-248c-46c3-9b4d- 54936b7f815c</a:MessageID><a:ReplyTo><a:Address>http://www.w3.org/2005/08/addressing/anonymous</a:Address></a:ReplyTo><a:To s:mustUnderstand=\"1\">[stsUrl]</a:To><o:Security s:mustUnderstand=\"1\" xmlns:o=\"http://docs.oasis- open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd\"><u:Timestamp u:Id=\"_0\"><u:Created>[Created] </u:Created><u:Expires>[Expires]</u:Expires></u:Timestamp><o:UsernameToken u:Id=\"uuid-e273c018-1da7-466e-8671-86f6bfe7ce3c- 17\"><o:Username>[username]</o:Username><o:Password Type=\"http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-username- token-profile-1.0#PasswordText\">[password] </o:Password></o:UsernameToken></o:Security></s:Header><s:Body><trust:RequestSecurityToken xmlns:trust=\"http://docs.oasis- open.org/ws-sx/ws-trust/200512\"><wsp:AppliesTo xmlns:wsp=\"http://schemas.xmlsoap.org/ws/2004/09/policy \"><wsa:EndpointReference xmlns:wsa=\"http://www.w3.org/2005/08/addressing \"><wsa:Address>https://mbplatform/</wsa:Address></wsa:EndpointReference></wsp:AppliesTo><trust:RequestType>http://docs.oasis- open.org/ws-sx/ws-trust/200512/Issue</trust:RequestType><trust:TokenType>http://docs.oasis-open.org/wss/oasis-wss-saml-token- profile-1.1#SAMLV2.0</trust:TokenType></trust:RequestSecurityToken></s:Body></s:Envelope>";
}
private String encode(String key, String data) throws Exception {
Mac sha256_HMAC = Mac.getInstance("HmacSHA256");
SecretKeySpec secret_key = new SecretKeySpec(key.getBytes("UTF-8"), "HmacSHA256");
sha256_HMAC.init(secret_key);
return Base64.getEncoder().encodeToString(sha256_HMAC.doFinal(data.getBytes("UTF-8")));
}
private void getProperties() throws Exception {
Properties prop = new Properties();
String fileName = _workingDirectory;
InputStream is = new FileInputStream(fileName);
prop.load(is);
_username = prop.getProperty("app.username");
_password = prop.getProperty("app.password");
_platformUrl = prop.getProperty("app.platformUrl");
_stsUrl = prop.getProperty("app.stsUrl");
}
}
and here is example usage:
SecurityService mbss = new SecurityService(true);
mbss.sendPostToSts();
System.out.println("CONTACTING AZURE SECURITY TOKEN SERVICE");
System.out.println("\nSending 'POST' request to URL : " + mbss.getStsUrl());
System.out.println("\nPost parameters : \n" + mbss.getSoapBody());
System.out.println("\nResponse Code : " + mbss.getResponseCode());
System.out.println("\nHERE IS THE SAML RESPONSE\n");
System.out.println(mbss.getResponse());
System.out.println("\nHERE IS THE BINARY SECRET\n");
System.out.println(mbss.getBinarySecret());
System.out.println("\nHERE IS THE SAML ASSERTION\n");
System.out.println(mbss.getSamlAssertion());
System.out.println("\nHERE IS THE ENCODED SAML ASSERTION\n");
System.out.println(mbss.getSamlEncoded());
System.out.println("\nHERE IS THE X-PROOF SIGNATURE\n");
System.out.println(mbss.getXProofSignature());
System.out.println("\nNOW CONTACTING WCF SERVICES WITH SECURITY HEADER\n");
mbss.sendAuthenticatedGet();
System.out.println("\nSending 'GET' request to URL : " + mbss.gePlatFormUrl());
System.out.println("Response Code : " + mbss.getPlatformResponseCode());
System.out.println("\nHERE ARE THE RESULTS FOLKS...ENJOY\n");
System.out.println(mbss.getPlatformResponse());
using .NET SDK v.1.5.21.0
I'm trying to upload a large file (63Mb) and I'm following the example at:
http://docs.aws.amazon.com/AmazonS3/latest/dev/LLuploadFileDotNet.html
But using a helper instead the hole code and using jQuery File Upload
https://github.com/blueimp/jQuery-File-Upload/blob/master/basic-plus.html
what I have is:
string bucket = "mybucket";
long totalSize = long.Parse(context.Request.Headers["X-File-Size"]),
maxChunkSize = long.Parse(context.Request.Headers["X-File-MaxChunkSize"]),
uploadedBytes = long.Parse(context.Request.Headers["X-File-UloadedBytes"]),
partNumber = uploadedBytes / maxChunkSize + 1,
fileSize = partNumber * inputStream.Length;
bool lastPart = inputStream.Length < maxChunkSize;
// http://docs.aws.amazon.com/AmazonS3/latest/dev/LLuploadFileDotNet.html
if (partNumber == 1) // initialize upload
{
iView.Utilities.Amazon_S3.S3MultipartUpload.InitializePartToCloud(fileName, bucket);
}
try
{
// upload part
iView.Utilities.Amazon_S3.S3MultipartUpload.UploadPartToCloud(fs, fileName, bucket, (int)partNumber, uploadedBytes, maxChunkSize);
if (lastPart)
// wrap it up and go home
iView.Utilities.Amazon_S3.S3MultipartUpload.CompletePartToCloud(fileName, bucket);
}
catch (System.Exception ex)
{
// Huston, we have a problem!
//Console.WriteLine("Exception occurred: {0}", exception.Message);
iView.Utilities.Amazon_S3.S3MultipartUpload.AbortPartToCloud(fileName, bucket);
}
and
public static class S3MultipartUpload
{
private static string accessKey = System.Configuration.ConfigurationManager.AppSettings["AWSAccessKey"];
private static string secretAccessKey = System.Configuration.ConfigurationManager.AppSettings["AWSSecretKey"];
private static AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(accessKey, secretAccessKey);
public static InitiateMultipartUploadResponse initResponse;
public static List<UploadPartResponse> uploadResponses;
public static void InitializePartToCloud(string destinationFilename, string destinationBucket)
{
// 1. Initialize.
uploadResponses = new List<UploadPartResponse>();
InitiateMultipartUploadRequest initRequest =
new InitiateMultipartUploadRequest()
.WithBucketName(destinationBucket)
.WithKey(destinationFilename.TrimStart('/'));
initResponse = client.InitiateMultipartUpload(initRequest);
}
public static void UploadPartToCloud(Stream fileStream, string destinationFilename, string destinationBucket, int partNumber, long uploadedBytes, long maxChunkedBytes)
{
// 2. Upload Parts.
UploadPartRequest request = new UploadPartRequest()
.WithBucketName(destinationBucket)
.WithKey(destinationFilename.TrimStart('/'))
.WithUploadId(initResponse.UploadId)
.WithPartNumber(partNumber)
.WithPartSize(maxChunkedBytes)
.WithFilePosition(uploadedBytes)
.WithInputStream(fileStream) as UploadPartRequest;
uploadResponses.Add(client.UploadPart(request));
}
public static void CompletePartToCloud(string destinationFilename, string destinationBucket)
{
// Step 3: complete.
CompleteMultipartUploadRequest compRequest =
new CompleteMultipartUploadRequest()
.WithBucketName(destinationBucket)
.WithKey(destinationFilename.TrimStart('/'))
.WithUploadId(initResponse.UploadId)
.WithPartETags(uploadResponses);
CompleteMultipartUploadResponse completeUploadResponse =
client.CompleteMultipartUpload(compRequest);
}
public static void AbortPartToCloud(string destinationFilename, string destinationBucket)
{
// abort.
client.AbortMultipartUpload(new AbortMultipartUploadRequest()
.WithBucketName(destinationBucket)
.WithKey(destinationFilename.TrimStart('/'))
.WithUploadId(initResponse.UploadId));
}
}
my maxChunckedSize is 6Mb (6 * (1024*1024)) as I have read that the minimum is 5Mb...
why am I getting "Your proposed upload is smaller than the minimum allowed size" exception? What am I doing wrong?
The error is:
<Error>
<Code>EntityTooSmall</Code>
<Message>Your proposed upload is smaller than the minimum allowed size</Message>
<ETag>d41d8cd98f00b204e9800998ecf8427e</ETag>
<MinSizeAllowed>5242880</MinSizeAllowed>
<ProposedSize>0</ProposedSize>
<RequestId>C70E7A23C87CE5FC</RequestId>
<HostId>pmhuMXdRBSaCDxsQTHzucV5eUNcDORvKY0L4ZLMRBz7Ch1DeMh7BtQ6mmfBCLPM2</HostId>
<PartNumber>1</PartNumber>
</Error>
How can I get ProposedSize if I'm passing the stream and stream length?
Here is a working solution for the latest Amazon SDK (as today: v.1.5.37.0)
Amazon S3 Multipart Upload works like:
Initialize the request using client.InitiateMultipartUpload(initRequest)
Send chunks of the file (loop until the end) using client.UploadPart(request)
Complete the request using client.CompleteMultipartUpload(compRequest)
If anything goes wrong, remember to dispose the client and request, as well fire the abort command using client.AbortMultipartUpload(abortMultipartUploadRequest)
I keep the client in Session as we need this for each chunk upload as well, keep an hold of the ETags that are now used to complete the process.
You can see an example and simple way of doing this in Amazon Docs itself, I ended up having a class to do everything, plus, I have integrated with the lovely jQuery File Upload plugin (Handler code below as well).
The S3MultipartUpload is as follow
public class S3MultipartUpload : IDisposable
{
string accessKey = System.Configuration.ConfigurationManager.AppSettings.Get("AWSAccessKey");
string secretAccessKey = System.Configuration.ConfigurationManager.AppSettings.Get("AWSSecretKey");
AmazonS3 client;
public string OriginalFilename { get; set; }
public string DestinationFilename { get; set; }
public string DestinationBucket { get; set; }
public InitiateMultipartUploadResponse initResponse;
public List<PartETag> uploadPartETags;
public string UploadId { get; private set; }
public S3MultipartUpload(string destinationFilename, string destinationBucket)
{
if (client == null)
{
System.Net.WebRequest.DefaultWebProxy = null; // disable proxy to make upload quicker
client = Amazon.AWSClientFactory.CreateAmazonS3Client(accessKey, secretAccessKey, new AmazonS3Config()
{
RegionEndpoint = Amazon.RegionEndpoint.EUWest1,
CommunicationProtocol = Protocol.HTTP
});
this.OriginalFilename = destinationFilename.TrimStart('/');
this.DestinationFilename = string.Format("{0:yyyy}{0:MM}{0:dd}{0:HH}{0:mm}{0:ss}{0:fffff}_{1}", DateTime.UtcNow, this.OriginalFilename);
this.DestinationBucket = destinationBucket;
this.InitializePartToCloud();
}
}
private void InitializePartToCloud()
{
// 1. Initialize.
uploadPartETags = new List<PartETag>();
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest();
initRequest.BucketName = this.DestinationBucket;
initRequest.Key = this.DestinationFilename;
// make it public
initRequest.AddHeader("x-amz-acl", "public-read");
initResponse = client.InitiateMultipartUpload(initRequest);
}
public void UploadPartToCloud(Stream fileStream, long uploadedBytes, long maxChunkedBytes)
{
int partNumber = uploadPartETags.Count() + 1; // current part
// 2. Upload Parts.
UploadPartRequest request = new UploadPartRequest();
request.BucketName = this.DestinationBucket;
request.Key = this.DestinationFilename;
request.UploadId = initResponse.UploadId;
request.PartNumber = partNumber;
request.PartSize = fileStream.Length;
//request.FilePosition = uploadedBytes // remove this line?
request.InputStream = fileStream; // as UploadPartRequest;
var up = client.UploadPart(request);
uploadPartETags.Add(new PartETag() { ETag = up.ETag, PartNumber = partNumber });
}
public string CompletePartToCloud()
{
// Step 3: complete.
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest();
compRequest.BucketName = this.DestinationBucket;
compRequest.Key = this.DestinationFilename;
compRequest.UploadId = initResponse.UploadId;
compRequest.PartETags = uploadPartETags;
string r = "Something went badly wrong";
using (CompleteMultipartUploadResponse completeUploadResponse = client.CompleteMultipartUpload(compRequest))
r = completeUploadResponse.ResponseXml;
return r;
}
public void AbortPartToCloud()
{
// abort.
client.AbortMultipartUpload(new AbortMultipartUploadRequest()
{
BucketName = this.DestinationBucket,
Key = this.DestinationFilename,
UploadId = initResponse.UploadId
});
}
public void Dispose()
{
if (client != null) client.Dispose();
if (initResponse != null) initResponse.Dispose();
}
}
I use DestinationFilename as the destination file so I can avoid the same name, but I keep the OriginalFilename as I needed later.
Using jQuery File Upload Plugin, all works inside a Generic Handler, and the process is something like this:
// Upload partial file
private void UploadPartialFile(string fileName, HttpContext context, List<FilesStatus> statuses)
{
if (context.Request.Files.Count != 1)
throw new HttpRequestValidationException("Attempt to upload chunked file containing more than one fragment per request");
var inputStream = context.Request.Files[0].InputStream;
string contentRange = context.Request.Headers["Content-Range"]; // "bytes 0-6291455/14130271"
int fileSize = int.Parse(contentRange.Split('/')[1]);,
maxChunkSize = int.Parse(context.Request.Headers["X-Max-Chunk-Size"]),
uploadedBytes = int.Parse(contentRange.Replace("bytes ", "").Split('-')[0]);
iView.Utilities.AWS.S3MultipartUpload s3Upload = null;
try
{
// ######################################################################################
// 1. Initialize Amazon S3 Client
if (uploadedBytes == 0)
{
HttpContext.Current.Session["s3-upload"] = new iView.Utilities.AWS.S3MultipartUpload(fileName, awsBucket);
s3Upload = (iView.Utilities.AWS.S3MultipartUpload)HttpContext.Current.Session["s3-upload"];
string msg = System.String.Format("Upload started: {0} ({1:N0}Mb)", s3Upload.DestinationFilename, (fileSize / 1024));
this.Log(msg);
}
// cast current session object
if (s3Upload == null)
s3Upload = (iView.Utilities.AWS.S3MultipartUpload)HttpContext.Current.Session["s3-upload"];
// ######################################################################################
// 2. Send Chunks
s3Upload.UploadPartToCloud(inputStream, uploadedBytes, maxChunkSize);
// ######################################################################################
// 3. Complete Upload
if (uploadedBytes + maxChunkSize > fileSize)
{
string completeRequest = s3Upload.CompletePartToCloud();
this.Log(completeRequest); // log S3 response
s3Upload.Dispose(); // dispose all objects
HttpContext.Current.Session["s3-upload"] = null; // we don't need this anymore
}
}
catch (System.Exception ex)
{
if (ex.InnerException != null)
while (ex.InnerException != null)
ex = ex.InnerException;
this.Log(string.Format("{0}\n\n{1}", ex.Message, ex.StackTrace)); // log error
s3Upload.AbortPartToCloud(); // abort current upload
s3Upload.Dispose(); // dispose all objects
statuses.Add(new FilesStatus(ex.Message));
return;
}
statuses.Add(new FilesStatus(s3Upload.DestinationFilename, fileSize, ""));
}
Keep in mind that to have a Session object inside a Generic Handler, you need to implement IRequiresSessionState so your handler will look like:
public class UploadHandlerSimple : IHttpHandler, IRequiresSessionState
Inside fileupload.js (under _initXHRData) I have added an extra header called X-Max-Chunk-Size so I can pass this to Amazon and calculate if it's the last part of the uploaded file.
Fell free to comment and make smart edits for everyone to use.
I guess you didn't set the content-length of the part inside the UploadPartToCloud() function.