We Keep Coding

Why is it not possible to delete files quickly (<60s) across threads in aspnet?

I get error
System.IO.IOException: 'The process cannot access the file 'xxx' because it is being used by another process.'
when I try to delete a temp file in a background worker service in aspnet core.
I am eventually allowed to delete the file after about a minute (52s, 73s).
If I change garbage collection to workstation mode, I may instead delete after ~1s (but still, a delay).
I have tried a combination of FileOptions to no avail, including FileOptions.WriteThrough.
When the controller writes the file, I use
FlushAsync(), Close(), Dispose() and 'using' (I know it's overkill.)
I also tried using just File.WriteAllBytesAsync, with same result.
In the background reader, I as well use Close() and Dispose().
(hint: background reader will not allow me to use DeleteOnClose,
which would have been ideal.)
As I search stackoverflow for similar 'used by another process' issues,
all those I have found eventually resolve to
'argh it turns out I/he still had an extra open instance/reference
he forgot about',
but I have not been able to figure out that I am doing that.
Another hint:
In the writing controller, I am able to delete the file immediately
after writing it, I presume because I am still on the same thread?
Is there some secret knowledge I should read somewhere,
about being able to delete recently open files, across threads?
UPDATE: Here relevant(?) code snippets:
// (AspNet Controller)
[RequestSizeLimit(9999999999)]
[DisableFormValueModelBinding]
[RequestFormLimits(MultipartBodyLengthLimit = MaxFileSize)]
[HttpPost("{sessionId}")]
public async Task<IActionResult> UploadRevisionChunk(Guid sessionId) {
log.LogWarning($"UploadRevisionChunk: {sessionId}");
string uploadFolder = UploadFolder.sessionFolderPath(sessionId);
if (!Directory.Exists(uploadFolder)) { throw new Exception($"chunk-upload failed"); }
var cr = parseContentRange(Request);
if (cr == null) { return this.BadRequest("no content range header specified"); }
string chunkName = $"{cr.From}-{cr.To}";
string saveChunkPath = Path.Combine(uploadFolder,chunkName);
await streamToChunkFile_WAB(saveChunkPath); // write-all-bytes.
//await streamToChunkFile_MAN(saveChunkPath); // Manual.
long crTo = cr.To ?? 0;
long crFrom = cr.From ?? 0;
long expected = (crTo - crFrom) + 1;
var fi = new FileInfo(saveChunkPath);
var dto = new ChunkResponse { wrote = fi.Length, expected = expected, where = "?" };
string msg = $"at {crFrom}, wrote {dto.wrote} bytes (expected {dto.expected}) to {dto.where}";
log.LogWarning(msg);
return Ok(dto);
}
private async Task streamToChunkFile_WAB(string saveChunkPath) {
using (MemoryStream ms = new MemoryStream()) {
Request.Body.CopyTo(ms);
byte[] allBytes = ms.ToArray();
await System.IO.File.WriteAllBytesAsync(saveChunkPath, allBytes);
}
}
// stream reader in the backgroundService:
public class MyMultiStream : Stream {
string[] filePaths;
FileStream curStream = null;
IEnumerator<string> i;
ILogger log;
QueueItem qItem;
public MyMultiStream(string[] filePaths_, Stream[] streams_, ILogger log_, QueueItem qItem_) {
qItem = qItem_;
log = log_;
filePaths = filePaths_;
log.LogWarning($"filepaths has #items: {filePaths.Length}");
IEnumerable<string> enumerable = filePaths;
i = enumerable.GetEnumerator();
i.MoveNext();// necessary to prime the iterator.
}
public override bool CanRead { get { return true; } }
public override bool CanWrite { get { return false; } }
public override bool CanSeek { get { return false; } }
public override long Length { get { throw new Exception("dont get length"); } }
public override long Position {
get { throw new Exception("dont get Position"); }
set { throw new Exception("dont set Position"); }
}
public override void SetLength(long value) { throw new Exception("dont set length"); }
public override long Seek(long offset, SeekOrigin origin) { throw new Exception("dont seek"); }
public override void Write(byte[] buffer, int offset, int count) { throw new Exception("dont write"); }
public override void Flush() { throw new Exception("dont flush"); }
public static int openStreamCounter = 0;
public static int closedStreamCounter = 0;
string curFileName = "?";
private FileStream getNextStream() {
string nextFileName = i.Current;
if (nextFileName == null) { throw new Exception("getNextStream should not be called past file list"); }
//tryDelete(nextFileName,log);
FileStream nextStream = new FileStream(
path:nextFileName,
mode: FileMode.Open,
access: FileAccess.Read,
share: FileShare.ReadWrite| FileShare.Delete,
bufferSize:4096, // apparently default.
options: 0
| FileOptions.Asynchronous
| FileOptions.SequentialScan
// | FileOptions.DeleteOnClose // (1) this ought to be possible, (2) we should fix this approach (3) if we can fix this, our issue is solved, and our code much simpler.
); // None); // ReadWrite); // None); // ReadWrite); //| FileShare.Read);
log.LogWarning($"TELLUS making new stream [{nextFileName}] opened:[{++openStreamCounter}] closed:[{closedStreamCounter}]");
curFileName = nextFileName;
++qItem.chunkCount;
return nextStream;
}
public override int Read(byte[] buffer, int offset, int count) {
int bytesRead = 0;
while (true) {
bytesRead = 0;
if (curStream == null) { curStream = getNextStream(); }
try {
bytesRead = curStream.Read(buffer, offset, count);
log.LogWarning($"..bytesRead:{bytesRead} [{Path.GetFileName(curFileName)}]"); // (only show a short name.)
} catch (Exception e) {
log.LogError($"failed reading [{curFileName}] [{e.Message}]",e);
}
if (bytesRead > 0) { break; }
curStream.Close();
curStream.Dispose();
curStream = null;
log.LogWarning($"TELLUS closing stream [{curFileName}] opened:[{openStreamCounter}] closed:[{++closedStreamCounter}]");
//tryDelete(curFileName); Presumably we can't delete so soon.
bool moreFileNames = i.MoveNext();
log.LogWarning($"moreFileNames?{moreFileNames}");
if (!moreFileNames) {
break;
}
}
return bytesRead;
}
..
// Background worker operating multistream:
public class BackgroundChunkWorker: BackgroundService {
ILogger L;
ChunkUploadQueue q;
public readonly IServiceScopeFactory scopeFactory;
public BackgroundChunkWorker(ILogger<int> log_, ChunkUploadQueue q_, IServiceScopeFactory scopeFactory_) {
q = q_; L = log_;
scopeFactory = scopeFactory_;
}
override protected async Task ExecuteAsync(CancellationToken cancel) { await BackgroundProcessing(cancel); }
private async Task BackgroundProcessing(CancellationToken cancel) {
while (!cancel.IsCancellationRequested) {
try {
await Task.Delay(1000,cancel);
bool ok = q.q.TryDequeue(out var item);
if (!ok) { continue; }
L.LogInformation($"item found! {item}");
await treatItemScope(item);
} catch (Exception ex) {
L.LogCritical("An error occurred when processing. Exception: {#Exception}", ex);
}
}
}
private async Task<bool> treatItemScope(QueueItem Qitem) {
using (var scope = scopeFactory.CreateScope()) {
var ris = scope.ServiceProvider.GetRequiredService<IRevisionIntegrationService>();
return await treatItem(Qitem, ris);
}
}
private async Task<bool> treatItem(QueueItem Qitem, IRevisionIntegrationService ris) {
await Task.Delay(0);
L.LogWarning($"TryAddValue from P {Qitem.sessionId}");
bool addOK = q.p.TryAdd(Qitem.sessionId, Qitem);
if (!addOK) {
L.LogError($"why couldnt we add session {Qitem.sessionId} to processing-queue?");
return false;
}
var startTime = DateTime.UtcNow;
Guid revisionId = Qitem.revisionId;
string[] filePaths = getFilePaths(Qitem.sessionId);
Stream[] streams = filePaths.Select(fileName => new FileStream(fileName, FileMode.Open)).ToArray();
MyMultiStream multiStream = new MyMultiStream(filePaths, streams, this.L, Qitem);
BimRevisionStatus brs = await ris.UploadRevision(revisionId, multiStream, startTime);
// (launchDeletes is my current hack/workaround,
// it is not part of the problem)
// await multiStream.launchDeletes();
Qitem.status = brs;
return true;
}
..

Loading XML document loads the same group twice

Some classes to start, I'm writing them all so you can reproduce my problem:
public class PermissionObject
{
public string permissionName;
public string permissionObject;
public bool permissionGranted;
public PermissionObject()
{
permissionName = "";
permissionObject = "";
permissionGranted = true;
}
public PermissionObject(string name, string obj, bool granted)
{
permissionName = name;
permissionObject = obj;
permissionGranted = granted;
}
}
public class Config
{
public string cmsDataPath = "";
public string cmsIP = "";
public List<UserClass> usersCMS = new List<UserClass>();
static public string pathToConfig = #"E:\testconpcms.xml";
public string cardServerAddress = "";
public void Save()
{
XmlSerializer serializer = new XmlSerializer(typeof(Config));
using (Stream fileStream = new FileStream(pathToConfig, FileMode.Create))
{
serializer.Serialize(fileStream, this);
}
}
public static Config Load()
{
if (File.Exists(pathToConfig))
{
XmlSerializer serializer = new XmlSerializer(typeof(Config));
try
{
using (Stream fileStream = new FileStream(pathToConfig, FileMode.Open))
{
return (Config)serializer.Deserialize(fileStream);
}
}
catch (Exception ex)
{
return new Config();
}
}
else
{
return null;
}
}
}
public class UserClass
{
public string Name;
public string Login;
public string Password;
public PCMS2 PermissionsList; // OR new PCMS1, as I will explain in a bit
public UserClass()
{
this.Name = "Admin";
this.Login = "61-64-6D-69-6E";
this.Password = "61-64-6D-69-6E";
this.PermissionsList = new PCMS2(); // OR new PCMS1, as I will explain in a bit
}
}
The problematic bit: consider two implementations of PCMS class, PCMS1 and PCMS2:
public class PCMS1
{
public PermissionObject p1, p2;
public PCMS1()
{
p1 = new PermissionObject("ImportConfigCMS", "tsmiImportCMSConfigFile", true);
p2 = new PermissionObject("ExportConfigCMS", "tsmiExportCMSConfigFile", true);
}
}
public class PCMS2
{
public List<PermissionObject> listOfPermissions = new List<PermissionObject>();
public PCMS2()
{
listOfPermissions.Add(new PermissionObject("ImportConfigCMS", "tsmiImportCMSConfigFile", true));
listOfPermissions.Add(new PermissionObject("ExportConfigCMS", "tsmiExportCMSConfigFile", true));
}
}
And finally main class:
public partial class Form1 : Form
{
private Config Con;
public Form1()
{
InitializeComponent();
}
private void Form1_Load(object sender, EventArgs e)
{
Con = Config.Load();
if (Con == null)
{
Con = new Config();
Con.cmsDataPath = #"E:\testconpcms.xml";
Con.Save();
}
if (Con.usersCMS.Count == 0)
{
UserClass adminDefault = new UserClass();
Con.usersCMS.Add(adminDefault);
Con.Save();
}
}
}
Now, using either PCMS1 or PCMS2, the config file generates properly - one user with 2 permissions.
However, when config file is present, calling Con = Config.Load() in the main class gives different results.
Using PCMS1, the Con object is as expected - 1 user with 2 permissions.
However, using PCMS2, the Con object is 1 user with 4 (four) permissions. It doubles that field (it's basically p1, p2, p1, p2). Put a BP to see Con after Load().
I guess the list (PCMS2) implementation is doing something wonky during load which I'm not aware of, but I can't seem to find the issue.

You creates your permission objects in constructor of PMCS2 you do it in the constructor of PMCS1 too, but there you do have two properties that will be overwritten by serializer.
In case of of PMCS2 your constructor adds two items to List and than serializer adds the items it has deserilized to the same list.
I don't know exactly your usecase but i would suggest to move init of the permissions to separated method:
public class PCMS1
{
public PermissionObject p1, p2;
public void Init()
{
p1 = new PermissionObject("ImportConfigCMS", "tsmiImportCMSConfigFile", true);
p2 = new PermissionObject("ExportConfigCMS", "tsmiExportCMSConfigFile", true);
}
}
public class PCMS2
{
public List<PermissionObject> listOfPermissions = new List<PermissionObject>();
public void Init()
{
listOfPermissions.Add(new PermissionObject("ImportConfigCMS", "tsmiImportCMSConfigFile", true));
listOfPermissions.Add(new PermissionObject("ExportConfigCMS", "tsmiExportCMSConfigFile", true));
}
}
after that you could call it, if you want to get initial settings:
if (Con.usersCMS.Count == 0)
{
UserClass adminDefault = new UserClass();
adminDefault.PermissionsList.Init();
Con.usersCMS.Add(adminDefault);
Con.Save();
}

Load serialized data in multiple applications C#

Scenario:
I have a program which uses a simple class to generate game data. Using the said program, I write the data out using serialization and BinaryFormatter to a file to be used by a second program. Reading the data from this initial program works without issue.
Problem:
It's probably down to my ignorance to how serialized files are handled, but I cannot then load this data into a second program, the actual game itself.
saveGame code (in program 1):
static List<GameData> gameData;
static GameData currentData;
private void saveGame(Sudoku sdk) {
BinaryFormatter bf = new BinaryFormatter();
FileStream file = null;
try {
if(!File.Exists(gameDataFile[currentDifficulty])) {
file = File.Open(gameDataFile[currentDifficulty], FileMode.CreateNew);
} else {
file = File.Open(gameDataFile[currentDifficulty], FileMode.Append);
}
currentData = setGameData(sdk);
bf.Serialize(file, currentData);
savePuzzleLog();
} catch(Exception e) {
Debug.LogException("saveGame", e);
}
if(file != null) {
file.Close();
}
}
loadGameData: (in program 2)
public static List<GameData> gameData;
public bool loadGameData() {
if(gameData == null) {
gameData = new List<GameData>();
} else {
gameData.Clear();
}
bool loadData = true;
bool OK = false;
if(File.Exists(gameDataFile[currentDifficulty])) {
BinaryFormatter bf = new BinaryFormatter();
FileStream file = File.Open(gameDataFile[currentDifficulty], FileMode.Open);
while(loadData) {
try {
GameData gd = new GameData();
gd = (GameData)bf.Deserialize(file);
gameData.Add(gd);
OK = true;
if(file.Position == file.Length) {
loadData = false;
}
} catch(Exception e) {
Debug.LogException(e);
loadData = false;
OK = false;
}
}
if(file != null) {
file.Close();
}
} else {
Debug.LogWarning(gameDataFile[currentDifficulty] + " does not exist!");
}
return OK;
}
GameData Class: (1st program)
[Serializable]
class GameData {
private int gameID;
private List<int> contentArray;
private int difficultyValue;
public GameData(List<int> data = null) {
id = -1;
difficulty = -1;
if(content != null) {
content.Clear();
} else {
content = new List<int>();
}
if(data != null) {
foreach(int i in data) {
content.Add(i);
}
}
}
public int id {
get {
return this.gameID;
}
set {
this.gameID = value;
}
}
public int difficulty {
get {
return this.difficultyValue;
}
set {
this.difficultyValue = value;
}
}
public List<int> content {
get {
return this.contentArray;
}
set {
this.contentArray = value;
}
}
}
GameData Class: (2nd program) The only difference is declaring as public
[Serializable]
public class GameData {
private int gameID;
private List<int> contentArray;
private int difficultyValue;
public GameData(List<int> data = null) {
id = -1;
difficulty = -1;
if(content != null) {
content.Clear();
} else {
content = new List<int>();
}
if(data != null) {
foreach(int i in data) {
content.Add(i);
}
}
}
public int id {
get {
return this.gameID;
}
set {
this.gameID = value;
}
}
public int difficulty {
get {
return this.difficultyValue;
}
set {
this.difficultyValue = value;
}
}
public List<int> content {
get {
return this.contentArray;
}
set {
this.contentArray = value;
}
}
}
What my question is, is how do I save the data out in one program and be able to load it using a different program without getting serialization errors or do I need to use an alternate save/load method and/or class structure?

When I had to do it i made it this way :
An independant dll (assembly) containing the class holding the data (for you the GameData class), and utility methods to save/load from a file.
Your two other projects must then reference this dll (assembly) and you should be able to (de)serialize correctly.
What I think the issue is in your case is that the BinaryFormatter does not only save the data in the file, but also the complete Type of the serialized object.
When you try to deserialize in another similar object, even if the structure is the same, the full name of the class is not (because the assembly name is not).

OK, I've sorted it using the advice given. Instead of using BinaryFormatter I have used BinaryWriter and BinaryReader as follows...
In program 1 (the creator):
private byte[] setByteData(Sudoku sdk) {
List<int> clues = sdk.puzzleListData();
byte[] bd = new byte[puzzleSize];
SudokuSolver solver = new SudokuSolver();
List<int> solution = solver.Solve(sdk.Copy(), false, currentDifficulty).puzzleListData();
for(int i = 0; i < puzzleSize; i++) {
bd[i] = Convert.ToByte(solution[i] + (clues[i] == 0 ? 0xF0 : 0));
}
return bd;
}
private GameData setGameData(Sudoku sdk) {
List<int> clues = sdk.puzzleListData();
GameData gd = new GameData();
gd.id = puzzleList.Items.Count;
gd.difficulty = currentDifficulty;
SudokuSolver solver = new SudokuSolver();
List<int> solution = solver.Solve(sdk.Copy(), false, currentDifficulty).puzzleListData();
for(int i = 0; i < puzzleSize; i++) {
gd.content.Add(solution[i] + (clues[i] == 0 ? 0xF0 : 0));
}
return gd;
}
private List<int> getByteData(byte[] data) {
List<int> retVal = new List<int>();
foreach(byte i in data) {
if(i > 9) {
retVal.Add(i - 0xF0);
} else {
retVal.Add(0);
}
}
return retVal;
}
private string getGameData(List<int> data) {
string retVal = "";
foreach(int i in data) {
if(i > 9) {
retVal += (i - 0xF0).ToString();
} else {
retVal += i.ToString();
}
}
return retVal;
}
private void saveGame(Sudoku sdk) {
FileStream file = null;
BinaryWriter bw = null;
try {
if(!File.Exists(gameDataFile[currentDifficulty])) {
file = File.Open(gameDataFile[currentDifficulty], FileMode.CreateNew);
} else {
file = File.Open(gameDataFile[currentDifficulty], FileMode.Append);
}
bw = new BinaryWriter(file);
currentData = setGameData(sdk);
byte[] bd = setByteData(sdk);
bw.Write(currentData.id);
bw.Write(currentData.difficulty);
bw.Write(bd);
savePuzzleLog();
} catch(Exception e) {
Debug.LogException("saveGame", e);
}
if(file != null) {
if(bw != null) {
bw.Flush();
bw.Close();
}
file.Close();
}
}
In program 2: (the actual game)
public bool loadGameData() {
if(gameData == null) {
gameData = new List<GameData>();
} else {
gameData.Clear();
}
bool loadData = true;
bool OK = false;
if(File.Exists(gameDataFile[currentDifficulty])) {
FileStream file = File.Open(gameDataFile[currentDifficulty], FileMode.Open);
BinaryReader br = new BinaryReader(file);
while(loadData) {
try {
GameData gd = new GameData();
gd.id = br.ReadInt32();
gd.difficulty = br.ReadInt32();
gd.content = getByteData(br.ReadBytes(puzzleSize));
gameData.Add(gd);
OK = true;
if(file.Position == file.Length) {
loadData = false;
}
} catch(Exception e) {
Debug.LogException(e);
loadData = false;
OK = false;
}
}
if(file != null) {
file.Close();
}
} else {
Debug.LogWarning(gameDataFile[currentDifficulty] + " does not exist!");
}
return OK;
}
The class structure is the same as before but using this method has resolved my issue and I can now create the data files using my creator and load the data comfortably using the actual game.
Thanks all for your assistance and advice to help me get this sorted.

Correct way to store encryption key for SqlCipher database

I have a Xamarin application and have managed to download my data from my server to my device. I have also got it set up so that it can take a SqlCipher Encryption key to encrypt the data.
My question is where is the correct location to store my key that I use to encrypt this data? Is it to you KeyStore / KeyChain? Which mono classes should I be looking to use?

Due to the popularity of this question I am going to post my implementation of this:
PCL interface
public interface IAuth
{
void CreateStore();
IEnumerable<string> FindAccountsForService(string serviceId);
void Save(string pin,string serviceId);
void Delete(string serviceId);
}
Android
public class IAuthImplementation : IAuth
{
Context context;
KeyStore ks;
KeyStore.PasswordProtection prot;
static readonly object fileLock = new object();
const string FileName = "MyProg.Accounts";
static readonly char[] Password = null;
public void CreateStore()
{
this.context = Android.App.Application.Context;
ks = KeyStore.GetInstance(KeyStore.DefaultType);
prot = new KeyStore.PasswordProtection(Password);
try
{
lock (fileLock)
{
using (var s = context.OpenFileInput(FileName))
{
ks.Load(s, Password);
}
}
}
catch (Java.IO.FileNotFoundException)
{
//ks.Load (null, Password);
LoadEmptyKeyStore(Password);
}
}
public IEnumerable<string> FindAccountsForService(string serviceId)
{
var r = new List<string>();
var postfix = "-" + serviceId;
var aliases = ks.Aliases();
while (aliases.HasMoreElements)
{
var alias = aliases.NextElement().ToString();
if (alias.EndsWith(postfix))
{
var e = ks.GetEntry(alias, prot) as KeyStore.SecretKeyEntry;
if (e != null)
{
var bytes = e.SecretKey.GetEncoded();
var password = System.Text.Encoding.UTF8.GetString(bytes);
r.Add(password);
}
}
}
return r;
}
public void Delete(string serviceId)
{
var alias = MakeAlias(serviceId);
ks.DeleteEntry(alias);
Save();
}
public void Save(string pin, string serviceId)
{
var alias = MakeAlias(serviceId);
var secretKey = new SecretAccount(pin);
var entry = new KeyStore.SecretKeyEntry(secretKey);
ks.SetEntry(alias, entry, prot);
Save();
}
void Save()
{
lock (fileLock)
{
using (var s = context.OpenFileOutput(FileName, FileCreationMode.Private))
{
ks.Store(s, Password);
}
}
}
static string MakeAlias(string serviceId)
{
return "-" + serviceId;
}
class SecretAccount : Java.Lang.Object, ISecretKey
{
byte[] bytes;
public SecretAccount(string password)
{
bytes = System.Text.Encoding.UTF8.GetBytes(password);
}
public byte[] GetEncoded()
{
return bytes;
}
public string Algorithm
{
get
{
return "RAW";
}
}
public string Format
{
get
{
return "RAW";
}
}
}
static IntPtr id_load_Ljava_io_InputStream_arrayC;
void LoadEmptyKeyStore(char[] password)
{
if (id_load_Ljava_io_InputStream_arrayC == IntPtr.Zero)
{
id_load_Ljava_io_InputStream_arrayC = JNIEnv.GetMethodID(ks.Class.Handle, "load", "(Ljava/io/InputStream;[C)V");
}
IntPtr intPtr = IntPtr.Zero;
IntPtr intPtr2 = JNIEnv.NewArray(password);
JNIEnv.CallVoidMethod(ks.Handle, id_load_Ljava_io_InputStream_arrayC, new JValue[]
{
new JValue (intPtr),
new JValue (intPtr2)
});
JNIEnv.DeleteLocalRef(intPtr);
if (password != null)
{
JNIEnv.CopyArray(intPtr2, password);
JNIEnv.DeleteLocalRef(intPtr2);
}
}
Call Create Store in the main activity of Android app first. - This could possibly be improved and remove CreateStrore() from the interface by checking if ks == null in Save and Delete and calling the method if true
iOS
public class IAuthImplementation : IAuth
{
public IEnumerable<string> FindAccountsForService(string serviceId)
{
var query = new SecRecord(SecKind.GenericPassword);
query.Service = serviceId;
SecStatusCode result;
var records = SecKeyChain.QueryAsRecord(query, 1000, out result);
return records != null ?
records.Select(GetAccountFromRecord).ToList() :
new List<string>();
}
public void Save(string pin, string serviceId)
{
var statusCode = SecStatusCode.Success;
var serializedAccount = pin;
var data = NSData.FromString(serializedAccount, NSStringEncoding.UTF8);
//
// Remove any existing record
//
var existing = FindAccount(serviceId);
if (existing != null)
{
var query = new SecRecord(SecKind.GenericPassword);
query.Service = serviceId;
statusCode = SecKeyChain.Remove(query);
if (statusCode != SecStatusCode.Success)
{
throw new Exception("Could not save account to KeyChain: " + statusCode);
}
}
//
// Add this record
//
var record = new SecRecord(SecKind.GenericPassword);
record.Service = serviceId;
record.Generic = data;
record.Accessible = SecAccessible.WhenUnlocked;
statusCode = SecKeyChain.Add(record);
if (statusCode != SecStatusCode.Success)
{
throw new Exception("Could not save account to KeyChain: " + statusCode);
}
}
public void Delete(string serviceId)
{
var query = new SecRecord(SecKind.GenericPassword);
query.Service = serviceId;
var statusCode = SecKeyChain.Remove(query);
if (statusCode != SecStatusCode.Success)
{
throw new Exception("Could not delete account from KeyChain: " + statusCode);
}
}
string GetAccountFromRecord(SecRecord r)
{
return NSString.FromData(r.Generic, NSStringEncoding.UTF8);
}
string FindAccount(string serviceId)
{
var query = new SecRecord(SecKind.GenericPassword);
query.Service = serviceId;
SecStatusCode result;
var record = SecKeyChain.QueryAsRecord(query, out result);
return record != null ? GetAccountFromRecord(record) : null;
}
public void CreateStore()
{
throw new NotImplementedException();
}
}
WP
public class IAuthImplementation : IAuth
{
public IEnumerable<string> FindAccountsForService(string serviceId)
{
using (var store = IsolatedStorageFile.GetUserStoreForApplication())
{
string[] auths = store.GetFileNames("MyProg");
foreach (string path in auths)
{
using (var stream = new BinaryReader(new IsolatedStorageFileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, store)))
{
int length = stream.ReadInt32();
byte[] data = stream.ReadBytes(length);
byte[] unprot = ProtectedData.Unprotect(data, null);
yield return Encoding.UTF8.GetString(unprot, 0, unprot.Length);
}
}
}
}
public void Save(string pin, string serviceId)
{
byte[] data = Encoding.UTF8.GetBytes(pin);
byte[] prot = ProtectedData.Protect(data, null);
var path = GetAccountPath(serviceId);
using (var store = IsolatedStorageFile.GetUserStoreForApplication())
using (var stream = new IsolatedStorageFileStream(path, FileMode.Create, FileAccess.Write, FileShare.None, store))
{
stream.WriteAsync(BitConverter.GetBytes(prot.Length), 0, sizeof(int)).Wait();
stream.WriteAsync(prot, 0, prot.Length).Wait();
}
}
public void Delete(string serviceId)
{
var path = GetAccountPath(serviceId);
using (var store = IsolatedStorageFile.GetUserStoreForApplication())
{
store.DeleteFile(path);
}
}
private string GetAccountPath(string serviceId)
{
return String.Format("{0}", serviceId);
}
public void CreateStore()
{
throw new NotImplementedException();
}
}
This is an adaptation of the Xamarin.Auth library (Found Here) but removes the dependency from the Xamarin.Auth library to provide cross platform use through the interface in the PCL. For this reason I have simplified it to only save one string. This is probably not the best implementation but it works in my case. Feel free to expand upon this

There is a nuget package called KeyChain.NET that encapsulated this logic for iOs, Android and Windows Phone.
It's open source and you have find sample at its github repository
More info at this blog post

how to upload a large file with ASP.NET MVC4 Web Api with progressbar

how can i upload a large file with ASP.NET MVC4 Web Api
and also get a progress?
i saw this post and i understand how to handle the uploaded file but how i can get the progress data?
How To Accept a File POST
please don't send me links to upload products.
i want to understand how handle this in the MVC4 Web Api way...
here is an example code of handling a file upload in MVC4 WebApi
public async Task<HttpResponseMessage> Post()
{
if (Request.Content.IsMimeMultipartContent())
{
var path = HttpContext.Current.Server.MapPath("~/App_Data");
var provider = new MultipartFormDataStreamProvider(path);
await Request.Content.ReadAsMultipartAsync(provider).ContinueWith(t =>
{
if (t.IsFaulted || t.IsCanceled)
throw new HttpResponseException(HttpStatusCode.InternalServerError);
});
return Request.CreateResponse(HttpStatusCode.OK);
}
else
{
throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.NotAcceptable, "This request is not properly formatted"));
}
}
now when
await Request.Content.ReadAsMultipartAsync(provider)
how can i get how bytes loaded?

There is a limitation to the size of files to be uploaded by default at two places. One at the request level, and second , if you hosting on IIS, then on web server level. I added couple of configs as mentioned in this blog, and i was able to upload a 36mb file without any issues. I have posted the snippet below.
Basically
1.
<system.web>
<httpRuntime maxRequestLength="2097152"/>
</system.web>
2.
<system.webServer>
<security>
<requestFiltering>
<requestLimits maxAllowedContentLength="2147483648" />
</requestFiltering>
</security><system.webServer>
Its easy to find the size of the file loaded into the server if you wish. In your code
while reading through the filedata in the stream, for each item in you file data, you can read the local file name as shown below.
string savedFile = fileData.LocalFileName;
// use the file info class to derive properties of the uploaded file
FileInfo file = new FileInfo(savedFile);
//this will give the size of the uploaded file
long size = file.length/1024
Hope this helps. I wonder why this was marked down?

I use this solution:
public class UploadController : ApiController
{
private static ConcurrentDictionary<string, State> _state = new ConcurrentDictionary<string, State>();
public State Get(string id)
{
State state;
if (_state.TryGetValue(id, out state))
{
return state;
}
return null;
}
public async Task<HttpResponseMessage> Post([FromUri] string id)
{
if (Request.Content.IsMimeMultipartContent())
{
var state = new State(Request.Content.Headers.ContentLength);
if (!_state.TryAdd(id, state))
throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.Conflict));
var path = System.Web.Hosting.HostingEnvironment.MapPath("~/App_Data");
var provider = new FileMultipartStreamProvider(path, state.Start, state.AddBytes);
await Request.Content.ReadAsMultipartAsync(provider).ContinueWith(t =>
{
_state.TryRemove(id, out state);
if (t.IsFaulted || t.IsCanceled)
throw new HttpResponseException(HttpStatusCode.InternalServerError);
});
return Request.CreateResponse(HttpStatusCode.OK);
}
else
{
throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.NotAcceptable, "This request is not properly formatted"));
}
}
}
public class State
{
public long? Total { get; set; }
public long Received { get; set; }
public string Name { get; set; }
public State(long? total = null)
{
Total = total;
}
public void Start(string name)
{
Received = 0;
Name = name;
}
public void AddBytes(long size)
{
Received = size;
}
}
public class FileMultipartStreamProvider : MultipartStreamProvider
{
private string _rootPath;
private Action<string> _startUpload;
private Action<long> _uploadProgress;
public FileMultipartStreamProvider(string root_path, Action<string> start_upload, Action<long> upload_progress)
: base()
{
_rootPath = root_path;
_startUpload = start_upload;
_uploadProgress = upload_progress;
}
public override System.IO.Stream GetStream(HttpContent parent, System.Net.Http.Headers.HttpContentHeaders headers)
{
var name = (headers.ContentDisposition.Name ?? "undefined").Replace("\"", "").Replace("\\", "_").Replace("/", "_").Replace("..", "_");
_startUpload(name);
return new WriteFileStreamProxy(Path.Combine(_rootPath, name), _uploadProgress);
}
}
public class WriteFileStreamProxy : FileStream
{
private Action<long> _writeBytes;
public WriteFileStreamProxy(string file_path, Action<long> write_bytes)
: base(file_path, FileMode.Create, FileAccess.Write)
{
_writeBytes = write_bytes;
}
public override void EndWrite(IAsyncResult asyncResult)
{
base.EndWrite(asyncResult);
#if DEBUG
System.Threading.Thread.Sleep(100);
#endif
if (_writeBytes != null)
_writeBytes(base.Position);
}
public override void Write(byte[] array, int offset, int count)
{
base.Write(array, offset, count);
#if DEBUG
System.Threading.Thread.Sleep(100);
#endif
if (_writeBytes != null)
_writeBytes(base.Position);
}
}
and small configure for non-buffered input stream:
config.Services.Replace(typeof(IHostBufferPolicySelector), new CustomPolicy());
implemented this:
public class CustomPolicy : System.Web.Http.WebHost.WebHostBufferPolicySelector
{
public override bool UseBufferedInputStream(object hostContext)
{
return false;
}
}

I Ended Up using an HttpModule but even the HttpModule won't show the progress bar
I found out something very interesting it's seems that when i upload the file in a Secure Protocol(over https://) then the progress are working but in non secure protocl (http://) the progress is not working and the file is fully buffered i don't know way is like that i believe it's a bug somewhere between the IIS to Asp.net Framework when the Request are get Procced.
now because i success make it work over https with an HttpModule i believe it is possible to make it work also with Mvc Web Api but i currently don't have the time to check that.
for parsing Mutlipart form data i used Nancy HttpMultipart parser here:
https://github.com/NancyFx/Nancy/tree/master/src/Nancy
just grabbed the classes:
HttpMultipart.cs
HttpMultipartBoundary.cs
HttpMultipartBuffer.cs
HttpMultipartSubStream.cs
here is the HttpModule Source:
public class HttpUploadModule : IHttpModule
{
public static DateTime lastClean = DateTime.UtcNow;
public static TimeSpan cleanInterval = new TimeSpan(0,10,0);
public static readonly object cleanLocker = new object();
public static readonly Dictionary<Guid,UploadData> Uploads = new Dictionary<Guid,UploadData>();
public const int KB = 1024;
public const int MB = KB * 1024;
public static void CleanUnusedResources( HttpContext context)
{
if( lastClean.Add( cleanInterval ) < DateTime.UtcNow ) {
lock( cleanLocker )
{
if( lastClean.Add( cleanInterval ) < DateTime.UtcNow )
{
int maxAge = int.Parse(ConfigurationManager.AppSettings["HttpUploadModule.MaxAge"]);
Uploads.Where(u=> DateTime.UtcNow.AddSeconds(maxAge) > u.Value.createdDate ).ToList().ForEach(u=>{
Uploads.Remove(u.Key);
});
Directory.GetFiles(context.Server.MapPath(ConfigurationManager.AppSettings["HttpUploadModule.Folder"].TrimEnd('/'))).ToList().ForEach(f=>{
if( DateTime.UtcNow.AddSeconds(maxAge) > File.GetCreationTimeUtc(f)) File.Delete(f);
});
lastClean = DateTime.UtcNow;
}
}
}
}
public void Dispose()
{
}
public void Init(HttpApplication app)
{
app.BeginRequest += app_BeginRequest;
}
void app_BeginRequest(object sender, EventArgs e)
{
HttpContext context = ((HttpApplication)sender).Context;
Guid uploadId = Guid.Empty;
if (context.Request.HttpMethod == "POST" && context.Request.ContentType.ToLower().StartsWith("multipart/form-data"))
{
IServiceProvider provider = (IServiceProvider)context;
HttpWorkerRequest wr = (HttpWorkerRequest)provider.GetService(typeof(HttpWorkerRequest));
FileStream fs = null;
MemoryStream ms = null;
CleanUnusedResources(context);
string contentType = wr.GetKnownRequestHeader(HttpWorkerRequest.HeaderContentType);
NameValueCollection queryString = HttpUtility.ParseQueryString( wr.GetQueryString() );
UploadData upload = new UploadData { id = uploadId ,status = 0, createdDate = DateTime.UtcNow };
if(
!contentType.Contains("boundary=") ||
/*AT LAST 1KB */ context.Request.ContentLength < KB ||
/*MAX 5MB */ context.Request.ContentLength > MB*5 ||
/*IS UPLOADID */ !Guid.TryParse(queryString["upload_id"], out uploadId) || Uploads.ContainsKey( uploadId )) {
upload.id = uploadId;
upload.status = 2;
Uploads.Add(upload.id, upload);
context.Response.StatusCode = 400;
context.Response.StatusDescription = "Bad Request";
context.Response.End();
}
string boundary = Nancy.HttpMultipart.ExtractBoundary( contentType );
upload.id = uploadId;
upload.status = 0;
Uploads.Add(upload.id, upload);
try {
if (wr.HasEntityBody())
{
upload.bytesRemaining =
upload.bytesTotal = wr.GetTotalEntityBodyLength();
upload.bytesLoaded =
upload.BytesReceived = wr.GetPreloadedEntityBodyLength();
if (!wr.IsEntireEntityBodyIsPreloaded())
{
byte[] buffer = new byte[KB * 8];
int readSize = buffer.Length;
ms = new MemoryStream();
//fs = new FileStream(context.Server.MapPath(ConfigurationManager.AppSettings["HttpUploadModule.Folder"].TrimEnd('/')+'/' + uploadId.ToString()), FileMode.CreateNew);
while (upload.bytesRemaining > 0)
{
upload.BytesReceived = wr.ReadEntityBody(buffer, 0, readSize);
if(upload.bytesRemaining == upload.bytesTotal) {
}
ms.Write(buffer, 0, upload.BytesReceived);
upload.bytesLoaded += upload.BytesReceived;
upload.bytesRemaining -= upload.BytesReceived;
if (readSize > upload.bytesRemaining)
{
readSize = upload.bytesRemaining;
}
}
//fs.Flush();
//fs.Close();
ms.Position = 0;
//the file is in our hands
Nancy.HttpMultipart multipart = new Nancy.HttpMultipart(ms, boundary);
foreach( Nancy.HttpMultipartBoundary b in multipart.GetBoundaries()) {
if(b.Name == "data") {
upload.filename = uploadId.ToString()+Path.GetExtension( b.Filename ).ToLower();
fs = new FileStream(context.Server.MapPath(ConfigurationManager.AppSettings["HttpUploadModule.Folder"].TrimEnd('/')+'/' + upload.filename ), FileMode.CreateNew);
b.Value.CopyTo(fs);
fs.Flush();
fs.Close();
upload.status = 1;
context.Response.StatusCode = 200;
context.Response.StatusDescription = "OK";
context.Response.Write( context.Request.ApplicationPath.TrimEnd('/') + "/images/temp/" + upload.filename );
}
}
}
}
}
catch(Exception ex) {
upload.ex = ex;
}
if(upload.status != 1)
{
upload.status = 2;
context.Response.StatusCode = 400;
context.Response.StatusDescription = "Bad Request";
}
context.Response.End();
}
}
}
public class UploadData {
public Guid id { get;set; }
public string filename {get;set;}
public int bytesLoaded { get; set; }
public int bytesTotal { get; set; }
public int BytesReceived {get; set;}
public int bytesRemaining { get;set; }
public int status { get;set; }
public Exception ex { get;set; }
public DateTime createdDate { get;set; }
}