We Keep Coding

Importing files into SQLite database not using all cores

Context
I am importing IMDB database files into an SQLite database with the help of EntityFrameworkCore. In fact, two files, the titles.basics and the titles.akas (which is linked to basics via its movie ID).
At first, I had a single thread reading lines from basics and loop through akas until it changes of ID. Though, there was an issue there and most of all, it was too slow. So, I decided to create a multithread code that would read both files at the same time and another combining akas with the appropriate movie.
I am currently importing so I still do not know if my issue is fixed (probably it is). Though, it is still too much slow for me.
Issue
The combining part is still very slow, but more importantly, I can see my process is only using around 12% of CPU which corresponds to only 1/8 of total usage and I have 8 physical cores. So, it really seems the process is only using 1 core.
I am not giving any code here, as having a minimal testable code wouldn't mean anything. Though, you can see both versions here:
https://cints.net/public/Imdb-MultiThread.cs.txt
using com.cyberinternauts.all.MediaRecognizer.Database;
using com.cyberinternauts.all.MediaRecognizer.Models.Metas;
using Microsoft.EntityFrameworkCore;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
namespace com.cyberinternauts.all.MediaRecognizer.MetaSources
{
class Imdb : MediaSource
{
private const string TITLES_FILE = "title.basics.tsv.gz";
private const string AKAS_FILE = "title.akas.tsv.gz";
private readonly string temporaryFolder = #"c:\temp\";
private readonly string baseUrl = "https://datasets.imdbws.com/";
private readonly WebClient webClient = new();
MediaRecognizerContext db = new();
private IQueryable<MetaMovie> imdbMovies = null;
private async Task<bool> GatherFilesAsync()
{
var totalFilesGathered = 0;
var filesToDownload = new string[] { AKAS_FILE, TITLES_FILE };
foreach(var fileToDownload in filesToDownload)
{
var compressedFile = temporaryFolder + fileToDownload;
if (!File.Exists(compressedFile) || !File.GetLastWriteTime(compressedFile).Date.Equals(DateTime.Today))
{
await GatherFileAsync(fileToDownload);
totalFilesGathered++;
}
}
return totalFilesGathered != 0;
}
private async Task GatherFileAsync(string fileName)
{
var compressedFile = temporaryFolder + fileName;
var uncompressedFile = temporaryFolder + Path.GetFileNameWithoutExtension(compressedFile);
await webClient.DownloadFileTaskAsync(baseUrl + fileName, compressedFile);
using Stream fd = File.Create(uncompressedFile);
using Stream fs = File.OpenRead(compressedFile);
using Stream csStream = new GZipStream(fs, CompressionMode.Decompress);
var buffer = new byte[1024];
int nRead;
while ((nRead = await csStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
await fd.WriteAsync(buffer, 0, nRead);
}
}
private async Task LoadMetaDataAsync()
{
//return; //TODO: Remove this line
//TODO: Reactivate this line
//if (!await GatherFilesAsync()) return;
var titlesFile = temporaryFolder + Path.GetFileNameWithoutExtension(TITLES_FILE);
var akasFile = temporaryFolder + Path.GetFileNameWithoutExtension(AKAS_FILE);
var dbLock = new SemaphoreSlim(1);
var akasLock = new SemaphoreSlim(1);
var currentTitlesAkasLock = new SemaphoreSlim(1);
var associateLock = new SemaphoreSlim(1);
using (var db = new MediaRecognizerContext())
{
db.ChangeTracker.AutoDetectChangesEnabled = false;
var titles = new ConcurrentDictionary<string, MetaMovie>();
var readTitles = Task.Factory.StartNew(() =>
{
Parallel.ForEach(File.ReadLines(titlesFile), (titleLine, _, readingIndex) =>
{
if (readingIndex == 0) return; // Skipping columns titles line
var movieInfos = titleLine.Split("\t", StringSplitOptions.None);
dbLock.Wait();
MetaMovie metaMovie = db.MetaMovies.Where(m => m.ExternalId == movieInfos[0]).Include(m => m.Titles).FirstOrDefault();
dbLock.Release();
if (metaMovie == null)
{
int totalMinutes = -1;
if (!int.TryParse(movieInfos[7], out totalMinutes))
{
totalMinutes = -1;
}
metaMovie = new MetaMovie
{
ExternalId = movieInfos[0],
MetaSource = nameof(Imdb),
MovieType = movieInfos[1],
Title = movieInfos[3],
TotalMinutes = totalMinutes,
Genres = movieInfos[8]
};
metaMovie.Titles = new List<MetaTitle>();
if (int.TryParse(movieInfos[5], out int startYear))
{
metaMovie.StartYear = new DateTime(startYear, 1, 1);
}
else
{
metaMovie.StartYear = new DateTime(9999, 1, 1);
}
if (int.TryParse(movieInfos[6], out int endYear))
{
metaMovie.EndYear = new DateTime(endYear, 1, 1);
}
else
{
metaMovie.EndYear = metaMovie.StartYear;
}
}
titles.TryAdd(metaMovie.ExternalId, metaMovie);
});
});
var akas = new Dictionary<string, List<MetaTitle>>();
var currentTitlesAkas = new ConcurrentDictionary<string, int>();
var readAkas = Task.Factory.StartNew(() =>
{
Parallel.ForEach(File.ReadLines(akasFile), (akaLine, _, readingIndex) =>
{
if (readingIndex == 0) return; // Skipping columns titles line
currentTitlesAkasLock.Wait();
var titleInfos = akaLine.Split("\t", StringSplitOptions.None);
var externalId = titleInfos[0];
if (!currentTitlesAkas.ContainsKey(externalId))
{
currentTitlesAkas.TryAdd(externalId, 1);
}
else
{
currentTitlesAkas[externalId]++;
}
currentTitlesAkasLock.Release();
var metaTitle = new MetaTitle
{
MetaMovie = null,
Text = titleInfos[2],
Region = titleInfos[3],
Language = titleInfos[4]
};
akasLock.Wait();
List<MetaTitle> titleAkas;
if (!akas.ContainsKey(externalId))
{
titleAkas = new List<MetaTitle>();
akas.Add(externalId, titleAkas);
}
else
{
titleAkas = akas[externalId];
}
titleAkas.Add(metaTitle);
akasLock.Release();
currentTitlesAkasLock.Wait();
currentTitlesAkas[externalId]--;
currentTitlesAkasLock.Release();
});
});
var savingCounter = 0;
var associate = Task.Factory.StartNew(() =>
{
Parallel.For(1, Environment.ProcessorCount * 10, async (_) =>
{
var isAssociating = true;
do
{
var externalId = string.Empty;
var currentTitleAkaRemoved = false;
currentTitlesAkasLock.Wait();
foreach (var curExternalId in currentTitlesAkas.Keys.OrderBy(t => t))
{
if (currentTitlesAkas[curExternalId] == 0)
{
externalId = curExternalId;
break;
}
}
if (externalId != String.Empty)
{
currentTitleAkaRemoved = currentTitlesAkas.TryRemove(externalId, out int useless0); // Removing so other threads won't take it
}
isAssociating = !readAkas.IsCompleted || !readTitles.IsCompleted || !currentTitlesAkas.IsEmpty;
currentTitlesAkasLock.Release();
if (String.IsNullOrEmpty(externalId) || !currentTitleAkaRemoved) continue;
if (titles.TryGetValue(externalId, out MetaMovie metaMovie))
{
akasLock.Wait();
var titleAkas = akas[externalId];
akas.Remove(externalId);
akasLock.Release();
var changedMovie = false;
var movieAkas = metaMovie.Titles.Select(t => t).ToList(); // Clone list
foreach (var metaTitle in titleAkas)
{
var existingTitle = movieAkas.Where(t => t.Text == metaTitle.Text && t.Region == metaTitle.Region && t.Language == metaTitle.Language).FirstOrDefault();
if (existingTitle == null)
{
changedMovie = true;
metaMovie.Titles.Add(metaTitle);
}
else
{
movieAkas.Remove(existingTitle);
}
}
foreach (var movieTitle in movieAkas)
{
changedMovie = true;
metaMovie.Titles.Remove(movieTitle);
}
dbLock.Wait();
if (metaMovie.Id == 0)
{
db.Add(metaMovie);
}
else if (changedMovie)
{
db.Update(metaMovie);
}
dbLock.Release();
currentTitlesAkasLock.Wait();
currentTitlesAkas.TryRemove(externalId, out int uselessOut); // Free memory
isAssociating = !readAkas.IsCompleted || !readTitles.IsCompleted || !currentTitlesAkas.IsEmpty;
currentTitlesAkasLock.Release();
titles.TryRemove(externalId, out MetaMovie uselessOut2); // Free memory
associateLock.Wait();
savingCounter++;
var localSavingCounter = savingCounter;
associateLock.Release();
if (localSavingCounter != 0 && localSavingCounter % 1000 == 0)
{
var ttt = currentTitlesAkas.Where(t => t.Value > 0);
dbLock.Wait();
await db.SaveChangesAsync();
dbLock.Release();
Console.WriteLine("Saved " + localSavingCounter);
}
}
else if (!readTitles.IsCompleted) // If reading titles is not ended, then maybe it was not read yet... otherwise, it doesn't exist
{
currentTitlesAkasLock.Wait();
currentTitlesAkas.TryAdd(externalId, 0); // Readd because still no movie associated
currentTitlesAkasLock.Release();
}
} while (isAssociating);
});
});
Task.WaitAll(readTitles, readAkas, associate);
await db.SaveChangesAsync();
}
}
public async override Task<IEnumerable<MetaMovie>> FindMediasAsync(DirectoryInfo directory)
{
await LoadMetaDataAsync();
var movie = await ExtractInfosAsync(directory);
if (movie == null) return null;
if (imdbMovies == null)
{
imdbMovies = db.MetaMovies.Where(m => m.MetaSource == nameof(Imdb) && m.MovieType == "movie");
}
return FindCorrespondances(imdbMovies, movie);
}
}
}
https://cints.net/public/Imdb-SingleThread.cs.txt
using com.cyberinternauts.all.MediaRecognizer.Database;
using com.cyberinternauts.all.MediaRecognizer.Models.Metas;
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
namespace com.cyberinternauts.all.MediaRecognizer.MetaSources
{
class Imdb : MediaSource
{
private const string TITLES_FILE = "title.basics.tsv.gz";
private const string AKAS_FILE = "title.akas.tsv.gz";
private readonly string temporaryFolder = #"c:\temp\";
private readonly string baseUrl = "https://datasets.imdbws.com/";
private readonly WebClient webClient = new();
MediaRecognizerContext db = new();
private IQueryable<MetaMovie> imdbMovies = null;
private async Task<bool> GatherFilesAsync()
{
var totalFilesGathered = 0;
var filesToDownload = new string[] { AKAS_FILE, TITLES_FILE };
foreach(var fileToDownload in filesToDownload)
{
var compressedFile = temporaryFolder + fileToDownload;
if (!File.Exists(compressedFile) || !File.GetLastWriteTime(compressedFile).Date.Equals(DateTime.Today))
{
await GatherFileAsync(fileToDownload);
totalFilesGathered++;
}
}
return totalFilesGathered != 0;
}
private async Task GatherFileAsync(string fileName)
{
var compressedFile = temporaryFolder + fileName;
var uncompressedFile = temporaryFolder + Path.GetFileNameWithoutExtension(compressedFile);
await webClient.DownloadFileTaskAsync(baseUrl + fileName, compressedFile);
using Stream fd = File.Create(uncompressedFile);
using Stream fs = File.OpenRead(compressedFile);
using Stream csStream = new GZipStream(fs, CompressionMode.Decompress);
var buffer = new byte[1024];
int nRead;
while ((nRead = await csStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
await fd.WriteAsync(buffer, 0, nRead);
}
}
private async Task LoadMetaDataAsync()
{
//return; //TODO: Remove this line
//TODO: Reactivate this line
//if (!await GatherFilesAsync()) return;
var titlesFile = temporaryFolder + Path.GetFileNameWithoutExtension(TITLES_FILE);
var akasFile = temporaryFolder + Path.GetFileNameWithoutExtension(AKAS_FILE);
var titlesLines = File.ReadLines(titlesFile);
var akasLines = File.ReadLines(akasFile);
var titlesIterator = titlesLines.GetEnumerator();
titlesIterator.MoveNext(); // Skip columns headers
var akasIterator = akasLines.GetEnumerator();
akasIterator.MoveNext();
akasIterator.MoveNext(); // Done twice to skip columns headers
var currentAka = akasIterator.Current;
var savingCounter = 0;
using (var db = new MediaRecognizerContext())
{
db.ChangeTracker.AutoDetectChangesEnabled = false;
while (titlesIterator.MoveNext())
{
var titleLine = titlesIterator.Current;
var movieInfos = titleLine.Split("\t", StringSplitOptions.None);
MetaMovie metaMovie = db.MetaMovies.Where(m => m.ExternalId == movieInfos[0]).FirstOrDefault();
var isNewMovie = false;
if (metaMovie == null)
{
int totalMinutes = -1;
if (!int.TryParse(movieInfos[7], out totalMinutes))
{
totalMinutes = -1;
}
isNewMovie = true;
metaMovie = new MetaMovie
{
ExternalId = movieInfos[0],
MetaSource = nameof(Imdb),
MovieType = movieInfos[1],
Title = movieInfos[3],
TotalMinutes = totalMinutes,
Genres = movieInfos[8]
};
metaMovie.Titles = new List<MetaTitle>();
if (int.TryParse(movieInfos[5], out int startYear))
{
metaMovie.StartYear = new DateTime(startYear, 1, 1);
}
else
{
metaMovie.StartYear = new DateTime(9999, 1, 1);
}
if (int.TryParse(movieInfos[6], out int endYear))
{
metaMovie.EndYear = new DateTime(endYear, 1, 1);
}
else
{
metaMovie.EndYear = metaMovie.StartYear;
}
}
var movieAkasIds = metaMovie.Titles.Select(t => t.Id).ToList();
var titleInfos = currentAka?.Split("\t", StringSplitOptions.None);
while (currentAka != null && int.Parse(titleInfos[0][2..]) <= int.Parse(metaMovie.ExternalId[2..]))
{
if (titleInfos[0] == metaMovie.ExternalId)
{
var metaTitle = new MetaTitle
{
MetaMovie = metaMovie,
Text = titleInfos[2],
Region = titleInfos[3],
Language = titleInfos[4]
};
var existingTitle = metaMovie.Titles.Where(t => t.Text == metaTitle.Text && t.Region == metaTitle.Region && t.Language == metaTitle.Language).FirstOrDefault();
if (existingTitle == null)
{
metaMovie.Titles.Add(metaTitle);
}
else
{
movieAkasIds.Remove(existingTitle.Id);
}
}
else
{
var a = 1;
}
akasIterator.MoveNext();
currentAka = akasIterator.Current;
titleInfos = currentAka.Split("\t", StringSplitOptions.None);
}
foreach(var movieTitleId in movieAkasIds)
{
metaMovie.Titles.Remove(metaMovie.Titles.Where(t => t.Id == movieTitleId).FirstOrDefault());
}
if (isNewMovie)
{
db.Add(metaMovie);
}
else
{
db.Update(metaMovie);
}
savingCounter++;
if (savingCounter % 10000 == 0)
{
await db.SaveChangesAsync();
Console.WriteLine("Saved " + savingCounter);
}
}
await db.SaveChangesAsync();
}
}
public async override Task<IEnumerable<MetaMovie>> FindMediasAsync(DirectoryInfo directory)
{
await LoadMetaDataAsync();
var movie = await ExtractInfosAsync(directory);
if (movie == null) return null;
if (imdbMovies == null)
{
imdbMovies = db.MetaMovies.Where(m => m.MetaSource == nameof(Imdb) && m.MovieType == "movie");
}
return FindCorrespondances(imdbMovies, movie);
}
}
}
In the multithread version, the slow part is in the method LoadMetaDataAsync and more precisely in var associate = Task.Factory.StartNew(() => code part.
This is in development and cleaning, splitting will done after I have the appropriate result/speed.

Case closed. I returned to the single thread version and I found my initial issue (my code was supposing the files were in order, which they were partially).
Thank you for all people that participated.

C# Roslyn CompletionService where to get method overload information

I've got a method that returning back from CompletionService.GetDescriptionAsync(Document, CompletionItem) gives me the following description:
void SQL.GetSQLiteDB(string url) (+ 1 overload)
This is a method I made on a Xamarin project, here are both method signatures:
public static void GetSQLiteDB(string url);
public static string GetSQLiteDB(string url, string name);
What's the Roslyn way to get information on both?
Here's how I'm setting up completions:
async Task InitCodeCompletion()
{
host = MefHostServices.Create(MefHostServices.DefaultAssemblies);
workspace = new AdhocWorkspace(host);
Type[] types =
{
typeof(object),
typeof(System.Linq.Enumerable),
typeof(System.Collections.IEnumerable),
typeof(Console),
typeof(System.Reflection.Assembly),
typeof(List<>),
typeof(Type),
typeof(SQL)
};
imports = types.Select(x => x.Namespace).Distinct().ToImmutableArray();
assemblies = types.Select(x => x.Assembly).Distinct().ToImmutableArray();
references = assemblies.Select(t => MetadataReference.CreateFromFile(t.Location) as MetadataReference).ToImmutableArray();
compilationOptions = new CSharpCompilationOptions(
OutputKind.DynamicallyLinkedLibrary,
usings: imports);
projectInfo = ProjectInfo.Create(ProjectId.CreateNewId(), VersionStamp.Create(), "Script", "Script", LanguageNames.CSharp, isSubmission: true)
.WithMetadataReferences(references).WithCompilationOptions(compilationOptions);
project = workspace.AddProject(projectInfo);
documentInfo = DocumentInfo.Create(DocumentId.CreateNewId(project.Id), "Script", sourceCodeKind: SourceCodeKind.Script,
loader: TextLoader.From(TextAndVersion.Create(SourceText.From(""), VersionStamp.Create())));
document = workspace.AddDocument(documentInfo);
var services = workspace.Services;
completionService = CompletionService.GetService(document);
}
async Task<CodeCompletionResults> GetCompletions(string code)
{
string codeModified = "using SQL = XamTestNET5.Services.SQLiteGeneratorService; " + Environment.NewLine;
codeModified += "using HtmlSvc = XamTestNET5.Services.HtmlRetrievalService;" + Environment.NewLine;
// ^^^ The above two lines set up some simple namespace aliases in my project, if you know how to put this in a separate project document and use it in code completion please let me know in comments as otherwise doing so gives me an exception that you can't have multiple syntax trees
codeModified += code;
var source = SourceText.From(codeModified);
document = document.WithText(source);
// cursor position is at the end
var position = source.Length;
var completions = await completionService.GetCompletionsAsync(document, position);
return new CodeCompletionResults() { InputCode = code, ModifiedCode = codeModified, Completions = completions };
}
Here's how I'm getting them now and putting them in a browser control:
private async void CSharpShellEnvironment_EntryCodeCompletionEntry(object sender, CSharpShellEnvironment.EntryEventArgs e)
{
if (e.Value != "")
{
CodeCompletionResults results = await GetCompletions(e.Value);
CompletionList list = results.Completions;
if (list != null)
{
if (list.Items != null)
{
StringBuilder sb = new StringBuilder();
foreach (var item in list.Items)
{
string spanText = (item.Span.Start != item.Span.End) ? results.ModifiedCode.Substring(item.Span.Start, item.Span.Length) : "";
bool recommended = spanText == "" ? true : item.DisplayText.StartsWith(spanText);
if (recommended)
{
string fText = item.DisplayText.Substring(spanText.Length);
string props = "";
foreach(var p in item.Properties)
{
props += $"<span data-key=\"{p.Key}\" data-value=\"{p.Value}\"></span>";
}
string tags = "";
foreach(var t in item.Tags)
{
tags += $"<span data-tag=\"{t}\"></span>";
}
string descStr = "";
if (item.Tags != null)
{
if (item.Tags.Where(x => x.ToLower() == "method").FirstOrDefault() != null && item.Tags.Where(x => x.ToLower() == "public").FirstOrDefault() != null)
{
var desc = await completionService.GetDescriptionAsync(document, item);
descStr += $"<span data-desc=\"{desc.Text}\">";
foreach(var part in desc.TaggedParts)
{
descStr += $"<span data-desc-part-tag=\"{part.Tag}\" data-desc-part-text=\"{part.Text}\"></span>";
}
descStr += "</span>";
}
}
sb.AppendLine($"<div class=\"codecompleteentry\" data-display-text=\"{item.DisplayText}\" data-span-text=\"{spanText}\" data-final-text=\"{fText}\">{props}{tags}{descStr}{fText}</div>");
}
}
string scriptInputClick = "Array.prototype.forEach.call(document.getElementsByClassName('codecompleteentry'), function(el) { el.addEventListener('click', function(elem) { var text = { MessageType: 'CodeCompletion', Parameters: JSON.stringify({ DataDisplayText: el.getAttribute('data-display-text'), DataSpanText: el.getAttribute('data-span-text'), DataFinalText: el.getAttribute('data-final-text') }), Message: el.innerText }; window.chrome.webview.postMessage(text); } ); });";
sb.AppendLine($"<script type=\"text/javascript\">{scriptInputClick}</script>");
env.EnterCodeCompletionResponse(sb.ToString());
}
else
{
env.EnterCodeCompletionResponse(strNoSuggestions);
}
}
else
{
env.EnterCodeCompletionResponse(strNoSuggestions);
}
}
else
{
env.EnterCodeCompletionResponse(strNoSuggestions);
}
}

It seems on the surface that CompletionSurface has everything you need, but it doesn't, you need to reference the Document's SemanticModel in order to get all of the signature overloads of a method when the user types ( on a method during code completion.
It wasn't very obvious to me until I started looking through the RoslynPad source, which I recommend doing for a practical example: https://github.com/aelij/RoslynPad
List<IEnumerable<ReferencedSymbol>> allMethodRefs = new List<IEnumerable<ReferencedSymbol>>();
async Task<CodeCompletionResults> GetCompletions(string code)
{
string codeModified = "using SQL = XamTestNET5.Services.SQLiteGeneratorService; " + Environment.NewLine;
codeModified += "using HtmlSvc = XamTestNET5.Services.HtmlRetrievalService;" + Environment.NewLine;
// ^^^ I put my namespace aliases in the same SyntaxTree for now,
// I'd like a better solution though.
codeModified += code;
var source = SourceText.From(codeModified);
document = document.WithText(source);
// cursor position is at the end
var position = source.Length;
var completions = await completionService.GetCompletionsAsync(document, position);
syntaxRoot = await document.GetSyntaxRootAsync();
semanticModel = await document.GetSemanticModelAsync();
var methods = syntaxRoot.DescendantNodes().OfType<InvocationExpressionSyntax>();
allMethodRefs = new List<IEnumerable<ReferencedSymbol>>();
if (methods != null)
{
if (methods.Count() > 0)
{
foreach(var m in methods)
{
var info = semanticModel.GetSymbolInfo(m);
if (info.Symbol != null)
{
allMethodRefs.Add(await SymbolFinder.FindReferencesAsync(info.Symbol, solution));
}
else
{
foreach(var symbol in info.CandidateSymbols)
{
allMethodRefs.Add(await SymbolFinder.FindReferencesAsync(symbol, solution));
}
}
}
}
}
return new CodeCompletionResults() { InputCode = code, ModifiedCode = codeModified, Completions = completions };
}

Reading from CSV file with CsvHelper

I managed to write contents from a List to a CSV file. Now I'm trying to read from the created CSV file and show the content in a Console Application. I'm getting a exception which lead me to some possible solutions on the internet. None of them are working for my Console Application. I hope you can help me.
My code from Program.cs which calls the method in FileOperations.cs:
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using CsvHelper;
namespace GimpiesConsoleOOcsvListUI
{
class Program
{
static void Main(string[] args)
{
// First line to skip a row and position all users correctly under the right headers
User user0 = new User("", "", "", "");
// Create user default instances
User user1 = new User("Beheer", "beheer#gimpies.nl", "123", "admin");
User user2 = new User("Inkoop", "inkoop#gimpies.nl", "123", "purchase");
User user3 = new User("Verkoop", "verkoop#gimpies.nl", "123", "sales");
// List of default users (with a list you can add, get and remove items in the list)
List<User> users = new List<User>();
users.Add(user0);
users.Add(user1);
users.Add(user2);
users.Add(user3);
// Create login instance
LoginManager loginMgr = new LoginManager();
// Create stock instance
Stock stock = new Stock();
// Create UserList instance
UserList usrlst = new UserList();
// Call method in UserList.cs
usrlst.Users(users);
Start:
// Welcome message
Console.WriteLine("Welcome to the Gimpies Console Application! Choose 1 to login or 2 to register as guest:");
// Get input from user
string input = Console.ReadLine();
bool successfull = false;
while (!successfull)
{
if(input == "1")
{
Console.WriteLine("Enter your username:");
string username = Console.ReadLine();
Console.WriteLine("Enter your password:");
string password = Console.ReadLine();
foreach (User user in users)
{
if (username == user.UserName && password == user.PassWord && user.UserRole == "admin")
{
// Create Admin instance to be able to call methods in that class
Admin am = new Admin();
// Calling the method in Admin.cs to start Menu logic
am.AdminLoggedIn();
successfull = true;
break;
}
if (username == user.UserName && password == user.PassWord && user.UserRole == "purchase")
{
// Create Purchase instance to be able to call methods in that class
Purchase pc = new Purchase();
// Calling the method in Purchase.cs to start Menu logic
pc.PurchaseLoggedIn();
successfull = true;
break;
}
if (username == user.UserName && password == user.PassWord && user.UserRole == "sales")
{
// Create Sales instance to be able to call methods in that class
Sales sl = new Sales();
// Calling the method in Sales.cs to start Menu logic
sl.SalesLoggedIn();
successfull = true;
break;
}
if (username == user.UserName && password == user.PassWord && user.UserRole == "guest")
{
// Create Guest instance to be able to call methods in that class
Guest gt = new Guest();
// Calling the method in Guest.cs to start Menu logic
gt.GuestLoggedIn();
successfull = true;
break;
}
}
if (!successfull)
{
Console.WriteLine("Your username or password is incorrect, try again !!!");
}
}
else if (input == "2")
{
// Create instance to go to method in class RegisterManager.cs
RegisterManager rm = new RegisterManager();
rm.Register(users);
successfull = true;
goto Start;
}
else if (input == "3")
{
FileOperations fo = new FileOperations();
// Calling the method from FileOperations.cs to write the List here to a CSV file
fo.WriteUsersToCSV(users);
goto Start;
}
else if (input == "4")
{
FileOperations fo = new FileOperations();
// Calling the method from FileOperations.cs to write the List here to a CSV file
fo.ReadUsersFromCSV(users);
goto Start;
}
else
{
Console.WriteLine("Try again !!!");
break;
}
}
// // Loop over stored users within instances inside the list where users are added
// foreach (User user in users)
// {
// if(loginMgr.Login(user))
// {
// // Login successfull
// Console.WriteLine("Login user " + user.UserName);
// }
// else
// {
// // Not successfull
// }
// }
}
}
}
In Program.cs it's about the else if with input == "4".
Then the method ReadUsersFromCSV is called inside FileOperations.cs:
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Globalization;
using System.Linq;
using CsvHelper;
namespace GimpiesConsoleOOcsvListUI
{
// Handles CRUD within CSV files and able to save them
public class FileOperations
{
// Writes to CSV file from List
public void WriteUsersToCSV(List<User> users)
{
// using (var mem = new MemoryStream())
// using (var writer = new StreamWriter(mem))
using (var writer = new StreamWriter("users.csv"))
using (var csvWriter = new CsvWriter(writer, CultureInfo.InvariantCulture))
{
csvWriter.Configuration.Delimiter = ";";
csvWriter.Configuration.HasHeaderRecord = true;
csvWriter.Configuration.AutoMap<User>();
csvWriter.WriteHeader<User>();
csvWriter.WriteRecords(users);
writer.Flush();
// var result = Encoding.UTF8.GetString(mem.ToArray());
// Console.WriteLine(result);
Console.WriteLine("Data saved to users.csv");
}
}
// Reads from CSV file and displays content from it (Work in progress...)
public void ReadUsersFromCSV(List<User> users)
{
// using (var mem = new MemoryStream())
// using (var writer = new StreamWriter(mem))
using (var reader = new StreamReader("users.csv"))
using (var csvReader = new CsvReader(reader, CultureInfo.InvariantCulture))
{
// csvReader.Configuration.Delimiter = ";";
// csvReader.Configuration.HasHeaderRecord = true;
// csvReader.Configuration.AutoMap<User>();
// csvReader.ReaderHeader<User>();
// var records = csvReader.GetRecords<dynamic>();
// reader.Flush();
// var result = Encoding.UTF8.GetString(mem.ToArray());
csvReader.Read();
csvReader.ReadHeader();
csvReader.Configuration.HeaderValidated = null;
csvReader.Configuration.MissingFieldFound = null;
var records = csvReader.GetRecords<User>();
foreach (var record in records)
{
Console.WriteLine(records);
}
// Console.WriteLine(records);
}
}
// Writes to CSV file from List
public void SaveGimpiesToCSV(List<Gimpies> gimpies)
{
// using (var mem = new MemoryStream())
// using (var writer = new StreamWriter(mem))
using (var writer = new StreamWriter("gimpies.csv"))
using (var csvWriter = new CsvWriter(writer, CultureInfo.InvariantCulture))
{
csvWriter.Configuration.Delimiter = ";";
csvWriter.Configuration.HasHeaderRecord = true;
csvWriter.Configuration.AutoMap<Gimpies>();
csvWriter.WriteHeader<Gimpies>();
csvWriter.WriteRecords(gimpies);
writer.Flush();
// var result = Encoding.UTF8.GetString(mem.ToArray());
// Console.WriteLine(result);
}
}
}
}
The exception:
Unhandled exception. CsvHelper.MissingFieldException: Field with name 'username' does not exist. You can ignore missing fields by setting MissingFieldFound to null.
at CsvHelper.Configuration.ConfigurationFunctions.MissingFieldFound(String[] headerNames, Int32 index, ReadingContext context)
at CsvHelper.CsvReader.GetFieldIndex(String[] names, Int32 index, Boolean isTryGet, Boolean isOptional)
at CsvHelper.CsvReader.GetFieldIndex(String name, Int32 index, Boolean isTryGet)
at CsvHelper.Expressions.ExpressionManager.CreateConstructorArgumentExpressionsForMapping(ClassMap map, List`1 argumentExpressions)
at CsvHelper.Expressions.ObjectRecordCreator.CreateCreateRecordDelegate(Type recordType)
at CsvHelper.Expressions.RecordCreator.GetCreateRecordDelegate(Type recordType)
at CsvHelper.Expressions.RecordCreator.Create[T]()
at CsvHelper.Expressions.RecordManager.Create[T]()
at CsvHelper.CsvReader.GetRecords[T]()+MoveNext()
at GimpiesConsoleOOcsvListUI.FileOperations.ReadUsersFromCSV(List`1 users) in /home/pascalmariany/Projects/Csharp/GimpiesConsoleOOcsvList/GimpiesConsoleOOcsvListUI/FileOperations.cs:line 55
at GimpiesConsoleOOcsvListUI.Program.Main(String[] args) in /home/pascalmariany/Projects/Csharp/GimpiesConsoleOOcsvList/GimpiesConsoleOOcsvListUI/Program.cs:line 132
The contents of the CSV file:
My User.cs:
namespace GimpiesConsoleOOcsvListUI
{
public class User
{
// Constructor
public User(string username, string email, string password, string userrole)
{
_UserName = username;
_Email = email;
_Password = password;
_UserRole = userrole;
}
private string _UserName;
private string _Email;
private string _Password;
private string _UserRole;
public string UserName
{
get { return _UserName; }
set { _UserName = value; }
}
public string Email
{
get { return _Email; }
set { _Email = value; }
}
public string PassWord
{
get { return _Password; }
set { _Password = value; }
}
public string UserRole
{
get { return _UserRole; }
set { _UserRole = value; }
}
}
}
What would I need to change?
Update:
I read all the feedback and I changed some thing in the method within FileOperations.cs:
// Reads from CSV file and displays content from it (Work in progress...)
public void ReadUsersFromCSV(List<User> users)
{
// using (var mem = new MemoryStream())
// using (var writer = new StreamWriter(mem))
using (var reader = new StreamReader("users.csv"))
using (var csvReader = new CsvReader(reader, CultureInfo.InvariantCulture))
{
try
{
csvReader.Configuration.Delimiter = ";";
csvReader.Configuration.IgnoreBlankLines = true;
csvReader.Configuration.HasHeaderRecord = true;
csvReader.Configuration.PrepareHeaderForMatch = (string header, int index) => header.ToLower();
csvReader.Configuration.MissingFieldFound = null;
// csvReader.Configuration.AutoMap<User>();
// csvReader.ReaderHeader<User>();
// var records = csvReader.GetRecords<dynamic>();
// reader.Flush();
// var result = Encoding.UTF8.GetString(mem.ToArray());
csvReader.Read();
csvReader.ReadHeader();
// csvReader.Configuration.HeaderValidated = null;
// Store all content inside a new List as objetcs
var records = csvReader.GetRecords<User>().ToList();
// users.ForEach(Console.WriteLine);
foreach (var record in records)
{
Console.WriteLine(record);
}
}
catch (CsvHelper.HeaderValidationException exception)
{
Console.WriteLine(exception);
}
}
}
I don't get an exception now, but I get this as output:
GimpiesConsoleOOcsvListUI.User
GimpiesConsoleOOcsvListUI.User
GimpiesConsoleOOcsvListUI.User
GimpiesConsoleOOcsvListUI.User
It's the correct amount of rows, but as you can see it doesn't display the contents which is stored from the CSV file into a List as objects.
Am I missing something?

I managed to solve it. I needed to adjust the foreach loop:
public void ReadUsersFromCSV(List<User> users)
{
// using (var mem = new MemoryStream())
// using (var writer = new StreamWriter(mem))
using (var reader = new StreamReader("users.csv"))
using (var csvReader = new CsvReader(reader, CultureInfo.InvariantCulture))
{
try
{
csvReader.Configuration.Delimiter = ";";
csvReader.Configuration.IgnoreBlankLines = true;
csvReader.Configuration.HasHeaderRecord = true;
csvReader.Configuration.PrepareHeaderForMatch = (string header, int index) => header.ToLower();
csvReader.Configuration.MissingFieldFound = null;
csvReader.Read();
csvReader.ReadHeader();
// Store all content inside a new List as objetcs
var records = csvReader.GetRecords<User>().ToList();
// Loop through the List and show them in Console
foreach (var record in records)
{
Console.WriteLine($"{record.username } {record.email} {record.password} {record.userrole}");
}
}
catch (CsvHelper.HeaderValidationException exception)
{
Console.WriteLine(exception);
}
}
}
Thanks, but the most I learned from this video: Link for Lists

Replicating C metadata example from ffmpeg in C# using ffmpeg-autogen - av_dict_get fails

I am trying to replicate https://ffmpeg.org/doxygen/3.0/doc_2examples_2metadata_8c_source.html in C# using the wrapper from https://github.com/Ruslan-B/FFmpeg.AutoGen
I can open and read some properties of the file just fine, however tag is always null, even after the call to av_dict_get
My code is as follows
using System;
using System.IO;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text;
using FFmpeg.AutoGen;
namespace ffmpeg_test
{
class Program
{
static void Main(string[] args)
{
var currentPath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
var libPath = Path.Combine(currentPath, "lib");
SetDllDirectory(libPath);
ffmpeg.av_register_all();
ffmpeg.avcodec_register_all();
DoRiskyThings();
}
private static unsafe void DoRiskyThings()
{
var pFormatContext = ffmpeg.avformat_alloc_context();
if (ffmpeg.avformat_open_input(&pFormatContext, "01 - 999,999.opus", null, null) != 0)
throw new ApplicationException(#"Could not open file.");
ffmpeg.avformat_find_stream_info(pFormatContext, null);
AVStream* pStream = null;
pStream = pFormatContext->streams[0];
var codecContext = *pStream->codec;
Console.WriteLine($"codec name: {ffmpeg.avcodec_get_name(codecContext.codec_id)}");
Console.WriteLine($"number of streams: {pFormatContext->nb_streams}");
//attempting to replicate https://ffmpeg.org/doxygen/3.0/doc_2examples_2metadata_8c_source.html
AVDictionaryEntry* tag = null;
tag = ffmpeg.av_dict_get(pFormatContext->metadata, "", null, 2);
while (tag != null)
{
tag = ffmpeg.av_dict_get(pFormatContext->metadata, "", tag, 2);
Console.WriteLine(Encoding.UTF8.GetString(tag->key,100));
//tag->key and //tag->value are byte pointers
}
}
[DllImport("kernel32", SetLastError = true)]
private static extern bool SetDllDirectory(string lpPathName);
}
}

You need to marshal strings. This is works like a charm:
var url = #"http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4";
var pFormatContext = ffmpeg.avformat_alloc_context();
if (ffmpeg.avformat_open_input(&pFormatContext, url, null, null) != 0)
throw new ApplicationException(#"Could not open file.");
if (ffmpeg.avformat_find_stream_info(pFormatContext, null) != 0)
throw new ApplicationException(#"Could not find stream info");
AVDictionaryEntry* tag = null;
while ((tag = ffmpeg.av_dict_get(pFormatContext->metadata, "", tag, ffmpeg.AV_DICT_IGNORE_SUFFIX)) != null)
{
var key = Marshal.PtrToStringAnsi((IntPtr) tag->key);
var value = Marshal.PtrToStringAnsi((IntPtr) tag->value);
Console.WriteLine($"{key} = {value}");
}

Merge XML files using XmlReader and XmlWriter

I am trying to merge multiple XML files in into one using XmlReader and XmlWriter though my final file only contains the data from the last file.
I am using XmlReader and XmlWriter because the XML files to merge are large in size.
What am I doing wrong in the code below?
class Program
{
static void Main(string[] args)
{
string folder = #"C:\Temp\";
string output = folder + "_all.xml";
Encoding readEncoding = System.Text.Encoding.Default;
XmlWriterSettings writerSettings = new XmlWriterSettings();
writerSettings.Encoding = Encoding.UTF8;
writerSettings.ConformanceLevel = ConformanceLevel.Fragment;
XmlWriter writer = XmlWriter.Create(new StreamWriter(output, false), writerSettings);
bool firstFile = true;
foreach (FileInfo file in new DirectoryInfo(folder).GetFiles("*.xml").Where(f => f.Name != "_all.xml"))
{
XmlReader reader = XmlReader.Create(new StreamReader(file.FullName, readEncoding));
while(reader.Read())
{
switch (reader.NodeType)
{
case XmlNodeType.Element:
if (firstFile && reader.Name == "CYPHS:CYPHS")
{
writer.WriteStartElement(reader.Prefix, reader.LocalName, reader.NamespaceURI);
writer.WriteAttributes(reader, true);
}
else if (firstFile && reader.Name == "CYP000")
writer.WriteStartElement(reader.Name);
else if (firstFile && reader.Name.StartsWith("C000"))
writer.WriteNode(reader, false);
else if (!firstFile && reader.Name != "CYPHS:CYPHS" && reader.Name != "CYP000" && !reader.Name.StartsWith("C000"))
writer.WriteNode(reader, false);
break;
default:
break;
}
}
firstFile = false;
reader.Close();
}
writer.WriteEndElement();
writer.WriteEndElement();
writer.Close();
Console.WriteLine("Done!");
Console.ReadLine();
}
}
File 1
<CYPHS:CYPHS xsi:schemaLocation="http://www.datadictionary.nhs.uk/messages/CYPHS-v1-5 CYPHSCYPHS_XMLSchema-v1-5.xsd"
xmlns:CYPHS="http://www.datadictionary.nhs.uk/messages/CYPHS-v1-5"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<CYP000>
<C000010>File 1</C000010>
<CYP001>
<C001901>File 1</C001901>
<CYP101>
<C101902>File 1</C101902>
<CYP102>
<C102902>File 1</C102902>
</CYP102>
</CYP101>
<CYP002>
<C002901>File 1</C002901>
</CYP002>
</CYP001>
</CYP000>
</CYPHS:CYPHS>
File 2
<CYPHS:CYPHS xsi:schemaLocation="http://www.datadictionary.nhs.uk/messages/CYPHS-v1-5 CYPHSCYPHS_XMLSchema-v1-5.xsd"
xmlns:CYPHS="http://www.datadictionary.nhs.uk/messages/CYPHS-v1-5"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<CYP000>
<C000010>File 2</C000010>
<CYP001>
<C001901>File 2</C001901>
<CYP101>
<C101902>File 2</C101902>
<CYP102>
<C102902>File 2</C102902>
</CYP102>
</CYP101>
<CYP002>
<C002901>File 2</C002901>
</CYP002>
</CYP001>
</CYP000>
</CYPHS:CYPHS>
Should be merged into file as so:
<CYPHS:CYPHS xsi:schemaLocation="http://www.datadictionary.nhs.uk/messages/CYPHS-v1-5 CYPHSCYPHS_XMLSchema-v1-5.xsd"
xmlns:CYPHS="http://www.datadictionary.nhs.uk/messages/CYPHS-v1-5"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<CYP000>
<C000010>File 1</C000010>
<CYP001>
<C001901>File 1</C001901>
<CYP101>
<C101902>File 1</C101902>
<CYP102>
<C102902>File 1</C102902>
</CYP102>
</CYP101>
<CYP002>
<C002901>File 1</C002901>
</CYP002>
</CYP001>
<CYP001>
<C001901>File 2</C001901>
<CYP101>
<C101902>File 2</C101902>
<CYP102>
<C102902>File 2</C102902>
</CYP102>
</CYP101>
<CYP002>
<C002901>File 2</C002901>
</CYP002>
</CYP001>
</CYP000>
</CYPHS:CYPHS>

Like This
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml;
using System.Xml.Linq;
namespace ConsoleApplication53
{
class Program
{
static void Main(string[] args)
{
string file1 =
"<CYPHS:CYPHS xsi:schemaLocation=\"http://www.datadictionary.nhs.uk/messages/CYPHS-v1-5 CYPHSCYPHS_XMLSchema-v1-5.xsd\"" +
" xmlns:CYPHS=\"http://www.datadictionary.nhs.uk/messages/CYPHS-v1-5\"" +
" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">" +
"<CYP000>" +
"<C000010>File 1</C000010>" +
"<CYP001>" +
"<C001901>File 1</C001901>" +
"<CYP101>" +
"<C101902>File 1</C101902>" +
"<CYP102>" +
"<C102902>File 1</C102902>" +
"</CYP102>" +
"</CYP101>" +
"<CYP002>" +
"<C002901>File 1</C002901>" +
"</CYP002>" +
"</CYP001>" +
"</CYP000>" +
"</CYPHS:CYPHS>";
XDocument doc1 = XDocument.Parse(file1);
XElement doc1_CYP000 = doc1.Descendants("CYP000").FirstOrDefault();
string file2 =
"<CYPHS:CYPHS xsi:schemaLocation=\"http://www.datadictionary.nhs.uk/messages/CYPHS-v1-5 CYPHSCYPHS_XMLSchema-v1-5.xsd\"" +
" xmlns:CYPHS=\"http://www.datadictionary.nhs.uk/messages/CYPHS-v1-5\"" +
" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">" +
"<CYP000>" +
"<C000010>File 2</C000010>" +
"<CYP001>" +
"<C001901>File 2</C001901>" +
"<CYP101>" +
"<C101902>File 2</C101902>" +
"<CYP102>" +
"<C102902>File 2</C102902>" +
"</CYP102>" +
"</CYP101>" +
"<CYP002>" +
"<C002901>File 2</C002901>" +
"</CYP002>" +
"</CYP001>" +
"</CYP000>" +
"</CYPHS:CYPHS>";
XDocument doc2 = XDocument.Parse(file2);
XElement doc2_CYP000 = doc2.Descendants("CYP000").FirstOrDefault();
doc1_CYP000.Add(doc2_CYP000.Descendants());
}
}
}

I'm not entirely sure where you went wrong, but it seems most straightforward to check the Depth, LocalName and NamespaceURI properties of XmlReader when combining your XML files. I strongly recommend against hardcoding the namespace prefixes since the prefix can be replaced with any other prefix without changing the semantics of the XML file.
One thing to note: XmlWriter.WriteNode(XmlReader, bool) advances the reader to the beginning of the next node, so if you subsequently call Read() and there is no whitespace in the file you'll skip over the next element. With this in mind, when working directly with XmlReader, it's better to test both with and without spacing.
Thus:
public class XmlConcatenate
{
public static void ConcatenateAllFiles()
{
string folder = "C:\\Temp\\";
string output = folder + "_all.xml";
Encoding readEncoding = System.Text.Encoding.Default; // WHY NOT Encoding.UTF8 !?
var files = new DirectoryInfo(folder).GetFiles("*.xml").Where(f => f.Name != "_all.xml").Select(f => f.FullName).Select(n => (TextReader)new StreamReader(n, readEncoding));
using (var textWriter = new StreamWriter(output, false))
{
Concatenate(files, textWriter);
}
}
public static void Concatenate(IEnumerable<TextReader> inputs, TextWriter output)
{
var writerSettings = new XmlWriterSettings() { Encoding = Encoding.UTF8, ConformanceLevel = ConformanceLevel.Fragment };
var whiteSpace = new StringBuilder();
int indent = 0;
using (var writer = XmlWriter.Create(output, writerSettings))
{
var writeDepth = 0;
var first = true;
foreach (var input in inputs)
{
using (input)
using (var reader = XmlReader.Create(input))
{
bool alreadyRead = false;
while (!reader.EOF && (alreadyRead || reader.Read()))
{
alreadyRead = false;
switch (reader.NodeType)
{
case XmlNodeType.Element:
{
if (reader.Depth == 0 && reader.LocalName == "CYPHS" && reader.NamespaceURI == "http://www.datadictionary.nhs.uk/messages/CYPHS-v1-5")
{
if (writeDepth == 0)
{
writer.WriteWhitespace(whiteSpace.ToString());
writer.WriteStartElement(reader.Prefix, reader.LocalName, reader.NamespaceURI);
writer.WriteAttributes(reader, true);
writeDepth++;
}
}
else if (reader.Depth == 1 && reader.LocalName == "CYP000" && reader.NamespaceURI == "")
{
if (writeDepth == 1)
{
indent = whiteSpace.ToString().Replace("\n", "").Replace("\r", "").Length;
writer.WriteWhitespace(whiteSpace.ToString());
writer.WriteStartElement(reader.LocalName, reader.NamespaceURI);
writeDepth++;
}
}
else if (reader.Depth == 2)
{
if (reader.LocalName.StartsWith("C000") && reader.NamespaceURI == "")
{
if (first)
{
first = false;
writer.WriteWhitespace(whiteSpace.ToString());
writer.WriteNode(reader, false);
alreadyRead = true;
}
}
else
{
writer.WriteWhitespace(whiteSpace.ToString());
writer.WriteNode(reader, false);
alreadyRead = true;
}
}
whiteSpace.Length = 0; // Clear accumulated whitespace.
}
break;
case XmlNodeType.Whitespace:
{
whiteSpace.Append(reader.Value);
}
break;
default:
break;
}
}
}
}
while (writeDepth-- > 0)
{
if (indent > 0)
writer.WriteWhitespace("\n" + new string(' ', indent * writeDepth));
writer.WriteEndElement();
}
}
}
}
Bit of a nuisance getting the spacing to merge, if you don't care about preserving the spacing you can simplify the code substantially.
Working fiddle.
You might not want to use System.Text.Encoding.Default for reading your XML files. From the docs:
Because all Default encodings lose data, you might use UTF8 instead. UTF-8 is often identical in the U+00 to U+7F range, but can encode other characters without loss.

A different solution could be to use a custom XmlReader-implementation to concat the files while reading them.
Then use this custom reader along with an XmlWriter to create the merged file.
The custom XmlReader keeps internal XmlReaders for each file.
The intro/end is only read from the first file.
Only the relevant (to-be-appended) elements are read from the other files.
create an XmlReader for the first file
read up to the point where elements should be appended
for each subsequent file
create a new XmlReader
skip ahead to the first relevant element
read the relevant elements
dispose the reader
read the rest of the first file (resume the reader from step 1)
dispose the reader
Example implementation
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Xml;
public static class XmlConcatenator
{
// first: pause reading at the end of this element, will resume after subsequent streams are read
// subsequent: stop reading at the end of this element
private const string StopAtEndOf = "CYP000";
// first: (ignores this)
// subsequent: skip ahead to the first instance of this element
private const string ResumeAtFirst = "CYP001";
private static readonly XmlReaderSettings XmlReaderSettings = new XmlReaderSettings() { DtdProcessing = DtdProcessing.Ignore };
private static readonly XmlWriterSettings XmlWriterSettings = new XmlWriterSettings() { Encoding = Encoding.UTF8, Indent = true };
public static void Concat(Stream outStream, Stream[] fileStreams)
{
using var reader = XmlConcatReader.Create(fileStreams);
using var writer = XmlWriter.Create(outStream, XmlWriterSettings);
writer.WriteNode(reader, true);
}
private class XmlConcatReader : XmlReader
{
private readonly XmlReader _firstReader;
private readonly IEnumerator<Stream> _streams;
private XmlReader _currentReader;
private XmlConcatReader(Stream first, IEnumerable<Stream> streams)
{
_firstReader = XmlReader.Create(first, XmlReaderSettings);
_streams = streams.GetEnumerator();
_currentReader = _firstReader;
}
public static XmlReader Create(Stream[] inputStreams)
{
if (!(inputStreams?.Length > 1))
{
throw new InvalidOperationException($"{nameof(inputStreams)} must contain at least two streams");
}
return new XmlConcatReader(inputStreams[0], inputStreams.Skip(1));
}
public override bool Read()
{
var b = _currentReader.Read();
if (_currentReader.NodeType == XmlNodeType.EndElement && _currentReader.LocalName == StopAtEndOf)
{
// note: _firstReader is disposed at the end. See: Dispose(bool)
if (!ReferenceEquals(_currentReader, _firstReader))
{
_currentReader.Dispose();
}
if (_streams.MoveNext())
{
_currentReader = XmlReader.Create(_streams.Current, XmlReaderSettings);
while (_currentReader.Read())
{
if (_currentReader.LocalName == ResumeAtFirst)
{
return true;
}
}
}
else
{
_currentReader = _firstReader;
return true;
}
}
return b;
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
_firstReader?.Dispose();
}
base.Dispose(disposing);
}
public override XmlNodeType NodeType => _currentReader.NodeType;
public override string LocalName => _currentReader.LocalName;
public override string NamespaceURI => _currentReader.NamespaceURI;
public override string Prefix => _currentReader.Prefix;
public override string Value => _currentReader.Value;
public override int Depth => _currentReader.Depth;
public override string BaseURI => _currentReader.BaseURI;
public override bool IsEmptyElement => _currentReader.IsEmptyElement;
public override int AttributeCount => _currentReader.AttributeCount;
public override bool EOF => _currentReader.EOF;
public override ReadState ReadState => _currentReader.ReadState;
public override XmlNameTable NameTable => _currentReader.NameTable;
public override string GetAttribute(string name) => _currentReader.GetAttribute(name);
public override string GetAttribute(string name, string namespaceURI) => _currentReader.GetAttribute(name, namespaceURI);
public override string GetAttribute(int i) => _currentReader.GetAttribute(i);
public override string LookupNamespace(string prefix) => _currentReader.LookupNamespace(prefix);
public override bool MoveToAttribute(string name) => _currentReader.MoveToAttribute(name);
public override bool MoveToAttribute(string name, string ns) => _currentReader.MoveToAttribute(name, ns);
public override bool MoveToElement() => _currentReader.MoveToElement();
public override bool MoveToFirstAttribute() => _currentReader.MoveToFirstAttribute();
public override bool MoveToNextAttribute() => _currentReader.MoveToNextAttribute();
public override bool ReadAttributeValue() => _currentReader.ReadAttributeValue();
public override void ResolveEntity() => _currentReader.ResolveEntity();
}
}
Example of use
using System.IO;
using System.Linq;
internal static class Program
{
private static void Main()
{
var input = new[] { "in1.xml", "in2.xml" };
var output = "output.xml";
var inputStreams = input.Select(p => File.Open(p, FileMode.Open)).ToArray();
using var outputStream = File.Create(output);
XmlConcatenator.Concat(outputStream, inputStreams);
foreach (var stream in inputStreams)
{
stream.Dispose();
}
}
}