We Keep Coding

How to handle multiple file downloads in Playwright?

I have a button that when clicked will start downloading multiple files (this button will also open a chrome://downloads tab and closes it immediately.
The page.download event handler for downloads will not fire.
The page.WaitForDownloadAsync() returns only one of these files.
I do not know the file names that will be downloaded, I also do not know if more than 1 file will be downloaded, there is always the possibility that only 1 file will be downloaded, but also the possibility that multiple files will be downloaded.
How can I handle this in playwright? I would like to return a list of all the downloaded files paths.

So I resolved this with the following logic.
I created two variables:
List<string> downloadedFiles = new List<string>();
List<string> fileDownloadSession = new();
I then created a method to add as a handler to the page.Download that looks like this:
private async void downloadHandler(object sender, IDownload download)
{
fileDownloadSession.Add("Downloading...");
var waiter = await download.PathAsync();
downloadedFiles.Add(waiter);
fileDownloadSession.Remove(fileDownloadSession.First());
}
Afterwards, I created a public method to get the downloaded files that looks like this:
public List<string> GetDownloadedFiles()
{
while (fileDownloadSession.Any())
{
}
var downloadedFilesList = downloadedFiles;
downloadedFiles = new List<string>();
return downloadedFilesList;
}
All these methods and planning are in a separate class of their own so that they can monitor the downloaded files properly, and also to freeze the main thread so it can grab all of the required files.
All in all it seems just as sketchy of a solution, similarly to how you would implement it in Selenium, nothing much has changed in terms of junkyard implementations in the new frameworks.
You can find my custom class here: https://paste.mod.gg/rztmzncvtagi/0, enjoy, there is no other topic that answers this specific question for playwright on C#.
Code here, in case it gets deleted from paste.mod.gg:
using System.Net;
using System.Runtime.InteropServices.JavaScript;
using Flanium;
using FlaUI.UIA3;
using Microsoft.Playwright;
using MoreLinq;
using Polly;
namespace Fight;
public class WebBrowser
{
private IBrowser _browser;
private IBrowserContext _context;
private IPage _page;
private bool _force;
private List<string> downloadedFiles = new List<string>();
private List<string> fileDownloadSession = new();
public void EagerMode()
{
_force = true;
}
public enum BrowserType
{
None,
Chrome,
Firefox,
}
public IPage GetPage()
{
return _page;
}
public WebBrowser(BrowserType browserType = BrowserType.Chrome, bool headlessMode = false)
{
var playwright = Playwright.CreateAsync().Result;
_browser = browserType switch
{
BrowserType.Chrome => playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions {Headless = headlessMode}).Result,
BrowserType.Firefox => playwright.Firefox.LaunchAsync(new BrowserTypeLaunchOptions {Headless = headlessMode}).Result,
_ => null
};
_context = _browser.NewContextAsync().Result;
_page = _context.NewPageAsync().Result;
_page.Download += downloadHandler;
Console.WriteLine("WebBrowser was successfully started.");
}
private async void downloadHandler(object sender, IDownload download)
{
fileDownloadSession.Add("Downloading...");
var waiter = await download.PathAsync();
downloadedFiles.Add(waiter);
fileDownloadSession.Remove(fileDownloadSession.First());
}
public List<string> GetDownloadedFiles()
{
while (fileDownloadSession.Any())
{
}
var downloadedFilesList = downloadedFiles;
downloadedFiles = new List<string>();
return downloadedFilesList;
}
public void Navigate(string url)
{
_page.GotoAsync(url).Wait();
}
public void Close(string containedURL)
{
var pages = _context.Pages.Where(x => x.Url.Contains(containedURL));
if (pages.Any())
pages.ForEach(x => x.CloseAsync().Wait());
}
public IElementHandle Click(string selector, int retries = 15, int retryInterval = 1)
{
var element = Policy.HandleResult<IElementHandle>(result => result == null)
.WaitAndRetry(retries, interval => TimeSpan.FromSeconds(retryInterval))
.Execute(() =>
{
var element = FindElement(selector);
if (element != null)
{
try
{
element.ClickAsync(new ElementHandleClickOptions() {Force = _force}).Wait();
element.DisposeAsync();
return element;
}
catch (Exception e)
{
return null;
}
}
return null;
});
return element;
}
public IElementHandle FindElement(string selector)
{
IElementHandle element = null;
var Pages = _context.Pages.ToArray();
foreach (var w in Pages)
{
//============================================================
element = w.QuerySelectorAsync(selector).Result;
if (element != null)
{
return element;
}
//============================================================
var iframes = w.Frames.ToList();
var index = 0;
for (; index < iframes.Count; index++)
{
var frame = iframes[index];
element = frame.QuerySelectorAsync(selector).Result;
if (element is not null)
{
return element;
}
var children = frame.ChildFrames;
if (children.Count > 0 && iframes.Any(x => children.Any(y => y.Equals(x))) == false)
{
iframes.InsertRange(index + 1, children);
index--;
}
}
}
return element;
}
}

How to prevent recursive in copying files and dirs?

I bumped into the problem with copying files/dirs. Been struggling almost for whole day.
I have to copy from root dir files and its dires with files and subdirs.
Actually, i've made something. However, every time i run in the stackoverflow error.
abstract class SystemOperations {
public virtual void SearchFiles() { }
public virtual void SearchDirectories() { }
public abstract void CreateDirectory(string DIR);
public abstract void CloneContent(string DIR);
public abstract void CreateJSON(string DIR);
public void ExecuteCopying(string DIR) {
CreateDirectory(DIR);
CloneContent(DIR);
CreateJSON(DIR);
}
}
class FileOperations : SystemOperations {
DirectoryInfo _MainPath;
public DirectoryInfo MainPath {
get { return _MainPath; }
set { _MainPath = value; }
}
public FileOperations(DirectoryInfo MainPath) {
this.MainPath = MainPath;
}
#region Unnecessary for current task
public override void SearchFiles() {
string path = "";
FileInfo[] files = MainPath.GetFiles();
foreach (FileInfo file in files) {
path = file.Name;
}
}
public override void SearchDirectories() {
string path = "";
DirectoryInfo[] directories = MainPath.GetDirectories();
foreach (DirectoryInfo directory in directories) {
path = directory.Name;
}
}
#endregion
public override void CreateDirectory(string DIR) {
string newFolder = Path.Combine(MainPath + "", DIR);
Directory.CreateDirectory(newFolder);
}
public override void CloneContent(string DIR) {
foreach (var directory in Directory.GetDirectories(MainPath + "")) {
string dir = Path.GetFileName(directory);
CloneContent(Path.Combine(MainPath + "", dir));
}
foreach (var file in Directory.GetFiles(MainPath + "")) {
File.Copy(file, Path.Combine(MainPath + "", Path.GetFileName(file)), true);
}
}
public override void CreateJSON(string DIR) {
if (!Directory.Exists(DIR)) {
var asd = new DirectoryInfo(DIR);
}
}
}
class Program {
static void Main() {
SystemOperations task = new FileOperations(new DirectoryInfo(#"D:\\LAK"));
task.ExecuteCopying("COPY");
}
}
So, the function CloneContent has to copy in each dir/subdirs files. But its recursive func and as i written above, i run to the error. And dont know how to fix this one. Thank u!

There is some kind of problem with the way you are trying to determine which directory you need to search next, the use of MainPath looks wrong to me.
Personally i also always prefer to have a secondary stop condition to avoid a StackOverflowException, like the maxrunCount i use below.
If you want a recursive directory lookup you should rewrite your code to something like
void Main()
{
string MainPath = "D:\\LAK";
// unless your directory is actually named \LAK:) you should use either #"D:\LAK" or "d:\\LAK"
CloneContent(MainPath,1000);
}
public void CloneContent(string directoryToSearch, int maxrunCount)
{
if(maxrunCount==0)
return;
System.Diagnostics.Debug.Print(directoryToSearch);
string[] directories = null;
try
{
directories = Directory.GetDirectories(directoryToSearch);
}
catch(UnauthorizedAccessException ex) {
System.Diagnostics.Debug.Print($"No access to dir {directoryToSearch}");
directories = new string[0];
}
// ensure you have access to the current directoryToSearch
foreach (var directory in directories)
{
CloneContent(directory,--maxrunCount);
}
System.Diagnostics.Debug.Print($"cloning {directoryToSearch}");
// .... do the actual cloning here,
// you will end up here when there are no more subdirectories on the current branch
}

For a recursive method to work, it must have at least one "exit" condition - the point at which it's done its job and can unwind the stack. In our case, it would be when there are no more direcories or files to copy from the source to the destination.
One way of writing this method would take in a source directory and a destination directory, and then it can recursively call itself for each sub-directory:
public static void CloneContent(string sourceDir, string destDir)
{
// If the source directory doesn't exist, return
if (!Directory.Exists(sourceDir)) return;
// Create destination if needed
Directory.CreateDirectory(destDir);
// Copy files from this directory to the new path
foreach (string file in Directory.GetFiles(sourceDir))
{
File.Copy(file, Path.Combine(destDir, Path.GetFileName(file)));
}
// Recursively call this method for each sub directory
foreach (string subDir in Directory.GetDirectories(sourceDir))
{
string dirName = Path.GetFileName(subDir);
string newSource = Path.Combine(sourceDir, dirName);
string newDest = Path.Combine(destDir, dirName);
CloneContent(newSource, newDest);
}
}

Extract an archive with progress bar?

How i can use an progress bar in this case?
void Client_DownloadFileCompleted(object sender, AsyncCompletedEventArgs e)
{
//System.Windows.MessageBox.Show("Update Complete!", "Message", MessageBoxButton.OK, MessageBoxImage.Information);
Uri uri = new Uri(url);
string filename = System.IO.Path.GetFileName(uri.AbsolutePath);
ZipFile.ExtractToDirectory(filePathDir + "/" + filename, filePathDir);
}
EDIT:
#Alessandro D'Andria , But in this case?:
WebClient wc = new WebClient();
Stream zipReadingStream = wc.OpenRead(url);
ZipArchive zip = new ZipArchive(zipReadingStream);
ZipFileExtensions.ExtractToDirectory(zip, filePathDir);

You can see the source of ExtractToDirectory on GitHub, the only thing you need to do is pass in a Progress<ZipProgress> and call it inside the foreach loop.
//This is a new class that represents a progress object.
public class ZipProgress
{
public ZipProgress(int total, int processed, string currentItem)
{
Total = total;
Processed = processed;
CurrentItem = currentItem;
}
public int Total { get; }
public int Processed { get; }
public string CurrentItem { get; }
}
public static class MyZipFileExtensions
{
public static void ExtractToDirectory(this ZipArchive source, string destinationDirectoryName, IProgress<ZipProgress> progress)
{
ExtractToDirectory(source, destinationDirectoryName, progress, overwrite: false);
}
public static void ExtractToDirectory(this ZipArchive source, string destinationDirectoryName, IProgress<ZipProgress> progress, bool overwrite)
{
if (source == null)
throw new ArgumentNullException(nameof(source));
if (destinationDirectoryName == null)
throw new ArgumentNullException(nameof(destinationDirectoryName));
// Rely on Directory.CreateDirectory for validation of destinationDirectoryName.
// Note that this will give us a good DirectoryInfo even if destinationDirectoryName exists:
DirectoryInfo di = Directory.CreateDirectory(destinationDirectoryName);
string destinationDirectoryFullPath = di.FullName;
int count = 0;
foreach (ZipArchiveEntry entry in source.Entries)
{
count++;
string fileDestinationPath = Path.GetFullPath(Path.Combine(destinationDirectoryFullPath, entry.FullName));
if (!fileDestinationPath.StartsWith(destinationDirectoryFullPath, StringComparison.OrdinalIgnoreCase))
throw new IOException("File is extracting to outside of the folder specified.");
var zipProgress = new ZipProgress(source.Entries.Count, count, entry.FullName);
progress.Report(zipProgress);
if (Path.GetFileName(fileDestinationPath).Length == 0)
{
// If it is a directory:
if (entry.Length != 0)
throw new IOException("Directory entry with data.");
Directory.CreateDirectory(fileDestinationPath);
}
else
{
// If it is a file:
// Create containing directory:
Directory.CreateDirectory(Path.GetDirectoryName(fileDestinationPath));
entry.ExtractToFile(fileDestinationPath, overwrite: overwrite);
}
}
}
}
This is used like
public class YourClass
{
public Progress<ZipProgress> _progress;
public YourClass()
{
// Create the progress object in the constructor, it will call it's ReportProgress using the sync context it was constructed on.
// If your program is a UI program that means you want to new it up on the UI thread.
_progress = new Progress<ZipProgress>();
_progress.ProgressChanged += Report
}
private void Report(object sender, ZipProgress zipProgress)
{
//Use zipProgress here to update the UI on the progress.
}
//I assume you have a `Task.Run(() => Download(url, filePathDir);` calling this so it is on a background thread.
public void Download(string url, string filePathDir)
{
WebClient wc = new WebClient();
Stream zipReadingStream = wc.OpenRead(url);
ZipArchive zip = new ZipArchive(zipReadingStream);
zip.ExtractToDirectory(filePathDir, _progress);
}
//...

Maybe something like this can work for you:
using (var archive = new ZipArchive(zipReadingStream))
{
var totalProgress = archive.Entries.Count;
foreach (var entry in archive.Entries)
{
entry.ExtractToFile(destinationFileName); // specify the output path of thi entry
// update progess there
}
}
It's simple a workaround to keep track of the progress.

Read x number of lines of a file at a time C#

I want to read and process 10+ lines at a time for GB files, but haven't found a solution to spit out 10 lines until the end.
My last attempt was :
int n = 10;
foreach (var line in File.ReadLines("path")
.AsParallel().WithDegreeOfParallelism(n))
{
System.Console.WriteLine(line);
Thread.Sleep(1000);
}
I've seen solutions that use buffer sizes but I want to read in the entire row.

The Default behavour is to read all the Line in one shot, if you want to read less than that you need to dig a little deeper into how it reads them and get a StreamReader which will then let you control the reading process
using (StreamReader sr = new StreamReader(path))
{
while (sr.Peek() >= 0)
{
Console.WriteLine(sr.ReadLine());
}
}
it also has a ReadLineAsync method that will return a task
if you contain these tasks in an ConcurrentBag you can very easily keep the processing running on 10 lines at a time.
var bag =new ConCurrentBag<Task>();
using (StreamReader sr = new StreamReader(path))
{
while(sr.Peek() >=0)
{
if(bag.Count < 10)
{
Task processing = sr.ReadLineAsync().ContinueWith( (read) => {
string s = read.Result;//EDIT Removed await to reflect Scots comment
//process line
});
bag.Add(processing);
}
else
{
Task.WaitAny(bag.ToArray())
//remove competed tasks from bag
}
}
}
note this code is for guidance only not to be used as is;
if all you want is the last ten lines then you can get that with the solution here
How to read a text file reversely with iterator in C#

This method would create "pages" of lines from your file.
public static IEnumerable<string[]> ReadFileAsLinesSets(string fileName, int setLen = 10)
{
using (var reader = new StreamReader(fileName))
while (!reader.EndOfStream)
{
var set = new List<string>();
for (var i = 0; i < setLen && !reader.EndOfStream; i++)
{
set.Add(reader.ReadLine());
}
yield return set.ToArray();
}
}
... More fun version...
class Example
{
static void Main(string[] args)
{
"YourFile.txt".ReadAsLines()
.AsPaged(10)
.Select(a=>a.ToArray()) //required or else you will get random data since "WrappedEnumerator" is not thread safe
.AsParallel()
.WithDegreeOfParallelism(10)
.ForAll(a =>
{
//Do your work here.
Console.WriteLine(a.Aggregate(new StringBuilder(),
(sb, v) => sb.AppendFormat("{0:000000} ", v),
sb => sb.ToString()));
});
}
}
public static class ToolsEx
{
public static IEnumerable<IEnumerable<T>> AsPaged<T>(this IEnumerable<T> items,
int pageLength = 10)
{
using (var enumerator = new WrappedEnumerator<T>(items.GetEnumerator()))
while (!enumerator.IsDone)
yield return enumerator.GetNextPage(pageLength);
}
public static IEnumerable<T> GetNextPage<T>(this IEnumerator<T> enumerator,
int pageLength = 10)
{
for (var i = 0; i < pageLength && enumerator.MoveNext(); i++)
yield return enumerator.Current;
}
public static IEnumerable<string> ReadAsLines(this string fileName)
{
using (var reader = new StreamReader(fileName))
while (!reader.EndOfStream)
yield return reader.ReadLine();
}
}
internal class WrappedEnumerator<T> : IEnumerator<T>
{
public WrappedEnumerator(IEnumerator<T> enumerator)
{
this.InnerEnumerator = enumerator;
this.IsDone = false;
}
public IEnumerator<T> InnerEnumerator { get; private set; }
public bool IsDone { get; private set; }
public T Current { get { return this.InnerEnumerator.Current; } }
object System.Collections.IEnumerator.Current { get { return this.Current; } }
public void Dispose()
{
this.InnerEnumerator.Dispose();
this.IsDone = true;
}
public bool MoveNext()
{
var next = this.InnerEnumerator.MoveNext();
this.IsDone = !next;
return next;
}
public void Reset()
{
this.IsDone = false;
this.InnerEnumerator.Reset();
}
}

How to use Microsoft.Build.Evaluation.Project.RemoveItem

I've successfuly added files programatically to my project using the following code:
var project = new Microsoft.Build.Evaluation.Project(projPath);
project.AddItem("Compile", filePath);
However, removing a file programatically is giving me a hard time.
Signature:
public bool RemoveItem(
ProjectItem item
)
How can I instantiate a ProjectItem? I couldn't find any examples.
Reference: https://msdn.microsoft.com/en-us/library/microsoft.build.evaluation.project.removeitem.aspx

you did
private static ProjectItem GetProjectItem(this Project project, string filePath)
{
var includePath = filePath.Substring(project.DirectoryPath.Length + 1);
var projectItem = project.GetItems(CompileType).FirstOrDefault(item => item.EvaluatedInclude.Equals(includePath));
return projectItem;
}
in your GetProjectItem method:
replace that:
var projectItem = project.GetItems(CompileType).FirstOrDefault(item => item.EvaluatedInclude.Equals(includePath));
with this:
var projectItem = project.GetItems("Compile").ToList()
.Where(item => item.EvaluatedInclude.Equals(includePath)).FirstOrDefault();
using .FirstOrDefault() will bring it to have just first item of all files. i used .ToList() and made it work with all my items which have same EvaluatedInclude. its totally worked for my.

This is the class I ended up writing. No simple solution for remove.
public static class SourceControlHelper
{
public static void CheckoutFile(string filePath)
{
TFSAction((workspace) => workspace.PendEdit(filePath), filePath);
}
public static void AddFile(this Project project, string filePath)
{
CheckoutFile(project.FullPath);
var projectItem = project.GetProjectItem(filePath);
if (projectItem != null)
{
return;
}
var includePath = filePath.Substring(project.DirectoryPath.Length + 1);
project.AddItem(CompileType, includePath);
project.Save();
TFSAction(workspace => workspace.PendAdd(filePath), filePath);
}
public static void DeleteFile(this Project project, string filePath)
{
CheckoutFile(project.FullPath);
var projectItem = project.GetProjectItem(filePath);
if (projectItem == null)
{
return;
}
project.RemoveItem(projectItem);
project.Save();
TFSAction(workspace => workspace.PendDelete(filePath), filePath);
}
private static ProjectItem GetProjectItem(this Project project, string filePath)
{
var includePath = filePath.Substring(project.DirectoryPath.Length + 1);
var projectItem = project.GetItems(CompileType).FirstOrDefault(item => item.EvaluatedInclude.Equals(includePath));
return projectItem;
}
private static void TFSAction(Action<Workspace> action, string filePath)
{
var workspaceInfo = Workstation.Current.GetLocalWorkspaceInfo(filePath);
if (workspaceInfo == null)
{
Console.WriteLine("Failed to initialize workspace info");
return;
}
using (var server = new TfsTeamProjectCollection(workspaceInfo.ServerUri))
{
var workspace = workspaceInfo.GetWorkspace(server);
action(workspace);
}
}
private static string CompileType
{
get { return CopyTool.Extension.Equals("ts") ? "TypeScriptCompile" : "Compile"; }
}
}