I'm uploading all my local files to s3 using this code:
static string bucketName = "s3bucket";
static string directoryPath = #"C:\data\";
private void btnUpload_Click(object sender, EventArgs e) {
try {
TransferUtility directoryTransferUtility =
new TransferUtility(new AmazonS3Client(Amazon.RegionEndpoint.USWest2));
TransferUtilityUploadDirectoryRequest request =
new TransferUtilityUploadDirectoryRequest {
BucketName = bucketName,
Directory = directoryPath,
SearchPattern = "*.xml"
};
directoryTransferUtility.UploadDirectory(request);
MessageBox.Show("Upload completed");
} catch (AmazonS3Exception ex) {
MessageBox.Show(ex.Message);
}
}
If I run the code again all files are re-uploaded to s3 and that's a bad idea if let's say we have 1000+ files in our local directory.
I know I can compare file by file because aws stores the md5 of each file. So my question is can I do this with a command that comes preinstalled? Do I have to recursively compare file by file? If the sync command exists on awscli bundle (aws s3 sync ./sourceDir s3://bucketname/) does it exists on c# as well?
All the files will be replaced. S3 docs , but if you still want to check files you can use some function like this
ListObjectsRequest request = new ListObjectsRequest()
{
BucketName = "Your Bucket name",
Delimiter = "/",
Prefix = "location"
};
public bool CheckFile(ListObjectsRequest request)
{
bool res = false;
try
{
ListObjectsResponse response = s3client.ListObjects(request);
if (response.S3Objects != null && response.S3Objects.Count > 0)
{
S3Object o = response.S3Objects.Where(x => x.Size != 0 && x.LastModified > DateTime.Now.AddHours(-24)).FirstOrDefault();
if (o != null)
{
//you can use thes fields
// o.Key; //o.Size, //o.LastModified
res = true;
}
}
else
{
res = false;
}
}
catch (Exception)
{
throw;
}
return res;
}
Related
First, thank you for reading my Post. I am currently working in my Thesis and need to implement a WPF-Application, which can send Data to a Android Device on the Public Folder. The app communicates with a App on the Android Device(exactly a Scanner). A Part of the WPF already existed, the WPF can read the Data, which are on the Scanners public folder stored. It is using the https://github.com/jtorjo/external_drive_lib. And i need help on how i can create a Method to send (Create) and update the Data on the PC, which i am inputing on the WPF with textboxes. So that they get stored on the Android Device and are visble.
Here is some Code from a AndroidDeviceService Class, which only reads the Data from the public folder:
//Gets all the connected android devices
private List<AndroidDevice> GetAndroidDevices()
{
List<IDrive> androidDrives = drive_root.inst.drives.Where(d => d.type.is_portable() && d.type.is_android() && d.is_connected()).ToList();
List<AndroidDevice> scannerAppDevices = new List<AndroidDevice>();
foreach (var drive in androidDrives)
{
scannerAppDevices.Add(new AndroidDevice(drive));
}
return scannerAppDevices;
}
//Tries to get a folder on the drive or returns null
public IFolder TryToGetFolder(IDrive drive, string folderPath)
{
if (drive.is_connected())
{
try
{
return drive.try_parse_folder(folderPath);
}
catch
{
return null;
}
}
else
return null;
}
//Tries to get a file in a folder or returns null
public IFile TryToGetFile(IFolder folder, string filename)
{
return folder.files.Where(f => f.name == filename).FirstOrDefault();
}
//Tries to read the text of a file or returns null
public string TryToGetFileText(IFile file)
{
//Uses the tempDir to copy the file to it
string tempDir = _tempDir + "temp-" + DateTime.Now.Ticks;
Directory.CreateDirectory(tempDir);
file.copy_sync(tempDir);
//Read file from tempDir
StreamReader reader = new StreamReader(tempDir + "\\" + file.name);
return reader.ReadToEnd();
}
//Tries to delete a file
public void TryToDeleteFile(IFile file)
{
if(file != null)
{
if (file.exists)
file.delete_sync();
}
}
Then i am using the methods in the ScannerAppCommunicationService:
/*
* Takes an AndroidDevice, gets the communication file, deserialize it and returns a List of CWCProducts
*/
public List<CWCProduct> GetProductsFromScanner(AndroidDevice scannerAppDevice)
{
IFile file = GetScannerAppFile(scannerAppDevice, CWC_FILENAME);
if (file == null)
throw new NoFileOnScannerException("No File, therefore no Products on the Scanner!");
try
{
string fileText = _deviceService.TryToGetFileText(file);
CWCFile cwcFile = Utilities.JsonDeserialize<CWCFile>(fileText);
return cwcFile.products.ToCWCProducts();
}
catch
{
throw new FileDecodeException($"Decoding Text from {CWC_FILENAME} File failed!");
}
}
//Deletes the communication file from the AndroidDevice
public void DeleteProductsOnScanner(AndroidDevice scannerAppDevice)
{
IFile file = GetScannerAppFile(scannerAppDevice, CWC_FILENAME);
if (file == null)
throw new NoFileOnScannerException("No File, therefore no Products on the Scanner!");
_deviceService.TryToDeleteFile(file);
}
//Gets a file from the public ScannerApp folder on the AndroidDevice
private IFile GetScannerAppFile(AndroidDevice scannerAppDevice, string filename)
{
IFolder folder = _deviceService.TryToGetFolder(scannerAppDevice.Drive, SCANNERAPP_FOLDERNAME);
if (folder == null)
throw new ScannerAppNotFoundOnDeviceException("ScannerApp not found on this device or the device has been disconnected!");
return _deviceService.TryToGetFile(folder, filename);
}
I already tried some Methods:
`//Tries to Send a file from the WPF to the Android Device
public bool SendFileToAndroid(AndroidDevice device, string filePath, string destinationFolder)
{
if (!device.Drive.is_connected())
return false;
try
{
var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = "adb",
Arguments = $"push {filePath} {destinationFolder}",
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
}
};
process.Start();
process.WaitForExit();
return true;
}
catch
{
return false;
}
}
//Trys to Update a File on the Android Advice
public bool TryToUpdateFile(AndroidDevice scannerAppDevice, string destinationFolder)
{
string filePath = "";
IFolder folder = scannerAppDevice.Drive.try_parse_folder(filePath);
IFile file = TryToGetFile(folder, filePath);
if (file != null)
{
//Delete the old file
TryToDeleteFile(file);
}
//Send the new file
return SendFileToAndroid(scannerAppDevice, filePath, destinationFolder);
}`
//Creates a File in the Public Folder of the Adroid Advice
public void CreateFileInPublicFolder(AndroidDevice scannerAppDevice, object data)
{
// Serialize the data to json
string jsonData = Utilities.JsonSerialize(data);
// Create a temporary file to store the json data
string tempFilePath = Path.GetTempPath() + "tempJsonFile.json";
File.WriteAllText(tempFilePath, jsonData);
// Set the destination folder for the file
string destinationFolder = SCANNERAPP_FOLDERNAME;
// Send the file to the android device
if (!_deviceService.SendFileToAndroid(scannerAppDevice, tempFilePath, destinationFolder))
{
throw new FileTransferException("Error while trying to transfer the file to the android device");
}
// Delete the temporary file
File.Delete(tempFilePath);
}
public void UpdateFileOnScanner(AndroidDevice scannerAppDevice, string newFileContent)
{
IFile file = GetScannerAppFile(scannerAppDevice, CWC_FILENAME);
if (file == null)
throw new NoFileOnScannerException("File not found on the scanner!");
try
{
_deviceService.TryToUpdateFile(scannerAppDevice, newFileContent);
}
catch
{
throw new FileUpdateException($"Failed to update file {CWC_FILENAME} on the scanner!");
}
}
I am trying to move data for example;
Source = "Uploads/Photos/" to Destination="Uploads/mytest/"
I am getting error like that but but when i give a specific file this works.
Basically, I want to move folder with all files.
My code is below;
public async Task<MoveResponse> MoveObject(MoveRequest moveRequest)
{
MoveResponse moveResponse = new MoveResponse();
CopyObjectRequest copyObjectRequest = new CopyObjectRequest
{
SourceBucket = moveRequest.BucketName,
DestinationBucket = moveRequest.BucketName + "/" + moveRequest.Destination,
SourceKey = moveRequest.Source,
DestinationKey = moveRequest.Source,
};
var response1 = await client.CopyObjectAsync(copyObjectRequest).ConfigureAwait(false);
if (response1.HttpStatusCode != System.Net.HttpStatusCode.OK)
{
moveResponse.IsError = true;
moveResponse.ErrorMessage = "Files could not moved to destination!";
return moveResponse;
}
return moveResponse;
}
I hope, you are using high level S3 API's.
Check out this sample code
private void uploadFolderToolStripMenuItem_Click(object sender, EventArgs e)
{
string directoryPath = textBoxBasePath.Text + listBoxFolder.SelectedItem.ToString().Replace("[", "").Replace("]", "");
string bucketName = comboBoxBucketNames.Text;
string FolderName = listBoxFolder.SelectedItem.ToString().Replace("[", "").Replace("]", "");
try
{
TransferUtility directoryTransferUtility = new TransferUtility(new AmazonS3Client(AwsAccessKeyID, AwsSecretAccessKey, RegionEndpoint.USEast1));
TransferUtilityUploadDirectoryRequest request = new TransferUtilityUploadDirectoryRequest
{
BucketName = bucketName,
KeyPrefix = FolderName,
StorageClass = S3StorageClass.StandardInfrequentAccess,
ServerSideEncryptionMethod = ServerSideEncryptionMethod.AES256,
Directory = directoryPath,
SearchOption = SearchOption.AllDirectories,
SearchPattern = "*.*",
CannedACL = S3CannedACL.AuthenticatedRead
};
ListMultipartUploadsRequest req1 = new ListMultipartUploadsRequest
{
BucketName = bucketName
};
var t = Task.Factory.FromAsync(directoryTransferUtility.BeginUploadDirectory, directoryTransferUtility.EndUploadDirectory, request, null);
t.Wait();
MessageBox.Show(string.Format("The Directory '{0}' is successfully uploaded", FolderName));
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
finally
{ }
}
I have a windows service that monitors a directory for new files and, when a file is created/copied into that directory, it renames it and does some processing.
I'm using File.Move to rename the file.
It works fine when I use the exact same code from a console application, but now when it runs as a Windows Service.
Code that initialize the file watcher(s)
public void StartFileSystemWatcher()
{
listFileSystemWatcher = new List<FileSystemWatcher>();
foreach (var customFolder in listFolders)
{
var dir = new DirectoryInfo(customFolder.FolderPath);
if (customFolder.FolderEnabled && dir.Exists)
{
var fileSWatch = new FileSystemWatcher
{
Filter = customFolder.FolderFilter,
Path = customFolder.FolderPath,
NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.FileName | NotifyFilters.DirectoryName,
EnableRaisingEvents = true
};
fileSWatch.Created += (senderObj, fileSysArgs) => FileSWatch_Created(senderObj, fileSysArgs, customFolder.VSTSUrl);
listFileSystemWatcher.Add(fileSWatch);
logger.Info(string.Format("Starting to monitor files with extension ({0}) in the folder ({1})", fileSWatch.Filter, fileSWatch.Path));
}
else
{
logger.Info(string.Format("File system monitor cannot start because the folder ({0}) does not exist", customFolder.FolderPath));
}
}
}
private void FileSWatch_Created(object sender, FileSystemEventArgs e, string vstsUrl)
{
PostFileInfoToVSTS(e.FullPath, vstsUrl);
}
Code that's run when the file is created
private void PostFileInfoToVSTS(string filePath, string vstsUrl)
{
try
{
var fileName = filePath.Substring(filePath.LastIndexOf("\\") + 1);
var fileParts = fileName.Split(new char[] { '.' });
if (fileParts.Length == 6)
{
//rename the file
var fileNewName = $"{fileParts[2]}.{fileParts[3]}.{fileParts[4]}.TXT";
var fileNewPath = $"{Path.GetDirectoryName(filePath)}\\oldfiles\\{fileNewName}";
File.Move(filePath, fileNewPath);
//prepare vsts request
var request = new VSTSRequest();
request.Variables.Region.Value = fileParts[0];
request.Variables.Sparc.Value = fileParts[1];
request.Variables.SystemName.Value = fileParts[2];
request.Variables.Type.Value = fileParts[3];
request.Variables.Name.Value = fileParts[4];
request.Variables.Filename.Value = fileNewName;
var httpWebRequest = (HttpWebRequest)WebRequest.Create(vstsUrl);
httpWebRequest.ContentType = "application/json";
httpWebRequest.Method = "POST";
httpWebRequest.Headers.Add("Authorization", ConfigurationManager.AppSettings["VSTSAuthentication"]);
//post to vsts
using (var streamWriter = new StreamWriter(httpWebRequest.GetRequestStream()))
{
streamWriter.Write(JsonConvert.SerializeObject(request));
streamWriter.Flush();
streamWriter.Close();
}
//read response
var httpResponse = (HttpWebResponse)httpWebRequest.GetResponse();
using (var streamReader = new StreamReader(httpResponse.GetResponseStream()))
{
var result = streamReader.ReadToEnd();
var json = JsonConvert.DeserializeObject<dynamic>(result);
logger.Info($"File '{fileNewPath}' processed. ReleaseId: {json.id}.");
}
}
else
{
logger.Info($"File '{fileName}' wasn't processed. Name is not in the correct format.");
}
}
catch (Exception ex)
{
logger.Error(ex);
}
}
The first time I copy a file into that directory, it works fine, but the following times, I get a System.IO.IOException: The process cannot access the file because it is being used by another process.
I have some data to save into a database.
I have created a web api post method to save data. Following is my post method:
[Route("PostRequirementTypeProcessing")]
public IEnumerable<NPAAddRequirementTypeProcessing> PostRequirementTypeProcessing(mdlAddAddRequirementTypeProcessing requTypeProcess)
{
mdlAddAddRequirementTypeProcessing rTyeProcessing = new mdlAddAddRequirementTypeProcessing();
rTyeProcessing.szDescription = requTypeProcess.szDescription;
rTyeProcessing.iRequirementTypeId = requTypeProcess.iRequirementTypeId;
rTyeProcessing.szRequirementNumber = requTypeProcess.szRequirementNumber;
rTyeProcessing.szRequirementIssuer = requTypeProcess.szRequirementIssuer;
rTyeProcessing.szOrganization = requTypeProcess.szOrganization;
rTyeProcessing.dIssuedate = requTypeProcess.dIssuedate;
rTyeProcessing.dExpirydate = requTypeProcess.dExpirydate;
rTyeProcessing.szSignedBy = requTypeProcess.szSignedBy;
rTyeProcessing.szAttachedDocumentNo = requTypeProcess.szAttachedDocumentNo;
if (String.IsNullOrEmpty(rTyeProcessing.szAttachedDocumentNo))
{
}
else
{
UploadFile();
}
rTyeProcessing.szSubject = requTypeProcess.szSubject;
rTyeProcessing.iApplicationDetailsId = requTypeProcess.iApplicationDetailsId;
rTyeProcessing.iEmpId = requTypeProcess.iEmpId;
NPAEntities context = new NPAEntities();
Log.Debug("PostRequirementTypeProcessing Request traced");
var newRTP = context.NPAAddRequirementTypeProcessing(requTypeProcess.szDescription, requTypeProcess.iRequirementTypeId,
requTypeProcess.szRequirementNumber, requTypeProcess.szRequirementIssuer, requTypeProcess.szOrganization,
requTypeProcess.dIssuedate, requTypeProcess.dExpirydate, requTypeProcess.szSignedBy,
requTypeProcess.szAttachedDocumentNo, requTypeProcess.szSubject, requTypeProcess.iApplicationDetailsId,
requTypeProcess.iEmpId);
return newRTP.ToList();
}
There is a field called 'szAttachedDocumentNo' which is a document that's being saved in the database as well.
After saving all data, I want the physical file of the 'szAttachedDocumentNo' to be saved on the server. So i created a method called "UploadFile" as follows:
[HttpPost]
public void UploadFile()
{
if (HttpContext.Current.Request.Files.AllKeys.Any())
{
// Get the uploaded file from the Files collection
var httpPostedFile = HttpContext.Current.Request.Files["UploadedFile"];
if (httpPostedFile != null)
{
// Validate the uploaded image(optional)
string folderPath = HttpContext.Current.Server.MapPath("~/UploadedFiles");
//string folderPath1 = Convert.ToString(ConfigurationManager.AppSettings["DocPath"]);
//Directory not exists then create new directory
if (!Directory.Exists(folderPath))
{
Directory.CreateDirectory(folderPath);
}
// Get the complete file path
var fileSavePath = Path.Combine(folderPath, httpPostedFile.FileName);
// Save the uploaded file to "UploadedFiles" folder
httpPostedFile.SaveAs(fileSavePath);
}
}
}
Before running the project, i debbugged the post method, so when it comes to "UploadFile" line, it takes me to its method.
From the file line, it skipped the remaining lines and went to the last line; what means it didn't see any file.
I am able to save everything to the database, just that i didn't see the physical file in the specified location.
Any help would be much appreciated.
Regards,
Somad
Makes sure the request "content-type": "multipart/form-data" is set
[HttpPost()]
public async Task<IHttpActionResult> UploadFile()
{
if (!Request.Content.IsMimeMultipartContent())
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
try
{
MultipartMemoryStreamProvider provider = new MultipartMemoryStreamProvider();
await Request.Content.ReadAsMultipartAsync(provider);
if (provider.Contents != null && provider.Contents.Count == 0)
{
return BadRequest("No files provided.");
}
foreach (HttpContent file in provider.Contents)
{
string filename = file.Headers.ContentDisposition.FileName.Trim('\"');
byte[] buffer = await file.ReadAsByteArrayAsync();
using (MemoryStream stream = new MemoryStream(buffer))
{
// save the file whereever you want
}
}
return Ok("files Uploded");
}
catch (Exception ex)
{
return InternalServerError(ex);
}
}
how do i get a full path of a folder with google drive api c#. Lets say i want a .net list filled with Folder class and Folder being a class with 2 properties. URL and folder name. Iam new to this so sorry if the question is bad/dumb. Any thing would help at this point.
There is a fantastic command line for working with google drive available on github.com/prasmussen/gdrive/
The logic exists in that codebase to walk the directory tree up from each file and construct the full path.
I've followed the .NET Quickstart instructions, then converted the relevant go-lang code from path.go into the C# equivalent below.
using Google.Apis.Auth.OAuth2;
using Google.Apis.Drive.v3;
using Google.Apis.Services;
using Google.Apis.Util.Store;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
namespace DriveQuickstart
{
class Program
{
// If modifying these scopes, delete your previously saved credentials
// at ~/.credentials/drive-dotnet-quickstart.json
static string[] Scopes = { DriveService.Scope.DriveReadonly };
static string ApplicationName = "Drive API .NET Quickstart";
static DriveService service;
static Dictionary<string, Google.Apis.Drive.v3.Data.File> files = new Dictionary<string, Google.Apis.Drive.v3.Data.File>();
static void Main(string[] args)
{
UserCredential credential;
using (var stream =
new FileStream("client_secret.json", FileMode.Open, FileAccess.Read))
{
string credPath = System.Environment.GetFolderPath(
System.Environment.SpecialFolder.Personal);
credPath = Path.Combine(credPath, ".credentials/drive-dotnet-quickstart.json");
credential = GoogleWebAuthorizationBroker.AuthorizeAsync(
GoogleClientSecrets.Load(stream).Secrets,
Scopes,
"user",
CancellationToken.None,
new FileDataStore(credPath, true)).Result;
Console.WriteLine("Credential file saved to: " + credPath);
}
// Create Drive API service.
service = new DriveService(new BaseClientService.Initializer()
{
HttpClientInitializer = credential,
ApplicationName = ApplicationName,
});
// Define parameters of request.
FilesResource.ListRequest listRequest = service.Files.List();
listRequest.PageSize = 10;
listRequest.Fields = "nextPageToken, files(id, name, parents)";
// List files.
IList<Google.Apis.Drive.v3.Data.File> files = listRequest.Execute()
.Files;
Console.WriteLine("Files:");
if (files != null && files.Count > 0)
{
foreach (var file in files)
{
var absPath = AbsPath(file);
Console.WriteLine("{0} ({1})", absPath, file.Id);
}
}
else
{
Console.WriteLine("No files found.");
}
Console.Read();
}
private static object AbsPath(Google.Apis.Drive.v3.Data.File file)
{
var name = file.Name;
if (file.Parents.Count() == 0)
{
return name;
}
var path = new List<string>();
while (true)
{
var parent = GetParent(file.Parents[0]);
// Stop when we find the root dir
if (parent.Parents == null || parent.Parents.Count() == 0)
{
break;
}
path.Insert(0, parent.Name);
file = parent;
}
path.Add(name);
return path.Aggregate((current, next) => Path.Combine(current, next));
}
private static Google.Apis.Drive.v3.Data.File GetParent(string id)
{
// Check cache
if (files.ContainsKey(id))
{
return files[id];
}
// Fetch file from drive
var request = service.Files.Get(id);
request.Fields = "name,parents";
var parent = request.Execute();
// Save in cache
files[id] = parent;
return parent;
}
}
}
Folder and Files both are considered as a File in Google Drive hence the following code will work for both the scenarios.
Create a function to return Full Path and Two other functions required in the same Task :
private IList<string> GetFullPath(Google.Apis.Drive.v3.Data.File file, IList<Google.Apis.Drive.v3.Data.File> files)
{
IList<string> Path = new List<string>();
if (file.Parents == null || file.Parents.Count == 0)
{
return Path;
}
Google.Apis.Drive.v3.Data.File Mainfile = file;
while (GetParentFromID(file.Parents[0], files) != null)
{
Path.Add(GetFolderNameFromID(GetParentFromID(file.Parents[0], files).Id, files));
file = GetParentFromID(file.Parents[0], files);
}
return Path;
}
private Google.Apis.Drive.v3.Data.File GetParentFromID(string FileID, IList<Google.Apis.Drive.v3.Data.File> files)
{
if (files != null && files.Count > 0)
{
foreach (var file in files)
{
if (file.Parents != null && file.Parents.Count > 0)
{
if (file.Id == FileID)
{
return file;
}
}
}
}
return null;
}
private string GetFolderNameFromID(string FolderID, IList<Google.Apis.Drive.v3.Data.File> files)
{
string FolderName = "";
if (files != null && files.Count > 0)
{
foreach (var file in files)
{
if (file.Id == FolderID)
{
FolderName = file.Name;
}
}
}
return FolderName;
}
Now you may call the function as :
string Path = "My Drive";
foreach (string Item in GetFullPath(file, files).Reverse())
{
Path += " / " + Item;
}
here, two parameters are passed -
1. file - it is the file whose path you are trying to find.
2. files - the list of files on your drive.