I am using the DataMovement Nuget to download a directory from Azure using a presigned URL since I don't want account details on the client. Thus I am generating a Presigned URL from the server and sending it to the client. Using C#
Downloading a single file works, but not a directory. The presinged URL is correct.
Here is the code:
private static async Task Main(string[] args)
{
string containerName = "main";
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
var desitination = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), "Temp");
var directoryName = "videos";
var url = GetClientDownloadURL(containerName, directoryName, blobClient);
Stopwatch stopWatch = Stopwatch.StartNew();
var checkpoint = await DownloadFromAzurePauseResume(url, desitination, containerName, "videos");
if (canceled)
{
var resuming = false;
Console.WriteLine("Press R to resume");
while (!resuming)
{
var keyInfo = Console.ReadKey();
if (keyInfo.Key == ConsoleKey.R)
{
resuming = true;
}
}
if (resuming)
await ResumeDownload(url, desitination, checkpoint);
}
stopWatch.Stop();
Console.WriteLine("\nTransfer operation completed in " + stopWatch.Elapsed.TotalSeconds + " seconds.");
}
public static DirectoryTransferContext GetDirectoryTransferContext(TransferCheckpoint checkpoint)
{
DirectoryTransferContext context = new DirectoryTransferContext(checkpoint);
context.ProgressHandler = new Progress<TransferStatus>((progress) =>
{
Console.Write("\rBytes transferred: {0}", progress.BytesTransferred);
});
return context;
}
public static async Task<TransferCheckpoint> DownloadFromAzurePauseResume(string directoryUri, string destination, string containerName, string folderName)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($"dirUrl: {directoryUri}");
Console.ForegroundColor = ConsoleColor.White;
CloudBlobClient cbc = new CloudBlobClient(new Uri(directoryUri));
CloudBlobContainer container = cbc.GetContainerReference(containerName);
CloudBlobDirectory cloudBlobDir = container.GetDirectoryReference(folderName);
// track transfer progress
TransferCheckpoint? checkPoint = null;
var context = GetDirectoryTransferContext(checkPoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource();
Console.WriteLine("\nTransfer started...\nPress 'c' to temporarily cancel your transfer...\n");
var downloadOption = new DownloadDirectoryOptions() { Recursive = true };
Task task;
ConsoleKeyInfo keyinfo;
try
{
//task = TransferManager.DownloadAsync(blob, destination, null, context, cancellationSource.Token);
task = TransferManager.DownloadDirectoryAsync(cloudBlobDir, destination, downloadOption, context, cancellationSource.Token);
while (!task.IsCompleted)
{
if (Console.KeyAvailable)
{
keyinfo = Console.ReadKey(true);
if (keyinfo.Key == ConsoleKey.C)
{
cancellationSource.Cancel();
canceled = true;
}
}
}
await task;
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
return context.LastCheckpoint;
}
This is the method from the server and is working correctly:
private static string GetClientDownloadURL(string containerName, string directoryName, CloudBlobClient blobClient)
{
var primaryStorageAccountURL = blobClient.BaseUri.AbsoluteUri;
// TODO : Add presigned URL
var container = blobClient.GetContainerReference(containerName);
var fullURL = $"{container.Uri.AbsoluteUri}/{directoryName}";
var accessStartTime = DateTime.Now;
fullURL += container.GetSharedAccessSignature(new SharedAccessBlobPolicy
{
Permissions = SharedAccessBlobPermissions.Read,
SharedAccessExpiryTime = accessStartTime.Add(TimeSpan.FromHours(6)),
SharedAccessStartTime = accessStartTime,
}, "PrivateAccess");
return fullURL;
}
I am testing this in a console application, but eventually want to move it to an app im building.
Related
I'm trying to generate a zip file of pdfs asynchronously to speed things up as follows:
var files = new Dictionary<string, byte[]>();
var fileTasks = new List<Task<Library.Models.Helpers.File>>();
foreach (var i in groups)
{
var task = Task.Run(async () =>
{
var fileName = $"{i.Key.Title.Replace('/', '-')} - Records.pdf";
ViewBag.GroupName= i.Key.Title;
var html = await this.RenderViewAsync("~/Views/Report/_UserRecordsReport.cshtml", i.ToList(), true);
return await _fileUtilityService.HtmlToPDF2(html, null, fileName);
});
fileTasks.Add(task);
}
var completedTaskFiles = await Task.WhenAll(fileTasks);
foreach(var item in completedTaskFiles)
{
files.Add($"{item.FileName}", item.FileResult);
}
return _fileUtilityService.GenerateZIP(files);
I'm generating all my html to pdf file tasks and waiting for them to be completed - then trying to synchronously loop through the completed tasks and add them to my dictionary for zipping but I keep getting the following error:
An item with the same key has already been added
There is no duplicate key in the list of items being added.
EDIT - so the current idea is that because its a scoped service, thats why i'm running into thread issues (attached the file utility service for information)
public class FileUtilityService : IFileUtilityService
{
private readonly IHttpClientFactory _clientFactory;
public FileUtilityService(IHttpClientFactory clientFactory)
{
public async Task<byte[]> HtmlToPDF(string html = null, string url = null)
{
try
{
byte[] res = null;
if (html is null && url != null)
{
var client = _clientFactory.CreateClient();
var requestResp = await client.GetAsync(url);
using var sr = new StreamReader(await requestResp.Content.ReadAsStreamAsync());
html = HttpUtility.HtmlDecode(await sr.ReadToEndAsync());
}
using(var ms = new MemoryStream())
{
HtmlConverter.ConvertToPdf(html, ms);
res = ms.ToArray();
}
return res;
}catch(Exception ex)
{
throw ex;
}
}
public async Task<Library.Models.Helpers.File> HtmlToPDF(string html = null, string url = null, string fileName = "")
{
return new Library.Models.Helpers.File() { FileName = fileName, FileResult = await HtmlToPDF(html, url) };
}
I have a condition set to where if there's a cached token, it will use that or else it will either look for passed in credentials or prompt for credentials. Even after entering my credentials once via AcquireTokenInteractive() and running the program again, it still prompts for credentials. If I only have AcquireTokenSilent(), I get a reference error for the token variable because nothing is passed to it. It is either not caching my token or it is not fetching my cached token properly, but I'm not sure how to troubleshoot either. According to the MSDocs https://learn.microsoft.com/en-us/azure/active-directory/develop/msal-acquire-cache-tokens it should do the caching automatically.
For the time being, I commented out the AcquireTokenByUsernamePassword() for troubleshooting purposes.
static async Task<GraphServiceClient> Auth()
{
string[] scopes = new string[] { "user.read" };
string token = null;
IPublicClientApplication app;
app = PublicClientApplicationBuilder.Create(ConfigurationManager.AppSettings["clientId"])
.Build();
AuthenticationResult result = null;
var accounts = await app.GetAccountsAsync();
if (accounts.Any())
{
result = await app.AcquireTokenSilent(scopes, accounts.FirstOrDefault())
.ExecuteAsync();
}
else
{
try
{
///var securePassword = new SecureString();
///foreach (char c in "dummy") // you should fetch the password
/// securePassword.AppendChar(c); // keystroke by keystroke
/// result = await app.AcquireTokenByUsernamePassword(scopes,"joe#contoso.com",securePassword).ExecuteAsync();
}
catch (MsalException)
{
///
}
try
{
result = await app.AcquireTokenInteractive(scopes).ExecuteAsync();
}
catch (MsalException)
{
///
}
}
token = result.AccessToken;
// Use the token
GraphServiceClient graphClient = new GraphServiceClient(
"https://graph.microsoft.com/v1.0",
new DelegateAuthenticationProvider(
async (requestMessage) =>
{
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("bearer", token);
}));
return graphClient;
}
Yes, you are right. The SDK will do the caching automatically. I tested it by creating a console app:
class Program
{
static void Main(string[] args)
{
string flag = "Y";
IPublicClientApplication app = PublicClientApplicationBuilder.Create("60ef0838-f145-4f00-b7ba-a5af7f31cc3c").Build();
string[] scopes = new string[] { "api://4a673722-5437-4663-bba7-5f49372e06ba/Group.All","openid" };
do
{
List<IAccount> list = app.GetAccountsAsync().GetAwaiter().GetResult().ToList();
Console.WriteLine("Checking cache");
if (list.Count > 0)
{
Console.WriteLine($"Find {list.Count}");
list.ForEach(_ =>
{
Console.WriteLine($"Acquire a new token for {_.Username}");
AuthenticationResult result = app.AcquireTokenSilent(scopes, _).WithForceRefresh(true).WithAuthority("https://login.microsoftonline.com/e4c9ab4e-bd27-40d5-8459-230ba2a757fb/").ExecuteAsync().Result;
Console.WriteLine($"New token -> {result.AccessToken}");
});
}
else
{
Console.WriteLine("No cache");
try
{
var securePassword = new SecureString();
// Test AcquireTokenByUsernamePassword
foreach (char c in "**********") securePassword.AppendChar(c);
AuthenticationResult result = app.AcquireTokenByUsernamePassword(scopes, "jack#hanxia.onmicrosoft.com", securePassword).WithAuthority("https://login.microsoftonline.com/e4c9ab4e-bd27-40d5-8459-230ba2a757fb/").ExecuteAsync().GetAwaiter().GetResult();
// Test AcquireTokenInteractive
//AuthenticationResult result = app.AcquireTokenInteractive(scopes).WithAuthority("https://login.microsoftonline.com/e4c9ab4e-bd27-40d5-8459-230ba2a757fb/").ExecuteAsync().Result;
Console.WriteLine(result.Account.Username + " -> " + result.AccessToken);
}
catch (Exception e)
{
Console.Error.WriteLine(e.Message);
Console.Error.WriteLine(e.StackTrace);
}
}
Console.Write("Continue? Y/N:");
flag = Console.ReadLine();
} while (flag.Trim().ToUpper().Equals("Y"));
Console.ReadLine();
}
}
The result:
I use the tool Octokit.net to make an own GitHub client and my question is how can I delete the files in the repository when they aren't in the local folder anymore?
This is the code I already have:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using System.Reactive.Linq;
using System.IO;
using Octokit;
using Octokit.Internal;
using Octokit.Reactive;
class Program
{
static void Main(string[] args)
{
MainAsync(args[0], args[1], args[2], args[3]).Wait();
}
static async Task MainAsync(string rootPath, string owner, string repo, string branch)
{
var github = new GitHubClient(new ProductHeaderValue("repository"));
var user = await github.User.Get("username");
github.Credentials = new Credentials("token");
var contents = await github.Repository.Content.GetAllContents(owner, repo);
foreach(var githubFilePath in contents)
{
Console.WriteLine(githubFilePath);
}
Console.WriteLine(contents);
UpdateFull().Wait();
// Update a whole folder
async Task UpdateFull()
{
updateGithub(rootPath).Wait();
async Task updateGithub(string path)
{
var localContents = Directory.GetFiles(path);
var localContentFolders = Directory.GetDirectories(path);
var headMasterRef = "heads/master";
var masterReference = await github.Git.Reference.Get(owner, repo, headMasterRef);
var latestCommit = await github.Git.Commit.Get(owner, repo, masterReference.Object.Sha);
var nt = new NewTree { BaseTree = latestCommit.Tree.Sha };
// Add items based on blobs
AddDirectoryToTree(path, path + "\\", nt).Wait();
var rootTree = await github.Git.Tree.Create(owner, repo, nt);
// Create Commit
var newCommit = new NewCommit("Commit test with several files", rootTree.Sha, masterReference.Object.Sha);
var commit = await github.Git.Commit.Create(owner, repo, newCommit);
await github.Git.Reference.Update(owner, repo, headMasterRef, new ReferenceUpdate(commit.Sha));
}
async Task AddDirectoryToTree(string DirectoryPath, string RootPath, NewTree Tree)
{
// all direct files in the folder
//
var localContents = Directory.GetFiles(DirectoryPath);
foreach (var filePath in localContents)
{
string sExtension = Path.GetExtension(filePath).ToUpper();
string sFilename = filePath.Replace(RootPath, "").Replace(#"\", "/");
Console.WriteLine("File(Local): " + sFilename);
switch (sExtension)
{
default:
// Create text blob
var textBlob = new NewBlob { Encoding = EncodingType.Utf8, Content = File.ReadAllText(filePath) };
var textBlobRef = await github.Git.Blob.Create(owner, repo, textBlob);
Tree.Tree.Add(new NewTreeItem { Path = sFilename, Mode = "100644", Type = TreeType.Blob, Sha = textBlobRef.Sha });
break;
case ".PNG":
case ".JPEG":
case ".JPG":
// For image, get image content and convert it to base64
var imgBase64 = Convert.ToBase64String(File.ReadAllBytes(filePath));
// Create image blob
var imgBlob = new NewBlob { Encoding = EncodingType.Base64, Content = (imgBase64) };
var imgBlobRef = await github.Git.Blob.Create(owner, repo, imgBlob);
Tree.Tree.Add(new NewTreeItem { Path = sFilename, Mode = "100644", Type = TreeType.Blob, Sha = imgBlobRef.Sha });
break;
}
}
// subfolder
//
var localContentFolders = Directory.GetDirectories(DirectoryPath);
foreach (string subDirPath in localContentFolders)
{
AddDirectoryToTree(subDirPath, RootPath, Tree).Wait();
}
}
}
Console.WriteLine("-----------------------------------");
Console.WriteLine("Successfully commited Files");
Console.ReadKey();
Do I maybe need to add a new foreach for the Github repository and make then an if statement to look at the repository and local folder? :o
Would appreciateif you guys could help me with that! :D
I'm trying to get the Bing Speech API to work in C# via WebSockets. I've looked through the implementation in Javascript here and have been following the protocol instructions here, but I've come up against a complete brick wall. I can't use the existing C# service because I'm running in a Linux container, so I need to use an implementation on .net Core. Annoyingly, the existing service is closed-source!
I can connect to the web socket successfully, but I can't ever get the server to respond to my connection. I'm expecting to receive a turn.start text message from the server, but I get booted off the server as soon as I've sent a few bytes of an audio file. I know the audio file is in the right format because I've got it directly from the C# service sample here.
I feel like I’ve exhausted the options here. The only thing I can think of now is that I’m not sending the audio chunks correctly. Currently, I’m just sending the audio file in consecutive 4096 bytes. I know the first audio message contains the RIFF header which is only 36 bytes, and then I'm just sending this along with the next (4096-36) bytes.
Here is my code in full. You should just be able to run it as a .net core or .net framework console application, and will need an audio file and an API key.
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.WebSockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace ConsoleApp3
{
class Program
{
static void Main(string[] args)
{
Task.Run(async () =>
{
var bingService = new BingSpeechToTextService();
var audioFilePath = #"FILEPATH GOES HERE";
var authenticationKey = #"BING AUTHENTICATION KEY GOES HERE";
await bingService.RegisterJob(audioFilePath, authenticationKey);
}).Wait();
}
}
public class BingSpeechToTextService
{
/* #region Private Static Methods */
private static async Task Receiving(ClientWebSocket client)
{
var buffer = new byte[128];
while (true)
{
var result = await client.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
var res = Encoding.UTF8.GetString(buffer, 0, result.Count);
if (result.MessageType == WebSocketMessageType.Text)
{
Console.WriteLine(Encoding.UTF8.GetString(buffer, 0, result.Count));
}
else if (result.MessageType == WebSocketMessageType.Close)
{
Console.WriteLine($"Closing ... reason {client.CloseStatusDescription}");
var description = client.CloseStatusDescription;
//await client.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, "", CancellationToken.None);
break;
}
else
{
Console.WriteLine("Other result");
}
}
}
/* #endregion Private Static Methods */
/* #region Public Static Methods */
public static UInt16 ReverseBytes(UInt16 value)
{
return (UInt16)((value & 0xFFU) << 8 | (value & 0xFF00U) >> 8);
}
/* #endregion Public Static Methods */
/* #region Interface: 'Unscrypt.Bing.SpeechToText.Client.Api.IBingSpeechToTextJobService' Methods */
public async Task<int?> RegisterJob(string audioFilePath, string authenticationKeyStr)
{
var authenticationKey = new BingSocketAuthentication(authenticationKeyStr);
var token = authenticationKey.GetAccessToken();
/* #region Connect web socket */
var cws = new ClientWebSocket();
var connectionId = Guid.NewGuid().ToString("N");
var lang = "en-US";
cws.Options.SetRequestHeader("X-ConnectionId", connectionId);
cws.Options.SetRequestHeader("Authorization", "Bearer " + token);
Console.WriteLine("Connecting to web socket.");
var url = $"wss://speech.platform.bing.com/speech/recognition/interactive/cognitiveservices/v1?format=simple&language={lang}";
await cws.ConnectAsync(new Uri(url), new CancellationToken());
Console.WriteLine("Connected.");
/* #endregion*/
/* #region Receiving */
var receiving = Receiving(cws);
/* #endregion*/
/* #region Sending */
var sending = Task.Run(async () =>
{
/* #region Send speech.config */
dynamic speechConfig =
new
{
context = new
{
system = new
{
version = "1.0.00000"
},
os = new
{
platform = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
name = "Browser",
version = ""
},
device = new
{
manufacturer = "SpeechSample",
model = "SpeechSample",
version = "1.0.00000"
}
}
};
var requestId = Guid.NewGuid().ToString("N");
var speechConfigJson = JsonConvert.SerializeObject(speechConfig, Formatting.None);
StringBuilder outputBuilder = new StringBuilder();
outputBuilder.Append("path:speech.config\r\n"); //Should this be \r\n
outputBuilder.Append($"x-timestamp:{DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffK")}\r\n");
outputBuilder.Append($"content-type:application/json\r\n");
outputBuilder.Append("\r\n\r\n");
outputBuilder.Append(speechConfigJson);
var strh = outputBuilder.ToString();
var encoded = Encoding.UTF8.GetBytes(outputBuilder.ToString());
var buffer = new ArraySegment<byte>(encoded, 0, encoded.Length);
if (cws.State != WebSocketState.Open) return;
Console.WriteLine("Sending speech.config");
await cws.SendAsync(buffer, WebSocketMessageType.Text, true, new CancellationToken());
Console.WriteLine("Sent.");
/* #endregion*/
/* #region Send audio parts. */
var fileInfo = new FileInfo(audioFilePath);
var streamReader = fileInfo.OpenRead();
for (int cursor = 0; cursor < fileInfo.Length; cursor++)
{
outputBuilder.Clear();
outputBuilder.Append("path:audio\r\n");
outputBuilder.Append($"x-requestid:{requestId}\r\n");
outputBuilder.Append($"x-timestamp:{DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffK")}\r\n");
outputBuilder.Append($"content-type:audio/x-wav");
var headerBytes = Encoding.ASCII.GetBytes(outputBuilder.ToString());
var headerbuffer = new ArraySegment<byte>(headerBytes, 0, headerBytes.Length);
var str = "0x" + (headerBytes.Length).ToString("X");
var headerHeadBytes = BitConverter.GetBytes((UInt16)headerBytes.Length);
var isBigEndian = !BitConverter.IsLittleEndian;
var headerHead = !isBigEndian ? new byte[] { headerHeadBytes[1], headerHeadBytes[0] } : new byte[] { headerHeadBytes[0], headerHeadBytes[1] };
//Audio should be pcm 16kHz, 16bps mono
var byteLen = 8192 - headerBytes.Length - 2;
var fbuff = new byte[byteLen];
streamReader.Read(fbuff, 0, byteLen);
var arr = headerHead.Concat(headerBytes).Concat(fbuff).ToArray();
var arrSeg = new ArraySegment<byte>(arr, 0, arr.Length);
Console.WriteLine($"Sending data from {cursor}");
if (cws.State != WebSocketState.Open) return;
cursor += byteLen;
var end = cursor >= fileInfo.Length;
await cws.SendAsync(arrSeg, WebSocketMessageType.Binary, true, new CancellationToken());
Console.WriteLine("Data sent");
var dt = Encoding.ASCII.GetString(arr);
}
await cws.SendAsync(new ArraySegment<byte>(), WebSocketMessageType.Binary, true, new CancellationToken());
streamReader.Dispose();
/* #endregion*/
{
var startWait = DateTime.UtcNow;
while ((DateTime.UtcNow - startWait).TotalSeconds < 30)
{
await Task.Delay(1);
}
if (cws.State != WebSocketState.Open) return;
}
});
/* #endregion*/
/* #region Wait for tasks to complete */
await Task.WhenAll(sending, receiving);
if (sending.IsFaulted)
{
var err = sending.Exception;
throw err;
}
if (receiving.IsFaulted)
{
var err = receiving.Exception;
throw err;
}
/* #endregion*/
return null;
}
/* #endregion Interface: 'Unscrypt.Bing.SpeechToText.Client.Api.IBingSpeechToTextJobService' Methods */
public class BingSocketAuthentication
{
public static readonly string FetchTokenUri = "https://api.cognitive.microsoft.com/sts/v1.0";
private string subscriptionKey;
private string token;
private Timer accessTokenRenewer;
//Access token expires every 10 minutes. Renew it every 9 minutes.
private const int RefreshTokenDuration = 9;
public BingSocketAuthentication(string subscriptionKey)
{
this.subscriptionKey = subscriptionKey;
this.token = FetchToken(FetchTokenUri, subscriptionKey).Result;
// renew the token on set duration.
accessTokenRenewer = new Timer(new TimerCallback(OnTokenExpiredCallback),
this,
TimeSpan.FromMinutes(RefreshTokenDuration),
TimeSpan.FromMilliseconds(-1));
}
public string GetAccessToken()
{
return this.token;
}
private void RenewAccessToken()
{
this.token = FetchToken(FetchTokenUri, this.subscriptionKey).Result;
Console.WriteLine("Renewed token.");
}
private void OnTokenExpiredCallback(object stateInfo)
{
try
{
RenewAccessToken();
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Failed renewing access token. Details: {0}", ex.Message));
}
finally
{
try
{
accessTokenRenewer.Change(TimeSpan.FromMinutes(RefreshTokenDuration), TimeSpan.FromMilliseconds(-1));
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Failed to reschedule the timer to renew access token. Details: {0}", ex.Message));
}
}
}
private async Task<string> FetchToken(string fetchUri, string subscriptionKey)
{
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", subscriptionKey);
UriBuilder uriBuilder = new UriBuilder(fetchUri);
uriBuilder.Path += "/issueToken";
var result = await client.PostAsync(uriBuilder.Uri.AbsoluteUri, null);
Console.WriteLine("Token Uri: {0}", uriBuilder.Uri.AbsoluteUri);
return await result.Content.ReadAsStringAsync();
}
}
}
}
}
I knew it was going to be simple.
After a frustrating few hours of coding, I've found the problem. I've been forgetting to send a request id along with the speech.config call.
I am trying to connect with c#.
Here is the class that submits hive queries successfully to my remote HDInsight cluster. what do i need to change here to connect to the local emulator
public class HadoopImporter : IImporter
{
public static readonly Logger log = LogManager.GetCurrentClassLogger();
public void Import(string _query)
{
try
{
log.Warn("Inside Hive submission method");
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert =
store.Certificates.Cast<X509Certificate2>()
.First(item => item.Thumbprint == "MYCERTTUMBPRINT");
if (cert == null)
log.Error("no cert found");
log.Warn(cert.FriendlyName);
log.Warn("got the cert with thumbprint ", cert.Thumbprint.ToString())
;
log.Warn("trying to create credentials from cert");
var creds = new JobSubmissionCertificateCredential(new Guid("MYSUBSCRIPTIONID"),
cert, "MYSTORAGECONTAINER");
log.Warn("trying to connect with cert");
var jobClient = JobSubmissionClientFactory.Connect(creds);
log.Warn("Setting Hive job parameters");
var hiveJob = new HiveJobCreateParameters()
{
Query = _query,
StatusFolder = "/samplequeryoutput"
};
var jobResults = jobClient.CreateHiveJob(hiveJob);
log.Warn("Executing wait for jhive results");
WaitForJobCompletion(jobResults, jobClient);
using (var stream = jobClient.GetJobOutput(jobResults.JobId))
{
var reader = new StreamReader(stream);
var res = reader.ReadToEnd();
log.Warn("trying to get the job results " + res.ToString());
}
}
catch (Exception exp)
{
log.Error(exp);
}
}
private static void WaitForJobCompletion(JobCreationResults jobDetails, IJobSubmissionClient client)
{
var jobInProgress = client.GetJob(jobDetails.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
log.Warn("Inside the while loop waiting for hive job to complete");
jobInProgress = client.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromSeconds(10));
}
log.Trace("HIVE Job has Imported " + jobDetails.JobId);
}
}
You should be able to connect to a local one-box using the REST implementation of the client.
You're looking for the WebHCatHttpClient interface. The code below runs a basic query against my local one-box.
var httpClient = new WebHCatHttpClient(new Uri("http://localhost:50111/"), "username", "password");
string outputDir = "basichivejob";
var task = httpClient.CreateHiveJob(#"select * from iris;", null, null, outputDir, null);
task.Wait();
var response = task.Result;
var output = response.Content.ReadAsAsync<JObject>();
output.Wait();
response.EnsureSuccessStatusCode();
string id = output.Result.GetValue("id").ToString();
httpClient.WaitForJobToCompleteAsync(id).Wait();
See the SDK docs for more info.