I am trying to connect with c#.
Here is the class that submits hive queries successfully to my remote HDInsight cluster. what do i need to change here to connect to the local emulator
public class HadoopImporter : IImporter
{
public static readonly Logger log = LogManager.GetCurrentClassLogger();
public void Import(string _query)
{
try
{
log.Warn("Inside Hive submission method");
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert =
store.Certificates.Cast<X509Certificate2>()
.First(item => item.Thumbprint == "MYCERTTUMBPRINT");
if (cert == null)
log.Error("no cert found");
log.Warn(cert.FriendlyName);
log.Warn("got the cert with thumbprint ", cert.Thumbprint.ToString())
;
log.Warn("trying to create credentials from cert");
var creds = new JobSubmissionCertificateCredential(new Guid("MYSUBSCRIPTIONID"),
cert, "MYSTORAGECONTAINER");
log.Warn("trying to connect with cert");
var jobClient = JobSubmissionClientFactory.Connect(creds);
log.Warn("Setting Hive job parameters");
var hiveJob = new HiveJobCreateParameters()
{
Query = _query,
StatusFolder = "/samplequeryoutput"
};
var jobResults = jobClient.CreateHiveJob(hiveJob);
log.Warn("Executing wait for jhive results");
WaitForJobCompletion(jobResults, jobClient);
using (var stream = jobClient.GetJobOutput(jobResults.JobId))
{
var reader = new StreamReader(stream);
var res = reader.ReadToEnd();
log.Warn("trying to get the job results " + res.ToString());
}
}
catch (Exception exp)
{
log.Error(exp);
}
}
private static void WaitForJobCompletion(JobCreationResults jobDetails, IJobSubmissionClient client)
{
var jobInProgress = client.GetJob(jobDetails.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
log.Warn("Inside the while loop waiting for hive job to complete");
jobInProgress = client.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromSeconds(10));
}
log.Trace("HIVE Job has Imported " + jobDetails.JobId);
}
}
You should be able to connect to a local one-box using the REST implementation of the client.
You're looking for the WebHCatHttpClient interface. The code below runs a basic query against my local one-box.
var httpClient = new WebHCatHttpClient(new Uri("http://localhost:50111/"), "username", "password");
string outputDir = "basichivejob";
var task = httpClient.CreateHiveJob(#"select * from iris;", null, null, outputDir, null);
task.Wait();
var response = task.Result;
var output = response.Content.ReadAsAsync<JObject>();
output.Wait();
response.EnsureSuccessStatusCode();
string id = output.Result.GetValue("id").ToString();
httpClient.WaitForJobToCompleteAsync(id).Wait();
See the SDK docs for more info.
Related
Trying to make use of the AndroidPublisherService from Play Developer API Client.
I can list active tracks and the releases in those tracks, but when I try to upload a new build there seems to be no way of attaching the authentication already made previously to read data.
I've authenticated using var googleCredentials = GoogleCredential.FromStream(keyDataStream) .CreateWithUser(serviceUsername); where serviceUsername is the email for my service account.
private static void Execute(string packageName, string aabfile, string credfile, string serviceUsername)
{
var credentialsFilename = credfile;
if (string.IsNullOrWhiteSpace(credentialsFilename))
{
// Check env. var
credentialsFilename =
Environment.GetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS",
EnvironmentVariableTarget.Process);
}
Console.WriteLine($"Using credentials {credfile} with package {packageName} for aab file {aabfile}");
var keyDataStream = File.OpenRead(credentialsFilename);
var googleCredentials = GoogleCredential.FromStream(keyDataStream)
.CreateWithUser(serviceUsername);
var credentials = googleCredentials.UnderlyingCredential as ServiceAccountCredential;
var service = new AndroidPublisherService();
var edit = service.Edits.Insert(new AppEdit { ExpiryTimeSeconds = "3600" }, packageName);
edit.Credential = credentials;
var activeEditSession = edit.Execute();
Console.WriteLine($"Edits started with id {activeEditSession.Id}");
var tracksList = service.Edits.Tracks.List(packageName, activeEditSession.Id);
tracksList.Credential = credentials;
var tracksResponse = tracksList.Execute();
foreach (var track in tracksResponse.Tracks)
{
Console.WriteLine($"Track: {track.TrackValue}");
Console.WriteLine("Releases: ");
foreach (var rel in track.Releases)
Console.WriteLine($"{rel.Name} version: {rel.VersionCodes.FirstOrDefault()} - Status: {rel.Status}");
}
using var fileStream = File.OpenRead(aabfile);
var upload = service.Edits.Bundles.Upload(packageName, activeEditSession.Id, fileStream, "application/octet-stream");
var uploadProgress = upload.Upload();
if (uploadProgress == null || uploadProgress.Exception != null)
{
Console.WriteLine($"Failed to upload. Error: {uploadProgress?.Exception}");
return;
}
Console.WriteLine($"Upload {uploadProgress.Status}");
var tracksUpdate = service.Edits.Tracks.Update(new Track
{
Releases = new List<TrackRelease>(new[]
{
new TrackRelease
{
Name = "Roswell - Grenis Dev Test",
Status = "completed",
VersionCodes = new List<long?>(new[] {(long?) upload?.ResponseBody?.VersionCode})
}
})
}, packageName, activeEditSession.Id, "internal");
tracksUpdate.Credential = credentials;
var trackResult = tracksUpdate.Execute();
Console.WriteLine($"Track {trackResult?.TrackValue}");
var commitResult = service.Edits.Commit(packageName, activeEditSession.Id);
Console.WriteLine($"{commitResult.EditId} has been committed");
}
And as the code points out, all action objects such as tracksList.Credential = credentials; can be given the credentials generated from the service account.
BUT the actual upload action var upload = service.Edits.Bundles.Upload(packageName, activeEditSession.Id, fileStream, "application/octet-stream"); does not expose a .Credential object, and it always fails with:
The service androidpublisher has thrown an exception: Google.GoogleApiException: Google.Apis.Requests.RequestError
Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project. [401]
Errors [
Message[Login Required.] Location[Authorization - header] Reason[required] Domain[global]
]
at Google.Apis.Upload.ResumableUpload`1.InitiateSessionAsync(CancellationToken cancellationToken)
at Google.Apis.Upload.ResumableUpload.UploadAsync(CancellationToken cancellationToken)
So, how would I go about providing the actual Upload action with the given credentials here?
Managed to figure this out during the day, I was missing one call to CreateScoped() when creating the GoogleCredential object as well as a call to InitiateSession() on the upload object.
var googleCredentials = GoogleCredential.FromStream(keyDataStream)
.CreateWithUser(serviceUsername)
.CreateScoped(AndroidPublisherService.Scope.Androidpublisher);
Once that was done I could then get a valid oauth token by calling
var googleCredentials = GoogleCredential.FromStream(keyDataStream)
.CreateWithUser(serviceUsername)
.CreateScoped(AndroidPublisherService.Scope.Androidpublisher);
var credentials = googleCredentials.UnderlyingCredential as ServiceAccountCredential;
var oauthToken = credentials?.GetAccessTokenForRequestAsync(AndroidPublisherService.Scope.Androidpublisher).Result;
And I can now use that oauth token in the upload request:
upload.OauthToken = oauthToken;
_ = await upload.InitiateSessionAsync();
var uploadProgress = await upload.UploadAsync();
if (uploadProgress == null || uploadProgress.Exception != null)
{
Console.WriteLine($"Failed to upload. Error: {uploadProgress?.Exception}");
return;
}
The full code example for successfully uploading a new aab file to google play store internal test track thus looks something like this:
private async Task UploadGooglePlayRelease(string fileToUpload, string changeLogFile, string serviceUsername, string packageName)
{
var serviceAccountFile = ResolveServiceAccountCertificateInfoFile();
if (!serviceAccountFile.Exists)
throw new ApplicationException($"Failed to find the service account certificate file. {serviceAccountFile.FullName}");
var keyDataStream = File.OpenRead(serviceAccountFile.FullName);
var googleCredentials = GoogleCredential.FromStream(keyDataStream)
.CreateWithUser(serviceUsername)
.CreateScoped(AndroidPublisherService.Scope.Androidpublisher);
var credentials = googleCredentials.UnderlyingCredential as ServiceAccountCredential;
var oauthToken = credentials?.GetAccessTokenForRequestAsync(AndroidPublisherService.Scope.Androidpublisher).Result;
var service = new AndroidPublisherService();
var edit = service.Edits.Insert(new AppEdit { ExpiryTimeSeconds = "3600" }, packageName);
edit.Credential = credentials;
var activeEditSession = await edit.ExecuteAsync();
_logger.LogInformation($"Edits started with id {activeEditSession.Id}");
var tracksList = service.Edits.Tracks.List(packageName, activeEditSession.Id);
tracksList.Credential = credentials;
var tracksResponse = await tracksList.ExecuteAsync();
foreach (var track in tracksResponse.Tracks)
{
_logger.LogInformation($"Track: {track.TrackValue}");
_logger.LogInformation("Releases: ");
foreach (var rel in track.Releases)
_logger.LogInformation($"{rel.Name} version: {rel.VersionCodes.FirstOrDefault()} - Status: {rel.Status}");
}
var fileStream = File.OpenRead(fileToUpload);
var upload = service.Edits.Bundles.Upload(packageName, activeEditSession.Id, fileStream, "application/octet-stream");
upload.OauthToken = oauthToken;
_ = await upload.InitiateSessionAsync();
var uploadProgress = await upload.UploadAsync();
if (uploadProgress == null || uploadProgress.Exception != null)
{
Console.WriteLine($"Failed to upload. Error: {uploadProgress?.Exception}");
return;
}
_logger.LogInformation($"Upload {uploadProgress.Status}");
var releaseNotes = await File.ReadAllTextAsync(changeLogFile);
var tracksUpdate = service.Edits.Tracks.Update(new Track
{
Releases = new List<TrackRelease>(new[]
{
new TrackRelease
{
Name = $"{upload?.ResponseBody?.VersionCode}",
Status = "completed",
InAppUpdatePriority = 5,
CountryTargeting = new CountryTargeting { IncludeRestOfWorld = true },
ReleaseNotes = new List<LocalizedText>(new []{ new LocalizedText { Language = "en-US", Text = releaseNotes } }),
VersionCodes = new List<long?>(new[] {(long?) upload?.ResponseBody?.VersionCode})
}
})
}, packageName, activeEditSession.Id, "internal");
tracksUpdate.Credential = credentials;
var trackResult = await tracksUpdate.ExecuteAsync();
_logger.LogInformation($"Track {trackResult?.TrackValue}");
var commitResult = service.Edits.Commit(packageName, activeEditSession.Id);
commitResult.Credential = credentials;
await commitResult.ExecuteAsync();
_logger.LogInformation($"{commitResult.EditId} has been committed");
}
When my AWS Credentials File (see docs) is updated by an external process the AmazonSQSClient doesn't re-read it, SendMessageAsync fails with a security/token error.
We use a custom powershell script to refresh the local AWS cred's file periodically. The script works fine, the file is refreshed prior to the credentials expiring on AWS. However, if my app is running when the file is refreshed the new credentials are not re-read from the file, the "client" will show that the previous credentials are still in use.
The AWS docs list several AWSCredential providers but none of them seem to be the correct choice...I think..
Restarting the app works, the new credentials are read correctly and messages are sent until the next time the cred's file is updated.
using (var client = new AmazonSQSClient(Amazon.RegionEndpoint.EUWest1))
{
return client.SendMessageAsync(request);
}
I don't think there is a way for a running app to pick up the default credentials being refreshed in credentials file. There is a solution for Node.js loading credentials from a JSON file. You can create a similar solution in C#. You can also run a local DB to store credentials so whenever credentials file is updated DB table or JSON file is also updated. You will need to use access key and secret key in your SQS client constructor as opposed to using default credentials.
// Load these from JSON file or DB.
var accessKey = "";
var secretKey = "";
using (var client = new AmazonSQSClient(accessKey, secretKey, Amazon.RegionEndpoint.EUWest1))
{
return client.SendMessageAsync(request);
}
The following works "ok" but I've only tested it with one profile and the file watcher is not as timely as you'd like so I'd recommend you wrap your usage inside a Retry mechanism.
// Usage..
var credentials = new AwsCredentialsFile();
using (var client = new AmazonSQSClient(credentials, Amazon.RegionEndpoint.EUWest1))
{
return client.SendMessageAsync(request);
}
public class AwsCredentialsFile : AWSCredentials
{
// https://docs.aws.amazon.com/sdk-for-net/v2/developer-guide/net-dg-config-creds.html#creds-file
private const string DefaultProfileName = "default";
private static ConcurrentDictionary<string, ImmutableCredentials> _credentials = new ConcurrentDictionary<string, ImmutableCredentials>(StringComparer.OrdinalIgnoreCase);
private static FileSystemWatcher _watcher = BuildFileSystemWatcher();
private readonly System.Text.Encoding _encoding;
private readonly string _profileName;
public AwsCredentialsFile()
: this(AwsCredentialsFile.DefaultProfileName, System.Text.Encoding.UTF8)
{
}
public AwsCredentialsFile(string profileName)
: this(profileName, System.Text.Encoding.UTF8)
{
}
public AwsCredentialsFile(string profileName, System.Text.Encoding encoding)
{
_profileName = profileName;
_encoding = encoding;
}
private static FileSystemWatcher BuildFileSystemWatcher()
{
var watcher = new FileSystemWatcher
{
Path = Path.GetDirectoryName(GetDefaultCredentialsFilePath()),
NotifyFilter = NotifyFilters.LastWrite,
Filter = "credentials"
};
watcher.Changed += (object source, FileSystemEventArgs e) => { _credentials?.Clear(); };
watcher.EnableRaisingEvents = true;
return watcher;
}
public static string GetDefaultCredentialsFilePath()
{
return System.Environment.ExpandEnvironmentVariables(#"C:\Users\%USERNAME%\.aws\credentials");
}
public static (string AccessKey, string SecretAccessKey, string Token) ReadCredentialsFromFile(string profileName, System.Text.Encoding encoding)
{
var profile = $"[{profileName}]";
string awsAccessKeyId = null;
string awsSecretAccessKey = null;
string token = null;
var lines = File.ReadAllLines(GetDefaultCredentialsFilePath(), encoding);
for (int i = 0; i < lines.Length; i++)
{
var text = lines[i];
if (text.Equals(profile, StringComparison.OrdinalIgnoreCase))
{
awsAccessKeyId = lines[i + 1].Replace("aws_access_key_id = ", string.Empty);
awsSecretAccessKey = lines[i + 2].Replace("aws_secret_access_key = ", string.Empty);
if (lines.Length >= i + 3)
{
token = lines[i + 3].Replace("aws_session_token = ", string.Empty);
}
break;
}
}
var result = (AccessKey: awsAccessKeyId, SecretAccessKey: awsSecretAccessKey, Token: token);
return result;
}
public override ImmutableCredentials GetCredentials()
{
if (_credentials.TryGetValue(_profileName, out ImmutableCredentials value))
{
return value;
}
else
{
var (AccessKey, SecretAccessKey, Token) = ReadCredentialsFromFile(_profileName, _encoding);
var credentials = new ImmutableCredentials(AccessKey, SecretAccessKey, Token);
_credentials.TryAdd(_profileName, credentials);
return credentials;
}
}
}
I have the following controller:
[Authorize]
public class SetupController : ApiController
{
[HttpPost]
public Task async SetupPartnerPackAsync(SetupInformation info)
{
if (info.SslCertificateGenerate)
{
SetupManager.CreateX509Certificate(info);
}
else
{
SetupManager.LoadX509Certificate(info);
}
info.SslCertificateThumbprint = SetupManager.GetX509CertificateThumbprint(info);
info.AzureAppKeyCredential = SetupManager.GetX509CertificateInformation(info);
await SetupManager.RegisterAzureADApplication(info);
}
}
But I have the following 2 error which seems simple:
Severity Code Description Project File Line Suppression State
Error CS1520 Method must have a return
type InnovationInABoxWebApi H:\InnovationInABoxWebApi\InnovationInABoxWebApi\Controllers\SetupController.cs 24 Active
Severity Code Description Project File Line Suppression State
Error CS4033 The 'await' operator can only be used within an async
method. Consider marking this method with the 'async' modifier and
changing its return type to
'Task'. InnovationInABoxWebApi H:\InnovationInABoxWebApi\InnovationInABoxWebApi\Controllers\SetupController.cs 39 Active
However I am not sure how to fix this, as the operation can take some time to complete, it really needs to be asybnc
and the setupmanager
using CERTENROLLLib;
using Microsoft.Identity.Client;
using Microsoft.Online.SharePoint.TenantAdministration;
using Microsoft.SharePoint.Client;
using Newtonsoft.Json;
using OfficeDevPnP.Core;
using OfficeDevPnP.Core.Entities;
using OfficeDevPnP.Core.Framework.Provisioning.Model;
using OfficeDevPnP.Core.Framework.Provisioning.ObjectHandlers;
using OfficeDevPnP.Core.Framework.Provisioning.Providers.Xml;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Configuration;
using System.IO;
using System.Linq;
using System.Resources;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using System.Xml.Linq;
namespace InnovationInABoxWebApi.Components
{
public static class SetupManager
{
public static String GetX509CertificateThumbprint(SetupInformation info)
{
var certificate = info.AuthenticationCertificate;
return (certificate.Thumbprint.ToUpper());
}
public static String GetX509CertificateInformation(SetupInformation info)
{
// var basePath = String.Format(#"{0}..\..\..\..\Scripts\", AppDomain.CurrentDomain.BaseDirectory);
var certificate = info.AuthenticationCertificate;
//var certificate = new X509Certificate2();
//if (info.SslCertificateGenerate)
//{
// certificate.Import($#"{basePath}{info.SslCertificateCommonName}.cer");
//}
//else
//{
// certificate = new X509Certificate2(info.SslCertificateFile, info.SslCertificatePassword);
//}
var rawCert = certificate.GetRawCertData();
var base64Cert = System.Convert.ToBase64String(rawCert);
var rawCertHash = certificate.GetCertHash();
var base64CertHash = System.Convert.ToBase64String(rawCertHash);
var KeyId = System.Guid.NewGuid().ToString();
var keyCredential =
"{" +
"\"customKeyIdentifier\": \"" + base64CertHash + "\"," +
"\"keyId\": \"" + KeyId + "\"," +
"\"type\": \"AsymmetricX509Cert\"," +
"\"usage\": \"Verify\"," +
"\"key\": \"" + base64Cert + "\"" +
"}";
return (keyCredential);
}
public static void CreateX509Certificate(SetupInformation info)
{
var certificate = CreateSelfSignedCertificate(info.SslCertificateCommonName.ToLower(),
info.SslCertificateStartDate, info.SslCertificateEndDate, info.SslCertificatePassword);
SaveCertificateFiles(info, certificate);
}
public static void LoadX509Certificate(SetupInformation info)
{
var certificate = new X509Certificate2(info.SslCertificateFile, info.SslCertificatePassword);
info.AuthenticationCertificate = certificate;
info.SslCertificateCommonName = certificate.SubjectName.Name;
}
public static void SaveCertificateFiles(SetupInformation info, X509Certificate2 certificate)
{
info.AuthenticationCertificate = certificate;
//var basePath = String.Format(#"{0}..\..\..\..\Scripts\", AppDomain.CurrentDomain.BaseDirectory);
//info.SslCertificateFile = $#"{basePath}{info.SslCertificateCommonName}.pfx";
//var pfx = certificate.Export(X509ContentType.Pfx, info.SslCertificatePassword);
//System.IO.File.WriteAllBytes(info.SslCertificateFile, pfx);
//var cer = certificate.Export(X509ContentType.Cert);
//System.IO.File.WriteAllBytes($#"{basePath}{info.SslCertificateCommonName}.cer", cer);
}
public static X509Certificate2 CreateSelfSignedCertificate(string subjectName, DateTime startDate, DateTime endDate, String password)
{
// Create DistinguishedName for subject and issuer
var name = new CX500DistinguishedName();
name.Encode("CN=" + subjectName, X500NameFlags.XCN_CERT_NAME_STR_NONE);
// Create a new Private Key for the certificate
CX509PrivateKey privateKey = new CX509PrivateKey();
privateKey.ProviderName = "Microsoft RSA SChannel Cryptographic Provider";
privateKey.KeySpec = X509KeySpec.XCN_AT_KEYEXCHANGE;
privateKey.Length = 2048;
privateKey.SecurityDescriptor = "D:PAI(A;;0xd01f01ff;;;SY)(A;;0xd01f01ff;;;BA)(A;;0x80120089;;;NS)";
privateKey.MachineContext = true;
privateKey.ExportPolicy = X509PrivateKeyExportFlags.XCN_NCRYPT_ALLOW_EXPORT_FLAG;
privateKey.Create();
// Define the hashing algorithm
var serverauthoid = new CObjectId();
serverauthoid.InitializeFromValue("1.3.6.1.5.5.7.3.1"); // Server Authentication
var ekuoids = new CObjectIds();
ekuoids.Add(serverauthoid);
var ekuext = new CX509ExtensionEnhancedKeyUsage();
ekuext.InitializeEncode(ekuoids);
// Create the self signing request
var cert = new CX509CertificateRequestCertificate();
cert.InitializeFromPrivateKey(X509CertificateEnrollmentContext.ContextMachine, privateKey, String.Empty);
cert.Subject = name;
cert.Issuer = cert.Subject;
cert.NotBefore = startDate;
cert.NotAfter = endDate;
cert.X509Extensions.Add((CX509Extension)ekuext);
cert.Encode();
// Enroll the certificate
var enroll = new CX509Enrollment();
enroll.InitializeFromRequest(cert);
string certData = enroll.CreateRequest(EncodingType.XCN_CRYPT_STRING_BASE64HEADER);
enroll.InstallResponse(InstallResponseRestrictionFlags.AllowUntrustedCertificate,
certData, EncodingType.XCN_CRYPT_STRING_BASE64HEADER, String.Empty);
var base64encoded = enroll.CreatePFX(password, PFXExportOptions.PFXExportChainWithRoot);
// Instantiate the target class with the PKCS#12 data
return new X509Certificate2(
System.Convert.FromBase64String(base64encoded), password,
System.Security.Cryptography.X509Certificates.X509KeyStorageFlags.Exportable);
}
public async static Task RegisterAzureADApplication(SetupInformation info)
{
// Fix the App URL
if (!info.AzureWebAppUrl.EndsWith("/"))
{
info.AzureWebAppUrl = info.AzureWebAppUrl + "/";
}
// Load the App Manifest template
//Stream stream = typeof(SetupManager)
// .Assembly
// .GetManifestResourceStream("OfficeDevPnP.PartnerPack.Setup.Resources.azure-ad-app-manifest.json");
using (StreamReader sr = new StreamReader("Resources\azure-ad-app-manifest.json"))
{
// Get the JSON manifest
var jsonApplication = sr.ReadToEnd();
var application = JsonConvert.DeserializeObject<AzureAdApplication>(jsonApplication);
var keyCredential = JsonConvert.DeserializeObject<KeyCredential>(info.AzureAppKeyCredential);
application.displayName = info.ApplicationName;
application.homepage = info.AzureWebAppUrl;
application.identifierUris = new List<String>();
application.identifierUris.Add(info.ApplicationUniqueUri);
application.keyCredentials = new List<KeyCredential>();
application.keyCredentials.Add(keyCredential);
application.replyUrls = new List<String>();
application.replyUrls.Add(info.AzureWebAppUrl);
// Generate the Application Shared Secret
var startDate = DateTime.Now;
Byte[] bytes = new Byte[32];
using (var rand = System.Security.Cryptography.RandomNumberGenerator.Create())
{
rand.GetBytes(bytes);
}
info.AzureAppSharedSecret = System.Convert.ToBase64String(bytes);
application.passwordCredentials = new List<object>();
application.passwordCredentials.Add(new AzureAdApplicationPasswordCredential
{
CustomKeyIdentifier = null,
StartDate = startDate.ToString("o"),
EndDate = startDate.AddYears(2).ToString("o"),
KeyId = Guid.NewGuid().ToString(),
Value = info.AzureAppSharedSecret,
});
// Get an Access Token to create the application via Microsoft Graph
var office365AzureADAccessToken = await AzureManagementUtility.GetAccessTokenSilentAsync(
AzureManagementUtility.MicrosoftGraphResourceId,
ConfigurationManager.AppSettings["O365:ClientId"]);
var azureAdApplicationCreated = false;
// Create the Azure AD Application
try
{
await CreateAzureADApplication(info, application, office365AzureADAccessToken);
azureAdApplicationCreated = true;
}
catch (ApplicationException ex)
{
var graphError = JsonConvert.DeserializeObject<GraphError>(((HttpException)ex.InnerException).Message);
if (graphError != null && graphError.error.code == "Request_BadRequest" &&
graphError.error.message.Contains("identifierUris already exists"))
{
// We need to remove the existing application
// Thus, retrieve it
String jsonApplications = await HttpHelper.MakeGetRequestForStringAsync(
String.Format("{0}applications?$filter=identifierUris/any(c:c+eq+'{1}')",
AzureManagementUtility.MicrosoftGraphBetaBaseUri,
HttpUtility.UrlEncode(info.ApplicationUniqueUri)),
office365AzureADAccessToken);
var applications = JsonConvert.DeserializeObject<AzureAdApplications>(jsonApplications);
var applicationToUpdate = applications.Applications.FirstOrDefault();
if (applicationToUpdate != null)
{
// Remove it
await HttpHelper.MakeDeleteRequestAsync(
String.Format("{0}applications/{1}",
AzureManagementUtility.MicrosoftGraphBetaBaseUri,
applicationToUpdate.Id),
office365AzureADAccessToken);
// And add it again
await CreateAzureADApplication(info, application, office365AzureADAccessToken);
azureAdApplicationCreated = true;
}
}
}
if (azureAdApplicationCreated)
{
// TODO: We should upload the logo
// property mainLogo: stream of the application via PATCH
}
}
}
public static async Task CreateAzureADApplication(SetupInformation info, AzureAdApplication application, string office365AzureADAccessToken)
{
String jsonResponse = await HttpHelper.MakePostRequestForStringAsync(
String.Format("{0}applications",
AzureManagementUtility.MicrosoftGraphBetaBaseUri),
application,
"application/json", office365AzureADAccessToken);
var azureAdApplication = JsonConvert.DeserializeObject<AzureAdApplication>(jsonResponse);
info.AzureAppClientId = azureAdApplication.AppId.HasValue ? azureAdApplication.AppId.Value : Guid.Empty;
}
}
}
You are defining the method with async word after the return type Task, async must be before Task.
public async Task SetupPartnerPackAsync(SetupInformation info)
{
.
.
.
I'm trying to get the Bing Speech API to work in C# via WebSockets. I've looked through the implementation in Javascript here and have been following the protocol instructions here, but I've come up against a complete brick wall. I can't use the existing C# service because I'm running in a Linux container, so I need to use an implementation on .net Core. Annoyingly, the existing service is closed-source!
I can connect to the web socket successfully, but I can't ever get the server to respond to my connection. I'm expecting to receive a turn.start text message from the server, but I get booted off the server as soon as I've sent a few bytes of an audio file. I know the audio file is in the right format because I've got it directly from the C# service sample here.
I feel like I’ve exhausted the options here. The only thing I can think of now is that I’m not sending the audio chunks correctly. Currently, I’m just sending the audio file in consecutive 4096 bytes. I know the first audio message contains the RIFF header which is only 36 bytes, and then I'm just sending this along with the next (4096-36) bytes.
Here is my code in full. You should just be able to run it as a .net core or .net framework console application, and will need an audio file and an API key.
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.WebSockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace ConsoleApp3
{
class Program
{
static void Main(string[] args)
{
Task.Run(async () =>
{
var bingService = new BingSpeechToTextService();
var audioFilePath = #"FILEPATH GOES HERE";
var authenticationKey = #"BING AUTHENTICATION KEY GOES HERE";
await bingService.RegisterJob(audioFilePath, authenticationKey);
}).Wait();
}
}
public class BingSpeechToTextService
{
/* #region Private Static Methods */
private static async Task Receiving(ClientWebSocket client)
{
var buffer = new byte[128];
while (true)
{
var result = await client.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
var res = Encoding.UTF8.GetString(buffer, 0, result.Count);
if (result.MessageType == WebSocketMessageType.Text)
{
Console.WriteLine(Encoding.UTF8.GetString(buffer, 0, result.Count));
}
else if (result.MessageType == WebSocketMessageType.Close)
{
Console.WriteLine($"Closing ... reason {client.CloseStatusDescription}");
var description = client.CloseStatusDescription;
//await client.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, "", CancellationToken.None);
break;
}
else
{
Console.WriteLine("Other result");
}
}
}
/* #endregion Private Static Methods */
/* #region Public Static Methods */
public static UInt16 ReverseBytes(UInt16 value)
{
return (UInt16)((value & 0xFFU) << 8 | (value & 0xFF00U) >> 8);
}
/* #endregion Public Static Methods */
/* #region Interface: 'Unscrypt.Bing.SpeechToText.Client.Api.IBingSpeechToTextJobService' Methods */
public async Task<int?> RegisterJob(string audioFilePath, string authenticationKeyStr)
{
var authenticationKey = new BingSocketAuthentication(authenticationKeyStr);
var token = authenticationKey.GetAccessToken();
/* #region Connect web socket */
var cws = new ClientWebSocket();
var connectionId = Guid.NewGuid().ToString("N");
var lang = "en-US";
cws.Options.SetRequestHeader("X-ConnectionId", connectionId);
cws.Options.SetRequestHeader("Authorization", "Bearer " + token);
Console.WriteLine("Connecting to web socket.");
var url = $"wss://speech.platform.bing.com/speech/recognition/interactive/cognitiveservices/v1?format=simple&language={lang}";
await cws.ConnectAsync(new Uri(url), new CancellationToken());
Console.WriteLine("Connected.");
/* #endregion*/
/* #region Receiving */
var receiving = Receiving(cws);
/* #endregion*/
/* #region Sending */
var sending = Task.Run(async () =>
{
/* #region Send speech.config */
dynamic speechConfig =
new
{
context = new
{
system = new
{
version = "1.0.00000"
},
os = new
{
platform = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
name = "Browser",
version = ""
},
device = new
{
manufacturer = "SpeechSample",
model = "SpeechSample",
version = "1.0.00000"
}
}
};
var requestId = Guid.NewGuid().ToString("N");
var speechConfigJson = JsonConvert.SerializeObject(speechConfig, Formatting.None);
StringBuilder outputBuilder = new StringBuilder();
outputBuilder.Append("path:speech.config\r\n"); //Should this be \r\n
outputBuilder.Append($"x-timestamp:{DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffK")}\r\n");
outputBuilder.Append($"content-type:application/json\r\n");
outputBuilder.Append("\r\n\r\n");
outputBuilder.Append(speechConfigJson);
var strh = outputBuilder.ToString();
var encoded = Encoding.UTF8.GetBytes(outputBuilder.ToString());
var buffer = new ArraySegment<byte>(encoded, 0, encoded.Length);
if (cws.State != WebSocketState.Open) return;
Console.WriteLine("Sending speech.config");
await cws.SendAsync(buffer, WebSocketMessageType.Text, true, new CancellationToken());
Console.WriteLine("Sent.");
/* #endregion*/
/* #region Send audio parts. */
var fileInfo = new FileInfo(audioFilePath);
var streamReader = fileInfo.OpenRead();
for (int cursor = 0; cursor < fileInfo.Length; cursor++)
{
outputBuilder.Clear();
outputBuilder.Append("path:audio\r\n");
outputBuilder.Append($"x-requestid:{requestId}\r\n");
outputBuilder.Append($"x-timestamp:{DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffK")}\r\n");
outputBuilder.Append($"content-type:audio/x-wav");
var headerBytes = Encoding.ASCII.GetBytes(outputBuilder.ToString());
var headerbuffer = new ArraySegment<byte>(headerBytes, 0, headerBytes.Length);
var str = "0x" + (headerBytes.Length).ToString("X");
var headerHeadBytes = BitConverter.GetBytes((UInt16)headerBytes.Length);
var isBigEndian = !BitConverter.IsLittleEndian;
var headerHead = !isBigEndian ? new byte[] { headerHeadBytes[1], headerHeadBytes[0] } : new byte[] { headerHeadBytes[0], headerHeadBytes[1] };
//Audio should be pcm 16kHz, 16bps mono
var byteLen = 8192 - headerBytes.Length - 2;
var fbuff = new byte[byteLen];
streamReader.Read(fbuff, 0, byteLen);
var arr = headerHead.Concat(headerBytes).Concat(fbuff).ToArray();
var arrSeg = new ArraySegment<byte>(arr, 0, arr.Length);
Console.WriteLine($"Sending data from {cursor}");
if (cws.State != WebSocketState.Open) return;
cursor += byteLen;
var end = cursor >= fileInfo.Length;
await cws.SendAsync(arrSeg, WebSocketMessageType.Binary, true, new CancellationToken());
Console.WriteLine("Data sent");
var dt = Encoding.ASCII.GetString(arr);
}
await cws.SendAsync(new ArraySegment<byte>(), WebSocketMessageType.Binary, true, new CancellationToken());
streamReader.Dispose();
/* #endregion*/
{
var startWait = DateTime.UtcNow;
while ((DateTime.UtcNow - startWait).TotalSeconds < 30)
{
await Task.Delay(1);
}
if (cws.State != WebSocketState.Open) return;
}
});
/* #endregion*/
/* #region Wait for tasks to complete */
await Task.WhenAll(sending, receiving);
if (sending.IsFaulted)
{
var err = sending.Exception;
throw err;
}
if (receiving.IsFaulted)
{
var err = receiving.Exception;
throw err;
}
/* #endregion*/
return null;
}
/* #endregion Interface: 'Unscrypt.Bing.SpeechToText.Client.Api.IBingSpeechToTextJobService' Methods */
public class BingSocketAuthentication
{
public static readonly string FetchTokenUri = "https://api.cognitive.microsoft.com/sts/v1.0";
private string subscriptionKey;
private string token;
private Timer accessTokenRenewer;
//Access token expires every 10 minutes. Renew it every 9 minutes.
private const int RefreshTokenDuration = 9;
public BingSocketAuthentication(string subscriptionKey)
{
this.subscriptionKey = subscriptionKey;
this.token = FetchToken(FetchTokenUri, subscriptionKey).Result;
// renew the token on set duration.
accessTokenRenewer = new Timer(new TimerCallback(OnTokenExpiredCallback),
this,
TimeSpan.FromMinutes(RefreshTokenDuration),
TimeSpan.FromMilliseconds(-1));
}
public string GetAccessToken()
{
return this.token;
}
private void RenewAccessToken()
{
this.token = FetchToken(FetchTokenUri, this.subscriptionKey).Result;
Console.WriteLine("Renewed token.");
}
private void OnTokenExpiredCallback(object stateInfo)
{
try
{
RenewAccessToken();
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Failed renewing access token. Details: {0}", ex.Message));
}
finally
{
try
{
accessTokenRenewer.Change(TimeSpan.FromMinutes(RefreshTokenDuration), TimeSpan.FromMilliseconds(-1));
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Failed to reschedule the timer to renew access token. Details: {0}", ex.Message));
}
}
}
private async Task<string> FetchToken(string fetchUri, string subscriptionKey)
{
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", subscriptionKey);
UriBuilder uriBuilder = new UriBuilder(fetchUri);
uriBuilder.Path += "/issueToken";
var result = await client.PostAsync(uriBuilder.Uri.AbsoluteUri, null);
Console.WriteLine("Token Uri: {0}", uriBuilder.Uri.AbsoluteUri);
return await result.Content.ReadAsStringAsync();
}
}
}
}
}
I knew it was going to be simple.
After a frustrating few hours of coding, I've found the problem. I've been forgetting to send a request id along with the speech.config call.
For a specific VM, I want to be able to retrieve the public IP address.
I know how to get all public IP addresses for a resource group, I also know how to get a nic-id for a specific VM - but I can't figure out how to connect the two.
This is what I have:
var resourceGroupName = "My-Resource-Group";
var vmName = "MyVM";
var subscriptionId = "bzz-bzz-bzz-bzz-bzz-bzz";
var tenantId = "bar-bar-bar-bar-bar-bar";
string clientId = "foo-foo-foo-foo-foo-foo";
string clientSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
var token = GetAccessTokenAsync(tenantId, clientId, clientSecret);
var credential = new TokenCredentials(token.Result.AccessToken);
var computeManagementClient = new ComputeManagementClient(credential) { SubscriptionId = subscriptionId };
var vmResult = await computeManagementClient.VirtualMachines.GetAsync(resourceGroupName, vmName, InstanceViewTypes.InstanceView);
//Get the NIC ID for the VM:
foreach (NetworkInterfaceReference nic in vmResult.NetworkProfile.NetworkInterfaces)
{
Console.WriteLine(" networkInterface id: " + nic.Id);
}
this gives me something like this:
/subscriptions/[guid]/resourceGroups/My-Resource-Group/providers/Microsoft.Network/networkInterfaces/myvm123
To get all public IPs for the resource group, I can do this:
using (var client = new NetworkManagementClient(credential))
{
client.SubscriptionId = subscriptionId;
foreach (var publicIpAddress in client.PublicIPAddresses.ListAll())
{
Console.WriteLine(publicIpAddress.IpAddress);
}
}
...But inspecting the properties of the nic-id and the public ip object, there are no obvious ways to get from one to the other.
Question:
How do I get from the nic-id string, to the actual public IP address for that VM/nic?
Helper function:
private static async Task<AuthenticationResult> GetAccessTokenAsync(string tenantId, string clientId, string clientSecret)
{
var cc = new ClientCredential(clientId, clientSecret);
var context = new AuthenticationContext($"https://login.windows.net/{tenantId}");
var token = context.AcquireToken("https://management.azure.com/", cc);
if (token == null)
{
throw new InvalidOperationException("Could not get the token");
}
return token;
}
I found a workaround. Not pretty, but it works.
It assumes you already have a Microsoft.Azure.Management.Compute.Models.VirtualMachine object from something like this:
VirtualMachine vmResult = await computeManagementClient.VirtualMachines.GetAsync(resourceGroupName, vmName, InstanceViewTypes.InstanceView);
Then you can take the first NIC, get the last part of that as an ID:
var firstNic = vmResult.NetworkProfile.NetworkInterfaces.First();
var nicNameParts = firstNic.Id.Split('/');
string networkIntefaceName = nicNameParts.Last();
using (var client = new NetworkManagementClient(credential))
{
client.SubscriptionId = subscriptionId;
string publicNicId = string.Empty;
//Query ALL Networkinterfaces in the client, and find the one with the matching NIC-name
var nic = client.NetworkInterfaces.ListAll().FirstOrDefault(x => x.Name == networkIntefaceName);
if (nic != null)
{
//If we find that, we can now use that to find the ID of the PublicIPAddress for said NIC
publicNicId = nic.IpConfigurations[0].PublicIPAddress.Id;
//...And when we have that, we can now query all public IP addresses for that specific public Nic ID
var publicIp = client.PublicIPAddresses.ListAll().FirstOrDefault(x => x.Id == publicNicId);
if (publicIp != null)
{
vmInfo.PublicIP = publicIp.IpAddress;
Console.WriteLine(" public ip: " + publicIp.IpAddress);
}
else
{
Console.WriteLine(" public ip: unknown");
}
}
}
Yes, it is not super elegant, it can be optimized etc - but it works, so that's a start. :)