Problem description
I need to receive JSON array of objects (with almost any shape) and store them to ES database using function IndexMany (or some similar bulk indexing function). I found some clumsy solution with one drawback - it doesn't set _id property in ES correctly (according to the id property in JSON object).
And also I would like to know if there is any more elegant way how to achieve my goal without casting every JArray item to string and back to ExpandoObject.
Additional info
Elasticsearch DB 7.5.1
NEST (7.6.1)
Newtonsoft.Json (12.0.3)
TLDR
Is there any elegant solution of following code:
var settings = new ConnectionSettings(new Uri("http://localhost:9200")).DefaultIndex("people");
var client = new ElasticClient(settings);
// this string represents incoming JSON message
string json = #"[
{
""id"": ""1"",
""name"": ""Andrej"",
""surname"": ""Burak"",
""dob"": ""1921-11-10T00:00:00+00:00""
},
{
""id"": ""2"",
""name"": ""Franta"",
""surname"": ""Dobrota"",
""dob"": ""1933-10-05T00:00:00+00:00""
},
{
""id"": ""3"",
""name"": ""Milos"",
""surname"": ""Ovcacek"",
""dob"": ""1988-05-05T00:00:00+00:00""
}
]";
JArray jArray = JArray.Parse(json);
foreach (var jtoken in jArray)
{
var jobj = (JObject)jtoken;
jobj.Add("DbCreated", JToken.FromObject(DateTime.UtcNow));
jobj.Add("DbCreatedBy", JToken.FromObject("authors name"));
}
//working, but seems to me a bit too clumsy to convert every item to string and then back to dynamic object
var converter = new ExpandoObjectConverter();
dynamic[] dlst = jArray.Select(t => (dynamic)JsonConvert.DeserializeObject<ExpandoObject>(t.ToString(), converter)).ToArray();
//not working cast
dynamic[] dlstNW = jArray.ToObject<dynamic[]>();
var indexManyResponse = client.IndexMany(dlst); //working partially (but not using ID as index)
var indexManyResponseNotWorking = client.IndexMany(jArray); //not working
var indexManyResponseNotWorking2 = client.IndexMany(dlstNW); //not working
// expected behavior
dynamic[] jsondyn = new dynamic[]
{
new { Id = "1", Name = "foo" },
new { Id = "2", Name = "bar" },
new { Id = "3", Name = "baz" },
};
var indexManyResponseWithIndex = client.IndexMany(jsondyn); //working perfectly, but don't know how to acieve this
After #gnud pointed me to low-level client I think I have found my answer. Important thing is to build request manually like it was explained here.
var settings = new ConnectionSettings(new Uri("http://localhost:9200")).DefaultIndex("people").DisableDirectStreaming();
var client = new ElasticClient(settings);
// this string represents incoming JSON message
string json = #"[
{
""id"": ""1"",
""name"": ""Andrej"",
""surname"": ""Burak"",
""dob"": ""1921-11-10T00:00:00+00:00""
},
{
""Id"": ""2"",
""name"": ""Franta"",
""surname"": ""Dobrota"",
""dob"": ""1933-10-05T00:00:00+00:00""
},
{
""Id"": ""3"",
""name"": ""Milos"",
""surname"": ""Ovcacek"",
""dob"": ""1988-05-05T00:00:00+00:00""
}
]";
JArray jArray = JArray.Parse(json);
// I have to build my request manually
List<string> esRequest = new List<string>();
foreach (var jtoken in jArray)
{
var jobj = (JObject)jtoken;
jobj.Add("DbCreated", JToken.FromObject(DateTime.UtcNow));
jobj.Add("DbCreatedBy", JToken.FromObject("authors name"));
string id;
// Ensure that Id is set even if we receive it in lowercase
if( jobj["Id"] == null)
{
if( jobj["id"] == null)
{
throw new Exception("missing Id");
}
id = jobj["id"].Value<string>();
}
else
{
id = jobj["Id"].Value<string>();
}
string indexName = "people";
esRequest.Add($"{{\"index\":{{\"_index\":\"{indexName}\",\"_id\":\"{id}\"}}}}");
esRequest.Add(jobj.ToString(Formatting.None));
}
PostData pd = PostData.MultiJson(esRequest);
var llres = client.LowLevel.Bulk<BulkResponse>(pd);
Hope it helps someone.
Related
So I.m storing data in dynamoDb as a list of maps. I can easily store the data just by using SaveAsync(ClassObject). But I cant fetched the data. This is the data I'm trying to fetch. But in the database it is Storing with AttributeValue such as [m:{s:{subservice : writer}}]
{
"Column" : "Writing",
"Requirements" : [
{
"subService" : "writer",
"description" : "info why writer"
},
{
"subService" : "blogger",
"description" : "info why blogger"
}
]
}
JArray req = new JArray();
string columnValue = data["Column"].ToObject<string>();
ScanRequest request = FilterRequest(tableName, "Column", columnValue );
//FilterRequest is just a scanrequest
var response = await dbClient.ScanAsync(request);
var serviceRequirements = response.Items[0];
foreach (var requirement in serviceRequirements["Requirements"])
{
JObject subCat = new JObject();
subCat.Add("subService", requirement["subService"].S.ToString());
subCat.Add("description", requirement["description"].S.ToString());
req.Add(requirement);
}
return req;
I'm stuck at this for two days. And I cannot find any reference to my problem. Im just simply want to fetch.
it took lot of hit and trial...but it works now
just want to share the answer, hope it helps someone
JArray req = new JArray();
string columnValue = data["Column"].ToObject<string>();
ScanRequest request = FilterRequest(tableName, "Column", columnValue );
//FilterRequest is just a scanrequest
var response = await dbClient.ScanAsync(request);
var serviceRequirements = response.Items[0]["Requirements"].L.ToArray();
var out1 = serviceRequirements[0].M.Values.ToArray();
foreach(var element in serviceRequirements)
{
JObject subCat = new JObject();
subCat.Add("subService",element.M.Values.ToArray()[0].S.ToString());
subCat.Add("description",element.M.Values.ToArray()[1].S.ToString());
req.Add(subCat);
}
return req;
this returns
[
{
"subService" : "writer",
"description" : "info why writer"
},
{
"subService" : "blogger",
"description" : "info why blogger"
}
]
I'm trying to add a JSON string to an existing one but still haven't been able to do it successfully.
This is the original string: originalJSONString
{
"properties": [
"property1",
"property2"
]
}
And I want to add this to the original string: JSONStringIwantToAdd
{
"filter": {
"Name": "some filter name",
"Parameters": {
"LookupKey": "somekey",
"LookupValue": "somevalue"
}
}
}
To make a resulting string like this: finalJSONString
{
"properties": [
"property1",
"property2"
],
"filter": {
"Name": "some filter name",
"Parameters": {
"LookupKey": "somekey",
"LookupValue": "somevalue"
}
}
}
This is my direction so far but I'm getting null in propertiesJObject and can't figure out afterwards.
Is this even the right direction I'm going?
var originalJObj = JObject.Parse(originalJSONString);
var tobeaddedJObj = JObject.Parse(JSONStringIwantToAdd);
var propertiesJObject = originalJObj["properties"] as JObject;
propertiesJObject.Add(tobeaddedJObj);
var serializer = new JsonSerializer { ContractResolver = new CamelCasePropertyNamesContractResolver() };
var finalJSONString = JObject.FromObject(originalJObj, serializer).ToString();
Can someone please help me with this?
Thank You for your time!
JSON.NET includes functionality to do exactly what you need: JContainer.Merge
Note that Merge modifies the original object, rather than returning a new one:
var original = JObject.Parse(#"{
""properties"": [
""property1"",
""property2""
]
}");
var toAdd = JObject.Parse(#"{
""filter"": {
""Name"": ""some filter name"",
""Parameters"": {
""LookupKey"": ""somekey"",
""LookupValue"": ""somevalue""
}
}
}");
original.Merge(toAdd, new JsonMergeSettings
{
// union array values together to avoid duplicates
MergeArrayHandling = MergeArrayHandling.Union
});
Fiddle link: https://dotnetfiddle.net/o51GuA
You can do this without explicitly using a serializer.
This code adds the first property from JSONStringIwantToAdd to originalJSONString:
var originalJson = "{\r\n \"properties\": [\r\n \"property1\",\r\n \"property2\"\r\n ]\r\n}";
var extraJson = "{\r\n \"filter\": {\r\n \"Name\": \"some filter name\",\r\n \"Parameters\": {\r\n \"LookupKey\": \"somekey\",\r\n \"LookupValue\": \"somevalue\"\r\n }\r\n }\r\n}";
var original = JObject.Parse(originalJson);
var extra = JObject.Parse(extraJson);
var newProperty = extra.Children().First() as JProperty;
original.Add(newProperty.Name, newProperty.Value);
var newJson = original.ToString();
Output:
{
"properties": [
"property1",
"property2"
],
"filter": {
"Name": "some filter name",
"Parameters": {
"LookupKey": "somekey",
"LookupValue": "somevalue"
}
}
}
This should work, the main issue was not accessing the filter property when adding it to the jobject. I would recommend adding some safeguards like checking for existing properties, this may not be as performant as possible. If speed is important you may have to write your own serializer/deserializer for json.net.
[Test]
public void AugmentJsonObjectTest()
{
// Given
var originalString = "{ \"properties\": [ \"property1\", \"property2\" ]}";
var stringToBeAdded = "{ \"filter\": { \"Name\": \"some filter name\", \"Parameters\": { \"LookupKey\": \"somekey\", \"LookupValue\": \"somevalue\" } }}";
// When
var originalObject = JObject.Parse(originalString);
var objectToBeAdded = JObject.Parse(stringToBeAdded);
originalObject.Add("filter", objectToBeAdded["filter"]);
var mergedObjectAsString = originalObject.ToString();
// Then
var expectedResult =
"{ \"properties\": [ \"property1\", \"property2\" ], \"filter\": { \"Name\": \"some filter name\", \"Parameters\": { \"LookupKey\": \"somekey\", \"LookupValue\": \"somevalue\" } }}";
// Parsing and toString to avoid any formatting issues
Assert.AreEqual(JObject.Parse(expectedResult).ToString(), mergedObjectAsString);
}
using Expando
using System.Collections.Generic;
using System.Dynamic;
using System.IO;
using Newtonsoft.Json;
namespace jsonConcat {
class Program {
static void Main (string[] args) {
ExpandoObject inputData;
ExpandoObject appendData;
using (TextReader inputReader = new StreamReader ("originalData.json"))
using (TextReader appendReader = new StreamReader ("appendData.json")) {
inputData = JsonConvert.DeserializeObject<ExpandoObject> (inputReader.ReadToEnd ());
appendData = JsonConvert.DeserializeObject<ExpandoObject> (appendReader.ReadToEnd ());
}
foreach (var item in appendData) {
inputData.TryAdd (item.Key.ToString (), item.Value);
}
using (TextWriter outputWriter = new StreamWriter ("outputData.json")) {
outputWriter.Write (JsonConvert.SerializeObject (inputData));
}
}
}
}
I am using Json.Net to parse my JSON
This is my JSON:
"OptionType": {
"C": [
"C",
"Call"
],
"P": [
"P",
"Put"
]
}
Before this step, when processed, as a result, I would get a value from any of this.
For example Option Type: Call
Whatever value I get, I need it to be transcodified according to the above JSON.
Example: Option Type: C
First of all your sample data is not a valid JSON. You need to wrap it to the curvy braces.
If your sample is a part of the JSON object, you can map OptionType to the Dictionary<string, List<string>>.
To find valid option you will need to iterate this dictionary like in the sample below:
var valueToCheck = "Call";
string type = null;
foreach (var kvp in optionTypes)
{
if (kvp.Value.Contains(valueToCheck))
{
type = kvp.Key;
break;
}
}
Same using JObject with fixed JSON data:
var json = #"{
""OptionType"":{
""C"": [
""C"",
""Call""
],
""P"": [
""P"",
""Put""
]
}
}";
var valueToCheck = "Call";
string type = null;
var ot = JObject.Parse(json);
var objectType = ot["OptionType"];
foreach (var token in objectType)
{
var prop = token.ToObject<JProperty>();
var key = prop.Name;
var values = prop.Value.ToObject<List<string>>();
if (values.Contains(valueToCheck))
{
type = key;
break;
}
}
Code is not perfect but it is just an idea what to do.
You need to iterate over properties of JObject and then search your option type and then replace your search option with its parent key.
This is custom function can do above task.
public static JObject GetJObject(JObject jObject, List<string> optionType)
{
foreach (var type in jObject["OptionType"])
{
var key = type.ToObject<JProperty>().Name;
var values = type.ToObject<JProperty>().Value.ToObject<List<string>>();
foreach (var option in optionType)
{
if (values.Contains(option))
{
int index = values.IndexOf(option);
values.RemoveAt(index);
values.Insert(index, key);
}
}
JToken jToken = JToken.FromObject(values);
jObject.SelectToken("OptionType." + key).Replace(jToken);
}
return jObject;
}
And you can use above custom function this like
string json = File.ReadAllText(#"Path to your json file");
JObject jObject = JObject.Parse(json);
List<string> optionType = new List<string> { "Call", "Put" };
JObject result = GetJObject(jObject, optionType);
Output:
My API returns
{
"result": [
{
"id": "51473",
"name": "serv-vc",
"modifydate": "2014-10-09 18:29:48.033",
"expirationoff": "false",
"createdate": "",
"scheduleoff": "false",
}
],
"status": 0
}
which I've stored as a JObject reponseobj
I'm having trouble figuring out how to access responseobj["result"][0]["id"].
Every time I try that, it gives an array about being out of bounds.
What am I missing?
I also tried
JArray resultarr = (JArray)responseobj.SelectToken("result");
resultarr[0]["id"]
but have the same results.
Assuming the response is in a string variable called response, this would do it:
JObject responseobj = JObject.Parse(response);
JObject result = (JObject)(responseobj.SelectToken("result") as JArray).First();
int id = result.Value<int>("id");
Try using:
JObject jObject = JObject.Parse( "{\"result\": [{\"id\": \"51473\", \"name\": \"serv-vc\", \"modifydate\": \"2014-10-09 18:29:48.033\", \"expirationoff\": \"false\", \"createdate\": \"\", \"scheduleoff\": \"false\", } ], \"status\": 0 }" );
And to access to the different nodes, you can use:
string name = jObject["result"]["name"].ToString();
string expirationoff = jObject["result"]["expirationoff"].ToString();
Or you can convert result in a new json a work on it
And to access to result you can do:
var result = jObject["result"][0];
Remember that you can have 0, 1, 2... x numbers of results in your json, then you need do reference to the first position.
Not sure what's your issue, but this seems to work for me :
static void Main(string[] args)
{
JObject j = JObject.Parse(
"{\"result\": [{\"id\": \"51473\", \"name\": \"serv-vc\", \"modifydate\": \"2014-10-09 18:29:48.033\", \"expirationoff\": \"false\", \"createdate\": \"\", \"scheduleoff\": \"false\", } ], \"status\": 0 }" );
var res = j["result"];
Console.Out.WriteLine(res);
// show an arrays
var maybe = j["result"][0];
Console.Out.WriteLine(maybe);
// shows the first object in the array
var fail = j["result"][0]["id"];
Console.Out.WriteLine(fail);
// shows 51473
}
var Jobj = ((JObject)RequestObject)["data"];
foreach (JObject content in Jobj.Children<JObject>()) {
foreach (JProperty prop in content.Properties()) {
Console.WriteLine(prop.Name);//the column name
Console.WriteLine(prop.Value.ToString());// column value
}
}
I have some JSON Data which looks like this:
{
"response":{
"_token":"StringValue",
"code":"OK",
"user":{
"userid":"2630944",
"firstname":"John",
"lastname":"Doe",
"reference":"999999999",
"guid":"StringValue",
"domainid":"99999",
"username":"jdoe",
"email":"jdoe#jdoe.edu",
"passwordquestion":"",
"flags":"0",
"lastlogindate":"2013-02-05T17:54:06.31Z",
"creationdate":"2011-04-15T14:40:07.22Z",
"version":"3753",
"data":{
"aliasname":{
"$value":"John Doe"
},
"smsaddress":{
"$value":"5555555555#messaging.sprintpcs.com"
},
"blti":{
"hideemail":"false",
"hidefullname":"false"
},
"notify":{
"grades":{
"$value":"0"
},
"messages":{
"$value":"1"
}
},
"beta_component_courseplanexpress_1":{
"$value":"true"
}
}
}
}
I am using C# with JSON.NET to parse through the data. I've been able to sucessfully get data using this algorithm:
User MyUser = new User();
JToken data = JObject.Parse(json);
MyUser.FirstName = (string) data.SelectToken("response.user.firstname");
//The same for all the other properties.
The problem is with the data field. This field is based on user preferences mostly and data is only inserted as it is used. The fields are all custom and developers can put in as many as they want without restrictions. Essentially, it's all free form data. Also as you notice they can be nested really far with data.
I've tried to run:
MyUser.Data = JsonConvert.DeserializeObject<List<JToken>>((string) data.SelectToken("response.user.data");
which doesn't work.
How would you go about converting it to be used in a C# object?
You can access it via the JToken / JArray / JObject methods. For example, this will list all of the keys under the data:
public class StackOverflow_14714085
{
const string JSON = #"{
""response"": {
""_token"": ""StringValue"",
""code"": ""OK"",
""user"": {
""userid"": ""2630944"",
""firstname"": ""John"",
""lastname"": ""Doe"",
""reference"": ""999999999"",
""guid"": ""StringValue"",
""domainid"": ""99999"",
""username"": ""jdoe"",
""email"": ""jdoe#jdoe.edu"",
""passwordquestion"": """",
""flags"": ""0"",
""lastlogindate"": ""2013-02-05T17:54:06.31Z"",
""creationdate"": ""2011-04-15T14:40:07.22Z"",
""version"": ""3753"",
""data"": {
""aliasname"": {
""$value"": ""John Doe""
},
""smsaddress"": {
""$value"": ""5555555555#messaging.sprintpcs.com""
},
""blti"": {
""hideemail"": ""false"",
""hidefullname"": ""false""
},
""notify"": {
""grades"": {
""$value"": ""0""
},
""messages"": {
""$value"": ""1""
}
},
""beta_component_courseplanexpress_1"": {
""$value"": ""true""
}
}
}
}
}";
public static void Test()
{
var jo = JObject.Parse(JSON);
var data = (JObject)jo["response"]["user"]["data"];
foreach (var item in data)
{
Console.WriteLine("{0}: {1}", item.Key, item.Value);
}
}
}
Json.NET can actually parse to a dynamic if that is useful to you.
Which means you can do something like.
dynamic parsedObject = JsonConvert.DeserializeObject("{ test: \"text-value\" }");
parsedObject["test"]; // "text-value"
parsedObject.test; // "text-value"
parsedObject.notHere; // null
Edit: might be more suitable for you to iterate the values if you don't know what you are looking for though.
dynamic parsedObject = JsonConvert.DeserializeObject("{ test: { inner: \"text-value\" } }");
foreach (dynamic entry in parsedObject)
{
string name = entry.Name; // "test"
dynamic value = entry.Value; // { inner: "text-value" }
}