This MongoDB native query works just fine:
db.collection.aggregate([
{$project:{hour:{$hour:'$CreatedAt'},day:{$dayOfMonth:'$CreatedAt'},month:{$month:'$CreatedAt'},year:{$year:'$CreatedAt'}}},
{$group:{_id:{hour:'$hour',day:'$day',month:'$month',year:'$year'}, count: {$sum:1}}}
])
...but when I try to reproduce the same query in C#, the result is 0 for every sum (which is not the case when running the native query in Robomongo).
This is the C# code. I am failing to see any differences between both...
public Dictionary<DateTime, Int32> GetSumPerHour()
{
var project = new BsonDocument
{
{
"$project",
new BsonDocument{
{"hour", new BsonDocument{{"$hour","$CreatedAt"}}},
{"day", new BsonDocument{{"$dayOfMonth","$CreatedAt"}}},
{"month", new BsonDocument{{"$month","$CreatedAt"}}},
{"year", new BsonDocument{{"$year","$CreatedAt"}}},
}
}
};
var group = new BsonDocument
{
{
"$group",
new BsonDocument
{
{"_id", new BsonDocument {{"hour","$hour"},{"day","$day"},{"month","$month"},{"year","$year"}}},
{"count", new BsonDocument {{"$sum","1"}} }
}
}
};
var pipeline = new[] { project, group };
var arguments = new AggregateArgs { Pipeline = pipeline };
var results = this.collection.Aggregate(arguments);
var timeValueDictionary = new Dictionary<DateTime, Int32>();
foreach (BsonDocument result in results)
{
int hour = result["_id"]["hour"].AsInt32;
int day = result["_id"]["day"].AsInt32;
int month = result["_id"]["month"].AsInt32;
int year = result["_id"]["year"].AsInt32;
DateTime time = new DateTime(year, month, day, hour, 0, 0);
int count = result["count"].AsInt32;
timeValueDictionary.Add(time, count);
}
return timeValueDictionary;
}
Related
I just started working on csvhelper library today and new help with how to add entries in the next column. Say I have three datasets composed of (ABC Office, Sarah's office and Brian's office) These are Building names...For each building I have TaskName, PointName, DataName and then Local and Value data of rows.
Each building will have 2 columns, There can be 10 or 20 buildings (Dynamic)
and each building can be 1000's of data rows below local and Value row.
I have the following code
using (var writer = new StreamWriter(filePath))
using (var csv1 = new CsvWriter(writer))
{
foreach (var point in this.Points)
{
csv1.WriteField(point.BuildingName);
csv1.WriteField(" ");
csv1.NextRecord();
csv1.WriteField(point.TaskName);
csv1.WriteField(" ");
csv1.NextRecord();
csv1.WriteField(point.PointName);
csv1.WriteField(" ");
csv1.NextRecord();
csv1.WriteField(point.DataName);
csv1.WriteField(" ");
csv1.NextRecord();
}
}
The problem is that it puts all the points information in first column and then ' ' in the second field.
What I need is something like this eventually.
After the point information added, I want to be able to append a data set of Local and Value column under each point.
Is this formatting in CSV possible using CSVHelper?
Update:
So far I have gotten this far.. Using String builder added first 5 rows of data.
Now I need to be able to append a dataset of columns Local and Value under each Office. Is there a way in CSVHeper library that I can specify where to append the dataset. First 5 rows I am getting like this.
var row1 = new StringBuilder();
var row2 = new StringBuilder();
var row3 = new StringBuilder();
var row4 = new StringBuilder();
foreach (var point in this.Points)
{
row1.Append($"{ point.BuildingName},,");
row2.Append($"{point.TaskName},,");
row3.Append($"{ point.PointName},,");
row4.Append($"{ point.DataName},,");
}
//StreamWriter sw = new StreamWriter("filePath", true);
using (StreamWriter sw = new StreamWriter(filePath1))
{
sw.Write(row1.ToString());
sw.WriteLine();
sw.Write(row2.ToString());
sw.WriteLine();
sw.Write(row3.ToString());
sw.WriteLine();
sw.Write(row4.ToString());
sw.WriteLine();
sw.Close();
}
What you are trying create is not the standard CSV file that CsvHelper was built to create. A CSV file has one row of headers followed by rows of data. You appear to be creating an Excel report using comma separated values. CsvHelper can help you write the fields, but you are going to have to supply the logic for writing the report format. The following is one way that you could do it.
public static void Main(string[] args)
{
var points = new List<Point>
{
new Point
{
BuildingName = "ABC Office",
TaskName = "Temperature",
PointName = "14",
DataName = "Temperature: Degrees F",
Results = new List<Result> {
new Result { Local = new DateTime(2019, 1,1), Value = 2},
new Result { Local = new DateTime(2019, 1, 2), Value = 23}
}
},
new Point
{
BuildingName = "Sarah's Office",
TaskName = "Fan",
PointName = "33",
DataName = "0=Stop;1=Run",
Results = new List<Result> {
new Result { Local = new DateTime(2019, 1,1), Value = 2},
new Result { Local = new DateTime(2019, 1, 2), Value = 23},
new Result { Local = new DateTime(2019, 1, 3), Value = 45},
new Result { Local = new DateTime(2019, 1, 4), Value = 34},
new Result { Local = new DateTime(2019, 1, 5), Value = 36}
}
},
new Point
{
BuildingName = "Brian's Office",
TaskName = "Fan",
PointName = "35",
DataName = "Humidity",
Results = new List<Result> {
new Result { Local = new DateTime(2019, 1,1), Value = 2},
new Result { Local = new DateTime(2019, 1, 2), Value = 23},
new Result { Local = new DateTime(2019, 1, 3), Value = 45},
new Result { Local = new DateTime(2019, 1, 4), Value = 34},
new Result { Local = new DateTime(2019, 1, 5), Value = 36},
new Result { Local = new DateTime(2019, 1, 6), Value = 56},
new Result { Local = new DateTime(2019, 1, 7), Value = 92}
}
},
};
using (var writer = new StreamWriter(filePath))
using (var csv = new CsvWriter(writer))
{
// Print buildings
foreach (var point in points)
{
csv.WriteField(point.BuildingName);
csv.WriteField("");
}
csv.NextRecord();
// Print Tasks
foreach (var point in points)
{
csv.WriteField(point.TaskName);
csv.WriteField("");
}
csv.NextRecord();
// Print Points
foreach (var point in points)
{
csv.WriteField(point.PointName);
csv.WriteField("");
}
csv.NextRecord();
// Print DataNames
foreach (var point in points)
{
csv.WriteField(point.DataName);
csv.WriteField("");
}
csv.NextRecord();
// Print value titles
foreach (var point in points)
{
csv.WriteField("Local");
csv.WriteField("Value");
}
csv.NextRecord();
var endReached = false;
var pointIndex = 0;
// Print values
while (!endReached)
{
endReached = true;
foreach (var point in points)
{
if (point.Results.Count > pointIndex)
{
csv.WriteField(point.Results[pointIndex].Local);
csv.WriteField(point.Results[pointIndex].Value);
if (point.Results.Count > pointIndex + 1)
{
endReached = false;
}
}
else
{
csv.WriteField("");
csv.WriteField("");
}
}
csv.NextRecord();
pointIndex += 1;
}
}
}
public class Point
{
public string BuildingName { get; set; }
public string TaskName { get; set; }
public string PointName { get; set; }
public string DataName { get; set; }
public List<Result> Results { get; set; }
}
public class Result
{
public DateTime Local { get; set; }
public int Value { get; set; }
}
I am trying to run or start an existing task definition within ECS but the documentation is lacking and I cant seem to find any examples online. I have hit a wall and I was wondering if anyone else has done a similar thing.
I am using the AWSSDK.ECS packages.
var request = JsonConvert.DeserializeObject<Request>(record.Sns.Message);
var task = new RunTaskRequest
{
Count = 1,
NetworkConfiguration = new NetworkConfiguration
{
AwsvpcConfiguration = new AwsVpcConfiguration
{
Subnets = new List<string>() { request.SubnetId},
SecurityGroups = new List<string>() { request.SecurityGroupId},
AssignPublicIp = AssignPublicIp.DISABLED
}
},
Cluster = request.Cluster,
LaunchType = LaunchType.FARGATE,
Overrides = new TaskOverride
{
ContainerOverrides = new List<ContainerOverride>
{
new ContainerOverride
{
Name = request.ContainerName,
Environment = request.EnvironmentVariables
.Select(kvp => new Amazon.ECS.Model.KeyValuePair()
{
Name = kvp.Key,
Value = kvp.Value
}).ToList()
}
}
},
TaskDefinition = request.TaskDefinitionUri
};
await new AmazonEcsClient().RunTaskAsync(task);
Check on your task definition. If the NetworkMode is awsvpc, then you must provide NetworkConfiguration parameter. This works for me:
var runTaskRequest = new ECSModel.RunTaskRequest
{
Cluster = "cluster-name",
Count = 1,
LaunchType = LaunchType.FARGATE,
TaskDefinition = "task-definition",
NetworkConfiguration = new ECSModel.NetworkConfiguration
{
AwsvpcConfiguration = new ECSModel.AwsVpcConfiguration
{
SecurityGroups = new List<string> { "security-group" },
Subnets = new List<string> { "subnet" }
}
},
Overrides = new ECSModel.TaskOverride
{
ContainerOverrides = new List<ECSModel.ContainerOverride>
{
new ECSModel.ContainerOverride
{
Name = "container-name",
Command = new List<string>
{
// In case if you need to pass parameters to your instance:
"parameter-one", "parameter-two", "etc"
}
}
}
}
};
await new AmazonEcsClient().RunTaskAsync(runTaskRequest);
I am trying to use AWS SDK for .NET Core.
Create a table to count views on videos.
Add a view count for a day.
Increment existing count for a day.
Query for video counts between two dates for a video.
.NET Core AWS SDK uses Async methods which are not documented in AWS. There is a feature request on their github page for this to happen.... but it is dated from last year. (https://github.com/aws/aws-sdk-net/issues/787)
CREATE THE TABLE
This works and creates a table on the AWS Console.
var ctRequest = new CreateTableRequest
{
AttributeDefinitions = new List<AttributeDefinition>()
{
new AttributeDefinition
{
AttributeName = "ViewUid",
AttributeType = ScalarAttributeType.S
},
new AttributeDefinition
{
AttributeName = "ViewDate",
AttributeType = ScalarAttributeType.S
}
},
KeySchema = new List<KeySchemaElement>
{
new KeySchemaElement
{
AttributeName = "ViewUid",
KeyType = KeyType.HASH //Partition key
},
new KeySchemaElement
{
AttributeName = "ViewDate",
KeyType = KeyType.RANGE
}
},
ProvisionedThroughput = new ProvisionedThroughput
{
ReadCapacityUnits = 5,
WriteCapacityUnits = 6
},
TableName = _settings.AWSDynamoDBViewCountTable
};
var response = _client.CreateTableAsync(ctRequest).Result;
UPDATE AND ITEM WITH AUTO-INCREMENT A FIELD
This, sadly, is where i hit issues. The old docs are found here under the Atomic Counter section. (https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/LowLevelDotNetItemCRUD.html)
Invalid ConditionExpression: Syntax error; token: \"SET\", near: \"SET
VC\"
var viewData = new Document();
viewData["ViewUid"] = videoUid; //Table entry UID
viewData["VideoId"] = videoId; // Video ID
viewData["ViewDate"] = date;
viewData["ViewCount"] = 0;
//Document result = await _viewCountTable.PutItemAsync(viewData);
Expression expr = new Expression();
expr.ExpressionStatement = "SET #VC = #VC + :val";
expr.ExpressionAttributeValues[":val"] = 1;
expr.ExpressionAttributeNames["#VC"] = "ViewCount";
var updateConfig = new UpdateItemOperationConfig() {
ConditionalExpression = expr,
ReturnValues = ReturnValues.UpdatedNewAttributes
};
var result = await _viewCountTable.UpdateItemAsync(viewData, updateConfig);
return result;
QUERY FOR DATE RANGE
Get one video's view count for a date range.
string queryTimeSpanStartString = dateFrom.ToString(AWSSDKUtils.ISO8601DateFormat);
string queryTimeSpanEndString = dateTo.ToString(AWSSDKUtils.ISO8601DateFormat);
var request = new QueryRequest
{
TableName = _settings.AWSDynamoDBViewCountTable,
KeyConditions = new Dictionary<string, Condition>()
{
{
"VideoId", new Condition()
{
ComparisonOperator = "EQ",
AttributeValueList = new List<AttributeValue>()
{
new AttributeValue { S = videoId }
}
}
},
{
"ViewDate",
new Condition
{
ComparisonOperator = "BETWEEN",
AttributeValueList = new List<AttributeValue>()
{
new AttributeValue { S = queryTimeSpanStartString },
new AttributeValue { S = queryTimeSpanEndString }
}
}
}
}
};
var response = await _client.QueryAsync(request);
Any help would be appreciated.
I was able to update the ViewCount with the following code:
string tableName = "videos";
var request = new UpdateItemRequest
{
Key = new Dictionary<string, AttributeValue>() { { "ViewUid", new AttributeValue { S = "replaceVideoIdhere" } } },
ExpressionAttributeNames = new Dictionary<string, string>()
{
{"#Q", "ViewCount"}
},
ExpressionAttributeValues = new Dictionary<string, AttributeValue>()
{
{":incr", new AttributeValue {N = "1"}}
},
UpdateExpression = "SET #Q = #Q + :incr",
TableName = tableName
};
var response = await _dynamoDbClient.UpdateItemAsync(request);
I created a table called "videos" with a partition key named "ViewUid" as string. Let me know if this works for you.
I have .NET Application that uses MongoDB. Current driver I am using is 1.9.2. I am trying to upgrade it to 2.7.0.
I am having some difficulty in getting an Aggregate query to work in the new version:
The working code in version 1.9.2 of the driver is:
public IEnumerable<Car> GetCarsModifiedInPeriod(DateTimeOffset dateFrom, DateTimeOffset dateTo)
{
var matchRequestFromDate = new BsonDocument
{
{
"$match",
new BsonDocument
{
{
// Filter out those too recently modified
"LastUpdatedOn.0", new BsonDocument {{"$gte", dateFrom.Ticks}}
}
}
}
};
var matchRequestToDate = new BsonDocument
{
{
"$match",
new BsonDocument
{
{
// Filter out those too recently modified
"LastUpdatedOn.0", new BsonDocument {{"$lte", dateTo.Ticks}}
}
}
}
};
var cars = collection.Aggregate(new AggregateArgs
{
Pipeline = new[] { matchRequestFromDate, matchRequestToDate},
AllowDiskUse = true,
// Setting the OutputMode to Cursor allows us to return Mongo Doc Size > 16 MB - in the case when a large date
// range is used or a large number of cars were modified in a short period of time
OutputMode = AggregateOutputMode.Cursor
}).Select(r => r.Values.Select(c => c.AsObjectId.ToString()).First());
var returnData = collection.AsQueryable().Where(c => cars.Contains(c.Id)).Select(c => c);
return returnData;
}
With a breakpoint set on returnData for the two periods specified I am getting a count of 25 cars which is what I expect.
This is how I have attempted to re-write for 2.7.0 version of driver:
public IEnumerable<Car> GetCarsModifiedInPeriod(DateTimeOffset dateFrom, DateTimeOffset dateTo)
{
var matchRequestFromDate = new BsonDocument
{
{
"$match",
new BsonDocument
{
{
// Filter out those too recently modified
"LastUpdatedOn.0", new BsonDocument {{"$gte", dateFrom.Ticks}}
}
}
}
};
var matchRequestToDate = new BsonDocument
{
{
"$match",
new BsonDocument
{
{
// Filter out those too recently modified
"LastUpdatedOn.0", new BsonDocument {{"$lte", dateTo.Ticks}}
}
}
}
};
var pipeline = new[] {matchRequestFromDate, matchRequestToDate};
//var mongoPipeline = new AggregateArgs { Pipeline = pipeline, AllowDiskUse = true, OutputMode = AggregateOutputMode.Cursor };
var aggregate = collection.Aggregate(); //.Match(mongoPipeline);
aggregate.Options.AllowDiskUse = true;
aggregate.Options.UseCursor = true;
foreach (var pipe in pipeline)
{
aggregate.AppendStage<BsonDocument>(pipe);
}
var returnData = aggregate.ToList();
return returnData;
}
If I set a breakpoint in returnData in this method I am getting a count of around 10K cars so it doesnt look like I am correctly applying the same matches
Is there a reason you are doing everything in BsonDocuments? There are methods that would make your life a lot easier, for example something like this.
collection.Aggregate(new AggregateOptions() { AllowDiskUse = true, UseCursor = true })
.Match(Builders<BsonDocument>.Filter.Gte("LastUpdatedOn.0", dateFrom.Ticks) & Builders<BsonDocument>.Filter.Lte("LastUpdatedOn.0", dateFrom.Ticks))
.ToListAsync()
You could tidy the filtering up more as well by using the right class for the collection and the builders.
Looking at the query, I'm not sure you even need to be using an aggregate unless you are doing more than a match. It could simply be a find.
I started out with Mongo client doing some nifty queries and aggretations.. but now that I want to use it in .NET/C#, I see that I can't simply run the query as text field..
Furthermore, after resorting to building an Aggregation Pipeline, and running the collection.Aggregate() function, I'm getting a result set, but I have no idea how to traverse it..
Can anyone help guide me here?
Here's my code:
var coll = db.GetCollection("animals");
var match = new BsonDocument {
{ "$match", new BsonDocument {{"category","cats"}} }
};
var group = new BsonDocument{
{
"$group", new BsonDocument{
{"_id", "$species"},
{"AvgWeight", new BsonDocument{{"$avg", "$weight"}}} }
}
};
var sort = new BsonDocument{{"$sort", new BsonDocument{{"AvgWeight", -1}}}};
var pipeline = new[] { match, group, sort };
var args = new AggregateArgs { Pipeline = pipeline };
var res = coll.Aggregate(args);
foreach (var obj in res)
{
// WHAT TO DO HERE??
}
Also, I should say that I'm a little rusty with C# / ASP.NET / MVC so any room for simplification would be much appreciated.
Your result is IEnumerable of BsonDocument, you can Serialize them to C# objects using the BSonSerializer. And this code snippet just writes them to your console, but you can see that you have typed objects
List<Average> returnValue = new List<Average>();
returnValue.AddRange(documents.Select(x=> BsonSerializer.Deserialize<Average>(x)));
foreach (var obj in returnValue)
{
Console.WriteLine("Species {0}, avg weight: {1}",returnValue._Id,returnValue.AvgWeight);
}
And then have a class called Average, where the property name match the names in the BSonDocument, if you want to rename then (because _Id is not so nice in c# terms concerning naming conventions), you can add a $project BsonDocument to your pipeline.
public class Average
{
public string _Id { get; set; }
public Double AvgWeight {get; set; }
}
$project sample (add this in your pipeline just before sort
var project = new BsonDocument
{
{
"$project",
new BsonDocument
{
{"_id", 0},
{"Species","$_id"},
{"AvgWeight", "$AvgWeight"},
}
}
};