I like to create a query with the latest .NET Elasticsearch client version 8.17.1.
What I want is:
groupBy:
sessionId AND destinationPort, sourcePort AND protocol AND
destinationPhysicalAddress AND frameProtocol AND projectId AND
sourcePhysicalAddress
aggregates:
packageNumber as sum in TotalPackageNumber,
length as sum in TotalLength,
timestamp sa min value in Timestamp
My index contains:
{
"destinationPort": 53,
"sourcePort": 54435,
"frameProtocol": "eth:ethertype:ip:udp:dns",
"length": 413,
"sessionId": "7cf5a952-250a-44b7-aa35-8bb316f36539",
"sourcePhysicalAddress": "22:e0:4c:a4:30:01",
"packageNumber": 2,
"destinationIp": "10.18.138.85",
"protocol": "udp",
"sourceIp": "10.114.8.2",
"destinationPhysicalAddress": "00:09:0f:09:00:02",
"projectId": 25,
"timestamp": "2024-06-13T08:36:47.229854Z"
}
And the index definition is:
{
"sessions": {
"aliases": {},
"mappings": {
"properties": {
"destinationIp": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"destinationPhysicalAddress": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"destinationPort": {
"type": "long"
},
"frameProtocol": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"icmpDestinationDeviceIp": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"icmpIdentifier": {
"type": "long"
},
"icmpOriginDeviceIp": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"icmpType": {
"type": "long"
},
"length": {
"type": "long"
},
"packageNumber": {
"type": "long"
},
"projectId": {
"type": "long"
},
"protocol": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sessionId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sourceIp": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sourcePhysicalAddress": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sourcePort": {
"type": "long"
},
"timestamp": {
"type": "date"
}
}
},
"settings": {
"index": {
"routing": {
"allocation": {
"include": {
"_tier_preference": "data_content"
}
}
},
"number_of_shards": "1",
"provided_name": "sessions",
"creation_date": "1738916432512",
"number_of_replicas": "1",
"uuid": "Sx6PiLANQuePPQiBTkFYvQ",
"version": {
"created": "8512000"
}
}
}
}
}
Thanks in advance
I like to create a query with the latest .NET Elasticsearch client version 8.17.1.
What I want is:
groupBy:
sessionId AND destinationPort, sourcePort AND protocol AND
destinationPhysicalAddress AND frameProtocol AND projectId AND
sourcePhysicalAddress
aggregates:
packageNumber as sum in TotalPackageNumber,
length as sum in TotalLength,
timestamp sa min value in Timestamp
My index contains:
{
"destinationPort": 53,
"sourcePort": 54435,
"frameProtocol": "eth:ethertype:ip:udp:dns",
"length": 413,
"sessionId": "7cf5a952-250a-44b7-aa35-8bb316f36539",
"sourcePhysicalAddress": "22:e0:4c:a4:30:01",
"packageNumber": 2,
"destinationIp": "10.18.138.85",
"protocol": "udp",
"sourceIp": "10.114.8.2",
"destinationPhysicalAddress": "00:09:0f:09:00:02",
"projectId": 25,
"timestamp": "2024-06-13T08:36:47.229854Z"
}
And the index definition is:
{
"sessions": {
"aliases": {},
"mappings": {
"properties": {
"destinationIp": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"destinationPhysicalAddress": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"destinationPort": {
"type": "long"
},
"frameProtocol": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"icmpDestinationDeviceIp": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"icmpIdentifier": {
"type": "long"
},
"icmpOriginDeviceIp": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"icmpType": {
"type": "long"
},
"length": {
"type": "long"
},
"packageNumber": {
"type": "long"
},
"projectId": {
"type": "long"
},
"protocol": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sessionId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sourceIp": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sourcePhysicalAddress": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sourcePort": {
"type": "long"
},
"timestamp": {
"type": "date"
}
}
},
"settings": {
"index": {
"routing": {
"allocation": {
"include": {
"_tier_preference": "data_content"
}
}
},
"number_of_shards": "1",
"provided_name": "sessions",
"creation_date": "1738916432512",
"number_of_replicas": "1",
"uuid": "Sx6PiLANQuePPQiBTkFYvQ",
"version": {
"created": "8512000"
}
}
}
}
}
Thanks in advance
Share Improve this question edited 2 days ago marc_s 755k184 gold badges1.4k silver badges1.5k bronze badges asked Feb 8 at 8:32 Vicente García DiezVicente García Diez 411 silver badge7 bronze badges1 Answer
Reset to default 0I think by applying CompositeAggregation you can correctly group your data.
Create your aggregation source like:
var compositeSources = new List<IDictionary<string, CompositeAggregationSource>>
{
// group by sessionId - use keyword for text
new Dictionary<string, CompositeAggregationSource>
{
{
"sessionId",
new CompositeAggregationSource
{
Terms = new CompositeTermsAggregation { Field = "sessionId.keyword" }
}
}
}
// Add other terms
}
Then you can apply your aggregation like:
var compositeAgg = new CompositeAggregation("grouped_sessions")
{
Sources = compositeSources,
Aggregations = new Dictionary<string, Aggregation>
{
{ "TotalPackageNumber", new SumAggregation("TotalPackageNumber", "packageNumber") },
{ "TotalLength", new SumAggregation("TotalLength", "length") },
{ "Timestamp", new MinAggregation("Timestamp", "timestamp") }
}
};
Then in your search:
var searchRequest = new SearchRequest("sessions")
{
Size = 0,
Aggregations = new AggregationDictionary
{
{ "grouped_sessions", compositeAgg }
}
};
var client = new ElasticClient(new ElasticsearchClientSettings(new Uri("http://localhost:9200")));
var response = client.Search<SessionRecord>(searchRequest);
Read CompositeAggregationSource for more information.