Summary
I want to to properly index (ingest) geo data (Geometry, GeometryCollection) as GeoShape in ElasticSearch using C#, Nest and NetTopologySuite (NTS) from GeoJson files or string representations.
I'm using the following stack:
ElasticSearch 7.10.1
NEST 7.10.1
NetTopologySuite 2.1.0
NetTopologySuite.IO.GeoJSON 2.0.4
In my GitHub GIST you can find the two sample files (postal-area.geojson and the geojson file as a sample for Scenario #7) along with the code presented bellow with what i've tried so far.
My attempts
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection.Metadata;
using System.Text;
using Bogus.DataSets;
using Elasticsearch.Net;
using ElasticSearch;
using GeoAPI.Geometries;
using Microsoft.Extensions.Configuration;
using Nest;
using Nest.JsonNetSerializer;
using NetTopologySuite.Features;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NetTopologySuite.IO;
using NetTopologySuite.Geometries;
using NetTopologySuite.IO.Converters;
using Newtonsoft.Json.Converters;
using Coordinate = NetTopologySuite.Geometries.Coordinate;
using GeometryCollection = NetTopologySuite.Geometries.GeometryCollection;
private static void Main()
{
try {
var defaultIndex = "my_shapes";
string cloudId = "cloudId";
string username = "username";
string password = "password";
var credentials = new BasicAuthenticationCredentials(username, password);
//var pool = new SingleNodeConnectionPool(new Uri($"http://localhost:9200"));
var pool = new CloudConnectionPool(cloudId, credentials);
var settings = new ConnectionSettings(pool, (c, s) =>
new JsonNetSerializer(c, s, contractJsonConverters: new JsonConverter[]
{
new AttributesTableConverter(),
new CoordinateConverter(),
new EnvelopeConverter(),
new FeatureConverter(),
new FeatureCollectionConverter(),
new GeometryConverter(),
new GeometryArrayConverter(),
new StringEnumConverter()
}))
.DefaultIndex(defaultIndex)
.DisableDirectStreaming()
.PrettyJson()
.OnRequestCompleted(callDetails => {
if (callDetails.RequestBodyInBytes != null) {
var json = JObject.Parse(Encoding.UTF8.GetString(callDetails.RequestBodyInBytes));
Console.WriteLine(
$"{callDetails.HttpMethod} {callDetails.Uri}
" +
$"{json.ToString(Newtonsoft.Json.Formatting.Indented)}");
}
else {
Console.WriteLine($"{callDetails.HttpMethod} {callDetails.Uri}");
}
Console.WriteLine();
if (callDetails.ResponseBodyInBytes != null) {
Console.WriteLine($"Status: {callDetails.HttpStatusCode}
" +
$"{Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}
" +
$"{new string('-', 30)}
");
}
else {
Console.WriteLine($"Status: {callDetails.HttpStatusCode}
" +
$"{new string('-', 30)}
");
}
});
var client = new ElasticClient(settings);
var createIndexResponse = client.Indices.Create(defaultIndex, c => c
.Map<MyDocument>(m => m
.Properties(p => p
.GeoShape(g => g
.Name(n => n.Geometry)
)
)
)
);
if (!createIndexResponse.IsValid) {
throw new Exception($"Error creating index: {createIndexResponse.DebugInformation}");
}
IndexResponse indexResponse;
MyDocument document;
Geometry geometryPolygon;
FeatureCollection featureCollection;
//Working Scenario #1: Geometry from mock Polygon -------------------works!!!!!!!!!!!
var polygon = new Polygon(new LinearRing(new [] {
new Coordinate(0, 0),
new Coordinate(0, 4),
new Coordinate(4, 4),
new Coordinate(4, 0),
new Coordinate(0, 0)
}));
document = new MyDocument(1, polygon);
indexResponse = client.IndexDocument(document);
//End of Scenario #1 -------------------
//Working Scenario #2: Geometry from FeatureCollection from real GeoJson file ------------------- works
var geojsonFileName = @"......\_GeoDataFilesGeoJSONsPostalArea.geojson";
var jsonData = File.ReadAllText(geojsonFileName);
featureCollection = new GeoJsonReader().Read<FeatureCollection>(jsonData);
if (featureCollection == null) return;
var geometry = featureCollection[0].Geometry;
document = new MyDocument(1, geometry);
indexResponse = client.IndexDocument(document);
//End of Scenario #2-------------------
//NOT Working Scenario #3: Geometry deserialized (with GeoJsonSerializer) from mock GeoJson string -------------------
//excluded coordinates arrays for clarity
var geoJsonPolygonStr1 = "{"type":"Polygon","coordinates":[ ... ]}";
var serializer = new NetTopologySuite.IO.GeoJsonSerializer();
using(var stringReader = new StringReader(geoJsonPolygonStr1))
using (var jsonReader = new JsonTextReader(stringReader))
{
/*Error:
{"Could not create an instance of type NetTopologySuite.Geometries.Geometry.
Type is an interface or abstract class and cannot be instantiated.
Path 'type', line 2, position 8."}*/
geometryPolygon = serializer.Deserialize<Geometry>(jsonReader);
}
document = new MyDocument(1, geometryPolygon);
indexResponse = client.IndexDocument(document);
//End of Scenario #3 -------------------
//NOT Working Scenario #4: Geometry deserialized (with JsonConvert) from mock GeoJson string -------------------
//excluded coordinates arrays for clarity
var geoJsonPolygonStr2 = "{"type":"Polygon","coordinates":[ ... ]}";
/*Error:
{"Could not create an instance of type NetTopologySuite.Geometries.Geometry.
Type is an interface or abstract class and cannot be instantiated.
Path 'type', line 2, position 8."}*/
geometryPolygon = JsonConvert.DeserializeObject<Geometry>(geoJsonPolygonStr2);
document = new MyDocument(1, geometryPolygon);
indexResponse = client.IndexDocument(document);
//End of Scenario #4 -------------------
//NOT Working Scenario #5: GeometryCollection deserialized (with JsonConvert) from mock GeoJson string -------------------
var geoCollectionMock =
@"{""type"": ""geometrycollection"",
""geometries"": ["
+ geoJsonPolygonStr1 +
","
+ geoJsonPolygonStr2 +
@"]
}";
/*Error:
{"Could not create an instance of type NetTopologySuite.Geometries.Geometry.
Type is an interface or abstract class and cannot be instantiated.
Path 'type', line 2, position 8."}*/
geometryPolygon = JsonConvert.DeserializeObject<Geometry>(geoCollectionMock);
document = new MyDocument(1, geometryPolygon);
indexResponse = client.IndexDocument(document);
//End of Scenario #5 -------------------
//Weired Scenario #6: GeometryCollection built from multiple Geometry objects from FeatureCollection from real GeoJson file -------------------
//Data ingested into ElasticSearch Index, BUT, polygons from GeometryCollection can't be seen on Kibana Maps as other simple Polygons can be seen
var geoCollectionObj = new NetTopologySuite.Geometries.GeometryCollection(new[]
{
featureCollection[0].Geometry,
featureCollection[1].Geometry,
featureCollection[2].Geometry
});
document = new MyDocument(1, geoCollectionObj);
indexResponse = client.IndexDocument(document);
//End of Scenario #6 -------------------
//Not working Scenario #7: Geometry from FeatureCollection from real GeoJson file - invalid Geometry -------------------
var isValid = featureCollection[0].Geometry.IsValid;//= false
/*Error:
"type" : "mapper_parsing_exception",
"reason" : "failed to parse field [geometry] of type [geo_shape]",
"caused_by" : {
"type" : "invalid_shape_exception",
"reason" : "Self-intersection at or near point [-3.173,57.545]"
}*/
document = new MyDocument(99, featureCollection[99].Geometry);
indexResponse = client.IndexDocument(document);
//End of Scenario #7 -------------------
if (!indexResponse.IsValid) {
throw new Exception($"Error indexinf document: {indexResponse.DebugInformation}");
}
}
catch (Exception ex)
{
Console.WriteLine($"General error: {ex}");
}
}
public class MyDocument {
public MyDocument(int id, Geometry geometry) {
Id = id;
Geometry = geometry;
}
public int Id { get; set; }
public Geometry Geometry { get; set; }
}
This is the GeoJson file from my GitHub GIST used as a sample for Scenario #7. It seems to be valid and it's displayed on other platforms (GitHub mapbox map preview, QGIS, geojson.io)
Questions
- Regarding not working scenarios (#3, #4, #5) how can i deserialize a GeoJson string into a Geometry object?
- Regarding Scenario #6, why GeometryCollection data isn't visibile on Kibana Map like simple Geometry (Polygons)?
