Merge branch 'develop' into feature/aggregations-top-hits

Author: Mpdreamz

docs/contents/nest/aggregations/advanced.markdown

@@ -0,0 +1,41 @@
+---
+template: layout.jade
+title: Aggregations
+menusection: aggregations
+menuitem: advanced
+---
+
+# Advanced usage
+
+## Creating multiple aggregations from an IEnumerable
+
+Premise:
+You have an IEnumerable<TestObject> MyFilters, and you want to build a Filter aggregation with a Term filter for each of the TestObjects you have.
+
+	class TestObject
+	{
+		public string Name { get; set; }
+		public string Term { get; set; }
+	}
+
+Then you could do something like this:
+
+	IEnumerable<TestObject> MyFilters;
+
+	var descriptor = new SearchDescriptor<MySearchType>().Query(q => q.MatchAll());
+
+	descriptor.Aggregations(aggr => 
+	{
+		foreach (var term in MyFilters)
+		{
+			aggr.Filter(term.Name, filter => filter
+				.Filter(f => f
+					.Term("TheTermField", term.Term)))
+		}
+
+		return aggr;
+	});
+
+	var result = client.Search<MySearchType>(descriptor);
+
+And then you can access the aggregation results as normal.

src/Connections/Elasticsearch.Net.Connection.HttpClient/HttpClientConnection.cs

@@ -3,6 +3,7 @@
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
+using System.Text;
 using System.Threading.Tasks;
 using Elasticsearch.Net.Connection.Configuration;
 
@@ -37,10 +38,14 @@ public HttpClientConnection(IConnectionConfigurationValues settings, HttpClientH
 
 			var innerHandler = handler ?? new WebRequestHandler();
 
+			if (settings.EnableCompressedResponses)
+				innerHandler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
+
 			if (innerHandler.SupportsProxy && !string.IsNullOrWhiteSpace(_settings.ProxyAddress))
 			{
 				innerHandler.Proxy = new WebProxy(_settings.ProxyAddress)
 				{
+
 					Credentials = new NetworkCredential(_settings.ProxyUsername, _settings.ProxyPassword),
 				};
 
@@ -51,6 +56,7 @@ public HttpClientConnection(IConnectionConfigurationValues settings, HttpClientH
 			{
 				Timeout = TimeSpan.FromMilliseconds(_settings.Timeout)
 			};
+
 		}
 
 		/// <summary>
@@ -115,24 +121,22 @@ public async Task<ElasticsearchResponse<Stream>> DoRequest(HttpMethod method, Ur
 			try
 			{
 				var request = new HttpRequestMessage(method, uri);
+				if (requestSpecificConfig != null && !string.IsNullOrWhiteSpace(requestSpecificConfig.ContentType))
+					request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(requestSpecificConfig.ContentType));
+				else if (!string.IsNullOrWhiteSpace(DefaultContentType))
+					request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(DefaultContentType));
+
+				if (!string.IsNullOrWhiteSpace(DefaultContentType))
+					request.Content.Headers.ContentType = new MediaTypeHeaderValue(DefaultContentType);
+
+				if (!string.IsNullOrEmpty(uri.UserInfo))
+				{
+					request.Headers.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes(uri.UserInfo)));
+				}
 
 				if (method != HttpMethod.Get && method != HttpMethod.Head && data != null && data.Length > 0)
 				{
 					request.Content = new ByteArrayContent(data);
-
-					if (requestSpecificConfig != null && !string.IsNullOrWhiteSpace(requestSpecificConfig.ContentType))
-					{
-						request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(requestSpecificConfig.ContentType));
-					}
-					else if (!string.IsNullOrWhiteSpace(DefaultContentType))
-					{
-						request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(DefaultContentType));
-					}
-
-					if (!string.IsNullOrWhiteSpace(DefaultContentType))
-					{
-						request.Content.Headers.ContentType = new MediaTypeHeaderValue(DefaultContentType);
-					}
 				}
 
 				var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);

src/Elasticsearch.Net/Connection/HttpConnection.cs

@@ -141,7 +141,7 @@ private static void ThreadTimeoutCallback(object state, bool timedOut)
 		protected virtual HttpWebRequest CreateHttpWebRequest(Uri uri, string method, byte[] data, IRequestConfiguration requestSpecificConfig)
 		{
 			var myReq = this.CreateWebRequest(uri, method, data, requestSpecificConfig);
-			this.SetBasicAuthorizationIfNeeded(myReq);
+			this.SetBasicAuthorizationIfNeeded(uri, myReq);
 			this.SetProxyIfNeeded(myReq);
 			return myReq;
 		}
@@ -164,15 +164,12 @@ private void SetProxyIfNeeded(HttpWebRequest myReq)
             }
 		}
 
-		private void SetBasicAuthorizationIfNeeded(HttpWebRequest myReq)
+		private void SetBasicAuthorizationIfNeeded(Uri uri, HttpWebRequest myReq)
 		{
-			//TODO figure out a way to cache this;
-
-			//if (this._ConnectionSettings.UriSpecifiedBasicAuth)
-			//{
-			myReq.Headers["Authorization"] =
-				"Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes(myReq.RequestUri.UserInfo));
-			//}
+			if (!uri.UserInfo.IsNullOrEmpty())
+			{
+				myReq.Headers["Authorization"] = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes(myReq.RequestUri.UserInfo));
+			}
 		}
 
 		protected virtual HttpWebRequest CreateWebRequest(Uri uri, string method, byte[] data, IRequestConfiguration requestSpecificConfig)

src/Elasticsearch.Net/Connection/RequestState/TransportRequestState.cs

@@ -5,6 +5,7 @@
 using System.Linq;
 using System.Text;
 using Elasticsearch.Net.Connection.Configuration;
+using Elasticsearch.Net.ConnectionPool;
 using PurifyNet;
 
 namespace Elasticsearch.Net.Connection.RequestState
@@ -29,6 +30,15 @@ public IRequestConfiguration RequestConfiguration
 			}
 		}
 
+		public bool UsingPooling
+		{
+			get
+			{
+				var pool = this.ClientSettings.ConnectionPool;
+				return pool != null && pool.GetType() != typeof(SingleNodeConnectionPool);
+			}
+		}
+
 		public string Method { get; private set; }
 
 		public string Path { get; private set; }

src/Elasticsearch.Net/Connection/Transport.cs

@@ -369,7 +369,7 @@ private ElasticsearchResponse<T> DoRequest<T>(TransportRequestState<T> requestSt
 			catch (Exception e)
 			{
 				requestState.SeenExceptions.Add(e);
-				if (maxRetries == 0 && retried == 0)
+				if (!requestState.UsingPooling || maxRetries == 0 && retried == 0)
 				{
 					if (_throwMaxRetry)
 						new MaxRetryException(e);
@@ -489,8 +489,8 @@ private Task<ElasticsearchResponse<T>> FinishOrRetryRequestAsync<T>(TransportReq
 					if (t.IsFaulted)
 					{
 						rq.Dispose();
-						if (maxRetries == 0 && retried == 0) throw t.Exception;
 						requestState.SeenExceptions.Add(t.Exception);
+						if (!requestState.UsingPooling || maxRetries == 0 && retried == 0) throw t.Exception;
 						return this.RetryRequestAsync<T>(requestState);
 					}
 
@@ -633,13 +633,13 @@ private ElasticsearchServerError ThrowOrGetErrorFromStreamResponse<T>(
 			TransportRequestState<T> requestState,
 			ElasticsearchResponse<Stream> streamResponse)
 		{
-			if (IsValidResponse(requestState, streamResponse))
-				return null;
-
 			if (((streamResponse.HttpStatusCode.HasValue && streamResponse.HttpStatusCode.Value <= 0)
 				|| !streamResponse.HttpStatusCode.HasValue) && streamResponse.OriginalException != null)
 				throw streamResponse.OriginalException;
 
+			if (IsValidResponse(requestState, streamResponse))
+				return null;
+
 			ElasticsearchServerError error = null;
 
 			var type = typeof(T);

src/Elasticsearch.Net/Domain/Response/ElasticsearchResponse.cs

@@ -1,6 +1,7 @@
 using System;
 using System.CodeDom;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.Configuration;
 using System.Globalization;
 using System.Linq;
@@ -12,6 +13,7 @@
 using System.Threading.Tasks;
 using Elasticsearch.Net;
 using Elasticsearch.Net.Connection;
+using Elasticsearch.Net.ConnectionPool;
 using Elasticsearch.Net.Exceptions;
 using Elasticsearch.Net.Serialization;
 
@@ -125,8 +127,12 @@ public bool SuccessOrKnownError
 		{
 			get
 			{
+				var pool = this.Settings.ConnectionPool;
+				var usingPool = pool != null && pool.GetType() != typeof(SingleNodeConnectionPool);
+
 				return this.Success ||
-					(this.HttpStatusCode.HasValue
+					(!usingPool && this.HttpStatusCode.GetValueOrDefault(1) < 0)
+					|| (this.HttpStatusCode.HasValue
 					&& this.HttpStatusCode.Value != 503 //service unavailable needs to be retried
 					&& this.HttpStatusCode.Value != 502 //bad gateway needs to be retried 
 					&& ((this.HttpStatusCode.Value >= 400 && this.HttpStatusCode.Value < 599)));
@@ -162,22 +168,26 @@ public static ElasticsearchResponse<T> CreateError(IConnectionConfigurationValue
 			return cs;
 		}
 
-		public static ElasticsearchResponse<T> Create(IConnectionConfigurationValues settings, int statusCode, string method, string path, byte[] request)
+		public static ElasticsearchResponse<T> Create(
+			IConnectionConfigurationValues settings, int statusCode, string method, string path, byte[] request, Exception innerException = null)
 		{
 			var cs = new ElasticsearchResponse<T>(settings, statusCode);
 			cs.Request = request;
 			cs.RequestUrl = path;
 			cs.RequestMethod = method;
+			cs.OriginalException = innerException;
 			return cs;
 		}
 
-		public static ElasticsearchResponse<T> Create(IConnectionConfigurationValues settings, int statusCode, string method, string path, byte[] request, T response)
+		public static ElasticsearchResponse<T> Create(
+			IConnectionConfigurationValues settings, int statusCode, string method, string path, byte[] request, T response, Exception innerException = null)
 		{
 			var cs = new ElasticsearchResponse<T>(settings, statusCode);
 			cs.Request = request;
 			cs.RequestUrl = path;
 			cs.RequestMethod = method;
 			cs.Response = response;
+			cs.OriginalException = innerException;
 			return cs;
 		}
 

src/Elasticsearch.Net/ElasticsearchClient.cs

@@ -50,14 +50,31 @@ public string Encoded(object o)
 			return Uri.EscapeDataString(this.Serializer.Stringify(o));
 		}
 
-
-		protected ElasticsearchResponse<T> DoRequest<T>(string method, string path, object data = null, IRequestParameters requestParameters = null)
+		
+		/// <summary>
+		/// Perform any request you want over the configured IConnection synchronously while taking advantage of the cluster failover.
+		/// </summary>
+		/// <typeparam name="T">The type representing the response JSON</typeparam>
+		/// <param name="method">the HTTP Method to use</param>
+		/// <param name="path">The path of the the url that you would like to hit</param>
+		/// <param name="data">The body of the request, string and byte[] are posted as is other types will be serialized to JSON</param>
+		/// <param name="requestParameters">Optionally configure request specific timeouts, headers</param>
+		/// <returns>An ElasticsearchResponse of T where T represents the JSON response body</returns>
+		public ElasticsearchResponse<T> DoRequest<T>(string method, string path, object data = null, IRequestParameters requestParameters = null)
 		{
 			return this.Transport.DoRequest<T>(method, path, data, requestParameters);
 		}
 
-
-		protected Task<ElasticsearchResponse<T>> DoRequestAsync<T>(string method, string path, object data = null, IRequestParameters requestParameters = null)
+		/// <summary>
+		/// Perform any request you want over the configured IConnection asynchronously while taking advantage of the cluster failover.
+		/// </summary>
+		/// <typeparam name="T">The type representing the response JSON</typeparam>
+		/// <param name="method">the HTTP Method to use</param>
+		/// <param name="path">The path of the the url that you would like to hit</param>
+		/// <param name="data">The body of the request, string and byte[] are posted as is other types will be serialized to JSON</param>
+		/// <param name="requestParameters">Optionally configure request specific timeouts, headers</param>
+		/// <returns>A task of ElasticsearchResponse of T where T represents the JSON response body</returns>
+		public Task<ElasticsearchResponse<T>> DoRequestAsync<T>(string method, string path, object data = null, IRequestParameters requestParameters = null)
 		{
 			return this.Transport.DoRequestAsync<T>(method, path, data, requestParameters);
 		}

src/Elasticsearch.Net/IElasticsearchClient.Generated.cs

@@ -19439,6 +19439,27 @@ public interface IElasticsearchClient
 		///</returns>
 
 		Task<ElasticsearchResponse<DynamicDictionary>> UpdateAsync(string index, string type, string id, object body, Func<UpdateRequestParameters, UpdateRequestParameters> requestParameters = null);
-		
+
+		/// <summary>
+		/// Perform any request you want over the configured IConnection synchronously while taking advantage of the cluster failover.
+		/// </summary>
+		/// <typeparam name="T">The type representing the response JSON</typeparam>
+		/// <param name="method">the HTTP Method to use</param>
+		/// <param name="path">The path of the the url that you would like to hit</param>
+		/// <param name="data">The body of the request, string and byte[] are posted as is other types will be serialized to JSON</param>
+		/// <param name="requestParameters">Optionally configure request specific timeouts, headers</param>
+		/// <returns>An ElasticsearchResponse of T where T represents the JSON response body</returns>
+		ElasticsearchResponse<T> DoRequest<T>(string method, string path, object data = null, IRequestParameters requestParameters = null);
+
+		/// <summary>
+		/// Perform any request you want over the configured IConnection asynchronously while taking advantage of the cluster failover.
+		/// </summary>
+		/// <typeparam name="T">The type representing the response JSON</typeparam>
+		/// <param name="method">the HTTP Method to use</param>
+		/// <param name="path">The path of the the url that you would like to hit</param>
+		/// <param name="data">The body of the request, string and byte[] are posted as is other types will be serialized to JSON</param>
+		/// <param name="requestParameters">Optionally configure request specific timeouts, headers</param>
+		/// <returns>A task of ElasticsearchResponse of T where T represents the JSON response body</returns>
+		Task<ElasticsearchResponse<T>> DoRequestAsync<T>(string method, string path, object data = null, IRequestParameters requestParameters = null);
 	}
 }

src/Nest/ElasticClient-DoRequest.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Elasticsearch.Net;
+
+namespace Nest
+{
+	public partial class ElasticClient
+	{
+		/// <summary>
+		/// Perform any request you want over the configured IConnection while taking advantage of the cluster failover.
+		/// </summary>
+		/// <typeparam name="T">The type representing the response JSON</typeparam>
+		/// <param name="method">the HTTP Method to use</param>
+		/// <param name="path">The path of the the url that you would like to hit</param>
+		/// <param name="data">The body of the request, string and byte[] are posted as is other types will be serialized to JSON</param>
+		/// <param name="requestParameters">Optionally configure request specific timeouts, headers</param>
+		/// <returns>An ElasticsearchResponse of T where T represents the JSON response body</returns>
+		public ElasticsearchResponse<T> DoRequest<T>(string method, string path, object data = null, IRequestParameters requestParameters = null)
+		{
+			return this.Raw.DoRequest<T>(method, path, data, requestParameters);
+		} 
+
+		/// <summary>
+		/// Perform any request you want over the configured IConnection asynchronously while taking advantage of the cluster failover.
+		/// </summary>
+		/// <typeparam name="T">The type representing the response JSON</typeparam>
+		/// <param name="method">the HTTP Method to use</param>
+		/// <param name="path">The path of the the url that you would like to hit</param>
+		/// <param name="data">The body of the request, string and byte[] are posted as is other types will be serialized to JSON</param>
+		/// <param name="requestParameters">Optionally configure request specific timeouts, headers</param>
+		/// <returns>A task of ElasticsearchResponse of T where T represents the JSON response body</returns>
+		public Task<ElasticsearchResponse<T>> DoRequestAsync<T>(string method, string path, object data = null, IRequestParameters requestParameters = null)
+		{
+			return this.Raw.DoRequestAsync<T>(method, path, data, requestParameters);
+		} 
+
+	}
+}

src/Nest/ElasticClient.cs

@@ -111,7 +111,7 @@ private static R CreateInvalidInstance<R>(IElasticsearchResponse response) where
 			return r;
 		}
 
-		internal Task<I> DispatchAsync<D, Q, R, I>(
+		private Task<I> DispatchAsync<D, Q, R, I>(
 			Func<D, D> selector
 			, Func<ElasticsearchPathInfo<Q>, D, Task<ElasticsearchResponse<R>>> dispatch
 			)