Exclude hidden documents from search (#2277)

* Exclude hidden documents from search

* Add mapping

Author: reakaleek

src/Elastic.Documentation/Search/DocumentationDocument.cs

@@ -65,4 +65,8 @@ public record DocumentationDocument
 
 	[JsonPropertyName("parents")]
 	public ParentDocument[] Parents { get; set; } = [];
+
+	[JsonPropertyName("hidden")]
+	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+	public bool Hidden { get; set; }
 }

src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchExporter.cs

@@ -199,6 +199,9 @@ protected static string CreateMapping(string? inferenceId) =>
 		          "prefix": { "type": "text", "analyzer" : "hierarchy_analyzer" }
 		        }
 		      },
+		      "hidden" : {
+		        "type" : "boolean"
+		      },
 		      "applies_to" : {
 		        "type" : "nested",
 		        "properties" : {

src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchMarkdownExporter.cs

@@ -451,7 +451,8 @@ public async ValueTask<bool> ExportAsync(MarkdownExportFileContext fileContext,
 				Title = i.NavigationTitle,
 				Url = i.Url
 			}).Reverse().ToArray(),
-			Headings = headings
+			Headings = headings,
+			Hidden = fileContext.NavigationItem.Hidden
 		};
 
 		AssignDocumentMetadata(doc);

src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Search/ElasticsearchGateway.cs

@@ -48,6 +48,9 @@ internal sealed record DocumentDto
 
 	[JsonPropertyName("applies_to")]
 	public ApplicableTo? Applies { get; init; }
+
+	[JsonPropertyName("hidden")]
+	public bool Hidden { get; init; } = false;
 }
 
 internal sealed record ParentDocumentDto
@@ -100,7 +103,8 @@ private static Query BuildLexicalQuery(string searchQuery) =>
 		 || new MatchQuery(Infer.Field<DocumentDto>(f => f.Parents.First().Title), searchQuery) { Boost = 2.0f }
 		 || new MatchQuery(Infer.Field<DocumentDto>(f => f.Title), searchQuery) { Fuzziness = 1, Boost = 1.0f }
 		)
-		&& !(Query)new TermsQuery(Infer.Field<DocumentDto>(f => f.Url.Suffix("keyword")), new TermsQueryField(["/docs", "/docs/", "/docs/404", "/docs/404/"]));
+		&& !(Query)new TermsQuery(Infer.Field<DocumentDto>(f => f.Url.Suffix("keyword")), new TermsQueryField(["/docs", "/docs/", "/docs/404", "/docs/404/"]))
+		&& !(Query)new TermQuery { Field = Infer.Field<DocumentDto>(f => f.Hidden), Value = true };
 
 	/// <summary>
 	/// Builds the semantic search query for the given search term.
@@ -110,7 +114,8 @@ private static Query BuildSemanticQuery(string searchQuery) =>
 		 || new SemanticQuery("abstract.semantic_text", searchQuery) { Boost = 3.0f }
 		)
 		&& !(Query)new TermsQuery(Infer.Field<DocumentDto>(f => f.Url.Suffix("keyword")),
-			new TermsQueryField(["/docs", "/docs/", "/docs/404", "/docs/404/"]));
+			new TermsQueryField(["/docs", "/docs/", "/docs/404", "/docs/404/"]))
+		&& !(Query)new TermQuery { Field = Infer.Field<DocumentDto>(f => f.Hidden), Value = true };
 
 	/// <summary>
 	/// Normalizes the search query by replacing "dotnet" with "net".