Skip to content

Commit fecdf8d

Browse files
deepdattadeepdatta
authored
Support index templates (#169)
* Add index_template versions of all the default templates and update Rakefile to generate that for ecs-1x * Add an option for legacy_template and use _index_template API when its set to false * Update unit test to check template_install Signed-off-by: Deep Datta <deedatta@amazon.com>
1 parent 179d611 commit fecdf8d

File tree

13 files changed

+16033
-32
lines changed

13 files changed

+16033
-32
lines changed

‎Rakefile‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@
77
# Modifications Copyright OpenSearch Contributors. See
88
# GitHub history for details.
99

10-
require "logstash/devutils/rake"
11-
1210
ECS_VERSIONS = {
1311
v1: 'v1.9.0'
1412
}
@@ -35,14 +33,37 @@ def download_ecs_schema(ecs_major_version, opensearch_major_version)
3533
template_directory = File.expand_path("../lib/logstash/outputs/opensearch/templates/ecs-#{ecs_major_version}", __FILE__)
3634
Dir.mkdir(template_directory) unless File.exists?(template_directory)
3735
template_file = File.join(template_directory, "/#{opensearch_major_version}x.json")
36+
template = replace_index_patterns(response.body, ECS_LOGSTASH_INDEX_PATTERNS)
3837
File.open(template_file, "w") do |handle|
39-
handle.write(replace_index_patterns(response.body, ECS_LOGSTASH_INDEX_PATTERNS))
38+
handle.write(JSON.pretty_generate(template))
39+
end
40+
index_template_file = File.join(template_directory, "/#{opensearch_major_version}x_index.json")
41+
template = transform_to_index_template(template)
42+
File.open(index_template_file, "w") do |handle|
43+
handle.write(JSON.pretty_generate(template))
4044
end
4145
end
4246
end
4347

4448
def replace_index_patterns(template_json, replacement_index_patterns)
4549
template_obj = JSON.load(template_json)
4650
template_obj.update('index_patterns' => replacement_index_patterns)
47-
JSON.pretty_generate(template_obj)
51+
template_obj
52+
end
53+
54+
def transform_to_index_template(template)
55+
if !template.key?("template")
56+
57+
# `order` is replaced with `priority`
58+
template.delete("order")
59+
template["priority"] = 10
60+
61+
# index_templates have `settings` and `mappings` under `template`
62+
template["template"] = {
63+
"settings" => template.delete("settings"),
64+
"mappings" => template.delete("mappings")
65+
}
66+
end
67+
template
4868
end
69+

‎lib/logstash/outputs/opensearch.rb‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,10 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
200200
# here like `pipeline => "%{INGEST_PIPELINE}"`
201201
config :pipeline, :validate => :string, :default => nil
202202

203+
# When set to true, use legacy templates via the _template API
204+
# When false, use index templates using the _index_template API
205+
config :legacy_template, :validate => :boolean, :default => true
206+
203207
attr_reader :client
204208
attr_reader :default_index
205209
attr_reader :default_template_name

‎lib/logstash/outputs/opensearch/http_client.rb‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -388,10 +388,16 @@ def template_put(name, template)
388388
@pool.put(path, nil, LogStash::Json.dump(template))
389389
end
390390

391+
def legacy_template?()
392+
# TODO: Also check Version and return true for < 7.8 even if :legacy_template=false
393+
# Need to figure a way to distinguish between OpenSearch, OpenDistro and other
394+
# variants, since they have version numbers in different ranges.
395+
client_settings.fetch(:legacy_template, true)
396+
end
397+
391398
def template_endpoint
392-
# TODO: Check Version < 7.8 and use index template for >= 7.8 & OpenSearch
393399
# https://opensearch.org/docs/opensearch/index-templates/
394-
'_template'
400+
legacy_template?() ? '_template' : '_index_template'
395401
end
396402

397403
# check whether rollover alias already exists

‎lib/logstash/outputs/opensearch/http_client_builder.rb‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ def self.build(logger, hosts, params)
1818
:pool_max_per_route => params["pool_max_per_route"],
1919
:check_connection_timeout => params["validate_after_inactivity"],
2020
:http_compression => params["http_compression"],
21-
:headers => params["custom_headers"] || {}
21+
:headers => params["custom_headers"] || {},
22+
:legacy_template => params["legacy_template"]
2223
}
2324

2425
client_settings[:proxy] = params["proxy"] if params["proxy"]

‎lib/logstash/outputs/opensearch/template_manager.rb‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,16 @@ def self.install_template(plugin)
1818
else
1919
plugin.logger.info("Using a default mapping template", :version => plugin.maximum_seen_major_version,
2020
:ecs_compatibility => plugin.ecs_compatibility)
21-
template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility)
21+
template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility, plugin.client.legacy_template?)
2222
end
2323

2424
plugin.logger.debug("Attempting to install template", template: template)
2525
install(plugin.client, template_name(plugin), template, plugin.template_overwrite)
2626
end
2727

2828
private
29-
def self.load_default_template(major_version, ecs_compatibility)
30-
template_path = default_template_path(major_version, ecs_compatibility)
29+
def self.load_default_template(major_version, ecs_compatibility, legacy_template)
30+
template_path = default_template_path(major_version, ecs_compatibility, legacy_template)
3131
read_template_file(template_path)
3232
rescue => e
3333
fail "Failed to load default template for OpenSearch v#{major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
@@ -45,9 +45,10 @@ def self.template_name(plugin)
4545
plugin.template_name
4646
end
4747

48-
def self.default_template_path(major_version, ecs_compatibility=:disabled)
48+
def self.default_template_path(major_version, ecs_compatibility=:disabled, legacy_template=true)
4949
template_version = major_version
50-
default_template_name = "templates/ecs-#{ecs_compatibility}/#{template_version}x.json"
50+
suffix = legacy_template ? "" : "_index"
51+
default_template_name = "templates/ecs-#{ecs_compatibility}/#{template_version}x#{suffix}.json"
5152
::File.expand_path(default_template_name, ::File.dirname(__FILE__))
5253
end
5354

‎lib/logstash/outputs/opensearch/templates/ecs-disabled/1x_index.json‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
"index_patterns": "logstash-*",
3+
"version": 60001,
4+
"priority": 10,
5+
"template": {
6+
"settings": {
7+
"index.refresh_interval": "5s",
8+
"number_of_shards": 1
9+
},
10+
"mappings": {
11+
"dynamic_templates": [
12+
{
13+
"message_field": {
14+
"path_match": "message",
15+
"match_mapping_type": "string",
16+
"mapping": {
17+
"type": "text",
18+
"norms": false
19+
}
20+
}
21+
},
22+
{
23+
"string_fields": {
24+
"match": "*",
25+
"match_mapping_type": "string",
26+
"mapping": {
27+
"type": "text",
28+
"norms": false,
29+
"fields": {
30+
"keyword": {
31+
"type": "keyword",
32+
"ignore_above": 256
33+
}
34+
}
35+
}
36+
}
37+
}
38+
],
39+
"properties": {
40+
"@timestamp": {
41+
"type": "date"
42+
},
43+
"@version": {
44+
"type": "keyword"
45+
},
46+
"geoip": {
47+
"dynamic": true,
48+
"properties": {
49+
"ip": {
50+
"type": "ip"
51+
},
52+
"location": {
53+
"type": "geo_point"
54+
},
55+
"latitude": {
56+
"type": "half_float"
57+
},
58+
"longitude": {
59+
"type": "half_float"
60+
}
61+
}
62+
}
63+
}
64+
}
65+
}
66+
}

‎lib/logstash/outputs/opensearch/templates/ecs-disabled/2x_index.json‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
"index_patterns": "logstash-*",
3+
"version": 60001,
4+
"priority": 10,
5+
"template": {
6+
"settings": {
7+
"index.refresh_interval": "5s",
8+
"number_of_shards": 1
9+
},
10+
"mappings": {
11+
"dynamic_templates": [
12+
{
13+
"message_field": {
14+
"path_match": "message",
15+
"match_mapping_type": "string",
16+
"mapping": {
17+
"type": "text",
18+
"norms": false
19+
}
20+
}
21+
},
22+
{
23+
"string_fields": {
24+
"match": "*",
25+
"match_mapping_type": "string",
26+
"mapping": {
27+
"type": "text",
28+
"norms": false,
29+
"fields": {
30+
"keyword": {
31+
"type": "keyword",
32+
"ignore_above": 256
33+
}
34+
}
35+
}
36+
}
37+
}
38+
],
39+
"properties": {
40+
"@timestamp": {
41+
"type": "date"
42+
},
43+
"@version": {
44+
"type": "keyword"
45+
},
46+
"geoip": {
47+
"dynamic": true,
48+
"properties": {
49+
"ip": {
50+
"type": "ip"
51+
},
52+
"location": {
53+
"type": "geo_point"
54+
},
55+
"latitude": {
56+
"type": "half_float"
57+
},
58+
"longitude": {
59+
"type": "half_float"
60+
}
61+
}
62+
}
63+
}
64+
}
65+
}
66+
}

‎lib/logstash/outputs/opensearch/templates/ecs-disabled/7x_index.json‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
"index_patterns": "logstash-*",
3+
"version": 60001,
4+
"priority": 10,
5+
"template": {
6+
"settings": {
7+
"index.refresh_interval": "5s",
8+
"number_of_shards": 1
9+
},
10+
"mappings": {
11+
"dynamic_templates": [
12+
{
13+
"message_field": {
14+
"path_match": "message",
15+
"match_mapping_type": "string",
16+
"mapping": {
17+
"type": "text",
18+
"norms": false
19+
}
20+
}
21+
},
22+
{
23+
"string_fields": {
24+
"match": "*",
25+
"match_mapping_type": "string",
26+
"mapping": {
27+
"type": "text",
28+
"norms": false,
29+
"fields": {
30+
"keyword": {
31+
"type": "keyword",
32+
"ignore_above": 256
33+
}
34+
}
35+
}
36+
}
37+
}
38+
],
39+
"properties": {
40+
"@timestamp": {
41+
"type": "date"
42+
},
43+
"@version": {
44+
"type": "keyword"
45+
},
46+
"geoip": {
47+
"dynamic": true,
48+
"properties": {
49+
"ip": {
50+
"type": "ip"
51+
},
52+
"location": {
53+
"type": "geo_point"
54+
},
55+
"latitude": {
56+
"type": "half_float"
57+
},
58+
"longitude": {
59+
"type": "half_float"
60+
}
61+
}
62+
}
63+
}
64+
}
65+
}
66+
}

0 commit comments

Comments
 (0)