Skip to content

Commit 25ffc15

Browse files
joegallojoegallo
authored
Support IPinfo database configurations (elastic#114548) (elastic#114663)
1 parent f61c9d0 commit 25ffc15

File tree

17 files changed

+449
-36
lines changed

17 files changed

+449
-36
lines changed

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.elasticsearch.common.CheckedSupplier;
2424
import org.elasticsearch.common.Strings;
2525
import org.elasticsearch.common.hash.MessageDigests;
26+
import org.elasticsearch.core.Nullable;
2627
import org.elasticsearch.core.TimeValue;
2728
import org.elasticsearch.core.Tuple;
2829
import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -236,7 +237,7 @@ boolean processDatabase(String id, DatabaseConfiguration database) throws IOExce
236237
logger.debug("Processing database [{}] for configuration [{}]", name, database.id());
237238

238239
try (ProviderDownload downloader = downloaderFor(database)) {
239-
if (downloader.validCredentials()) {
240+
if (downloader != null && downloader.validCredentials()) {
240241
// the name that comes from the enterprise downloader cluster state doesn't include the .mmdb extension,
241242
// but the downloading and indexing of database code expects it to be there, so we add it on here before continuing
242243
final String fileName = name + ".mmdb";
@@ -443,10 +444,17 @@ private void scheduleNextRun(TimeValue time) {
443444
}
444445
}
445446

447+
@Nullable
446448
private ProviderDownload downloaderFor(DatabaseConfiguration database) {
447-
assert database.provider() instanceof DatabaseConfiguration.Maxmind
448-
: "Attempt to use maxmind downloader with a provider of type" + database.provider().getClass();
449-
return new MaxmindDownload(database.name(), (DatabaseConfiguration.Maxmind) database.provider());
449+
if (database.provider() instanceof DatabaseConfiguration.Maxmind) {
450+
return new MaxmindDownload(database.name(), (DatabaseConfiguration.Maxmind) database.provider());
451+
} else if (database.provider() instanceof DatabaseConfiguration.Ipinfo) {
452+
// as a temporary implementation detail, null here means 'not actually supported *just yet*'
453+
return null;
454+
} else {
455+
assert false : "Attempted to use database downloader with unsupported provider type [" + database.provider().getClass() + "]";
456+
return null;
457+
}
450458
}
451459

452460
class MaxmindDownload implements ProviderDownload {

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER;
5656
import static org.elasticsearch.ingest.geoip.GeoIpProcessor.Factory.downloadDatabaseOnPipelineCreation;
5757
import static org.elasticsearch.ingest.geoip.GeoIpProcessor.GEOIP_TYPE;
58+
import static org.elasticsearch.ingest.geoip.GeoIpProcessor.IP_LOCATION_TYPE;
5859

5960
/**
6061
* Persistent task executor that is responsible for starting {@link GeoIpDownloader} after task is allocated by master node.
@@ -297,9 +298,18 @@ private static boolean hasAtLeastOneGeoipProcessor(Map<String, Object> processor
297298
return false;
298299
}
299300

300-
final Map<String, Object> processorConfig = (Map<String, Object>) processor.get(GEOIP_TYPE);
301-
if (processorConfig != null) {
302-
return downloadDatabaseOnPipelineCreation(GEOIP_TYPE, processorConfig, null) == downloadDatabaseOnPipelineCreation;
301+
{
302+
final Map<String, Object> processorConfig = (Map<String, Object>) processor.get(GEOIP_TYPE);
303+
if (processorConfig != null) {
304+
return downloadDatabaseOnPipelineCreation(GEOIP_TYPE, processorConfig, null) == downloadDatabaseOnPipelineCreation;
305+
}
306+
}
307+
308+
{
309+
final Map<String, Object> processorConfig = (Map<String, Object>) processor.get(IP_LOCATION_TYPE);
310+
if (processorConfig != null) {
311+
return downloadDatabaseOnPipelineCreation(IP_LOCATION_TYPE, processorConfig, null) == downloadDatabaseOnPipelineCreation;
312+
}
303313
}
304314

305315
return isProcessorWithOnFailureGeoIpProcessor(processor, downloadDatabaseOnPipelineCreation)

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
4242
+ "in a future version of Elasticsearch"; // TODO add a message about migration?
4343

4444
public static final String GEOIP_TYPE = "geoip";
45+
public static final String IP_LOCATION_TYPE = "ip_location";
4546

4647
private final String type;
4748
private final String field;
@@ -225,7 +226,7 @@ public Processor create(
225226
final Map<String, Object> config
226227
) throws IOException {
227228
String ipField = readStringProperty(type, processorTag, config, "field");
228-
String targetField = readStringProperty(type, processorTag, config, "target_field", "geoip");
229+
String targetField = readStringProperty(type, processorTag, config, "target_field", type);
229230
String databaseFile = readStringProperty(type, processorTag, config, "database_file", "GeoLite2-City.mmdb");
230231
List<String> propertyNames = readOptionalList(type, processorTag, config, "properties");
231232
boolean ignoreMissing = readBooleanProperty(type, processorTag, config, "ignore_missing", false);

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
import java.util.function.Predicate;
7272
import java.util.function.Supplier;
7373

74+
import static java.util.Map.entry;
7475
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
7576
import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER;
7677
import static org.elasticsearch.ingest.IngestService.INGEST_ORIGIN;
@@ -129,7 +130,10 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
129130
parameters.ingestService.getClusterService()
130131
);
131132
databaseRegistry.set(registry);
132-
return Map.of(GeoIpProcessor.GEOIP_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.GEOIP_TYPE, registry));
133+
return Map.ofEntries(
134+
entry(GeoIpProcessor.GEOIP_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.GEOIP_TYPE, registry)),
135+
entry(GeoIpProcessor.IP_LOCATION_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.IP_LOCATION_TYPE, registry))
136+
);
133137
}
134138

135139
@Override
@@ -239,6 +243,11 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
239243
DatabaseConfiguration.Maxmind.NAME,
240244
DatabaseConfiguration.Maxmind::new
241245
),
246+
new NamedWriteableRegistry.Entry(
247+
DatabaseConfiguration.Provider.class,
248+
DatabaseConfiguration.Ipinfo.NAME,
249+
DatabaseConfiguration.Ipinfo::new
250+
),
242251
new NamedWriteableRegistry.Entry(
243252
DatabaseConfiguration.Provider.class,
244253
DatabaseConfiguration.Local.NAME,

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java‎

Lines changed: 66 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
import java.io.IOException;
2828
import java.nio.charset.StandardCharsets;
29+
import java.util.Arrays;
2930
import java.util.Objects;
3031
import java.util.Set;
3132
import java.util.regex.Pattern;
@@ -78,8 +79,19 @@ public record DatabaseConfiguration(String id, String name, Provider provider) i
7879
// "GeoLite2-Country"
7980
);
8081

82+
public static final Set<String> IPINFO_NAMES = Set.of(
83+
// these file names are from https://ipinfo.io/developers/database-filename-reference
84+
"asn", // "Free IP to ASN"
85+
"country", // "Free IP to Country"
86+
// "country_asn" // "Free IP to Country + IP to ASN", not supported at present
87+
"standard_asn", // commercial "ASN"
88+
"standard_location", // commercial "IP Geolocation"
89+
"standard_privacy" // commercial "Privacy Detection" (sometimes "Anonymous IP")
90+
);
91+
8192
private static final ParseField NAME = new ParseField("name");
8293
private static final ParseField MAXMIND = new ParseField(Maxmind.NAME);
94+
private static final ParseField IPINFO = new ParseField(Ipinfo.NAME);
8395
private static final ParseField WEB = new ParseField(Web.NAME);
8496
private static final ParseField LOCAL = new ParseField(Local.NAME);
8597

@@ -89,12 +101,21 @@ public record DatabaseConfiguration(String id, String name, Provider provider) i
89101
(a, id) -> {
90102
String name = (String) a[0];
91103
Provider provider;
104+
105+
// one and only one provider object must be present
106+
final long numNonNulls = Arrays.stream(a, 1, a.length).filter(Objects::nonNull).count();
107+
if (numNonNulls != 1) {
108+
throw new IllegalArgumentException("Exactly one provider object must be specified, but [" + numNonNulls + "] were found");
109+
}
110+
92111
if (a[1] != null) {
93112
provider = (Maxmind) a[1];
94113
} else if (a[2] != null) {
95-
provider = (Web) a[2];
114+
provider = (Ipinfo) a[2];
115+
} else if (a[3] != null) {
116+
provider = (Web) a[3];
96117
} else {
97-
provider = (Local) a[3];
118+
provider = (Local) a[4];
98119
}
99120
return new DatabaseConfiguration(id, name, provider);
100121
}
@@ -107,6 +128,7 @@ public record DatabaseConfiguration(String id, String name, Provider provider) i
107128
(parser, id) -> Maxmind.PARSER.apply(parser, null),
108129
MAXMIND
109130
);
131+
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Ipinfo.PARSER.apply(parser, null), IPINFO);
110132
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Web.PARSER.apply(parser, null), WEB);
111133
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Local.PARSER.apply(parser, null), LOCAL);
112134
}
@@ -194,8 +216,16 @@ public ActionRequestValidationException validate() {
194216
err.addValidationError("invalid name [" + name + "]: cannot be empty");
195217
}
196218

197-
if (MAXMIND_NAMES.contains(name) == false) {
198-
err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + MAXMIND_NAMES + "])");
219+
// provider-specific name validation
220+
if (provider instanceof Maxmind) {
221+
if (MAXMIND_NAMES.contains(name) == false) {
222+
err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + MAXMIND_NAMES + "])");
223+
}
224+
}
225+
if (provider instanceof Ipinfo) {
226+
if (IPINFO_NAMES.contains(name) == false) {
227+
err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + IPINFO_NAMES + "])");
228+
}
199229
}
200230

201231
// important: the name must be unique across all configurations of this same type,
@@ -234,7 +264,7 @@ public String getWriteableName() {
234264

235265
private static final ParseField ACCOUNT_ID = new ParseField("account_id");
236266

237-
private static final ConstructingObjectParser<Maxmind, Void> PARSER = new ConstructingObjectParser<>("database", false, (a, id) -> {
267+
private static final ConstructingObjectParser<Maxmind, Void> PARSER = new ConstructingObjectParser<>("maxmind", false, (a, id) -> {
238268
String accountId = (String) a[0];
239269
return new Maxmind(accountId);
240270
});
@@ -247,10 +277,6 @@ public Maxmind(StreamInput in) throws IOException {
247277
this(in.readString());
248278
}
249279

250-
public static Maxmind parse(XContentParser parser) {
251-
return PARSER.apply(parser, null);
252-
}
253-
254280
@Override
255281
public void writeTo(StreamOutput out) throws IOException {
256282
out.writeString(accountId);
@@ -270,6 +296,37 @@ public boolean isReadOnly() {
270296
}
271297
}
272298

299+
public record Ipinfo() implements Provider {
300+
public static final String NAME = "ipinfo";
301+
302+
// this'll become a ConstructingObjectParser once we accept the token (securely) in the json definition
303+
private static final ObjectParser<Ipinfo, Void> PARSER = new ObjectParser<>("ipinfo", Ipinfo::new);
304+
305+
public Ipinfo(StreamInput in) throws IOException {
306+
this();
307+
}
308+
309+
@Override
310+
public void writeTo(StreamOutput out) throws IOException {}
311+
312+
@Override
313+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
314+
builder.startObject();
315+
builder.endObject();
316+
return builder;
317+
}
318+
319+
@Override
320+
public String getWriteableName() {
321+
return NAME;
322+
}
323+
324+
@Override
325+
public boolean isReadOnly() {
326+
return false;
327+
}
328+
}
329+
273330
public record Local(String type) implements Provider {
274331
public static final String NAME = "local";
275332

@@ -288,10 +345,6 @@ public Local(StreamInput in) throws IOException {
288345
this(in.readString());
289346
}
290347

291-
public static Local parse(XContentParser parser) {
292-
return PARSER.apply(parser, null);
293-
}
294-
295348
@Override
296349
public void writeTo(StreamOutput out) throws IOException {
297350
out.writeString(type);
@@ -325,10 +378,6 @@ public Web(StreamInput in) throws IOException {
325378
this();
326379
}
327380

328-
public static Web parse(XContentParser parser) {
329-
return PARSER.apply(parser, null);
330-
}
331-
332381
@Override
333382
public void writeTo(StreamOutput out) throws IOException {}
334383

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ public class RestDeleteDatabaseConfigurationAction extends BaseRestHandler {
2727

2828
@Override
2929
public List<Route> routes() {
30-
return List.of(new Route(DELETE, "/_ingest/geoip/database/{id}"));
30+
return List.of(new Route(DELETE, "/_ingest/ip_location/database/{id}"), new Route(DELETE, "/_ingest/geoip/database/{id}"));
3131
}
3232

3333
@Override

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,12 @@ public class RestGetDatabaseConfigurationAction extends BaseRestHandler {
2626

2727
@Override
2828
public List<Route> routes() {
29-
return List.of(new Route(GET, "/_ingest/geoip/database"), new Route(GET, "/_ingest/geoip/database/{id}"));
29+
return List.of(
30+
new Route(GET, "/_ingest/ip_location/database"),
31+
new Route(GET, "/_ingest/ip_location/database/{id}"),
32+
new Route(GET, "/_ingest/geoip/database"),
33+
new Route(GET, "/_ingest/geoip/database/{id}")
34+
);
3035
}
3136

3237
@Override

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public class RestPutDatabaseConfigurationAction extends BaseRestHandler {
2929

3030
@Override
3131
public List<Route> routes() {
32-
return List.of(new Route(PUT, "/_ingest/geoip/database/{id}"));
32+
return List.of(new Route(PUT, "/_ingest/ip_location/database/{id}"), new Route(PUT, "/_ingest/geoip/database/{id}"));
3333
}
3434

3535
@Override

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.elasticsearch.core.Nullable;
3030
import org.elasticsearch.core.Strings;
3131
import org.elasticsearch.core.Tuple;
32+
import org.elasticsearch.features.FeatureService;
3233
import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata;
3334
import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction.Request;
3435
import org.elasticsearch.injection.guice.Inject;
@@ -41,6 +42,8 @@
4142
import java.util.Map;
4243
import java.util.Optional;
4344

45+
import static org.elasticsearch.ingest.IngestGeoIpFeatures.PUT_DATABASE_CONFIGURATION_ACTION_IPINFO;
46+
4447
public class TransportPutDatabaseConfigurationAction extends TransportMasterNodeAction<Request, AcknowledgedResponse> {
4548

4649
private static final Logger logger = LogManager.getLogger(TransportPutDatabaseConfigurationAction.class);
@@ -58,6 +61,7 @@ public void taskSucceeded(UpdateDatabaseConfigurationTask task, Void unused) {
5861
}
5962
};
6063

64+
private final FeatureService featureService;
6165
private final MasterServiceTaskQueue<UpdateDatabaseConfigurationTask> updateDatabaseConfigurationTaskQueue;
6266

6367
@Inject
@@ -66,7 +70,8 @@ public TransportPutDatabaseConfigurationAction(
6670
ClusterService clusterService,
6771
ThreadPool threadPool,
6872
ActionFilters actionFilters,
69-
IndexNameExpressionResolver indexNameExpressionResolver
73+
IndexNameExpressionResolver indexNameExpressionResolver,
74+
FeatureService featureService
7075
) {
7176
super(
7277
PutDatabaseConfigurationAction.NAME,
@@ -79,6 +84,7 @@ public TransportPutDatabaseConfigurationAction(
7984
AcknowledgedResponse::readFrom,
8085
EsExecutors.DIRECT_EXECUTOR_SERVICE
8186
);
87+
this.featureService = featureService;
8288
this.updateDatabaseConfigurationTaskQueue = clusterService.createTaskQueue(
8389
"update-geoip-database-configuration-state-update",
8490
Priority.NORMAL,
@@ -89,6 +95,19 @@ public TransportPutDatabaseConfigurationAction(
8995
@Override
9096
protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
9197
final String id = request.getDatabase().id();
98+
99+
// if this is an ipinfo configuration, then make sure the whole cluster supports that feature
100+
if (request.getDatabase().provider() instanceof DatabaseConfiguration.Ipinfo
101+
&& featureService.clusterHasFeature(clusterService.state(), PUT_DATABASE_CONFIGURATION_ACTION_IPINFO) == false) {
102+
listener.onFailure(
103+
new IllegalArgumentException(
104+
"Unable to use ipinfo database configurations in mixed-clusters with nodes that do not support feature "
105+
+ PUT_DATABASE_CONFIGURATION_ACTION_IPINFO.id()
106+
)
107+
);
108+
return;
109+
}
110+
92111
updateDatabaseConfigurationTaskQueue.submitTask(
93112
Strings.format("update-geoip-database-configuration-[%s]", id),
94113
new UpdateDatabaseConfigurationTask(listener, request.getDatabase()),

0 commit comments

Comments
 (0)