Skip to content

Commit c45792e

Browse files
robinhood-jimrobinhood-jim
authored
Merge pull request #46 from robinhood-jim/develop
Develop
2 parents 3b82c9d + 1a3680a commit c45792e

File tree

42 files changed

+2170
-755
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+2170
-755
lines changed

‎common/pom.xml‎

Lines changed: 51 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -23,97 +23,91 @@
2323
<dependency>
2424
<groupId>org.slf4j</groupId>
2525
<artifactId>slf4j-log4j12</artifactId>
26+
<optional>true</optional>
2627
</dependency>
2728

2829
<dependency>
2930
<groupId>com.google.code.gson</groupId>
3031
<artifactId>gson</artifactId>
32+
<optional>true</optional>
3133
</dependency>
32-
<dependency>
33-
<groupId>commons-lang</groupId>
34-
<artifactId>commons-lang</artifactId>
35-
</dependency>
36-
<dependency>
37-
<groupId>mysql</groupId>
38-
<artifactId>mysql-connector-java</artifactId>
39-
</dependency>
34+
35+
4036
<dependency>
4137
<groupId>org.apache.commons</groupId>
4238
<artifactId>commons-pool2</artifactId>
39+
<optional>true</optional>
4340
</dependency>
44-
<dependency>
45-
<groupId>commons-io</groupId>
46-
<artifactId>commons-io</artifactId>
47-
</dependency>
48-
<dependency>
49-
<groupId>commons-codec</groupId>
50-
<artifactId>commons-codec</artifactId>
51-
</dependency>
52-
<dependency>
53-
<groupId>commons-beanutils</groupId>
54-
<artifactId>commons-beanutils</artifactId>
55-
</dependency>
41+
5642
<dependency>
5743
<groupId>commons-net</groupId>
5844
<artifactId>commons-net</artifactId>
45+
<optional>true</optional>
5946
</dependency>
6047
<dependency>
6148
<groupId>org.apache.commons</groupId>
6249
<artifactId>commons-vfs2</artifactId>
50+
<optional>true</optional>
6351
</dependency>
6452
<dependency>
6553
<groupId>org.apache.commons</groupId>
6654
<artifactId>commons-compress</artifactId>
55+
<optional>true</optional>
6756
</dependency>
6857
<dependency>
6958
<groupId>com.google.guava</groupId>
7059
<artifactId>guava</artifactId>
60+
<optional>true</optional>
7161
</dependency>
7262
<dependency>
7363
<groupId>junit</groupId>
7464
<artifactId>junit</artifactId>
7565
</dependency>
76-
7766
<dependency>
7867
<groupId>commons-dbutils</groupId>
7968
<artifactId>commons-dbutils</artifactId>
8069
</dependency>
81-
<dependency>
82-
<groupId>org.dom4j</groupId>
83-
<artifactId>dom4j</artifactId>
84-
</dependency>
70+
8571

8672
<dependency>
8773
<groupId>cglib</groupId>
8874
<artifactId>cglib-nodep</artifactId>
75+
<optional>true</optional>
8976
</dependency>
9077
<dependency>
9178
<groupId>org.springframework</groupId>
9279
<artifactId>spring-core</artifactId>
80+
<optional>true</optional>
9381
</dependency>
9482
<dependency>
9583
<groupId>org.springframework</groupId>
9684
<artifactId>spring-context</artifactId>
85+
<optional>true</optional>
9786
</dependency>
9887
<dependency>
9988
<groupId>org.springframework</groupId>
10089
<artifactId>spring-tx</artifactId>
90+
<optional>true</optional>
10191
</dependency>
10292
<dependency>
10393
<groupId>org.springframework</groupId>
10494
<artifactId>spring-orm</artifactId>
95+
<optional>true</optional>
10596
</dependency>
10697
<dependency>
10798
<groupId>org.springframework</groupId>
10899
<artifactId>spring-jdbc</artifactId>
100+
<optional>true</optional>
109101
</dependency>
110102
<dependency>
111103
<groupId>javax.persistence</groupId>
112104
<artifactId>javax.persistence-api</artifactId>
105+
<optional>true</optional>
113106
</dependency>
114107
<dependency>
115108
<groupId>org.xeustechnologies</groupId>
116109
<artifactId>jcl-core</artifactId>
110+
<optional>true</optional>
117111
</dependency>
118112
<dependency>
119113
<groupId>javax.servlet</groupId>
@@ -123,96 +117,128 @@
123117
<dependency>
124118
<groupId>net.sf.supercsv</groupId>
125119
<artifactId>super-csv</artifactId>
120+
<optional>true</optional>
126121
</dependency>
127122

128123
<dependency>
129124
<groupId>org.apache.poi</groupId>
130125
<artifactId>poi</artifactId>
126+
<optional>true</optional>
131127
</dependency>
132128
<dependency>
133129
<groupId>org.apache.poi</groupId>
134130
<artifactId>poi-ooxml</artifactId>
131+
<optional>true</optional>
135132
</dependency>
136133
<dependency>
137134
<groupId>org.apache.poi</groupId>
138135
<artifactId>poi-ooxml-schemas</artifactId>
136+
<optional>true</optional>
139137
</dependency>
140138
<dependency>
141139
<groupId>org.apache.poi</groupId>
142140
<artifactId>poi-scratchpad</artifactId>
141+
<optional>true</optional>
143142
</dependency>
144143
<dependency>
145144
<groupId>net.sf.json-lib</groupId>
146145
<artifactId>json-lib</artifactId>
147146
<classifier>jdk15</classifier>
147+
<optional>true</optional>
148148
</dependency>
149149
<dependency>
150150
<groupId>com.lowagie</groupId>
151151
<artifactId>itext</artifactId>
152+
<optional>true</optional>
152153
</dependency>
153154
<dependency>
154155
<groupId>com.lowagie</groupId>
155156
<artifactId>itext-rtf</artifactId>
157+
<optional>true</optional>
156158
</dependency>
157159
<dependency>
158160
<groupId>com.itextpdf</groupId>
159161
<artifactId>itext-asian</artifactId>
162+
<optional>true</optional>
160163
</dependency>
161164
<dependency>
162165
<groupId>net.sf.jtidy</groupId>
163166
<artifactId>jtidy</artifactId>
167+
<optional>true</optional>
164168
</dependency>
165169
<dependency>
166170
<groupId>org.xhtmlrenderer</groupId>
167171
<artifactId>core-renderer</artifactId>
172+
<optional>true</optional>
168173
</dependency>
169174

170175
<dependency>
171176
<groupId>com.jcraft</groupId>
172177
<artifactId>jsch</artifactId>
173178
<version>0.1.50</version>
179+
<optional>true</optional>
174180
</dependency>
175181
<dependency>
176182
<groupId>org.apache.avro</groupId>
177183
<artifactId>avro</artifactId>
184+
<optional>true</optional>
178185
</dependency>
179186
<dependency>
180187
<groupId>org.tmatesoft.svnkit</groupId>
181188
<artifactId>svnkit</artifactId>
189+
<optional>true</optional>
182190
</dependency>
183191
<dependency>
184192
<groupId>org.eclipse.jgit</groupId>
185193
<artifactId>org.eclipse.jgit</artifactId>
194+
<optional>true</optional>
186195
</dependency>
187196
<dependency>
188197
<groupId>redis.clients</groupId>
189198
<artifactId>jedis</artifactId>
190199
<version>2.1.0</version>
200+
<optional>true</optional>
191201
</dependency>
192202
<dependency>
193203
<groupId>org.tukaani</groupId>
194204
<artifactId>xz</artifactId>
195205
<version>1.6</version>
206+
<optional>true</optional>
196207
</dependency>
197208

198209
<dependency>
199210
<groupId>com.esri.geometry</groupId>
200211
<artifactId>esri-geometry-api</artifactId>
201212
<version>1.1</version>
213+
<optional>true</optional>
202214
</dependency>
203215
<dependency>
204216
<groupId>org.springframework.retry</groupId>
205217
<artifactId>spring-retry</artifactId>
206218
<version>1.2.2.RELEASE</version>
219+
<optional>true</optional>
207220
</dependency>
208221
<dependency>
209222
<groupId>org.yaml</groupId>
210223
<artifactId>snakeyaml</artifactId>
211224
<version>1.23</version>
225+
<optional>true</optional>
212226
</dependency>
213227
<dependency>
214228
<groupId>org.apache.calcite</groupId>
215229
<artifactId>calcite-core</artifactId>
230+
<optional>true</optional>
231+
</dependency>
232+
<dependency>
233+
<groupId>org.projectlombok</groupId>
234+
<artifactId>lombok</artifactId>
235+
<optional>true</optional>
236+
</dependency>
237+
<dependency>
238+
<groupId>com.zaxxer</groupId>
239+
<artifactId>HikariCP</artifactId>
240+
<version>2.6.0</version>
241+
<optional>true</optional>
216242
</dependency>
217243

218244

‎common/src/main/java/com/robin/comm/util/es/ESSchemaAwareUtil.java‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
package com.robin.comm.util.es;
2+
3+
import com.google.gson.Gson;
4+
import com.google.gson.internal.LinkedTreeMap;
5+
import com.google.gson.reflect.TypeToken;
6+
import com.robin.comm.util.http.HttpUtils;
7+
import com.robin.comm.util.json.GsonUtil;
8+
import com.robin.core.base.util.Const;
9+
import lombok.extern.slf4j.Slf4j;
10+
import org.apache.commons.lang3.StringUtils;
11+
import org.springframework.util.Assert;
12+
import org.springframework.util.Base64Utils;
13+
14+
import java.util.HashMap;
15+
import java.util.Iterator;
16+
import java.util.LinkedHashMap;
17+
import java.util.Map;
18+
19+
@Slf4j
20+
public class ESSchemaAwareUtil {
21+
private static Gson gson= GsonUtil.getGson();
22+
public static Map<String,Object> getIndexs(String httpUrl,String... params){
23+
String url=httpUrl;
24+
if(!url.endsWith("/")){
25+
url+="/_all";
26+
}else{
27+
url+="_all";
28+
}
29+
Map<String, String> requestHeaders = new HashMap<>();
30+
wrapSecurity(params,requestHeaders);
31+
HttpUtils.Response response=HttpUtils.doGet(url,"UTF-8",new HashMap<>());
32+
Map<String,Object> indexMap=new HashMap<>();
33+
if(response.getStatusCode()==200){
34+
LinkedHashMap<String,Object> map=gson.fromJson(response.getResponseData(),new TypeToken<LinkedHashMap<String,Object>>(){}.getType());
35+
//Iterator<Map.Entry<String,Object>> iter=map.entrySet().iterator();
36+
for(Map.Entry<String,Object> entry:map.entrySet()){
37+
//Map.Entry<String,Object> entry=iter.next();
38+
String indexName=entry.getKey();
39+
Map<String,Object> indexCfgMap=new HashMap<>();
40+
LinkedTreeMap<String,Object> mapping=(LinkedTreeMap<String,Object>)((LinkedTreeMap<String,Object>)entry.getValue()).get("mappings");
41+
//Iterator<Map.Entry<String,Object>> iter1=mapping.entrySet().iterator();
42+
for(Map.Entry<String,Object> entry1:mapping.entrySet()){
43+
//Map.Entry<String,Object> entry1=iter1.next();
44+
readMapping(indexCfgMap, mapping, entry1);
45+
}
46+
indexMap.put(indexName,indexCfgMap);
47+
}
48+
}
49+
return indexMap;
50+
}
51+
52+
private static void wrapSecurity(String[] params, Map<String, String> requestHeaders) {
53+
StringBuilder builder=new StringBuilder();
54+
55+
if(params.length>=2) {
56+
if (null != params[0] && !StringUtils.isEmpty(params[0]) && null != params[1] && !StringUtils.isEmpty(params[1])) {
57+
builder.append(params[0]).append(":").append(params[1]);
58+
}
59+
}
60+
if (builder.length() > 0) {
61+
requestHeaders.put("Authorization", "Basic " + Base64Utils.encodeToString(builder.toString().getBytes()));
62+
}
63+
}
64+
65+
private static void readMapping(Map<String, Object> indexCfgMap, LinkedTreeMap<String, Object> mapping, Map.Entry<String, Object> entry1) {
66+
String docType=entry1.getKey();
67+
indexCfgMap.put("doctype",docType);
68+
LinkedTreeMap<String,Object> propMap=(LinkedTreeMap<String,Object>)((LinkedTreeMap<String,Object>) mapping.get(docType)).get("properties");
69+
Map<String,Object> propsMap=new HashMap<>();
70+
for(Map.Entry<String,Object> propEntry:propMap.entrySet()){
71+
Map<String,Object> fieldMap=(Map<String,Object>)propEntry.getValue();
72+
Iterator<Map.Entry<String,Object>> fielditer=fieldMap.entrySet().iterator();
73+
while(fielditer.hasNext()){
74+
Map.Entry<String,Object> fieldEntry=fielditer.next();
75+
String fieldName=fieldEntry.getKey();
76+
if(fieldName.equalsIgnoreCase("type")) {
77+
String type = fieldEntry.getValue().toString();
78+
Map<String, Object> fieldCfgMap = new HashMap<>();
79+
fieldCfgMap.put("fieldName", propEntry.getKey());
80+
fieldCfgMap.put("type", type);
81+
propsMap.put(propEntry.getKey(), fieldCfgMap);
82+
}
83+
}
84+
}
85+
indexCfgMap.put("props",propsMap);
86+
}
87+
88+
public static Map<String,Object> getIndex(String httpUrl,String indexName,String... params){
89+
String url=httpUrl;
90+
if(!url.endsWith("/")){
91+
url+="/"+indexName+"/_mapping";
92+
}else {
93+
url+=indexName+"/_mapping";
94+
}
95+
Map<String,Object> indexCfgMap=new HashMap<>();
96+
Map<String, String> requestHeaders=new HashMap<>();
97+
wrapSecurity(params,requestHeaders);
98+
HttpUtils.Response response= HttpUtils.doGet(url,"UTF-8",requestHeaders);
99+
if(response.getStatusCode()==200) {
100+
LinkedHashMap<String, Object> map = gson.fromJson(response.getResponseData(), new TypeToken<LinkedHashMap<String, Object>>() {
101+
}.getType());
102+
for (Map.Entry<String, Object> entry : map.entrySet()) {
103+
LinkedTreeMap<String,Object> mapping=(LinkedTreeMap<String,Object>)((LinkedTreeMap<String,Object>)entry.getValue()).get("mappings");
104+
for(Map.Entry<String,Object> entry1:mapping.entrySet()){
105+
readMapping(indexCfgMap, mapping, entry1);
106+
}
107+
}
108+
}
109+
return indexCfgMap;
110+
}
111+
public static String translateEsType(String columnType) {
112+
Assert.notNull(columnType,"type is null");
113+
String retType= Const.META_TYPE_STRING;
114+
if (columnType.equalsIgnoreCase("auto") || columnType.equalsIgnoreCase("keyword") || columnType.equalsIgnoreCase("text")) {
115+
retType=Const.META_TYPE_STRING;
116+
}else if(columnType.equalsIgnoreCase("double")){
117+
retType=Const.META_TYPE_DOUBLE;
118+
}else if(columnType.equalsIgnoreCase("integer")){
119+
retType=Const.META_TYPE_INTEGER;
120+
}else if(columnType.equalsIgnoreCase("float")){
121+
retType=Const.META_TYPE_NUMERIC;
122+
}else if(columnType.equalsIgnoreCase("short")){
123+
retType=Const.META_TYPE_SHORT;
124+
}
125+
return retType;
126+
}
127+
public static void main(String[] args){
128+
Map<String,Object> map=getIndexs("http://localhost:9200");
129+
log.info("{}",map.size());
130+
log.info("{}",map);
131+
}
132+
}

0 commit comments

Comments
 (0)