您现在的位置是：首页 > 文章详情
elasticsearch 常见操作

日期：2016-03-10点击：347收藏
package com.topwalk.analysis.es.operation.impl; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import org.apache.log4j.Logger; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.bulk.BulkProcessor.Listener; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.count.CountRequestBuilder; import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.metrics.stats.Stats; import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.sort.SortOrder; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.topwalk.analysis.es.configuration.EsAggregation; import com.topwalk.analysis.es.configuration.EsAggregationParams; import com.topwalk.analysis.es.configuration.EsQueryParam; import com.topwalk.analysis.es.configuration.LogicOperator; import com.topwalk.analysis.es.configuration.Operator; import com.topwalk.analysis.es.configuration.Pager; import com.topwalk.analysis.es.factory.EsClient; import com.topwalk.analysis.es.operation.intf.EsService; public class EsServiceImpl implements EsService{ private Logger logger = Logger.getLogger(EsServiceImpl.class); @Override public boolean indexDataById(Client client, String index, String type, String id, String data) { // TODO Auto-generated method stub IndexResponse response = null; try { response = client.prepareIndex(index, type, id).setSource(data).execute().actionGet(); } catch (Exception e) { e.printStackTrace(); return false; } return response.isCreated(); } @Override public boolean bulkindexData(Client client, String index, String type, String data) { // TODO Auto-generated method stub BulkRequestBuilder bulkRequest = client.prepareBulk(); JSONArray arry = JSON.parseArray(data); for(int i = 0; i < arry.size(); i++) { JSONObject jobj = JSON.parseObject(arry.getString(i)); if(jobj.containsKey("_id")) { String _id = jobj.getString("_id"); jobj.remove("_id"); bulkRequest.add(client.prepareIndex(index, type, _id).setSource(arry.getString(i)) ); } else { bulkRequest.add(client.prepareIndex(index, type).setSource(arry.getString(i)) ); } } BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if(bulkResponse.hasFailures()) { logger.error("bulkinsert fail message...." + bulkResponse.buildFailureMessage()); return false; } else { return true; } } @Override public boolean bulkupdateData(Client client, String index, String type, String data, List<EsQueryParam> params) { // TODO Auto-generated method stub List<String> types = new ArrayList<String>(); types.add(type); EsServiceImpl esimpl = new EsServiceImpl(); Pager page = new Pager(); page.setFrom(0); int count = esimpl.getCountByFields(client, index, types, params); page.setNum(count); List<String> ids = esimpl.getIdByFields(client, index, type, params, page); if(ids.size() == 0 || count == 0) { return false; } Map<String, Object> fieldValue = (Map<String, Object>) JSON.parse(data); for (String id : ids ) { esimpl.updateIndexById(client, index, type, id, fieldValue); } return true; } @Override public boolean indexData(Client client, String index, String type, String data) { // TODO Auto-generated method stub IndexResponse response = null; try { response = client.prepareIndex(index, type).setSource(data).execute().actionGet(); } catch (Exception e) { e.printStackTrace(); return false; } return response.isCreated(); } @Override public String getIndexById(Client client, String index, String type, String id) { // TODO Auto-generated method stub GetResponse response = null ; String result = null; try { response = client.prepareGet(index, type, id).execute().actionGet(); result = response.getSourceAsString(); } catch (Exception e ) { e.printStackTrace(); } return result; } @Override public boolean isIndexExists(Client client, String index, String type, String id) { // TODO Auto-generated method stub GetResponse response = null ; boolean result = false; try { response = client.prepareGet(index, type, id).execute().actionGet(); result = response.isExists(); } catch (Exception e ) { e.printStackTrace(); return false; } return result; } @Override public boolean delIndexById(Client client, String index, String type, String id) { // TODO Auto-generated method stub DeleteResponse response = null ; boolean result = false; try { response = client.prepareDelete(index, type, id).execute().actionGet(); result = response.isFound(); } catch (Exception e ) { e.printStackTrace(); return result; } return result; } @Override public boolean delIndexByFields(Client client, String index, String type, List<EsQueryParam> params) { // TODO Auto-generated method stub BoolQueryBuilder db = QueryBuilders.boolQuery(); try { DeleteByQueryRequestBuilder dbq = client.prepareDeleteByQuery(index).setTypes(type); db = getQueryBuilder(params); dbq.setQuery(db); dbq.execute().actionGet(); } catch (Exception e ) { e.printStackTrace(); return true; } return true; } @Override public boolean updateIndexById(Client client, String index, String type, String id, Map<String, Object> fieldValue) { UpdateResponse response = null; UpdateRequest updateRequest = new UpdateRequest(); updateRequest.index(index); if(type != null) { updateRequest.type(type); } updateRequest.id(id); updateRequest.doc(fieldValue); updateRequest.retryOnConflict(5); try { response = client.update(updateRequest).get(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); return false; } catch (ExecutionException e) { // TODO Auto-generated catch block e.printStackTrace(); return false; } return true; } @Override public boolean addFieldToDoc(Client client, String index, String type, String id, Map<String, Object> fieldValue) { // TODO Auto-generated method stub Iterator<String> iter = fieldValue.keySet().iterator(); boolean isExist = false; while (iter.hasNext()) { String key = iter.next(); isExist = isFieldExist(client, index, type, id, key); if(isExist) { try { throw new Exception("the key [" + key + "] is exist!"); } catch (Exception e) { e.printStackTrace(); return false; } } } UpdateResponse response = null; UpdateRequest updateRequest = new UpdateRequest(); updateRequest.index(index).type(type).id(id); updateRequest.doc(fieldValue); try { response = client.update(updateRequest).get(); } catch (InterruptedException e) { e.printStackTrace(); return false; } catch (ExecutionException e) { e.printStackTrace(); return false; } return true; } @Override public boolean removeieldFromDoc(Client client, String index, String type, String id, String field) { boolean isExist = false; isExist = isFieldExist( client, index, type, id, field); if(! isExist) { try { throw new Exception("the key [" + field + "] is not exist!"); } catch (Exception e) { e.printStackTrace(); return false; } } try { client.prepareUpdate(index, type, id).setScript("ctx._source.remove(\"" + field+ "\")", ScriptService.ScriptType.INLINE).execute().actionGet(); } catch (Exception e) { e.printStackTrace(); return false; } return true; } public List<String> getIdByFields(Client client, String index, String type, List<EsQueryParam> params, Pager pager) { BoolQueryBuilder db = QueryBuilders.boolQuery(); List<String> list= new ArrayList<String>(); SearchRequestBuilder searchRequest = null; db = getQueryBuilder(params); if (params != null && params.size() > 0 ) { searchRequest = client.prepareSearch(index).setQuery(db).setTypes(type); } else { searchRequest = client.prepareSearch(index).setQuery(QueryBuilders.matchAllQuery()).setTypes(type); } if(pager != null && pager.getFrom() >= 0 && pager.getNum() > 0 ) { searchRequest.setFrom(pager.getFrom()).setSize(pager.getNum()); } else if(pager != null && pager.getSortField() != null && pager.getSortValue().equals( "ASC")) { searchRequest.addSort(pager.getSortField(), SortOrder.ASC); } else if(pager != null && pager.getSortField() != null && pager.getSortValue().equals("desc")) { searchRequest.addSort(pager.getSortField(), SortOrder.DESC); } SearchResponse response = searchRequest.execute().actionGet(); for(SearchHit hit : response.getHits()) { list.add(hit.getId()); } return list; } @Override public List<String> getIndexByFields(Client client, String index, List<String> types, List<EsQueryParam> params, Pager pager) { BoolQueryBuilder db = QueryBuilders.boolQuery(); List<String> list= new ArrayList<String>(); try { SearchRequestBuilder searchRequest = null; if (params != null && params.size() > 0 ) { db = getQueryBuilder(params); searchRequest = client.prepareSearch(index).setQuery(db); } else { searchRequest = client.prepareSearch(index).setQuery(QueryBuilders.matchAllQuery()); } if(pager != null && pager.getFrom() >= 0 && pager.getNum() > 0 ) { searchRequest.setFrom(pager.getFrom()).setSize(pager.getNum()); } else if(pager != null && pager.getSortField() != null && pager.getSortValue().equals("asc")) { searchRequest.addSort(pager.getSortField(), SortOrder.ASC); } else if(pager != null && pager.getSortField() != null && pager.getSortValue().equals("desc")) { searchRequest.addSort(pager.getSortField(), SortOrder.DESC); } if(types != null && types.size() > 0) { String[] a = new String[types.size()]; searchRequest.setTypes(types.toArray(a)); } SearchResponse response = searchRequest.execute().actionGet(); if(response.getHits().totalHits() > 0) { for(SearchHit hit : response.getHits()) { list.add(JSON.toJSONString(hit.getSource())); } } } catch (Exception e) { e.printStackTrace(); } return list; } @Override public List<String> scrollSearch(Client client, String index, String type, List<EsQueryParam> params, Pager pager) { BoolQueryBuilder db = QueryBuilders.boolQuery(); List<String> list= new ArrayList<String>(); try { SearchRequestBuilder searchRequest = null; if (params != null && params.size() > 0 ) { db = getQueryBuilder(params); searchRequest = client.prepareSearch(index).setSearchType(SearchType.SCAN).setScroll(new TimeValue(60000)).setQuery(db); } else { searchRequest = client.prepareSearch(index).setSearchType(SearchType.SCAN).setScroll(new TimeValue(60000)).setQuery(QueryBuilders.matchAllQuery()); } if (type != null) { searchRequest.setTypes(type); } if(pager != null) { searchRequest.setSize(pager.getNum()); } SearchResponse scrollResp = searchRequest.execute().actionGet(); while (true) { scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet(); if(scrollResp.getHits().getHits().length == 0) { break; } for(SearchHit hit: scrollResp.getHits()) { list.add(hit.getSourceAsString()); } } } catch(Exception e) { e.printStackTrace(); } return list; } @Override public boolean reindex(Client client, String index, String type,String newindex, String newtype, List<EsQueryParam> params) { BoolQueryBuilder db = QueryBuilders.boolQuery(); try { SearchRequestBuilder searchRequest = null; if (params != null && params.size() > 0 ) { db = getQueryBuilder(params); searchRequest = client.prepareSearch(index).setSearchType(SearchType.SCAN).setScroll(new TimeValue(60000)).setQuery(db); } else { searchRequest = client.prepareSearch(index).setSearchType(SearchType.SCAN).setScroll(new TimeValue(60000)).setQuery(QueryBuilders.matchAllQuery()); } if (type != null) { searchRequest.setTypes(type); } SearchResponse scrollResp = searchRequest.execute().actionGet(); BulkProcessor bulkProcessor = BulkProcessor.builder(client, createBulkProcessorListener()) .setBulkActions(10000) .setFlushInterval(TimeValue.timeValueSeconds(5)).build(); while (true) { scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet(); if(scrollResp.getHits().getHits().length == 0) { bulkProcessor.close(); break; } for(SearchHit hit: scrollResp.getHits()) { IndexRequest request = new IndexRequest(newindex, newtype, hit.id()); request.source(hit.getSourceAsString()); bulkProcessor.add(request); } bulkProcessor.flush(); } } catch(Exception e) { e.printStackTrace(); return false; } return true; } public Map<String, Object> getMappings(Client client, String index, String type) { Map<String, Object> result = null; IndexMetaData imd = null; try { ClusterState cs = client.admin() .cluster() .prepareState() .setIndices(index) .execute() .actionGet() .getState(); imd = cs.getMetaData().index(index); MappingMetaData mdd = imd.mapping(type); result = mdd.getSourceAsMap(); } catch (IndexMissingException | IOException e) { } return result; } public Listener createBulkProcessorListener() { return new BulkProcessor.Listener() { @Override public void beforeBulk(long executionId, BulkRequest request) { // TODO Auto-generated method stub } @Override public void afterBulk(long executionId, BulkRequest request, Throwable failure) { // TODO Auto-generated method stub } @Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { // TODO Auto-generated method stub } }; } @Override public int getCountByFields(Client client, String index, List<String> types, List<EsQueryParam> params) { CountResponse response = null; CountRequestBuilder countBuilder = null; BoolQueryBuilder db = null; try { db = getQueryBuilder(params); countBuilder = client.prepareCount(index); if(types != null && types.size() > 0) { String[] a = new String[types.size()]; countBuilder.setTypes(types.toArray(a)); } if (params != null && params.size() > 0) { response = countBuilder.setQuery(db).execute().actionGet(); } else { response = countBuilder.setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(); } } catch (Exception e) { e.printStackTrace(); } return (int) response.getCount(); } //check field isexist public boolean isFieldExist(Client client, String index, String type, String id, String key) { GetResponse response = null ; boolean result = false; try { response = client.prepareGet(index, type, id).execute().actionGet(); result = JSON.parseObject(response.getSourceAsString()).containsKey(key); } catch (Exception e ) { e.printStackTrace(); } return result; } @Override public int getTermsCountByAgg(Client client, String index, List<String> types, List<EsQueryParam> params, List<EsAggregationParams> aggList, Pager page) { BoolQueryBuilder db = null; if(params != null) { db = getQueryBuilder(params); } List<Map<String, String>> result = new ArrayList<Map<String, String>>(); AbstractAggregationBuilder parentBuilder = null; for (int i = aggList.size()-1; i >= 0; i--) { EsAggregationParams aggParams = aggList.get(i); if(i == aggList.size()-1) { if (aggParams.getEsAggregation() == EsAggregation.TERM) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(page.getNum()); } } else { if (aggParams.getEsAggregation() == EsAggregation.TERM) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(page.getNum()).subAggregation(parentBuilder); } } } SearchRequestBuilder requestBuilder = client.prepareSearch(index); if(types != null && types.size() > 0) { String[] a = new String[types.size()]; requestBuilder.setTypes(types.toArray(a)); } requestBuilder.addAggregation(parentBuilder); if(db != null) { requestBuilder.setQuery(db); } SearchResponse sr = requestBuilder.execute().actionGet(); Terms t = sr.getAggregations().get(aggList.get(0).getAggName()); Iterator< ? extends MultiBucketsAggregation.Bucket > topBucket = null; topBucket = t.getBuckets().iterator(); result = getMap(topBucket, aggList, 0, result, null); return result.size(); } @Override public List<Map<String, String>> AggregationByFields(Client client, String index, List<String> types, List<EsQueryParam> params, List<EsAggregationParams> aggList) { BoolQueryBuilder db = null; if(params != null && params.size() > 0) { db = getQueryBuilder(params); } List<Map<String, String>> result = new ArrayList<Map<String, String>>(); AbstractAggregationBuilder parentBuilder = null; for (int i = aggList.size()-1; i >= 0; i--) { EsAggregationParams aggParams = aggList.get(i); if(i == aggList.size()-1) { if (aggParams.getEsAggregation() == EsAggregation.STATS) { parentBuilder = AggregationBuilders.stats( aggParams.getAggName()) .field(aggParams.getField()); } else if (aggParams.getEsAggregation() == EsAggregation.DATE_HISTOGRAM) { DateHistogram.Order order = null; if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) { order = DateHistogram.Order.KEY_ASC; } else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) { order = DateHistogram.Order.KEY_DESC; }else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) { order = DateHistogram.Order.COUNT_ASC; } else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){ order = DateHistogram.Order.COUNT_DESC; } parentBuilder = AggregationBuilders.dateHistogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getDateInterval()).order(order); } else if (aggParams.getEsAggregation() == EsAggregation.TERM) { Terms.Order order = null; if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) { order = Terms.Order.term(true); } else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) { order = Terms.Order.term(false); }else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) { order = Terms.Order.count(true); } else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){ order = Terms.Order.count(false); } if( aggParams.isKeyOrder() == null && aggParams.isValOrder() == null) { if (aggParams.getAggSize() == 0) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()); } else { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()); } } else { if (aggParams.getAggSize() == 0) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).order(order); } else { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).order(order); } } } else if (aggParams.getEsAggregation() == EsAggregation.TERM_SCRIPTS) { Terms.Order order = null; if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) { order = Terms.Order.term(true); } else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) { order = Terms.Order.term(false); }else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) { order = Terms.Order.count(true); } else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){ order = Terms.Order.count(false); } if( aggParams.isKeyOrder() == null || aggParams.isValOrder() == null) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()); } else { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()).order(order); } } else if (aggParams.getEsAggregation() == EsAggregation.HISTOGRAM) { parentBuilder = AggregationBuilders.histogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getHisaggrateInterval()); } else if (aggParams.getEsAggregation() == EsAggregation.SUM) { parentBuilder = AggregationBuilders.sum(aggParams.getAggName()).field(aggParams.getField()); } } else { if (aggParams.getEsAggregation() == EsAggregation.TERM) { Terms.Order order = null; if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) { order = Terms.Order.term(true); } else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) { order = Terms.Order.term(false); }else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) { order = Terms.Order.count(true); } else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){ order = Terms.Order.count(false); } if( aggParams.isKeyOrder() == null && aggParams.isValOrder() == null) { if (aggParams.getAggSize() == 0 ) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).subAggregation(parentBuilder); } else { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).subAggregation(parentBuilder); } } else { if (aggParams.getAggSize() == 0 ) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).order(order).size(aggParams.getAggSize()).subAggregation(parentBuilder); } else { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).order(order).size(aggParams.getAggSize()).subAggregation(parentBuilder); } } } else if (aggParams.getEsAggregation() == EsAggregation.TERM_SCRIPTS) { Terms.Order order = null; if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) { order = Terms.Order.term(true); } else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) { order = Terms.Order.term(false); }else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) { order = Terms.Order.count(true); } else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){ order = Terms.Order.count(false); } if( aggParams.isKeyOrder() == null || aggParams.isValOrder() == null) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()).subAggregation(parentBuilder); } else { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()).order(order).subAggregation(parentBuilder); } } else if (aggParams.getEsAggregation() == EsAggregation.DATE_HISTOGRAM) { DateHistogram.Order order = null; if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) { order = DateHistogram.Order.KEY_ASC; } else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) { order = DateHistogram.Order.KEY_DESC; }else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) { order = DateHistogram.Order.COUNT_ASC; } else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){ order = DateHistogram.Order.COUNT_DESC; } parentBuilder = AggregationBuilders.dateHistogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getDateInterval()).subAggregation(parentBuilder).order(order); } else if (aggParams.getEsAggregation() == EsAggregation.HISTOGRAM) { parentBuilder = AggregationBuilders.histogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getHisaggrateInterval()).subAggregation(parentBuilder); } } } SearchRequestBuilder requestBuilder = client.prepareSearch(index); if(types != null && types.size() > 0) { String[] a = new String[types.size()]; requestBuilder.setTypes(types.toArray(a)); } requestBuilder.addAggregation(parentBuilder); if(db != null) { requestBuilder.setQuery(db); } logger.debug(requestBuilder.toString()); SearchResponse sr = requestBuilder.execute().actionGet(); Iterator< ? extends MultiBucketsAggregation.Bucket > topBucket = null; if(aggList.get(0).getEsAggregation() == EsAggregation.DATE_HISTOGRAM) { DateHistogram agg = sr.getAggregations().get(aggList.get(0).getAggName()); topBucket = agg.getBuckets().iterator(); } else if (aggList.get(0).getEsAggregation() == EsAggregation.TERM ||aggList.get(0).getEsAggregation() == EsAggregation.TERM_SCRIPTS ) { Terms t = sr.getAggregations().get(aggList.get(0).getAggName()); topBucket = t.getBuckets().iterator(); } else if (aggList.get(0).getEsAggregation() == EsAggregation.HISTOGRAM) { Histogram agg = sr.getAggregations().get(aggList.get(0).getAggName()); topBucket = agg.getBuckets().iterator(); } else if (aggList.get(0).getEsAggregation() == EsAggregation.SUM) { Sum agg = sr.getAggregations().get(aggList.get(0).getAggName()); Map<String, String> sumresult = new HashMap<String, String>(); sumresult.put(agg.getName(), String.valueOf(agg.getValue())); result.add(sumresult); return result; } else if(aggList.get(0).getEsAggregation() == EsAggregation.STATS) { Stats stats = sr.getAggregations().get(aggList.get(0).getAggName()); Map<String, String> statsresult = new HashMap<String, String>(); statsresult.put("count", String.valueOf(stats.getCount())); statsresult.put("avg", String.valueOf(stats.getAvg())); statsresult.put("max", String.valueOf(stats.getMax())); statsresult.put("min", String.valueOf(stats.getMin())); statsresult.put("sum", String.valueOf(stats.getSum())); result.add(statsresult); return result; } Map<String, String> map = new HashMap<String, String>(); result = getMap(topBucket, aggList, 0, result, map); return result; } @Override public SearchResponse getResponseByAggregationField(Client client, String index, List<String> types, List<EsQueryParam> params, List<EsAggregationParams> aggList) { BoolQueryBuilder db = null; if(params != null && params.size() > 0) { db = getQueryBuilder(params); } List<Map<String, String>> result = new ArrayList<Map<String, String>>(); AbstractAggregationBuilder parentBuilder = null; for (int i = aggList.size()-1; i >= 0; i--) { EsAggregationParams aggParams = aggList.get(i); if(i == aggList.size()-1) { if (aggParams.getEsAggregation() == EsAggregation.STATS) { parentBuilder = AggregationBuilders.stats( aggParams.getAggName()) .field(aggParams.getField()); } else if (aggParams.getEsAggregation() == EsAggregation.DATE_HISTOGRAM) { DateHistogram.Order order = null; if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) { order = DateHistogram.Order.KEY_ASC; } else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) { order = DateHistogram.Order.KEY_DESC; }else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) { order = DateHistogram.Order.COUNT_ASC; } else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){ order = DateHistogram.Order.COUNT_DESC; } parentBuilder = AggregationBuilders.dateHistogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getDateInterval()).order(order); } else if (aggParams.getEsAggregation() == EsAggregation.TERM) { Terms.Order order = null; if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) { order = Terms.Order.term(true); } else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) { order = Terms.Order.term(false); }else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) { order = Terms.Order.count(true); } else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){ order = Terms.Order.count(false); } if( aggParams.isKeyOrder() == null && aggParams.isValOrder() == null) { if (aggParams.getAggSize() == 0) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()); } else { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()); } } else { if (aggParams.getAggSize() == 0) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).order(order); } else { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).order(order); } } } else if (aggParams.getEsAggregation() == EsAggregation.TERM_SCRIPTS) { Terms.Order order = null; if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) { order = Terms.Order.term(true); } else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) { order = Terms.Order.term(false); }else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) { order = Terms.Order.count(true); } else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){ order = Terms.Order.count(false); } if( aggParams.isKeyOrder() == null || aggParams.isValOrder() == null) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()); } else { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()).order(order); } } else if (aggParams.getEsAggregation() == EsAggregation.HISTOGRAM) { parentBuilder = AggregationBuilders.histogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getHisaggrateInterval()); } else if (aggParams.getEsAggregation() == EsAggregation.SUM) { parentBuilder = AggregationBuilders.sum(aggParams.getAggName()).field(aggParams.getField()); } else if (aggParams.getEsAggregation() == EsAggregation.SUM_SCRIPTS) { parentBuilder = AggregationBuilders.sum(aggParams.getAggName()).script(aggParams.getField()); } } else { if (aggParams.getEsAggregation() == EsAggregation.TERM) { Terms.Order order = null; if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) { order = Terms.Order.term(true); } else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) { order = Terms.Order.term(false); }else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) { order = Terms.Order.count(true); } else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){ order = Terms.Order.count(false); } if( aggParams.isKeyOrder() == null && aggParams.isValOrder() == null) { if (aggParams.getAggSize() == 0 ) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).subAggregation(parentBuilder); } else { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).subAggregation(parentBuilder); } } else { if (aggParams.getAggSize() == 0 ) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).order(order).size(aggParams.getAggSize()).subAggregation(parentBuilder); } else { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).order(order).size(aggParams.getAggSize()).subAggregation(parentBuilder); } } } else if (aggParams.getEsAggregation() == EsAggregation.TERM_SCRIPTS) { Terms.Order order = null; if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) { order = Terms.Order.term(true); } else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) { order = Terms.Order.term(false); }else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) { order = Terms.Order.count(true); } else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){ order = Terms.Order.count(false); } if( aggParams.isKeyOrder() == null || aggParams.isValOrder() == null) { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()).subAggregation(parentBuilder); } else { parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()).order(order).subAggregation(parentBuilder); } } else if (aggParams.getEsAggregation() == EsAggregation.DATE_HISTOGRAM) { DateHistogram.Order order = null; if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) { order = DateHistogram.Order.KEY_ASC; } else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) { order = DateHistogram.Order.KEY_DESC; }else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) { order = DateHistogram.Order.COUNT_ASC; } else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){ order = DateHistogram.Order.COUNT_DESC; } parentBuilder = AggregationBuilders.dateHistogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getDateInterval()).subAggregation(parentBuilder).order(order); } else if (aggParams.getEsAggregation() == EsAggregation.HISTOGRAM) { parentBuilder = AggregationBuilders.histogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getHisaggrateInterval()).subAggregation(parentBuilder); } } } SearchRequestBuilder requestBuilder = client.prepareSearch(index); if(types != null && types.size() > 0) { String[] a = new String[types.size()]; requestBuilder.setTypes(types.toArray(a)); } requestBuilder.addAggregation(parentBuilder); if(db != null) { requestBuilder.setQuery(db); } System.out.println(requestBuilder.toString()); SearchResponse sr = requestBuilder.execute().actionGet(); return sr; } public List<Map<String, String>> getMap(Iterator< ? extends MultiBucketsAggregation.Bucket > topBucket , List<EsAggregationParams> aggList, int level, List<Map<String, String>> result, Map<String, String> map) { if (aggList.size() ==1&& level == 0) { while(topBucket.hasNext()) { map = new HashMap<String, String>(); MultiBucketsAggregation.Bucket b3= topBucket.next(); if (aggList.get(0).getEsAggregation() == EsAggregation.TERM_SCRIPTS) { map.put( b3.getKey(), String.valueOf(b3.getDocCount()) ); } else { map.put(aggList.get(0).getField(), b3.getKey()); map.put("doc_count", String.valueOf(b3.getDocCount())); } result.add(map); } return result; } else { level ++; } if(level < aggList.size() && aggList.size() != 1) { while(topBucket.hasNext()) { Map<String, String> map2 = new HashMap<String, String>(); MultiBucketsAggregation.Bucket b2= topBucket.next(); map2.put(aggList.get(level-1).getField(), b2.getKey()); map2.put("doc_count" , String.valueOf(b2.getDocCount()) ); if(aggList.get(level).getEsAggregation() == EsAggregation.TERM || aggList.get(level).getEsAggregation() == EsAggregation.TERM_SCRIPTS) { Terms t = b2.getAggregations().get(aggList.get(level ).getAggName()); map2.put(aggList.get(level).getField(), b2.getKey()); getMap(t.getBuckets().iterator(), aggList, level, result, map2); } else if (aggList.get(level).getEsAggregation() == EsAggregation.STATS) { Stats stats = b2.getAggregations().get(aggList.get(level ).getAggName()); map2.put("count", String.valueOf(stats.getCount())); map2.put("sum", String.valueOf(stats.getSum())); map2.put("avg", String.valueOf(stats.getAvg())); map2.put("max", String.valueOf(stats.getMax())); map2.put("min", String.valueOf(stats.getMin())); result.add(map2); } else if (aggList.get(level).getEsAggregation() == EsAggregation.SUM) { Sum agg = b2.getAggregations().get(aggList.get(level).getAggName()); map2.put(aggList.get(level ).getField(), String.valueOf(agg.getValue())); result.add(map2); } else if (aggList.get(level).getEsAggregation() == EsAggregation.DATE_HISTOGRAM) { DateHistogram datehis = b2.getAggregations().get(aggList.get(level ).getAggName()); map2.put(aggList.get(level ).getField(), b2.getKey()); getMap( datehis.getBuckets().iterator(), aggList, level, result, map2); } else if (aggList.get(level).getEsAggregation() == EsAggregation.HISTOGRAM) { Histogram his = b2.getAggregations().get(aggList.get(level ).getAggName()); map2.put(aggList.get(level ).getField(), b2.getKey()); getMap( his.getBuckets().iterator(), aggList, level, result, map2); } } return result; } else { List<Map<String, String>> list = new ArrayList<Map<String, String>>(); //List<String> list = new ArrayList<String>(); while(topBucket.hasNext()) { Map<String, String> m = new HashMap<String, String>(); MultiBucketsAggregation.Bucket b3= topBucket.next(); m.put(aggList.get(level -1 ).getField(), b3.getKey()); m.put("doc_count", String.valueOf(b3.getDocCount())); list.add(m); } map.put(aggList.get(level -1 ).getField(), JSON.toJSONString(list)); result.add(map); return result; } } public BoolQueryBuilder getQueryBuilder(List<EsQueryParam> params) { BoolQueryBuilder db = null; if (params != null) { db = QueryBuilders.boolQuery(); for (EsQueryParam param : params) { if(param.getLogicOperator() == LogicOperator.AND && param.getOperator() == Operator.EQUAL) { db = db.must(QueryBuilders.matchQuery(param.getField(), param.getValue())); } else if(param.getLogicOperator() == LogicOperator.AND && param.getOperator() == Operator.LIKE) { db = db.must(QueryBuilders.wildcardQuery(param.getField(), param.getValue())); } else if(param.getLogicOperator() == LogicOperator.OR && param.getOperator() == Operator.LIKE) { db = db.should(QueryBuilders.wildcardQuery(param.getField(), param.getValue())); } else if(param.getLogicOperator() == LogicOperator.MUST_NOT && param.getOperator() == Operator.LIKE) { db = db.mustNot(QueryBuilders.wildcardQuery(param.getField(), param.getValue())); } else if(param.getLogicOperator() == LogicOperator.MUST_NOT && param.getOperator() == Operator.EQUAL) { db = db.mustNot(QueryBuilders.matchQuery(param.getField(), param.getValue())); } else if(param.getOperator() == Operator.GT) { db = db.must(QueryBuilders.rangeQuery(param.getField()).gt(param.getValue())); } else if(param.getOperator() == Operator.GTE) { db = db.must(QueryBuilders.rangeQuery(param.getField()).gte(param.getValue())); } else if(param.getOperator() == Operator.LT) { db = db.must(QueryBuilders.rangeQuery(param.getField()).lt(param.getValue())); } else if(param.getOperator() == Operator.LTE) { db = db.must(QueryBuilders.rangeQuery(param.getField()).lte(param.getValue())); } else if (param.getLogicOperator() == LogicOperator.OR) { db = db.should(QueryBuilders.matchQuery(param.getField(), param.getValue())); } else if (param.getLogicOperator() == LogicOperator.AND ) { db = db.must(QueryBuilders.matchQuery(param.getField(), param.getValue())); } else if (param.getOperator() == Operator.EQUAL ) { db = db.must(QueryBuilders.matchQuery(param.getField(), param.getValue())); } else if (param.getOperator() == Operator.NOTEQUAL ) { db = db.mustNot(QueryBuilders.matchQuery(param.getField(), param.getValue())); } else if(param.getOperator() == Operator.TERMS) { db = db.must(QueryBuilders.termsQuery(param.getField(), param.getTermsList()).minimumMatch(1)); } else if (param.getLogicOperator() == null &&param.getOperator() == null ) { db = db.must(QueryBuilders.matchQuery(param.getField(), param.getValue())); } } } return db; } @Override public boolean createType(Client client, String index, String type, String mapping) { // TODO Auto-generated method stub IndicesExistsRequest indexrequest = new IndicesExistsRequest(index); boolean exist = client.admin().indices().exists(indexrequest).actionGet().isExists(); if (! exist) { client.admin().indices().create(new CreateIndexRequest(index)).actionGet(); } PutMappingRequest putMappingRequest = new PutMappingRequest(index); putMappingRequest.type(type); putMappingRequest.source(mapping); PutMappingResponse putMappingResponse = client.admin().indices().putMapping(putMappingRequest).actionGet(); return putMappingResponse.isAcknowledged(); } @Override public boolean isIndexExistsByFieldVal(Client client, String index, String type, String field, String fieldVal) { // TODO Auto-generated method stub SearchRequestBuilder searchRequest = null ; boolean result = false; try { searchRequest = client.prepareSearch(index).setTypes(type).setQuery(QueryBuilders.matchQuery(field, fieldVal)); SearchResponse response = searchRequest.execute().actionGet(); result = (response.getHits().hits().length > 0) ? true : false; } catch (Exception e ) { e.printStackTrace(); return false; } return result; } @Override public SearchHits termsQuery(Client client, String index, String type,String field,List<String> terms) { SearchRequestBuilder searchRequest = null ; QueryBuilder qb = QueryBuilders.termsQuery(field, terms).minimumMatch(1); searchRequest = client.prepareSearch(index).setTypes(type).setQuery(qb).setSize(1000000000); SearchResponse response = searchRequest.execute().actionGet(); return response.getHits(); //后续优化 } }