下面列出了org.apache.lucene.index.IndexFormatTooOldException#org.elasticsearch.common.xcontent.XContentFactory 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
private void appendElasticsearchStats(final StringBuilder buf) {
String stats = null;
try {
final FessEsClient esClient = ComponentUtil.getFessEsClient();
final NodesStatsResponse response = esClient.admin().cluster().prepareNodesStats().all().execute().actionGet(10000L);
final XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
response.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.endObject();
builder.flush();
try (OutputStream out = builder.getOutputStream()) {
stats = ((ByteArrayOutputStream) out).toString(Constants.UTF_8);
}
} catch (final Exception e) {
logger.debug("Failed to access Elasticsearch stats.", e);
}
buf.append("\"elasticsearch\":").append(stats).append(',');
}
public static void createIndex() throws Exception{
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
{
builder.startObject("properties");
{
builder.startObject("message");
{
builder.field("type", "text");
builder.field("analyzer", "ik_max_word");
builder.field("search_analyzer", "ik_max_word");
}
builder.endObject();
}
builder.endObject();
}
builder.endObject();
CreateIndexRequest request = new CreateIndexRequest("twitter");
request.mapping(builder);
client.indices().create(request, RequestOptions.DEFAULT);
}
public void testNonStandardnumber() throws Exception {
String mapping = copyToStringFromClasspath("mapping.json");
DocumentMapper docMapper = createIndex("some_index")
.mapperService().documentMapperParser()
.parse("someType", new CompressedXContent(mapping));
String sampleText = "Hello world";
BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder()
.startObject().field("someField", sampleText).endObject());
SourceToParse sourceToParse = SourceToParse.source("some_index", "someType", "1", json, XContentType.JSON);
ParseContext.Document doc = docMapper.parse(sourceToParse).rootDoc();
assertEquals(0, doc.getFields("someField").length);
// re-parse it
String builtMapping = docMapper.mappingSource().string();
logger.warn("testNonStandardnumber: built mapping =" + builtMapping);
DocumentMapper docMapper2 = createIndex("some_index2")
.mapperService().documentMapperParser()
.parse("someType", new CompressedXContent(builtMapping));
json = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.field("someField", sampleText).endObject());
sourceToParse = SourceToParse.source("some_index2", "someType", "1", json, XContentType.JSON);
doc = docMapper2.parse(sourceToParse).rootDoc();
assertEquals(0, doc.getFields("someField").length);
}
/**
* Associates a filter to the alias
*/
public Alias filter(QueryBuilder filterBuilder) {
if (filterBuilder == null) {
this.filter = null;
return this;
}
try {
XContentBuilder builder = XContentFactory.jsonBuilder();
filterBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.close();
this.filter = builder.string();
return this;
} catch (IOException e) {
throw new ElasticsearchGenerationException("Failed to build json for alias request", e);
}
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(NAME.getPreferredName(), name);
builder.field(PARAMS.getPreferredName(), queryParams);
builder.field(TEMPLATE_LANGUAGE.getPreferredName(), templateLanguage);
if (templateAsString) {
builder.field(TEMPLATE.getPreferredName(), template);
} else {
builder.field(TEMPLATE.getPreferredName());
// it's ok to use NamedXContentRegistry.EMPTY because we don't really parse we copy the structure...
XContentParser parser = XContentFactory.xContent(template).createParser(NamedXContentRegistry.EMPTY,
LoggingDeprecationHandler.INSTANCE, template);
builder.copyCurrentStructure(parser);
}
builder.endObject();
return builder;
}
public XContentBuilder internalGetMapping() throws IOException {
// Configure the RootObjectMapper:
RootObjectMapper.Builder rootObjectMapperBuilder = getRootObjectBuilder();
// Populate the Settings:
Settings.Builder settingsBuilder = getSettingsBuilder();
//new Mapping(arg0, arg1, arg2, arg3)getSourceTransforms(),
// Build the Mapping:
Mapping mapping = new Mapping(
version,
rootObjectMapperBuilder.build(new Mapper.BuilderContext(settingsBuilder.build(), new ContentPath(1))),
getMetaDataFieldMappers(),
getMeta());
// Turn it into JsonXContent:
return mapping.toXContent(XContentFactory.jsonBuilder().startObject(), new ToXContent.MapParams(emptyMap())).endObject();
}
@Override
public Boolean profilingExists(Map<String, String> tags) {
try {
List<BytesReference> refList = new ArrayList<>();
XContentBuilder xContent = XContentFactory.jsonBuilder();
xContent.map(tags);
refList.add(BytesReference.bytes(xContent));
PercolateQueryBuilder percolateQuery = new PercolateQueryBuilder(PercolatorMetricProfiling.QUERY_KEYWORD,
refList, XContentType.JSON);
val boolQueryBuilder = new BoolQueryBuilder();
boolQueryBuilder.filter(percolateQuery);
val searchSourceBuilder = elasticsearchUtil.getSourceBuilder(boolQueryBuilder);
searchSourceBuilder.timeout(new TimeValue(elasticSearchProperties.getConfig().getConnectionTimeout()));
searchSourceBuilder.size(500);
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(METRIC_PROFILING_INDEX);
val searchResponse = legacyElasticSearchClient.search(searchRequest, RequestOptions.DEFAULT);
return searchResponse.getHits().getHits().length > 0;
} catch (IOException e) {
log.error("Error ES lookup", e);
throw new RuntimeException(e);
}
}
/**
* Submit a search query to Elasticsearch.
*
* @param indexName The name of the index to search.
* @param query The query to search.
* @return The search response returned by Elasticsearch.
* @throws IOException
*/
private SearchResponse search(String indexName, String query) throws IOException {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(new NamedXContentRegistry(searchModule
.getNamedXContents()), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, query)) {
searchSourceBuilder.parseXContent(parser);
}
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
searchRequestBuilder.setIndices(indexName).setSource(searchSourceBuilder);
if (this.searchAllowPartialSearchResults != null)
searchRequestBuilder.setAllowPartialSearchResults(this.searchAllowPartialSearchResults);
if (this.searchBatchedReduceSize != null)
searchRequestBuilder.setBatchedReduceSize(this.searchBatchedReduceSize);
if (this.searchMaxConcurrentShardRequests != null)
searchRequestBuilder.setMaxConcurrentShardRequests(this.searchMaxConcurrentShardRequests);
if (this.searchPreFilterShardSize != null)
searchRequestBuilder.setPreFilterShardSize(this.searchPreFilterShardSize);
if (this.searchPreference != null)
searchRequestBuilder.setPreference(this.searchPreference);
if (this.searchRequestCache != null)
searchRequestBuilder.setRequestCache(this.searchRequestCache);
if (this.maxTimePerQuery != null)
searchRequestBuilder.setTimeout(TimeValue.parseTimeValue(this.maxTimePerQuery, "timeout"));
return searchRequestBuilder.execute().actionGet();
}
public XContentBuilder internalGetMapping() throws IOException {
// Configure the RootObjectMapper:
RootObjectMapper.Builder rootObjectMapperBuilder = getRootObjectBuilder();
// Populate the Settings:
Settings.Builder settingsBuilder = getSettingsBuilder();
//new Mapping(arg0, arg1, arg2, arg3)getSourceTransforms(),
// Build the Mapping:
Mapping mapping = new Mapping(
version,
rootObjectMapperBuilder.build(new Mapper.BuilderContext(settingsBuilder.build(), new ContentPath(1))),
getMetaDataFieldMappers(),
getMeta());
// Turn it into JsonXContent:
return mapping.toXContent(XContentFactory.jsonBuilder().startObject(), new ToXContent.MapParams(emptyMap())).endObject();
}
/**
* {@inheritDoc}
*/
@Override
public void softDelete(final String type, final String id, final MetacatRequestContext metacatRequestContext) {
try {
RETRY_ES_PUBLISH.call(() -> {
final XContentBuilder builder = XContentFactory.contentBuilder(contentType);
builder.startObject().field(ElasticSearchDoc.Field.DELETED, true)
.field(ElasticSearchDoc.Field.TIMESTAMP, java.time.Instant.now().toEpochMilli())
.field(ElasticSearchDoc.Field.USER,
metacatRequestContext.getUserName()).endObject();
client.prepareUpdate(esIndex, type, id)
.setRetryOnConflict(NO_OF_CONFLICT_RETRIES).setDoc(builder).get(esCallTimeout);
ensureMigrationByCopy(type, Collections.singletonList(id));
return null;
});
} catch (Exception e) {
handleException("ElasticSearchUtil.softDelete", type, id, e,
Metrics.CounterElasticSearchDelete.getMetricName());
}
}
/**
* https://github.com/medcl/elasticsearch-analysis-pinyin/tree/v1.7.5
* pinyin配置分析器
* @return
*/
public static String getIndexPinYinSetting() throws IOException {
XContentBuilder mapping = XContentFactory.jsonBuilder()
.startObject()
.startObject("analysis")
.startObject("analyzer")
.startObject("pinyin_analyzer")
.field("tokenizer", "my_pinyin")
.array("filter","nGram","word_delimiter")
.endObject()
.endObject()
.startObject("tokenizer")
.startObject("my_pinyin")
.field("type", "pinyin")
.field("first_letter","prefix")
.field("padding_char","")
.endObject()
.endObject()
.endObject()
.endObject();
return mapping.string();
}
@Test
public void testParseDynamicEmptyArray() throws Exception {
String mapping = Strings.toString(XContentFactory.jsonBuilder()
.startObject().startObject(TYPE).startObject("properties")
.endObject().endObject().endObject());
DocumentMapper mapper = mapper(INDEX, mapping);
// parse source with empty array
BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
.startObject()
.array("new_array_field")
.endObject());
SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
ParsedDocument doc = mapper.parse(sourceToParse);
assertThat(doc.docs().get(0).getField("new_array_field"), is(nullValue()));
assertThat(mapper.mappers().getMapper("new_array_field"), is(nullValue()));
}
/**
* 创建mapping分词IK索引
* Elasticsearch的mapping一旦创建,只能增加字段,而不能修改已经mapping的字段
* @param indexType
* @return
*/
private static XContentBuilder createIKMapping(String indexType,String field) {
XContentBuilder mapping = null;
try {
mapping = XContentFactory.jsonBuilder().startObject()
// 索引库名(类似数据库中的表)
.startObject(indexType)
//匹配全部
.startObject("properties")
//根据只对content这个Fields分词
.startObject(field).field("type","string").field("store","no")
.field("term_vector","with_positions_offsets").field("analyzer","ik_max_word")
.field("search_analyzer","ik_max_word").field("include_in_all","true").field("boost",8)
.endObject()
.endObject()
.endObject().endObject();
} catch (IOException e) {
log.error("创建mapping分词IK索引 error"+e.getLocalizedMessage());
}
return mapping;
}
@Test
public void testCopyToWithoutMetaIndices() throws Exception {
// regression test... this mapping used to cause an NPE
XContentBuilder builder = XContentFactory.jsonBuilder()
.startObject()
.startObject("properties")
.startObject("description")
.field("type", "string")
.array("copy_to", "description_ft")
.endObject()
.startObject("description_ft")
.field("type", "string")
.field("analyzer", "english")
.endObject()
.endObject()
.endObject();
IndexMetaData metaData = getIndexMetaData("test1", builder);
DocIndexMetaData md = newMeta(metaData, "test1");
assertThat(md.indices().size(), is(1));
assertThat(md.indices().keySet().iterator().next(), is(new ColumnIdent("description_ft")));
}
@Override
public Query parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
XContentParser parser = parseContext.parser();
XContentParser.Token token = parser.nextToken();
if (token != XContentParser.Token.FIELD_NAME) {
throw new QueryParsingException(parseContext, "[wrapper] query malformed");
}
String fieldName = parser.currentName();
if (!fieldName.equals("query")) {
throw new QueryParsingException(parseContext, "[wrapper] query malformed");
}
parser.nextToken();
byte[] querySource = parser.binaryValue();
try (XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(querySource)) {
final QueryParseContext context = new QueryParseContext(parseContext.index(), parseContext.indexQueryParserService());
context.reset(qSourceParser);
Query result = context.parseInnerQuery();
parser.nextToken();
parseContext.combineNamedQueries(context);
return result;
}
}
/**
* Adds mapping that will be added when the index gets created.
*
* @param type The mapping type
* @param source The mapping source
*/
@SuppressWarnings("unchecked")
public CreateIndexRequest mapping(String type, Map source) {
if (mappings.containsKey(type)) {
throw new IllegalStateException("mappings for type \"" + type + "\" were already defined");
}
// wrap it in a type map if its not
if (source.size() != 1 || !source.containsKey(type)) {
source = MapBuilder.<String, Object>newMapBuilder().put(type, source).map();
}
try {
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
builder.map(source);
return mapping(type, builder);
} catch (IOException e) {
throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
}
}
@Test
public void testLicenceKeyFromXContent() throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder();
// reflects the logic used to process custom metadata in the cluster state
builder.startObject();
builder.startObject(LicenseKey.WRITEABLE_TYPE)
.field("license_key", LICENSE_KEY)
.endObject();
builder.endObject();
XContentParser parser = JsonXContent.JSON_XCONTENT.createParser(xContentRegistry(),
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
BytesReference.toBytes(BytesReference.bytes(builder)));
parser.nextToken(); // start object
LicenseKey licenseKey2 = LicenseKey.fromXContent(parser);
assertEquals(createLicenseKey(), licenseKey2);
// a metadata custom must consume the surrounded END_OBJECT token, no token must be left
assertThat(parser.nextToken(), nullValue());
}
public XContentBuilder createStrictMappingDefinition() {
try {
// @formatter:off
//How to enable it in intellij see it here: http://stackoverflow.com/questions/3375307/how-to-disable-code-formatting-for-some-part-of-the-code-using-comments
return XContentFactory.jsonBuilder()
.startObject()
.field("dynamic", "strict")
.startObject("properties")
.startObject("field1")
.field("type", "string")
.endObject()
.endObject()
.endObject();
// @formatter:off
} catch (IOException e) {
throw new RuntimeException("Failed building index mappingDef", e);
}
}
@Test
public void givenContentBuilder_whenHelpers_thanIndexJson() throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder()
.startObject()
.field("fullName", "Test")
.field("salary", "11500")
.field("age", "10")
.endObject();
IndexRequest indexRequest = new IndexRequest("people");
indexRequest.source(builder);
IndexResponse response = client.index(indexRequest, RequestOptions.DEFAULT);
assertEquals(Result.CREATED, response.getResult());
}
/**
* 创建索引以及映射mapping,并给索引某些字段指定iK分词,以后向该索引中查询时,就会用ik分词。
* @Author lihaodong
* @Description
* @Date 20:19 2018/12/21
* @Param [indexName, esType]
* @return boolean
**/
public static boolean createIndex(String indexName, String esType) {
if (!isIndexExist(indexName)) {
log.info("Index is not exits!");
}
//创建映射
XContentBuilder mapping = null;
try {
mapping = XContentFactory.jsonBuilder()
.startObject()
.startObject("properties") // .startObject("m_id").field("type","keyword").endObject()
// title:字段名, type:文本类型 analyzer :分词器类型
.startObject("id").field("type", "text").field("analyzer", "standard").endObject() //该字段添加的内容,查询时将会使用ik_smart分词
.startObject("name").field("type", "text").field("analyzer", "standard").endObject() //ik_smart ik_max_word standard
.startObject("message").field("type", "text").field("analyzer", "standard").endObject()
.startObject("price").field("type", "float").endObject()
.startObject("creatDate").field("type", "date").endObject()
.endObject()
.endObject();
} catch (IOException e) {
log.error("执行建立失败:{}",e.getMessage());
}
//index:索引名 type:类型名
PutMappingRequest putmap = Requests.putMappingRequest(indexName).type(esType).source(mapping);
//创建索引
client.admin().indices().prepareCreate(indexName).execute().actionGet();
//为索引添加映射
PutMappingResponse indexresponse = client.admin().indices().putMapping(putmap).actionGet();
log.info("执行建立成功?" + indexresponse.isAcknowledged());
return indexresponse.isAcknowledged();
}
public AliasAction filter(QueryBuilder queryBuilder) {
if (queryBuilder == null) {
this.filter = null;
return this;
}
try {
XContentBuilder builder = XContentFactory.jsonBuilder();
queryBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.close();
this.filter = builder.string();
return this;
} catch (IOException e) {
throw new ElasticsearchGenerationException("Failed to build json for alias request", e);
}
}
@Test
public void testSetXBasedOnXAndPartitionedColumn() throws Exception {
SQLExecutor e = SQLExecutor.builder(clusterService)
.addPartitionedTable("create table t (x int, p int) partitioned by (p)",
new PartitionName(new RelationName("doc", "t"), Collections.singletonList("1")).asIndexName())
.build();
AnalyzedUpdateStatement update = e.analyze("update t set x = x + p");
Assignments assignments = Assignments.convert(update.assignmentByTargetCol(), e.functions());
DocTableInfo table = (DocTableInfo) update.table().tableInfo();
UpdateSourceGen updateSourceGen = new UpdateSourceGen(
e.functions(),
txnCtx,
table,
assignments.targetNames()
);
Map<String, Object> source = singletonMap("x", 1);
Map<String, Object> updatedSource = updateSourceGen.generateSource(
new Doc(
1,
table.concreteIndices()[0],
"1",
1,
1,
1,
source,
() -> {
try {
return Strings.toString(XContentFactory.jsonBuilder().map(source));
} catch (IOException e1) {
throw new RuntimeException(e1);
}
}
),
assignments.sources(),
new Object[0]
);
assertThat(updatedSource, is(Map.of("x", 2)));
}
@Override
public void login(final RestRequest request,
final ActionListener<String[]> listener) {
String username = request.param(usernameKey);
String password = request.param(passwordKey);
final BytesReference content = request.content();
final XContentType xContentType = XContentFactory.xContentType(content);
XContentParser parser = null;
try {
parser = XContentFactory.xContent(xContentType).createParser(
content);
final XContentParser.Token t = parser.nextToken();
if (t != null) {
final Map<String, Object> contentMap = parser.map();
username = MapUtil.getAsString(contentMap, usernameKey,
username);
password = MapUtil.getAsString(contentMap, passwordKey,
password);
}
} catch (final Exception e) {
listener.onFailure(e);
return;
} finally {
if (parser != null) {
parser.close();
}
}
if (username == null) {
listener.onResponse(new String[0]);
return;
}
processLogin(username, password, listener);
}
public void testNormal() throws IOException, JsonPathNotFoundException {
CronRequest request = new CronRequest();
CronNodeRequest nodeRequest = new CronNodeRequest();
BytesStreamOutput nodeRequestOut = new BytesStreamOutput();
nodeRequestOut.setVersion(Version.CURRENT);
nodeRequest.writeTo(nodeRequestOut);
StreamInput siNode = nodeRequestOut.bytes().streamInput();
CronNodeRequest nodeResponseRead = new CronNodeRequest(siNode);
CronNodeResponse nodeResponse1 = action.nodeOperation(nodeResponseRead);
CronNodeResponse nodeResponse2 = action.nodeOperation(new CronNodeRequest());
CronResponse response = action.newResponse(request, Arrays.asList(nodeResponse1, nodeResponse2), Collections.emptyList());
assertEquals(2, response.getNodes().size());
assertTrue(!response.hasFailures());
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
response.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.endObject();
String json = Strings.toString(builder);
Function<JsonElement, String> function = (s) -> {
try {
return JsonDeserializer.getTextValue(s, CronNodeResponse.NODE_ID);
} catch (Exception e) {
Assert.fail(e.getMessage());
}
return null;
};
assertArrayEquals(
JsonDeserializer.getArrayValue(json, function, CronResponse.NODES_JSON_KEY),
new String[] { localNodeID, localNodeID }
);
}
@PutMapping("update/book/novel")
public ResponseEntity add(@RequestParam(name = "id") String id,
@RequestParam(name = "title", required = false) String title,
@RequestParam(name = "author", required = false) String author,
@RequestParam(name = "wordCount", required = false) String wordCount,
@RequestParam(name = "publishDate", required = false) @DateTimeFormat(pattern = "yyyy-MM-dd") Date publishDate) {
try {
UpdateRequest updateRequest = new UpdateRequest(INDEX, TYPE, id);
// 构建文档
XContentBuilder contentBuilder = XContentFactory.jsonBuilder()
.startObject();
if (title != null) {
contentBuilder.field("title", title);
}
if (author != null) {
contentBuilder.field("author", author);
}
if (wordCount != null) {
contentBuilder.field("word_count", wordCount);
}
if (publishDate != null) {
contentBuilder.field("publish_date", publishDate.getTime());
}
contentBuilder.endObject();
updateRequest.doc(contentBuilder);
UpdateResponse result = transportClient.update(updateRequest).get();
return new ResponseEntity(result.getId(), HttpStatus.OK);
} catch (Exception e) {
LOGGER.error("update error", e);
return new ResponseEntity(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
public void updateClusterSettings(String settingKey, Object value) throws Exception {
XContentBuilder builder = XContentFactory
.jsonBuilder()
.startObject()
.startObject("persistent")
.field(settingKey, value)
.endObject()
.endObject();
Request request = new Request("PUT", "_cluster/settings");
request.setJsonEntity(Strings.toString(builder));
Response response = client().performRequest(request);
assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode()));
}
public void testToInstantWithNullToken() throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder().value((Long) null);
XContentParser parser = this.createParser(builder);
parser.nextToken();
XContentParser.Token token = parser.currentToken();
assertEquals(token, XContentParser.Token.VALUE_NULL);
Instant instant = ParseUtils.toInstant(parser);
assertNull(instant);
}
@Override
protected SuggestRequest beforeExecute(SuggestRequest request) {
try {
XContentBuilder builder = XContentFactory.contentBuilder(Requests.CONTENT_TYPE);
suggest.toXContent(builder, ToXContent.EMPTY_PARAMS);
request.suggest(builder.bytes());
} catch (IOException e) {
throw new ElasticsearchException("Unable to build suggestion request", e);
}
return request;
}
public static void appendField(XContentBuilder builder, String field,
byte[] data) throws IOException {
XContentType contentType = XContentFactory.xContentType(data);
if (contentType == null) {
addSimpleField(builder, field, data);
} else {
addComplexField(builder, field, contentType, data);
}
}
private String generateSetting(List<JobEvent> values) throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startArray();
for (JobEvent value : values) {
value.toXContent(builder, EMPTY_PARAMS);
}
builder.endArray();
return builder.string();
}