下面列出了怎么用org.elasticsearch.common.settings.Settings.Builder的API类实例代码及写法，或者点击链接到github查看源代码。
private Settings settings() {
Builder builder = Settings.builder()
.put("cluster.name", getClusterName());
if (getUser() == null && noPassword()) {
return builder.build();
}
if (getUser() == null || getPassword() == null) {
throw new IllegalArgumentException("Lacking user or password");
}
return builder
.put("xpack.security.user", getUser() + COLON + getPassword())
// .put("client.transport.sniff", clientTransportSniff)
// .put("client.transport.ignore_cluster_name", clientIgnoreClusterName)
// .put("client.transport.ping_timeout", clientPingTimeout)
// .put("client.transport.nodes_sampler_interval", clientNodesSamplerInterval)
.build();
}
private Builder applySettings() {
Builder settingsBuilder = Settings.builder();
settingsBuilder.put("client.transport.ping_timeout", "15s");
settingsBuilder.put("client.transport.nodes_sampler_interval", "5s");
// YG: to ensure reliable connection & resolve NoNodeAvailableException
settingsBuilder.put("client.transport.sniff", true);
settingsBuilder.put("network.bind_host", 0);
// YG: for supporting ES Auth with ES Shield
Optional.ofNullable(config.getElastic().getAuth())
.ifPresent(auth -> settingsBuilder.put("xpack.security.user", String.join(":", auth.getUser(), auth.getPwd())));
if (Objects.nonNull(config.getElastic().getClusterName())) {
settingsBuilder.put("cluster.name", config.getElastic().getClusterName());
} else {
settingsBuilder.put("client.transport.ignore_cluster_name", true);
}
return settingsBuilder;
}
@Before
public void setUp() throws Exception {
clusterName = "es-dynarank-" + System.currentTimeMillis();
runner = new ElasticsearchClusterRunner();
runner.onBuild(new ElasticsearchClusterRunner.Builder() {
@Override
public void build(final int number, final Builder settingsBuilder) {
settingsBuilder.put("dynarank.cache.clean_interval", "1s");
settingsBuilder.put("http.cors.enabled", true);
settingsBuilder.put("http.cors.allow-origin", "*");
settingsBuilder.put("discovery.type", "single-node");
// settingsBuilder.putList("discovery.seed_hosts", "127.0.0.1:9301");
// settingsBuilder.putList("cluster.initial_master_nodes", "127.0.0.1:9301");
}
}).build(newConfigs().numOfNode(1).clusterName(clusterName).pluginTypes(
"org.codelibs.elasticsearch.dynarank.DynamicRankingPlugin" + ",org.codelibs.elasticsearch.minhash.MinHashPlugin"));
runner.ensureGreen();
}
@Override
protected void setUp() throws Exception {
clusterName = "es-reindexing-" + System.currentTimeMillis();
// create runner instance
runner = new ElasticsearchClusterRunner();
// create ES nodes
runner.onBuild(new ElasticsearchClusterRunner.Builder() {
@Override
public void build(final int number, final Builder settingsBuilder) {
settingsBuilder.put("http.cors.enabled", true);
settingsBuilder.put("http.cors.allow-origin", "*");
settingsBuilder.put("index.number_of_shards", 3);
settingsBuilder.put("index.number_of_replicas", 0);
settingsBuilder.putArray("discovery.zen.ping.unicast.hosts",
"localhost:9301-9310");
settingsBuilder.put("plugin.types",
"org.codelibs.elasticsearch.reindex.ReindexingPlugin");
settingsBuilder
.put("index.unassigned.node_left.delayed_timeout", "0");
}
}).build(newConfigs().numOfNode(1).clusterName(clusterName));
// wait for yellow status
runner.ensureYellow();
}
@Override
protected void setUp() throws Exception {
clusterName = "es-minhash-" + System.currentTimeMillis();
// create runner instance
runner = new ElasticsearchClusterRunner();
// create ES nodes
runner.onBuild(new ElasticsearchClusterRunner.Builder() {
@Override
public void build(final int number, final Builder settingsBuilder) {
settingsBuilder.put("http.cors.enabled", true);
settingsBuilder.put("http.cors.allow-origin", "*");
settingsBuilder.put("discovery.type", "single-node");
}
}).build(newConfigs().clusterName(clusterName).numOfNode(1).pluginTypes("org.codelibs.elasticsearch.minhash.MinHashPlugin"));
// wait for yellow status
runner.ensureYellow();
}
@Override
protected void setUp() throws Exception {
clusterName = "es-taste-" + System.currentTimeMillis();
// create runner instance
runner = new ElasticsearchClusterRunner();
// create ES nodes
runner.onBuild(new ElasticsearchClusterRunner.Builder() {
@Override
public void build(final int number, final Builder settingsBuilder) {
settingsBuilder.put("http.cors.enabled", true);
settingsBuilder.put("http.cors.allow-origin", "*");
settingsBuilder.putArray("discovery.zen.ping.unicast.hosts", "localhost:9301-9305");
settingsBuilder.put("plugin.types", "org.codelibs.elasticsearch.taste.TastePlugin");
}
}).build(newConfigs().clusterName(clusterName).numOfNode(1));
// wait for yellow status
runner.ensureYellow();
}
@Override
protected void setUp() throws Exception {
clusterName = "es-cl-run-" + System.currentTimeMillis();
// create runner instance
runner = new ElasticsearchClusterRunner();
// create ES nodes
runner.onBuild(new ElasticsearchClusterRunner.Builder() {
@Override
public void build(final int number, final Builder settingsBuilder) {
settingsBuilder.put("http.cors.enabled", true);
settingsBuilder.put("http.cors.allow-origin", "*");
settingsBuilder.putList("discovery.seed_hosts", "127.0.0.1:9301", "127.0.0.1:9302");
settingsBuilder.putList("cluster.initial_master_nodes", "127.0.0.1:9301");
}
}).build(newConfigs().clusterName(clusterName).numOfNode(NUM_OF_NODES));
// wait for yellow status
runner.ensureYellow();
}
@BeforeClass
public static void setUp() throws IOException {
clusterName = "es-dataformat-" + System.currentTimeMillis();
// create runner instance
runner = new ElasticsearchClusterRunner();
// create ES nodes
runner.onBuild(new ElasticsearchClusterRunner.Builder() {
@Override
public void build(final int number, final Builder settingsBuilder) {
settingsBuilder.put("http.cors.enabled", true);
settingsBuilder.put("http.cors.allow-origin", "*");
settingsBuilder.put("discovery.type", "single-node");
// settingsBuilder.putList("discovery.seed_hosts", "127.0.0.1:9301");
// settingsBuilder.putList("cluster.initial_master_nodes", "127.0.0.1:9301");
}
}).build(newConfigs().clusterName(clusterName).numOfNode(1)
.pluginTypes("org.codelibs.elasticsearch.df.DataFormatPlugin"));
// wait for yellow status
runner.ensureYellow();
indexing();
node = runner.node();
}
@Override
protected void setUp() throws Exception {
// create runner instance
clusterName = "es-river-web-" + UUID.randomUUID().toString();
runner = new ElasticsearchClusterRunner();
runner.onBuild(new ElasticsearchClusterRunner.Builder() {
@Override
public void build(final int number, final Builder settingsBuilder) {
settingsBuilder.put("http.cors.enabled", true);
settingsBuilder.put("http.cors.allow-origin", "*");
settingsBuilder.put("index.number_of_shards", 3);
settingsBuilder.put("index.number_of_replicas", 0);
settingsBuilder.putArray("discovery.zen.ping.unicast.hosts", "localhost:9301-9310");
settingsBuilder.put("index.unassigned.node_left.delayed_timeout", "0");
settingsBuilder.put("network.host", "0");
}
}).build(newConfigs().clusterName(clusterName).numOfNode(numOfNode));
// wait for yellow status
runner.ensureYellow();
}
private void testAppliesNoMasterBlock(String noMasterBlockSetting, ClusterBlock expectedBlock) {
final Cluster cluster = new Cluster(3);
cluster.runRandomly();
cluster.stabilise();
final ClusterNode leader = cluster.getAnyLeader();
leader.submitUpdateTask("update NO_MASTER_BLOCK_SETTING", cs -> {
final Builder settingsBuilder = Settings.builder().put(cs.metaData().persistentSettings());
settingsBuilder.put(NO_MASTER_BLOCK_SETTING.getKey(), noMasterBlockSetting);
return ClusterState.builder(cs).metaData(MetaData.builder(cs.metaData()).persistentSettings(settingsBuilder.build())).build();
}, (source, e) -> {});
cluster.runFor(DEFAULT_CLUSTER_STATE_UPDATE_DELAY, "committing setting update");
leader.disconnect();
cluster.runFor(defaultMillis(FOLLOWER_CHECK_TIMEOUT_SETTING) + defaultMillis(FOLLOWER_CHECK_INTERVAL_SETTING)
+ DEFAULT_CLUSTER_STATE_UPDATE_DELAY, "detecting disconnection");
assertThat(leader.getLastAppliedClusterState().blocks().global(), hasItem(expectedBlock));
// TODO reboot the leader and verify that the same block is applied when it restarts
}
public void testFailsNodeThatDoesNotRespond() {
final Builder settingsBuilder = Settings.builder();
if (randomBoolean()) {
settingsBuilder.put(FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), randomIntBetween(1, 10));
}
if (randomBoolean()) {
settingsBuilder.put(FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), randomIntBetween(100, 100000) + "ms");
}
if (randomBoolean()) {
settingsBuilder.put(FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), randomIntBetween(1, 100000) + "ms");
}
final Settings settings = settingsBuilder.build();
testBehaviourOfFailingNode(settings, () -> null,
"followers check retry count exceeded",
(FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings) - 1) * FOLLOWER_CHECK_INTERVAL_SETTING.get(settings).millis()
+ FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings) * FOLLOWER_CHECK_TIMEOUT_SETTING.get(settings).millis());
}
public void testFailsNodeThatRejectsCheck() {
final Builder settingsBuilder = Settings.builder();
if (randomBoolean()) {
settingsBuilder.put(FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), randomIntBetween(1, 10));
}
if (randomBoolean()) {
settingsBuilder.put(FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), randomIntBetween(100, 100000) + "ms");
}
final Settings settings = settingsBuilder.build();
testBehaviourOfFailingNode(settings, () -> {
throw new ElasticsearchException("simulated exception");
},
"followers check retry count exceeded",
(FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings) - 1) * FOLLOWER_CHECK_INTERVAL_SETTING.get(settings).millis());
}
private Settings getSettings(int nodeOrdinal, long nodeSeed, Settings others) {
Builder builder = Settings.builder().put(defaultSettings)
.put(getRandomNodeSettings(nodeSeed));
Settings settings = nodeConfigurationSource.nodeSettings(nodeOrdinal);
if (settings != null) {
if (settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey()) != null) {
throw new IllegalStateException("Tests must not set a '" + ClusterName.CLUSTER_NAME_SETTING.getKey()
+ "' as a node setting set '" + ClusterName.CLUSTER_NAME_SETTING.getKey() + "': ["
+ settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey()) + "]");
}
builder.put(settings);
}
if (others != null) {
builder.put(others);
}
builder.put(ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName);
return builder.build();
}
@Test
public void completetlyInvalidConfigurationTest() throws Exception {
Builder settingsBuilder = Settings.builder();
settingsBuilder.put("opendistro_security.audit.type", TestAuditlogImpl.class.getName());
settingsBuilder.put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_TRANSPORT_CATEGORIES, "nonexistent");
settingsBuilder.put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, "nonexistent");
AuditLogImpl auditLog = new AuditLogImpl(settingsBuilder.build(), null, null, AbstractSecurityUnitTest.MOCK_POOL, null, cs);
logAll(auditLog);
auditLog.close();
String result = TestAuditlogImpl.sb.toString();
Assert.assertTrue(categoriesPresentInLog(result, filterComplianceCategories(Category.values())));
}
@Test
public void invalidConfigurationTest() {
Builder settingsBuilder = Settings.builder();
settingsBuilder.put("opendistro_security.audit.type", "debug");
settingsBuilder.put("opendistro_security.audit.config.disabled_categories", "nonexistant, bad_headers");
AuditLog auditLog = new AuditLogImpl(settingsBuilder.build(), null, null, AbstractSecurityUnitTest.MOCK_POOL, null, cs);
logAll(auditLog);
String result = TestAuditlogImpl.sb.toString();
Assert.assertFalse(categoriesPresentInLog(result, Category.BAD_HEADERS));
}
@Test
public void enableAllCategoryTest() throws Exception {
final Builder settingsBuilder = Settings.builder();
settingsBuilder.put("opendistro_security.audit.type", TestAuditlogImpl.class.getName());
settingsBuilder.put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_TRANSPORT_CATEGORIES, "NONE");
settingsBuilder.put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, "NONE");
// we use the debug output, no ES client is needed. Also, we
// do not need to close.
AuditLogImpl auditLog = new AuditLogImpl(settingsBuilder.build(), null, null, AbstractSecurityUnitTest.MOCK_POOL, null, cs);
logAll(auditLog);
// we're using the ExecutorService in AuditLogImpl, so we need to wait
// until all tasks are finished before we can check the result
auditLog.close();
String result = TestAuditlogImpl.sb.toString();
Assert.assertTrue(Category.values()+"#"+result, categoriesPresentInLog(result, filterComplianceCategories(Category.values())));
Assert.assertThat(result, containsString("testuser.transport.succeededlogin"));
Assert.assertThat(result, containsString("testuser.rest.succeededlogin"));
Assert.assertThat(result, containsString("testuser.rest.failedlogin"));
Assert.assertThat(result, containsString("testuser.transport.failedlogin"));
Assert.assertThat(result, containsString("privilege.missing"));
Assert.assertThat(result, containsString("action.indexattempt"));
Assert.assertThat(result, containsString("action.transport.ssl"));
Assert.assertThat(result, containsString("action.success"));
Assert.assertThat(result, containsString("Empty"));
}
protected void checkCategoriesDisabled(Category ... disabledCategories) throws Exception {
List<String> categoryNames = new LinkedList<>();
for (Category category : disabledCategories) {
categoryNames.add(category.name().toLowerCase());
}
String disabledCategoriesString = Joiner.on(",").join(categoryNames);
Builder settingsBuilder = Settings.builder();
settingsBuilder.put("opendistro_security.audit.type", TestAuditlogImpl.class.getName());
settingsBuilder.put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_TRANSPORT_CATEGORIES, disabledCategoriesString);
settingsBuilder.put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, disabledCategoriesString);
// we use the debug output, no ES client is needed. Also, we
// do not need to close.
AuditLog auditLog = new AuditLogImpl(settingsBuilder.build(), null, null, AbstractSecurityUnitTest.MOCK_POOL, null, cs);
logAll(auditLog);
auditLog.close();
String result = TestAuditlogImpl.sb.toString();
List<Category> allButDisablesCategories = new LinkedList<>(Arrays.asList(Category.values()));
allButDisablesCategories.removeAll(Arrays.asList(disabledCategories));
System.out.println(result+"###"+disabledCategoriesString);
Assert.assertFalse(categoriesPresentInLog(result, disabledCategories));
Assert.assertTrue(categoriesPresentInLog(result, filterComplianceCategories(allButDisablesCategories.toArray(new Category[] {}))));
}
private void testAuditType(String type, Class<? extends AuditLogSink> expectedClass) throws Exception {
Builder settingsBuilder = Settings.builder();
settingsBuilder.put("opendistro_security.audit.type", type);
settingsBuilder.put("path.home", ".");
AuditLogImpl auditLog = new AuditLogImpl(settingsBuilder.build(), null, null, null, null, null);
auditLog.close();
// if (expectedClass != null) {
// Assert.assertNotNull("delegate is null for type: "+type,auditLog.delegate);
// Assert.assertEquals(expectedClass, auditLog.delegate.getClass());
// } else {
// Assert.assertNull(auditLog.delegate);
// }
}
private void prepareClient() {
Builder settingsBuilder = applySettings();
try {
TransportAddress ista = new TransportAddress(InetAddress.getByName(config.getElastic().getHost()), config.getElastic().getPort());
client = new PreBuiltTransportClient(settingsBuilder.build())
.addTransportAddress(ista);
} catch (UnknownHostException ex) {
logger.error(ex.getMessage(), ex);
System.exit(-1);
}
}
@Before
public void setUp() throws Exception {
clusterName = "es-analysissynonym-" + System.currentTimeMillis();
runner = new ElasticsearchClusterRunner();
runner.onBuild(new ElasticsearchClusterRunner.Builder() {
@Override
public void build(final int number, final Builder settingsBuilder) {
settingsBuilder.put("http.cors.enabled", true);
settingsBuilder.put("http.cors.allow-origin", "*");
}
}).build(
newConfigs().numOfNode(numOfNode).clusterName(clusterName).pluginTypes("org.codelibs.elasticsearch.synonym.SynonymPlugin"));
synonymFiles = null;
}
public void testAutoShrinking() {
final String[] allNodes = new String[]{"a", "b", "c", "d", "e", "f", "g"};
final String[] liveNodes = new String[randomIntBetween(1, allNodes.length)];
randomSubsetOf(liveNodes.length, allNodes).toArray(liveNodes);
final String[] initialVotingNodes = new String[randomIntBetween(1, allNodes.length)];
randomSubsetOf(initialVotingNodes.length, allNodes).toArray(initialVotingNodes);
final Builder settingsBuilder = Settings.builder();
if (randomBoolean()) {
settingsBuilder.put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), true);
}
final Reconfigurator reconfigurator = makeReconfigurator(settingsBuilder.build());
final Set<DiscoveryNode> liveNodesSet = nodes(liveNodes);
final VotingConfiguration initialConfig = conf(initialVotingNodes);
final int quorumSize = Math.max(liveNodes.length / 2 + 1, initialVotingNodes.length < 3 ? 1 : 2);
final VotingConfiguration finalConfig = reconfigurator.reconfigure(liveNodesSet, emptySet(),
randomFrom(liveNodesSet), initialConfig);
final String description = "reconfigure " + liveNodesSet + " from " + initialConfig + " yielded " + finalConfig;
if (quorumSize > liveNodes.length) {
assertFalse(description + " without a live quorum", finalConfig.hasQuorum(Arrays.asList(liveNodes)));
} else {
final List<String> expectedQuorum = randomSubsetOf(quorumSize, liveNodes);
assertTrue(description + " with quorum[" + quorumSize + "] of " + expectedQuorum, finalConfig.hasQuorum(expectedQuorum));
}
}
public void testFailureCounterResetsOnSuccess() {
final Builder settingsBuilder = Settings.builder();
if (randomBoolean()) {
settingsBuilder.put(FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), randomIntBetween(2, 10));
}
if (randomBoolean()) {
settingsBuilder.put(FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), randomIntBetween(100, 100000) + "ms");
}
final Settings settings = settingsBuilder.build();
final int retryCount = FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings);
final int maxRecoveries = randomIntBetween(3, 10);
// passes just enough checks to keep it alive, up to maxRecoveries, and then fails completely
testBehaviourOfFailingNode(settings, new Supplier<Empty>() {
private int checkIndex;
private int recoveries;
@Override
public Empty get() {
checkIndex++;
if (checkIndex % retryCount == 0 && recoveries < maxRecoveries) {
recoveries++;
return Empty.INSTANCE;
}
throw new ElasticsearchException("simulated exception");
}
},
"followers check retry count exceeded",
(FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings) * (maxRecoveries + 1) - 1)
* FOLLOWER_CHECK_INTERVAL_SETTING.get(settings).millis());
}
private Settings getNodeSettings(final int nodeId, final long seed, final Settings extraSettings, final int defaultMinMasterNodes) {
final Settings settings = getSettings(nodeId, seed, extraSettings);
final String name = buildNodeName(nodeId, settings);
final Settings.Builder updatedSettings = Settings.builder();
updatedSettings.put(Environment.PATH_HOME_SETTING.getKey(), baseDir);
if (numDataPaths > 1) {
updatedSettings.putList(Environment.PATH_DATA_SETTING.getKey(), IntStream.range(0, numDataPaths).mapToObj(i ->
baseDir.resolve(name).resolve("d" + i).toString()).collect(Collectors.toList()));
} else {
updatedSettings.put(Environment.PATH_DATA_SETTING.getKey(), baseDir.resolve(name));
}
updatedSettings.put(Environment.PATH_SHARED_DATA_SETTING.getKey(), baseDir.resolve(name + "-shared"));
// allow overriding the above
updatedSettings.put(settings);
// force certain settings
updatedSettings.put("node.name", name);
updatedSettings.put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed);
if (autoManageMinMasterNodes) {
assertThat("if master nodes are automatically managed then nodes must complete a join cycle when starting",
updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()), nullValue());
}
return updatedSettings.build();
}
@Test
public void testTlsConfigurationNoFallback() throws Exception {
TestHttpHandler handler = new TestHttpHandler();
server = ServerBootstrap.bootstrap().setListenerPort(8083).setServerInfo("Test/1.1").setSslContext(createSSLContext()).registerHandler("*", handler).create();
server.start();
Builder builder = Settings.builder().loadFromPath(FileHelper.getAbsoluteFilePathFromClassPath("auditlog/endpoints/sink/configuration_tls.yml"));
builder.put("path.home", "/");
// replace some values with absolute paths for unit tests
builder.put("opendistro_security.audit.config.webhook.ssl.pemtrustedcas_filepath", FileHelper.getAbsoluteFilePathFromClassPath("auditlog/root-ca.pem"));
builder.put("opendistro_security.audit.endpoints.endpoint1.config.webhook.ssl.pemtrustedcas_filepath", FileHelper.getAbsoluteFilePathFromClassPath("auditlog/root-ca.pem"));
builder.put("opendistro_security.audit.endpoints.endpoint2.config.webhook.ssl.pemtrustedcas_content", FileHelper.loadFile("auditlog/root-ca.pem"));
SinkProvider provider = new SinkProvider(builder.build(), null, null, null);
WebhookSink defaultSink = (WebhookSink) provider.defaultSink;
Assert.assertEquals(true, defaultSink.verifySSL);
AuditMessage msg = MockAuditMessageFactory.validAuditMessage();
provider.allSinks.get("endpoint1").store(msg);
Assert.assertTrue(handler.method.equals("POST"));
Assert.assertTrue(handler.body != null);
Assert.assertTrue(handler.body.contains("{"));
assertStringContainsAllKeysAndValues(handler.body);
handler.reset();
provider.allSinks.get("endpoint2").store(msg);
Assert.assertTrue(handler.method.equals("POST"));
Assert.assertTrue(handler.body != null);
Assert.assertTrue(handler.body.contains("{"));
assertStringContainsAllKeysAndValues(handler.body);
handler.reset();
provider.defaultSink.store(msg);
Assert.assertTrue(handler.method.equals("POST"));
Assert.assertTrue(handler.body != null);
Assert.assertTrue(handler.body.contains("{"));
assertStringContainsAllKeysAndValues(handler.body);
server.stop();
}
public static void main(String args[]) throws Exception {
if (args.length != 2) {
System.out.printf("Usage: java KibanaExporter.jar %s %s%n", "localhost:9300", "my-cluster-name");
System.exit(0);
}
if (!args[0].contains(":")) {
throw new IllegalArgumentException(
String.format("%s not a valid format. Expected <hostname>:<port>.", args[0]));
}
// set ES address
String split[] = args[0].split(":");
// set cluster name
Builder tcSettings = Settings.builder();
tcSettings.put("cluster.name", args[1]);
System.out.println("Connection to " + args[1]);
client = new RestHighLevelClient(RestClient
.builder(new HttpHost(InetAddress.getByName(split[0]), Integer.parseInt(split[1], 10), "http")));
// search index pattern for needle
searchIndexPattern();
KibanaConfigHolderDto holder = new KibanaConfigHolderDto();
System.out.println("Reading .kibana index");
SearchResponse resp = client.search(new SearchRequest(".kibana").source(new SearchSourceBuilder().size(1000)),
RequestOptions.DEFAULT);
Arrays.asList(resp.getHits().getHits()).stream().map(KibanaExporter::parseSearchHit).forEach(holder::add);
System.out.println("Reading finished");
ObjectMapper mapper = new ObjectMapper();
// we love pretty things
mapper.enable(SerializationFeature.INDENT_OUTPUT);
File f = new File("kibana_config.json");
try (FileOutputStream out = new FileOutputStream(f, false)) {
mapper.writeValue(out, holder);
}
System.out.println("File outputted to: " + f.getAbsolutePath());
client.close();
}
@Before
public void setup() throws Exception {
clusterName = "es-taste-" + System.currentTimeMillis();
runner = new ElasticsearchClusterRunner();
runner.onBuild(new ElasticsearchClusterRunner.Builder() {
@Override
public void build(final int number, final Builder settingsBuilder) {
settingsBuilder.put("http.cors.enabled", true);
settingsBuilder.put("http.cors.allow-origin", "*");
settingsBuilder.put("index.number_of_shards", 3);
settingsBuilder.put("index.number_of_replicas", 0);
settingsBuilder.putArray("discovery.zen.ping.unicast.hosts", "localhost:9301-9310");
settingsBuilder.put("plugin.types",
"org.codelibs.elasticsearch.taste.TastePlugin");
settingsBuilder.put("index.unassigned.node_left.delayed_timeout", "0");
}
}).build(newConfigs().clusterName(clusterName).numOfNode(numOfNode));
runner.ensureYellow();
Client client = runner.client();
// Wait for Yellow status
client.admin().cluster().prepareHealth().setWaitForYellowStatus()
.setTimeout(TimeValue.timeValueMinutes(1)).execute()
.actionGet();
final CreateIndexResponse response = client.admin().indices()
.prepareCreate(TEST_INDEX).execute().actionGet();
if (!response.isAcknowledged()) {
throw new TasteException("Failed to create index: " + TEST_INDEX);
}
final XContentBuilder userBuilder = XContentFactory.jsonBuilder()//
.startObject()//
.startObject(TasteConstants.USER_TYPE)//
.startObject("properties")//
// @timestamp
.startObject(TasteConstants.TIMESTAMP_FIELD)//
.field("type", "date")//
.field("format", "date_optional_time")//
.endObject()//
// user_id
.startObject(TasteConstants.USER_ID_FIELD)//
.field("type", "long")//
.endObject()//
// id
.startObject("id")//
.field("type", "string")//
.field("index", "not_analyzed")//
.endObject()//
.endObject()//
.endObject()//
.endObject();
final PutMappingResponse userResponse = client.admin().indices()
.preparePutMapping(TEST_INDEX)
.setType(TasteConstants.USER_TYPE).setSource(userBuilder)
.execute().actionGet();
if (!userResponse.isAcknowledged()) {
throw new TasteException("Failed to create user mapping.");
}
final XContentBuilder itemBuilder = XContentFactory.jsonBuilder()//
.startObject()//
.startObject(TasteConstants.ITEM_TYPE)//
.startObject("properties")//
// @timestamp
.startObject(TasteConstants.TIMESTAMP_FIELD)//
.field("type", "date")//
.field("format", "date_optional_time")//
.endObject()//
// item_id
.startObject(TasteConstants.ITEM_ID_FIELD)//
.field("type", "long")//
.endObject()//
// id
.startObject("id")//
.field("type", "string")//
.field("index", "not_analyzed")//
.endObject()//
.endObject()//
.endObject()//
.endObject();
final PutMappingResponse itemResponse = client.admin().indices()
.preparePutMapping(TEST_INDEX)
.setType(TasteConstants.ITEM_TYPE).setSource(itemBuilder)
.execute().actionGet();
if (!itemResponse.isAcknowledged()) {
throw new TasteException("Failed to create item mapping.");
}
}
protected Client createHttpClient(final FessConfig fessConfig, final String host) {
final Builder builder = Settings.builder().putList("http.hosts", host).put("processors", fessConfig.availableProcessors());
return new HttpClient(builder.build(), null);
}
public void test_javascript_indexed() throws Exception {
// create runner instance
String clusterName = "es-river-web-" + UUID.randomUUID().toString();
ElasticsearchClusterRunner runner = new ElasticsearchClusterRunner();
runner.onBuild(new ElasticsearchClusterRunner.Builder() {
@Override
public void build(final int number, final Builder settingsBuilder) {
settingsBuilder.put("http.cors.enabled", true);
settingsBuilder.put("http.cors.allow-origin", "*");
settingsBuilder.put("index.number_of_shards", 3);
settingsBuilder.put("index.number_of_replicas", 0);
settingsBuilder.putArray("discovery.zen.ping.unicast.hosts", "localhost:9301-9310");
settingsBuilder.put("index.unassigned.node_left.delayed_timeout", "0");
}
}).build(newConfigs().clusterName(clusterName).numOfNode(1));
// wait for yellow status
runner.ensureYellow();
try {
esClient.setClusterName(clusterName);
esClient.setAddresses(new String[] { "localhost:" + runner.node().settings().get("transport.tcp.port") });
esClient.connect();
Map<String, Object> localVars = new HashMap<>();
String lang = "javascript";
ScriptType scriptType = ScriptType.INDEXED;
String script = "script1";
runner.insert(ScriptService.SCRIPT_INDEX, lang, script, "{\"script\":\"'test';\"}");
assertEquals("test", scriptService.execute(lang, script, scriptType, localVars));
script = "script2";
runner.insert(ScriptService.SCRIPT_INDEX, lang, script, "{\"script\":\"print('test');\"}");
assertNull(scriptService.execute(lang, script, scriptType, localVars));
localVars.put("testVar", "aaa");
script = "script3";
runner.insert(ScriptService.SCRIPT_INDEX, lang, script, "{\"script\":\"testVar;\"}");
assertEquals("aaa", scriptService.execute(lang, script, scriptType, localVars));
} finally {
// close runner
runner.close();
// delete all files
runner.clean();
}
}
public void testRetriesOnCorrectSchedule() {
final Builder settingsBuilder = Settings.builder();
final long initialTimeoutMillis;
if (randomBoolean()) {
initialTimeoutMillis = randomLongBetween(1, 10000);
settingsBuilder.put(ELECTION_INITIAL_TIMEOUT_SETTING.getKey(), initialTimeoutMillis + "ms");
} else {
initialTimeoutMillis = ELECTION_INITIAL_TIMEOUT_SETTING.get(Settings.EMPTY).millis();
}
if (randomBoolean()) {
settingsBuilder.put(ELECTION_BACK_OFF_TIME_SETTING.getKey(), randomLongBetween(1, 60000) + "ms");
}
if (ELECTION_MAX_TIMEOUT_SETTING.get(Settings.EMPTY).millis() < initialTimeoutMillis || randomBoolean()) {
settingsBuilder.put(ELECTION_MAX_TIMEOUT_SETTING.getKey(),
randomLongBetween(Math.max(200, initialTimeoutMillis), 180000) + "ms");
}
final long electionDurationMillis;
if (randomBoolean()) {
electionDurationMillis = randomLongBetween(1, 300000);
settingsBuilder.put(ELECTION_DURATION_SETTING.getKey(), electionDurationMillis + "ms");
} else {
electionDurationMillis = ELECTION_DURATION_SETTING.get(Settings.EMPTY).millis();
}
final Settings settings = settingsBuilder.put(NODE_NAME_SETTING.getKey(), "node").build();
final long initialTimeout = ELECTION_INITIAL_TIMEOUT_SETTING.get(settings).millis();
final long backOffTime = ELECTION_BACK_OFF_TIME_SETTING.get(settings).millis();
final long maxTimeout = ELECTION_MAX_TIMEOUT_SETTING.get(settings).millis();
final long duration = ELECTION_DURATION_SETTING.get(settings).millis();
final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
final ElectionSchedulerFactory electionSchedulerFactory
= new ElectionSchedulerFactory(settings, random(), deterministicTaskQueue.getThreadPool());
assertElectionSchedule(deterministicTaskQueue, electionSchedulerFactory, initialTimeout, backOffTime, maxTimeout, duration);
// do it again to show that the max is reset when the scheduler is restarted
assertElectionSchedule(deterministicTaskQueue, electionSchedulerFactory, initialTimeout, backOffTime, maxTimeout, duration);
}
private static Settings getRandomNodeSettings(long seed) {
Random random = new Random(seed);
Builder builder = Settings.builder();
builder.put(TransportSettings.TRANSPORT_COMPRESS.getKey(), rarely(random));
if (random.nextBoolean()) {
builder.put("cache.recycler.page.type", RandomPicks.randomFrom(random, PageCacheRecycler.Type.values()));
}
builder.put(EsExecutors.PROCESSORS_SETTING.getKey(), 1 + random.nextInt(3));
// randomize tcp settings
if (random.nextBoolean()) {
builder.put(TransportSettings.CONNECTIONS_PER_NODE_RECOVERY.getKey(), random.nextInt(2) + 1);
builder.put(TransportSettings.CONNECTIONS_PER_NODE_BULK.getKey(), random.nextInt(3) + 1);
builder.put(TransportSettings.CONNECTIONS_PER_NODE_REG.getKey(), random.nextInt(6) + 1);
}
if (random.nextBoolean()) {
builder.put(MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING.getKey(),
timeValueSeconds(RandomNumbers.randomIntBetween(random, 10, 30)).getStringRep());
}
if (random.nextInt(10) == 0) {
builder.put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_TYPE_SETTING.getKey(), "noop");
builder.put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING.getKey(), "noop");
}
if (random.nextBoolean()) {
if (random.nextInt(10) == 0) { // do something crazy slow here
builder.put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(),
new ByteSizeValue(RandomNumbers.randomIntBetween(random, 1, 10), ByteSizeUnit.MB));
} else {
builder.put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(),
new ByteSizeValue(RandomNumbers.randomIntBetween(random, 10, 200), ByteSizeUnit.MB));
}
}
if (random.nextBoolean()) {
builder.put(TransportSettings.PING_SCHEDULE.getKey(), RandomNumbers.randomIntBetween(random, 100, 2000) + "ms");
}
return builder.build();
}