public static void buildBroadcastShardsHeader(XContentBuilder builder, ToXContent.Params params, int total, int successful, int failed, ShardOperationFailedException[] shardFailures) throws IOException {
    builder.startObject(Fields._SHARDS);
    builder.field(Fields.TOTAL, total);
    builder.field(Fields.SUCCESSFUL, successful);
    builder.field(Fields.FAILED, failed);
    if (shardFailures != null && shardFailures.length > 0) {
        builder.startArray(Fields.FAILURES);
        final boolean group = params.paramAsBoolean("group_shard_failures", true); // we group by default
        for (ShardOperationFailedException shardFailure : group ? ExceptionsHelper.groupBy(shardFailures) : shardFailures) {
            builder.startObject();
            shardFailure.toXContent(builder, params);
            builder.endObject();
        }
        builder.endArray();
    }
    builder.endObject();
}
 

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeVInt(storeStatuses.size());
    for (ObjectObjectCursor<String, ImmutableOpenIntMap<List<StoreStatus>>> indexShards : storeStatuses) {
        out.writeString(indexShards.key);
        out.writeVInt(indexShards.value.size());
        for (IntObjectCursor<List<StoreStatus>> shardStatusesEntry : indexShards.value) {
            out.writeInt(shardStatusesEntry.key);
            out.writeVInt(shardStatusesEntry.value.size());
            for (StoreStatus storeStatus : shardStatusesEntry.value) {
                storeStatus.writeTo(out);
            }
        }
    }
    out.writeVInt(failures.size());
    for (ShardOperationFailedException failure : failures) {
        failure.writeTo(out);
    }
}
 

@Override
protected RecoveryResponse newResponse(RecoveryRequest request, int totalShards, int successfulShards, int failedShards, List<RecoveryState> responses, List<ShardOperationFailedException> shardFailures, ClusterState clusterState) {
    Map<String, List<RecoveryState>> shardResponses = Maps.newHashMap();
    for (RecoveryState recoveryState : responses) {
        if (recoveryState == null) {
            continue;
        }
        String indexName = recoveryState.getShardId().getIndex();
        if (!shardResponses.containsKey(indexName)) {
            shardResponses.put(indexName, new ArrayList<RecoveryState>());
        }
        if (request.activeOnly()) {
            if (recoveryState.getStage() != RecoveryState.Stage.DONE) {
                shardResponses.get(indexName).add(recoveryState);
            }
        } else {
            shardResponses.get(indexName).add(recoveryState);
        }
    }
    return new RecoveryResponse(totalShards, successfulShards, failedShards, request.detailed(), shardResponses, shardFailures);
}
 

@Override
protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
    builder.field("phase", phaseName);
    final boolean group = params.paramAsBoolean("group_shard_failures", true); // we group by default
    builder.field("grouped", group); // notify that it's grouped
    builder.field("failed_shards");
    builder.startArray();
    ShardOperationFailedException[] failures = params.paramAsBoolean("group_shard_failures", true) ? ExceptionsHelper.groupBy(shardFailures) : shardFailures;
    for (ShardOperationFailedException failure : failures) {
        builder.startObject();
        failure.toXContent(builder, params);
        builder.endObject();
    }
    builder.endArray();
    super.innerToXContent(builder, params);
}
 

private void finishAndNotifyListener(ActionListener listener, CopyOnWriteArrayList<ShardResponse> shardsResponses) {
    logger.trace("{}: got all shard responses", actionName);
    int successfulShards = 0;
    int failedShards = 0;
    int totalNumCopies = 0;
    List<ShardOperationFailedException> shardFailures = null;
    for (int i = 0; i < shardsResponses.size(); i++) {
        ActionWriteResponse shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            // non active shard, ignore
        } else {
            failedShards += shardResponse.getShardInfo().getFailed();
            successfulShards += shardResponse.getShardInfo().getSuccessful();
            totalNumCopies += shardResponse.getShardInfo().getTotal();
            if (shardFailures == null) {
                shardFailures = new ArrayList<>();
            }
            for (ActionWriteResponse.ShardInfo.Failure failure : shardResponse.getShardInfo().getFailures()) {
                shardFailures.add(new DefaultShardOperationFailedException(new BroadcastShardOperationFailedException(new ShardId(failure.index(), failure.shardId()), failure.getCause())));
            }
        }
    }
    listener.onResponse(newResponse(successfulShards, failedShards, totalNumCopies, shardFailures));
}
 

@Override
protected DfsOnlyResponse newResponse(DfsOnlyRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
    int successfulShards = 0;
    int failedShards = 0;
    List<ShardOperationFailedException> shardFailures = null;
    AtomicArray<DfsSearchResult> dfsResults = new AtomicArray<>(shardsResponses.length());
    for (int i = 0; i < shardsResponses.length(); i++) {
        Object shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            // simply ignore non active shards
        } else if (shardResponse instanceof BroadcastShardOperationFailedException) {
            failedShards++;
            if (shardFailures == null) {
                shardFailures = new ArrayList<>();
            }
            shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
        } else {
            dfsResults.set(i, ((ShardDfsOnlyResponse) shardResponse).getDfsSearchResult());
            successfulShards++;
        }
    }
    AggregatedDfs dfs = searchPhaseController.aggregateDfs(dfsResults);
    return new DfsOnlyResponse(dfs, shardsResponses.length(), successfulShards, failedShards, shardFailures, buildTookInMillis(request));
}
 

protected void validateRespose(final SearchResponse response) {
    final int totalShards = response.getTotalShards();
    final int successfulShards = response.getSuccessfulShards();
    if (totalShards != successfulShards) {
        throw new MissingShardsException(totalShards - successfulShards
                + " shards are failed.");
    }
    final ShardSearchFailure[] failures = response.getShardFailures();
    if (failures.length > 0) {
        final StringBuilder buf = new StringBuilder();
        for (final ShardOperationFailedException failure : failures) {
            buf.append('\n').append(failure.toString());
        }
        throw new OperationFailedException("Search Operation Failed: "
                + buf.toString());
    }
}
 

@Override
protected ExportResponse newResponse(ExportRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
    int successfulShards = 0;
    int failedShards = 0;
    List<ShardOperationFailedException> shardFailures = null;
    List<ShardExportResponse> responses = new ArrayList<ShardExportResponse>();
    for (int i = 0; i < shardsResponses.length(); i++) {
        Object shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            failedShards++;
        } else if (shardResponse instanceof BroadcastShardOperationFailedException) {
            failedShards++;
            if (shardFailures == null) {
                shardFailures = newArrayList();
            }
            shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
        } else {
            responses.add((ShardExportResponse) shardResponse);
            successfulShards++;
        }
    }
    return new ExportResponse(responses, shardsResponses.length(), successfulShards, failedShards, shardFailures);
}
 

private static String genMessage(ShardOperationFailedException[] shardFailures) {
    StringBuilder sb;

    if (shardFailures.length == 1) {
        sb = new StringBuilder("query failed on shard ");
    } else {
        sb = new StringBuilder("query failed on shards ");
    }

    List<String> errors = new ArrayList<>(shardFailures.length);
    String table = null;
    for (ShardOperationFailedException shardFailure : shardFailures) {
        if (shardFailure == null) {
            continue;
        }
        errors.add(shardFailure.shardId()+" ( "+shardFailure.reason()+" )");
        table = shardFailure.index();
    }

    if (errors.isEmpty() && table == null) {
        return "query failed on unknown shard / table";
    }
    sb.append(Joiner.on(", ").join(errors));
    if (table != null) {
        sb.append(" of table ").append(table);
    }
    return sb.toString();
}
 

/**
 * Deduplicate the failures by exception message and index.
 */
public static ShardOperationFailedException[] groupBy(ShardOperationFailedException[] failures) {
    List<ShardOperationFailedException> uniqueFailures = new ArrayList<>();
    Set<GroupBy> reasons = new HashSet<>();
    for (ShardOperationFailedException failure : failures) {
        GroupBy reason = new GroupBy(failure.getCause());
        if (reasons.contains(reason) == false) {
            reasons.add(reason);
            uniqueFailures.add(failure);
        }
    }
    return uniqueFailures.toArray(new ShardOperationFailedException[0]);
}
 

/**
 * Returns snapshot REST status
 */
public RestStatus status() {
    if (state == SnapshotState.FAILED) {
        return RestStatus.INTERNAL_SERVER_ERROR;
    }
    if (shardFailures.size() == 0) {
        return RestStatus.OK;
    }
    return RestStatus.status(successfulShards, totalShards, shardFailures.toArray(new ShardOperationFailedException[shardFailures.size()]));
}
 

@Override
protected ExistsResponse newResponse(ExistsRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
    int successfulShards = 0;
    int failedShards = 0;
    boolean exists = false;
    List<ShardOperationFailedException> shardFailures = null;

    // if docs do exist, the last response will have exists = true (since we early terminate the shard requests)
    for (int i = shardsResponses.length() - 1; i >= 0 ; i--) {
        Object shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            // simply ignore non active shards
        } else if (shardResponse instanceof BroadcastShardOperationFailedException) {
            failedShards++;
            if (shardFailures == null) {
                shardFailures = new ArrayList<>();
            }
            shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
        } else {
            successfulShards++;
            if ((exists = ((ShardExistsResponse) shardResponse).exists())) {
                successfulShards = shardsResponses.length() - failedShards;
                break;
            }
        }
    }
    return new ExistsResponse(exists, shardsResponses.length(), successfulShards, failedShards, shardFailures);
}
 

ValidateQueryResponse(boolean valid, List<QueryExplanation> queryExplanations, int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures) {
    super(totalShards, successfulShards, failedShards, shardFailures);
    this.valid = valid;
    this.queryExplanations = queryExplanations;
    if (queryExplanations == null) {
        this.queryExplanations = Collections.emptyList();
    }
}
 

@Override
protected ValidateQueryResponse newResponse(ValidateQueryRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
    int successfulShards = 0;
    int failedShards = 0;
    boolean valid = true;
    List<ShardOperationFailedException> shardFailures = null;
    List<QueryExplanation> queryExplanations = null;
    for (int i = 0; i < shardsResponses.length(); i++) {
        Object shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            // simply ignore non active shards
        } else if (shardResponse instanceof BroadcastShardOperationFailedException) {
            failedShards++;
            if (shardFailures == null) {
                shardFailures = new ArrayList<>();
            }
            shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
        } else {
            ShardValidateQueryResponse validateQueryResponse = (ShardValidateQueryResponse) shardResponse;
            valid = valid && validateQueryResponse.isValid();
            if (request.explain() || request.rewrite()) {
                if (queryExplanations == null) {
                    queryExplanations = new ArrayList<>();
                }
                queryExplanations.add(new QueryExplanation(
                        validateQueryResponse.getIndex(),
                        validateQueryResponse.isValid(),
                        validateQueryResponse.getExplanation(),
                        validateQueryResponse.getError()
                ));
            }
            successfulShards++;
        }
    }
    return new ValidateQueryResponse(valid, queryExplanations, shardsResponses.length(), successfulShards, failedShards, shardFailures);
}
 

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
    if (failures.size() > 0) {
        builder.startArray(Fields.FAILURES);
        for (ShardOperationFailedException failure : failures) {
            builder.startObject();
            failure.toXContent(builder, params);
            builder.endObject();
        }
        builder.endArray();
    }

    builder.startObject(Fields.INDICES);
    for (ObjectObjectCursor<String, ImmutableOpenIntMap<List<StoreStatus>>> indexShards : storeStatuses) {
        builder.startObject(indexShards.key);

        builder.startObject(Fields.SHARDS);
        for (IntObjectCursor<List<StoreStatus>> shardStatusesEntry : indexShards.value) {
            builder.startObject(String.valueOf(shardStatusesEntry.key));
            builder.startArray(Fields.STORES);
            for (StoreStatus storeStatus : shardStatusesEntry.value) {
                builder.startObject();
                storeStatus.toXContent(builder, params);
                builder.endObject();
            }
            builder.endArray();

            builder.endObject();
        }
        builder.endObject();

        builder.endObject();
    }
    builder.endObject();
    return builder;
}
 

PercolateResponse(int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures,
                         Match[] matches, long count, long tookInMillis, InternalAggregations aggregations) {
    super(totalShards, successfulShards, failedShards, shardFailures);
    if (tookInMillis < 0) {
        throw new IllegalArgumentException("tookInMillis must be positive but was: " + tookInMillis);
    }
    this.tookInMillis = tookInMillis;
    this.matches = matches;
    this.count = count;
    this.aggregations = aggregations;
}
 

PercolateResponse(int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures, long tookInMillis, Match[] matches) {
    super(totalShards, successfulShards, failedShards, shardFailures);
    if (tookInMillis < 0) {
        throw new IllegalArgumentException("tookInMillis must be positive but was: " + tookInMillis);
    }
    this.tookInMillis = tookInMillis;
    this.matches = matches;
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    totalShards = in.readVInt();
    successfulShards = in.readVInt();
    failedShards = in.readVInt();
    int size = in.readVInt();
    if (size > 0) {
        shardFailures = new ShardOperationFailedException[size];
        for (int i = 0; i < size; i++) {
            shardFailures[i] = readShardOperationFailed(in);
        }
    }
}
 

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeVInt(totalShards);
    out.writeVInt(successfulShards);
    out.writeVInt(failedShards);
    out.writeVInt(shardFailures.length);
    for (ShardOperationFailedException exp : shardFailures) {
        exp.writeTo(out);
    }
}
 

private final Response newResponse(
        Request request,
        AtomicReferenceArray responses,
        List<NoShardAvailableActionException> unavailableShardExceptions,
        Map<String, List<ShardRouting>> nodes,
        ClusterState clusterState) {
    int totalShards = 0;
    int successfulShards = 0;
    List<ShardOperationResult> broadcastByNodeResponses = new ArrayList<>();
    List<ShardOperationFailedException> exceptions = new ArrayList<>();
    for (int i = 0; i < responses.length(); i++) {
        if (responses.get(i) instanceof FailedNodeException) {
            FailedNodeException exception = (FailedNodeException) responses.get(i);
            totalShards += nodes.get(exception.nodeId()).size();
            for (ShardRouting shard : nodes.get(exception.nodeId())) {
                exceptions.add(new DefaultShardOperationFailedException(shard.getIndex(), shard.getId(), exception));
            }
        } else {
            NodeResponse response = (NodeResponse) responses.get(i);
            broadcastByNodeResponses.addAll(response.results);
            totalShards += response.getTotalShards();
            successfulShards += response.getSuccessfulShards();
            for (BroadcastShardOperationFailedException throwable : response.getExceptions()) {
                if (!TransportActions.isShardNotAvailableException(throwable)) {
                    exceptions.add(new DefaultShardOperationFailedException(throwable.getIndex(), throwable.getShardId().getId(), throwable));
                }
            }
        }
    }
    totalShards += unavailableShardExceptions.size();
    int failedShards = exceptions.size();
    return newResponse(request, totalShards, successfulShards, failedShards, broadcastByNodeResponses, exceptions, clusterState);
}
 

@Override
protected SuggestResponse newResponse(SuggestRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
    int successfulShards = 0;
    int failedShards = 0;

    final Map<String, List<Suggest.Suggestion>> groupedSuggestions = new HashMap<>();

    List<ShardOperationFailedException> shardFailures = null;
    for (int i = 0; i < shardsResponses.length(); i++) {
        Object shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            // simply ignore non active shards
        } else if (shardResponse instanceof BroadcastShardOperationFailedException) {
            failedShards++;
            if (shardFailures == null) {
                shardFailures = new ArrayList<>();
            }
            shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
        } else {
            Suggest suggest = ((ShardSuggestResponse) shardResponse).getSuggest();
            Suggest.group(groupedSuggestions, suggest);
            successfulShards++;
        }
    }

    return new SuggestResponse(new Suggest(Suggest.reduce(groupedSuggestions)), shardsResponses.length(), successfulShards, failedShards, shardFailures);
}
 

public FlushResponse flush(final BuilderCallback<FlushRequestBuilder> builder) {
    waitForRelocation();
    final FlushResponse actionGet = builder.apply(client().admin().indices().prepareFlush()).execute().actionGet();
    final ShardOperationFailedException[] shardFailures = actionGet.getShardFailures();
    if (shardFailures != null && shardFailures.length != 0) {
        final StringBuilder buf = new StringBuilder(100);
        for (final ShardOperationFailedException shardFailure : shardFailures) {
            buf.append(shardFailure.toString()).append('\n');
        }
        onFailure(buf.toString(), actionGet);
    }
    return actionGet;
}
 

public RefreshResponse refresh(final BuilderCallback<RefreshRequestBuilder> builder) {
    waitForRelocation();
    final RefreshResponse actionGet = builder.apply(client().admin().indices().prepareRefresh()).execute().actionGet();
    final ShardOperationFailedException[] shardFailures = actionGet.getShardFailures();
    if (shardFailures != null && shardFailures.length != 0) {
        final StringBuilder buf = new StringBuilder(100);
        for (final ShardOperationFailedException shardFailure : shardFailures) {
            buf.append(shardFailure.toString()).append('\n');
        }
        onFailure(buf.toString(), actionGet);
    }
    return actionGet;
}
 

public UpgradeResponse upgrade(final BuilderCallback<UpgradeRequestBuilder> builder) {
    waitForRelocation();
    final UpgradeResponse actionGet = builder.apply(client().admin().indices().prepareUpgrade()).execute().actionGet();
    final ShardOperationFailedException[] shardFailures = actionGet.getShardFailures();
    if (shardFailures != null && shardFailures.length != 0) {
        final StringBuilder buf = new StringBuilder(100);
        for (final ShardOperationFailedException shardFailure : shardFailures) {
            buf.append(shardFailure.toString()).append('\n');
        }
        onFailure(buf.toString(), actionGet);
    }
    return actionGet;
}
 

public ForceMergeResponse forceMerge(final BuilderCallback<ForceMergeRequestBuilder> builder) {
    waitForRelocation();
    final ForceMergeResponse actionGet = builder.apply(client().admin().indices().prepareForceMerge()).execute().actionGet();
    final ShardOperationFailedException[] shardFailures = actionGet.getShardFailures();
    if (shardFailures != null && shardFailures.length != 0) {
        final StringBuilder buf = new StringBuilder(100);
        for (final ShardOperationFailedException shardFailure : shardFailures) {
            buf.append(shardFailure.toString()).append('\n');
        }
        onFailure(buf.toString(), actionGet);
    }
    return actionGet;
}
 

public Observable<IndexRefreshCommandInfo> refreshAsync() {

        refreshIndexMeter.mark();
        final long start = System.currentTimeMillis();

        String[] indexes = getIndexes();
        if (indexes.length == 0) {
            if (logger.isTraceEnabled()) {
                logger.trace("Not refreshing indexes. none found");
            }
        }
        //Added For Graphite Metrics
        RefreshResponse response =
            esProvider.getClient().admin().indices().prepareRefresh(indexes).execute().actionGet();
        int failedShards = response.getFailedShards();
        int successfulShards = response.getSuccessfulShards();
        ShardOperationFailedException[] sfes = response.getShardFailures();
        if (sfes != null) {
            for (ShardOperationFailedException sfe : sfes) {
                logger.error("Failed to refresh index:{} reason:{}", sfe.index(), sfe.reason());
            }
        }
        if (logger.isTraceEnabled()) {
            logger.trace("Refreshed indexes: {},success:{} failed:{} ", StringUtils.join(indexes, ", "),
                successfulShards, failedShards);
        }

        IndexRefreshCommandInfo refreshResults = new IndexRefreshCommandInfo(failedShards == 0,
            System.currentTimeMillis() - start);


        return ObservableTimer.time(Observable.just(refreshResults), refreshTimer);
    }
 

/**
 * Validate the response doesn't contain errors, if it does, fail fast at the first error we encounter
 */
private void checkDeleteByQueryResponse( final QueryBuilder query, final DeleteByQueryResponse response ) {

    for ( IndexDeleteByQueryResponse indexDeleteByQueryResponse : response ) {
        final ShardOperationFailedException[] failures = indexDeleteByQueryResponse.getFailures();

        for ( ShardOperationFailedException failedException : failures ) {
            logger.error("Unable to delete by query {}. Failed with code {} and reason {} on shard {} in index {}",
                query.toString(),
                failedException.status().getStatus(), failedException.reason(),
                failedException.shardId(), failedException.index() );
        }
    }
}
 

@Override
protected SearchIntoResponse newResponse(SearchIntoRequest request,
        AtomicReferenceArray shardsResponses, ClusterState clusterState) {
    int successfulShards = 0;
    int failedShards = 0;
    List<ShardOperationFailedException> shardFailures = null;
    List<ShardSearchIntoResponse> responses = new
            ArrayList<ShardSearchIntoResponse>();
    for (int i = 0; i < shardsResponses.length(); i++) {
        Object shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            failedShards++;
        } else if (shardResponse instanceof
                BroadcastShardOperationFailedException) {
            failedShards++;
            if (shardFailures == null) {
                shardFailures = newArrayList();
            }
            shardFailures.add(new DefaultShardOperationFailedException(
                    (BroadcastShardOperationFailedException)
                            shardResponse));
        } else {
            responses.add((ShardSearchIntoResponse) shardResponse);
            successfulShards++;
        }
    }
    return new SearchIntoResponse(responses, shardsResponses.length(),
            successfulShards, failedShards, shardFailures);
}
 

public SearchIntoResponse(List<ShardSearchIntoResponse> responses,
        int totalShards, int successfulShards, int failedShards,
        List<ShardOperationFailedException> shardFailures) {
    super(totalShards, successfulShards, failedShards, shardFailures);
    this.responses = responses;
    for (ShardSearchIntoResponse r : this.responses) {
        totalWrites += r.getTotalWrites();
        succeededWrites += r.getSucceededWrites();
        failedWrites += r.getFailedWrites();
    }
}
 

public ExportResponse(List<ShardExportResponse> responses, int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures) {
    //To change body of created methods use File | Settings | File Templates.
    super(totalShards, successfulShards, failedShards, shardFailures);
    this.responses = responses;
    for (ShardExportResponse r : this.responses) {
        totalExported += r.getNumExported();
    }
}