下面列出了com.google.common.collect.ImmutableMap#Entry ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
/** Resolve the map of names to SourcePaths to a map of names to CxxSource objects. */
private static ImmutableMap<String, CxxSource> resolveCxxSources(
ImmutableMap<String, SourceWithFlags> sources) {
ImmutableMap.Builder<String, CxxSource> cxxSources = ImmutableMap.builder();
// For each entry in the input C/C++ source, build a CxxSource object to wrap
// it's name, input path, and output object file path.
for (ImmutableMap.Entry<String, SourceWithFlags> ent : sources.entrySet()) {
String extension = Files.getFileExtension(ent.getKey());
Optional<CxxSource.Type> type = CxxSource.Type.fromExtension(extension);
if (!type.isPresent()) {
throw new HumanReadableException("invalid extension \"%s\": %s", extension, ent.getKey());
}
cxxSources.put(
ent.getKey(),
CxxSource.of(type.get(), ent.getValue().getSourcePath(), ent.getValue().getFlags()));
}
return cxxSources.build();
}
private ImmutableMap<ShardId, ShardSnapshotStatus> processWaitingShards(ImmutableMap<ShardId, ShardSnapshotStatus> snapshotShards, RoutingTable routingTable) {
boolean snapshotChanged = false;
ImmutableMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableMap.builder();
for (ImmutableMap.Entry<ShardId, ShardSnapshotStatus> shardEntry : snapshotShards.entrySet()) {
ShardSnapshotStatus shardStatus = shardEntry.getValue();
if (shardStatus.state() == State.WAITING) {
ShardId shardId = shardEntry.getKey();
IndexRoutingTable indexShardRoutingTable = routingTable.index(shardId.getIndex());
if (indexShardRoutingTable != null) {
IndexShardRoutingTable shardRouting = indexShardRoutingTable.shard(shardId.id());
if (shardRouting != null && shardRouting.primaryShard() != null) {
if (shardRouting.primaryShard().started()) {
// Shard that we were waiting for has started on a node, let's process it
snapshotChanged = true;
logger.trace("starting shard that we were waiting for [{}] on node [{}]", shardEntry.getKey(), shardStatus.nodeId());
shards.put(shardEntry.getKey(), new ShardSnapshotStatus(shardRouting.primaryShard().currentNodeId()));
continue;
} else if (shardRouting.primaryShard().initializing() || shardRouting.primaryShard().relocating()) {
// Shard that we were waiting for hasn't started yet or still relocating - will continue to wait
shards.put(shardEntry);
continue;
}
}
}
// Shard that we were waiting for went into unassigned state or disappeared - giving up
snapshotChanged = true;
logger.warn("failing snapshot of shard [{}] on unassigned shard [{}]", shardEntry.getKey(), shardStatus.nodeId());
shards.put(shardEntry.getKey(), new ShardSnapshotStatus(shardStatus.nodeId(), State.FAILED, "shard is unassigned"));
} else {
shards.put(shardEntry);
}
}
if (snapshotChanged) {
return shards.build();
} else {
return null;
}
}
private static ImmutableMap<Path, ImmutableList<Path>> filterCurrentRuleInput(
Set<Path> mlInput, ImmutableMap<Path, ImmutableList<Path>> deps) {
ImmutableMap.Builder<Path, ImmutableList<Path>> builder = ImmutableMap.builder();
for (ImmutableMap.Entry<Path, ImmutableList<Path>> entry : deps.entrySet()) {
if (mlInput.contains(entry.getKey())) {
builder.put(
entry.getKey(),
FluentIterable.from(entry.getValue()).filter(mlInput::contains).toList());
}
}
return builder.build();
}
/**
* Checks if any of the deleted indices are still recovering and fails recovery on the shards of these indices
*
* @param event cluster changed event
*/
private void processDeletedIndices(ClusterChangedEvent event) {
RestoreInProgress restore = event.state().custom(RestoreInProgress.TYPE);
if (restore == null) {
// Not restoring - nothing to do
return;
}
if (!event.indicesDeleted().isEmpty()) {
// Some indices were deleted, let's make sure all indices that we are restoring still exist
for (RestoreInProgress.Entry entry : restore.entries()) {
List<ShardId> shardsToFail = null;
for (ImmutableMap.Entry<ShardId, ShardRestoreStatus> shard : entry.shards().entrySet()) {
if (!shard.getValue().state().completed()) {
if (!event.state().metaData().hasIndex(shard.getKey().getIndex())) {
if (shardsToFail == null) {
shardsToFail = new ArrayList<>();
}
shardsToFail.add(shard.getKey());
}
}
}
if (shardsToFail != null) {
for (ShardId shardId : shardsToFail) {
logger.trace("[{}] failing running shard restore [{}]", entry.snapshotId(), shardId);
updateRestoreStateOnMaster(new UpdateIndexShardRestoreStatusRequest(entry.snapshotId(), shardId, new ShardRestoreStatus(null, RestoreInProgress.State.FAILURE, "index was deleted")));
}
}
}
}
}
static void forEachModuleParam(
BuildTarget target,
ActionGraphBuilder actionGraphBuilder,
CxxPlatform cxxPlatform,
String parameter,
Path baseModule,
Iterable<SourceSortedSet> inputs,
BiConsumer<Path, SourcePath> consumer) {
for (SourceSortedSet input : inputs) {
ImmutableMap<String, SourcePath> namesAndSourcePaths;
if (input.getUnnamedSources().isPresent()) {
namesAndSourcePaths =
actionGraphBuilder
.getSourcePathResolver()
.getSourcePathNames(target, parameter, input.getUnnamedSources().get());
} else {
namesAndSourcePaths = input.getNamedSources().get();
}
for (ImmutableMap.Entry<String, SourcePath> entry : namesAndSourcePaths.entrySet()) {
consumer.accept(
baseModule.resolve(entry.getKey()),
CxxGenruleDescription.fixupSourcePath(
actionGraphBuilder, cxxPlatform, entry.getValue()));
}
}
}
@Override
public ImmutableList<Step> getBuildSteps(
BuildContext buildContext,
ProjectFilesystem filesystem,
OutputPathResolver outputPathResolver,
BuildCellRelativePathFactory buildCellPathFactory) {
ImmutableList.Builder<Step> steps = ImmutableList.builder();
Path outputPath = outputPathResolver.resolvePath(output);
Path srcPath = outputPathResolver.resolvePath(src);
Path jarPath = outputPathResolver.resolvePath(outputJar);
steps.add(MkdirStep.of(buildCellPathFactory.from(outputPath.getParent())));
StringBuilder fileContents = new StringBuilder();
for (ImmutableMap.Entry<String, String> entry : properties.entrySet()) {
fileContents.append(entry.getKey()).append("=").append(entry.getValue()).append('\n');
}
steps.add(new WriteFileStep(filesystem, fileContents.toString(), outputPath, false));
steps.add(RmStep.of(BuildCellRelativePath.of(jarPath)));
steps.add(
new ZipStep(
filesystem, jarPath, ImmutableSet.of(), false, ZipCompressionLevel.NONE, srcPath));
return steps.build();
}
public static ImmutableMap<String, String> withDefaults(
ImmutableMap<String, String> map, ImmutableMap<String, String> defaults) {
ImmutableMap.Builder<String, String> builder =
ImmutableMap.<String, String>builder().putAll(map);
for (ImmutableMap.Entry<String, String> entry : defaults.entrySet()) {
if (!map.containsKey(entry.getKey())) {
builder = builder.put(entry.getKey(), entry.getValue());
}
}
return builder.build();
}
private SortedMap<Path, Path> getReadOnlyBindMounts(
BlazeDirectories blazeDirs, Path sandboxExecRoot) throws UserExecException {
Path tmpPath = fileSystem.getPath("/tmp");
final SortedMap<Path, Path> bindMounts = Maps.newTreeMap();
if (blazeDirs.getWorkspace().startsWith(tmpPath)) {
bindMounts.put(blazeDirs.getWorkspace(), blazeDirs.getWorkspace());
}
if (blazeDirs.getOutputBase().startsWith(tmpPath)) {
bindMounts.put(blazeDirs.getOutputBase(), blazeDirs.getOutputBase());
}
for (ImmutableMap.Entry<String, String> additionalMountPath :
getSandboxOptions().sandboxAdditionalMounts) {
try {
final Path mountTarget = fileSystem.getPath(additionalMountPath.getValue());
// If source path is relative, treat it as a relative path inside the execution root
final Path mountSource = sandboxExecRoot.getRelative(additionalMountPath.getKey());
// If a target has more than one source path, the latter one will take effect.
bindMounts.put(mountTarget, mountSource);
} catch (IllegalArgumentException e) {
throw new UserExecException(
createFailureDetail(
String.format("Error occurred when analyzing bind mount pairs. %s", e.getMessage()),
Code.BIND_MOUNT_ANALYSIS_FAILURE));
}
}
for (Path inaccessiblePath : getInaccessiblePaths()) {
if (inaccessiblePath.isDirectory(Symlinks.NOFOLLOW)) {
bindMounts.put(inaccessiblePath, inaccessibleHelperDir);
} else {
bindMounts.put(inaccessiblePath, inaccessibleHelperFile);
}
}
validateBindMounts(bindMounts);
return bindMounts;
}
@VisibleForTesting
ImmutableMap<String, MySQLDataSources> getEnqueueableShards() {
ImmutableMap.Builder<String, MySQLDataSources> shardMapBuilder =
new ImmutableMap.Builder<String, MySQLDataSources>();
for (ImmutableMap.Entry<String, MySQLDataSources> shard : shardMapRef.get().entrySet()) {
if (!shard.getValue().isDequeueOnly()) {
shardMapBuilder.put(shard);
}
}
return shardMapBuilder.build();
}
private ImmutableList<SourcePath> generateMLBytecodeCompilation(
ImmutableMap<Path, ImmutableList<Path>> mlSources) {
ImmutableList.Builder<SourcePath> cmoFiles = ImmutableList.builder();
Map<Path, ImmutableSortedSet<BuildRule>> sourceToRule = new HashMap<>();
for (ImmutableMap.Entry<Path, ImmutableList<Path>> mlSource : mlSources.entrySet()) {
generateSingleMLBytecodeCompilation(
sourceToRule, cmoFiles, mlSource.getKey(), mlSources, ImmutableList.of());
}
return cmoFiles.build();
}
@Override
protected Set<String> getQueueNamesImpl() throws SQLException {
ImmutableMap.Entry<String, MySQLDataSources> randomShard = getRandomShard();
if (randomShard == null) {
return Sets.newHashSet();
}
Connection conn = null;
try {
conn = randomShard.getValue().getGeneralDataSource().getConnection();
return MySQLBackendUtils.getQueueNames(conn, randomShard.getKey());
} finally {
JdbcUtils.closeConnection(conn);
}
}
private ImmutableMap.Entry<String, MySQLDataSources> getRandomShard() {
ImmutableMap<String, MySQLDataSources> shardMap = shardMapRef.get();
if (shardMap == null || shardMap.isEmpty()) {
return null;
}
int rand = RANDOM.nextInt(shardMap.size());
return shardMap.entrySet().asList().get(rand);
}
@Override
public void run() {
final long runStartMillis = System.currentTimeMillis();
for (ImmutableMap.Entry<String, MySQLDataSources> shard : getShardMap().entrySet()) {
jobMonitorImpl(runStartMillis, shard, getMaxAutoRetries());
}
}
/** Applies function designated in {@code expression} ([-|*]<float>) to value. */
private Integer applyOperator(@Nullable String expression, Supplier<Integer> firstOperandSupplier)
throws OptionsParsingException {
if (expression == null) {
return firstOperandSupplier.get();
}
for (ImmutableMap.Entry<String, DoubleBinaryOperator> operator : OPERATORS.entrySet()) {
if (expression.startsWith(operator.getKey())) {
float secondOperand;
try {
secondOperand = Float.parseFloat(expression.substring(operator.getKey().length()));
} catch (NumberFormatException e) {
throw new OptionsParsingException(
String.format("'%s is not a float", expression.substring(operator.getKey().length())),
e);
}
return (int)
Math.round(
operator
.getValue()
.applyAsDouble((float) firstOperandSupplier.get(), secondOperand));
}
}
// This should never happen because we've checked for a valid operator already.
throw new OptionsParsingException(
String.format("Parameter value '%s' does not contain a valid operator.", expression));
}
private static ImmutableMap.Entry<String, String> extractHostAndPort(String nextHostname, int defaultPort) {
final String hostname;
final String port;
int idx = nextHostname.indexOf(':');
if (idx > 0) {
port = nextHostname.substring(idx + 1);
hostname = nextHostname.substring(0, idx);
} else {
hostname = nextHostname;
port = Integer.toString(defaultPort);
}
return Maps.immutableEntry(hostname, port);
}
private static void assertMountPair(
ImmutableMap.Entry<String, String> pathPair, String source, String target) {
assertThat(source).isEqualTo(pathPair.getKey());
assertThat(target).isEqualTo(pathPair.getValue());
}
@Override
protected String enqueueSingleJob(final String queueName, final PinLaterJob job,
int numAutoRetries) throws Exception {
final double currentTimeSeconds = System.currentTimeMillis() / 1000.0;
// Check whether the queue to enqueue exists.
if ((queueNames.get() == null) || !queueNames.get().contains(queueName)) {
reloadQueueNames();
if (!queueNames.get().contains(queueName)) {
Stats.incr("redis-queue-not-found-enqueue");
throw new PinLaterException(ErrorCode.QUEUE_NOT_FOUND, "Queue not found: " + queueName);
}
}
final ImmutableMap.Entry<String, RedisPools> shard = getRandomEnqueueableShard();
if (shard == null) {
throw new PinLaterException(ErrorCode.NO_HEALTHY_SHARDS, "Unable to find healthy shard");
}
try {
return RedisUtils.executeWithConnection(
shard.getValue().getGeneralRedisPool(),
new Function<Jedis, String>() {
@Override
public String apply(Jedis conn) {
String jobIdRedisKey = RedisBackendUtils.constructJobIdRedisKey(
queueName, shard.getKey());
String hashRedisKeyPrefix = RedisBackendUtils.constructHashRedisKeyPrefix(
queueName, shard.getKey());
String queueRedisKey = RedisBackendUtils.constructQueueRedisKey(
queueName, shard.getKey(), job.getPriority(), PinLaterJobState.PENDING);
List<String> keys = Lists.newArrayList(
jobIdRedisKey, hashRedisKeyPrefix, queueRedisKey);
double jobToRunTimestampSeconds;
if (job.getRunAfterTimestampMillis() != 0) {
jobToRunTimestampSeconds = job.getRunAfterTimestampMillis() / 1000.0;
} else {
jobToRunTimestampSeconds = currentTimeSeconds;
}
List<String> argv = Lists.newArrayList(
BytesUtil.stringFromByteBuffer(ByteBuffer.wrap(job.getBody())),
String.valueOf(job.getNumAttemptsAllowed()),
String.valueOf(currentTimeSeconds),
String.valueOf(jobToRunTimestampSeconds),
RedisBackendUtils.truncateCustomStatus(job.getCustomStatus())
);
Long jobId = (Long) conn.eval(RedisLuaScripts.ENQUEUE_JOB, keys, argv);
return new PinLaterJobDescriptor(
queueName, shard.getKey(), job.getPriority(), jobId).toString();
}
});
} catch (JedisConnectionException e) {
if (numAutoRetries > 0) {
// Retry the enqueue, potentially on a different shard.
Stats.incr("enqueue-failures-retry");
return enqueueSingleJob(queueName, job, numAutoRetries - 1);
}
String host = shard.getValue().getHost();
Stats.incr("shard_connection_failed_" + host);
LOG.error("Failed to get a redis connection.", e);
throw new PinLaterException(ErrorCode.SHARD_CONNECTION_FAILED,
String.format("Redis connection to %s failed", host));
}
}
@Override
protected void createQueueImpl(final String queueName) throws Exception {
for (ImmutableMap.Entry<String, MySQLDataSources> shard : shardMapRef.get().entrySet()) {
Connection conn = null;
String dbName = MySQLBackendUtils.constructDBName(queueName, shard.getKey());
try {
// We share the data source with enqueue.
conn = shard.getValue().getGeneralDataSource().getConnection();
// Create the database.
JdbcUtils.executeUpdate(
conn,
String.format(MySQLQueries.CREATE_DATABASE, dbName));
// Create the queue tables, one for each priority level.
for (int priority = 1;
priority <= numPriorityLevels;
priority++) {
JdbcUtils.executeUpdate(
conn,
String.format(
MySQLQueries.CREATE_JOBS_TABLE,
MySQLBackendUtils.constructJobsTableName(queueName, shard.getKey(), priority)));
}
} catch (SQLException e) {
// If database already exists, then just ignore this and move onto the next shard.
if (MySQLBackendUtils.isDatabaseAlreadyExistsException(e)) {
continue;
}
// Wrap any other recognized exceptions as a PinLaterException.
if (MySQLBackendUtils.isDatabaseNameTooLongException(e)) {
throw new PinLaterException(ErrorCode.QUEUE_NAME_TOO_LONG,
String.format(
"Queue name is too long by %d characters. Attempted to create queue DB '%s'"
+ " but it's longer than maximum allowed %d characters.",
dbName.length() - MySQLBackendUtils.MYSQL_MAX_DB_NAME_LENGTH,
dbName,
MySQLBackendUtils.MYSQL_MAX_DB_NAME_LENGTH));
}
throw e;
} finally {
JdbcUtils.closeConnection(conn);
}
}
}
@Override
protected String enqueueSingleJob(String queueName, PinLaterJob job, int numAutoRetries)
throws Exception {
final long currentTimeMillis = System.currentTimeMillis();
Connection conn = null;
PreparedStatement stmt = null;
ResultSet rs = null;
final ImmutableMap.Entry<String, MySQLDataSources> shard = getRandomEnqueueableShard();
try {
conn = shard.getValue().getGeneralDataSource().getConnection();
String jobsTableName =
MySQLBackendUtils.constructJobsTableName(queueName, shard.getKey(), job.getPriority());
stmt = conn.prepareStatement(
String.format(MySQLQueries.ENQUEUE_INSERT, jobsTableName),
Statement.RETURN_GENERATED_KEYS);
stmt.setInt(1, PinLaterJobState.PENDING.getValue());
stmt.setInt(2, job.getNumAttemptsAllowed());
stmt.setInt(3, job.getNumAttemptsAllowed());
stmt.setString(4, job.getCustomStatus());
stmt.setTimestamp(5, new Timestamp(currentTimeMillis));
stmt.setTimestamp(6, new Timestamp(job.isSetRunAfterTimestampMillis()
? job.getRunAfterTimestampMillis() : currentTimeMillis));
stmt.setBytes(7, job.getBody());
stmt.executeUpdate();
rs = stmt.getGeneratedKeys();
rs.next();
return new PinLaterJobDescriptor(
queueName, shard.getKey(), job.getPriority(), rs.getLong(1)).toString();
} catch (SQLException e) {
boolean shouldRetry = checkExceptionIsRetriable(e, shard.getKey(), "enqueue");
if (shouldRetry && numAutoRetries > 0) {
// Retry the enqueue, potentially on a different shard.
Stats.incr("enqueue-failures-retry");
return enqueueSingleJob(queueName, job, numAutoRetries - 1);
}
// Out of retries, throw the exception. Wrap it into a PinLaterException if the exception
// is recognized and return the appropriate error code.
if (MySQLBackendUtils.isDatabaseDoesNotExistException(e)) {
throw new PinLaterException(ErrorCode.QUEUE_NOT_FOUND, "Queue not found: " + queueName);
}
throw e;
} finally {
JdbcUtils.closeResultSet(rs);
JdbcUtils.closeStatement(stmt);
JdbcUtils.closeConnection(conn);
}
}
public List<String> build() {
Preconditions.checkNotNull(sandboxExecRoot, "sandboxExecRoot must be set");
Preconditions.checkState(!imageName.isEmpty(), "imageName must be set");
Preconditions.checkState(!commandArguments.isEmpty(), "commandArguments must be set");
ImmutableList.Builder<String> dockerCmdLine = ImmutableList.builder();
dockerCmdLine.add(dockerClient.getPathString());
dockerCmdLine.add("run");
dockerCmdLine.add("--rm");
if (createNetworkNamespace) {
dockerCmdLine.add("--network=none");
} else {
dockerCmdLine.add("--network=host");
}
if (privileged) {
dockerCmdLine.add("--privileged");
}
for (Map.Entry<String, String> env : environmentVariables.entrySet()) {
dockerCmdLine.add("-e", env.getKey() + "=" + env.getValue());
}
PathFragment execRootInsideDocker =
PathFragment.create("/execroot/").getRelative(sandboxExecRoot.getBaseName());
dockerCmdLine.add(
"-v", sandboxExecRoot.getPathString() + ":" + execRootInsideDocker.getPathString());
dockerCmdLine.add("-w", execRootInsideDocker.getPathString());
for (ImmutableMap.Entry<String, String> additionalMountPath : additionalMounts) {
final String mountTarget = additionalMountPath.getValue();
final String mountSource = additionalMountPath.getKey();
dockerCmdLine.add("-v", mountSource + ":" + mountTarget);
}
StringBuilder uidGidFlagBuilder = new StringBuilder();
if (uid != 0) {
uidGidFlagBuilder.append(uid);
}
if (gid != 0) {
uidGidFlagBuilder.append(":");
uidGidFlagBuilder.append(gid);
}
String uidGidFlag = uidGidFlagBuilder.toString();
if (!uidGidFlag.isEmpty()) {
dockerCmdLine.add("-u", uidGidFlagBuilder.toString());
}
if (!commandId.isEmpty()) {
dockerCmdLine.add("-l", "command_id=" + commandId);
}
if (uuid != null) {
dockerCmdLine.add("--name", uuid.toString());
}
dockerCmdLine.add(imageName);
dockerCmdLine.addAll(commandArguments);
ProcessWrapper.CommandLineBuilder processWrapperCmdLine =
processWrapper.commandLineBuilder(dockerCmdLine.build());
if (timeout != null) {
processWrapperCmdLine.setTimeout(timeout);
}
return processWrapperCmdLine.build();
}