下面列出了com.google.common.collect.ImmutableMap#builderWithExpectedSize ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
@Override
public ImmutableMap<String, String> exportFieldsToMap() {
ImmutableMap.Builder<String, String> retval = ImmutableMap.builderWithExpectedSize(16);
retval
.put("cas_downloads_count", Integer.toString(getCasDownloads()))
.put("cas_downloads_bytes", Long.toString(getCasDownloadSizeBytes()))
.put("cas_uploads_count", Integer.toString(getCasUploads()))
.put("cas_uploads_bytes", Long.toString(getCasUploadSizeBytes()))
.put("localfallback_totally_executed_rules", localFallbackTotalExecutions.toString())
.put("localfallback_locally_executed_rules", localFallbackLocalExecutions.toString())
.put(
"localfallback_locally_successful_executed_rules",
localFallbackSuccessfulLocalExecutions.toString())
.put("remote_cpu_time_ms", Long.toString(getRemoteCpuTimeMs()))
.put("remote_queue_time_ms", Long.toString(getRemoteQueueTimeMs()))
.put("remote_total_time_ms", Long.toString(getTotalRemoteTimeMs()));
for (ImmutableMap.Entry<State, Integer> entry : getActionsPerState().entrySet()) {
retval.put(
String.format("remote_state_%s_count", entry.getKey().getAbbreviateName()),
Integer.toString(entry.getValue()));
}
return retval.build();
}
private ImmutableMap<Field, Boolean> parameterFieldsFromFields(Class<?> clazz) {
Field[] fields = clazz.getDeclaredFields();
ImmutableMap.Builder<Field, Boolean> fieldsBuilder =
ImmutableMap.builderWithExpectedSize(fields.length);
Field ignoredField = null;
for (Field field : fields) {
field.setAccessible(true);
// static fields can be ignored and volatile fields are used for handling lazy field states.
if (Modifier.isStatic(field.getModifiers()) || Modifier.isVolatile(field.getModifiers())) {
ignoredField = field;
continue;
}
boolean isLazy = ignoredField != null && ignoredField.getName().endsWith("LAZY_INIT_BIT");
fieldsBuilder.put(field, isLazy);
}
return fieldsBuilder.build();
}
private NodeIdentifierWithPredicates readNodeIdentifierWithPredicates(final QName qname, final int size)
throws IOException {
if (size == 1) {
return NodeIdentifierWithPredicates.of(qname, readQName(), readLeafValue());
} else if (size > 1) {
final ImmutableMap.Builder<QName, Object> builder = ImmutableMap.builderWithExpectedSize(size);
for (int i = 0; i < size; ++i) {
builder.put(readQName(), readLeafValue());
}
return NodeIdentifierWithPredicates.of(qname, builder.build());
} else if (size == 0) {
return NodeIdentifierWithPredicates.of(qname);
} else {
throw new InvalidNormalizedNodeStreamException("Invalid predicate count " + size);
}
}
private Optional<ModificationApplyOperation> appendChild(
final ImmutableMap<PathArgument, ModificationApplyOperation> initial, final PathArgument identifier,
final ModificationApplyOperation computed) {
ImmutableMap<PathArgument, ModificationApplyOperation> previous = initial;
while (true) {
// Build up a new map based on observed snapshot and computed child
final Builder<PathArgument, ModificationApplyOperation> builder = ImmutableMap.builderWithExpectedSize(
previous.size() + 1);
builder.putAll(previous);
builder.put(identifier, computed);
final ImmutableMap<PathArgument, ModificationApplyOperation> updated = builder.build();
// Attempt to install the updated map
if (UPDATER.compareAndSet(this, previous, updated)) {
return Optional.of(computed);
}
// We have raced, acquire a new snapshot, recheck presence and retry if needed
previous = children;
final ModificationApplyOperation raced = previous.get(identifier);
if (raced != null) {
return Optional.of(raced);
}
}
}
@Before
public void setUp() {
eventBus = BuckEventBusForTests.newInstance();
capturingConsoleEventListener = new CapturingConsoleEventListener();
eventBus.register(capturingConsoleEventListener);
path = Paths.get("/foo/bar");
parserMock = EasyMock.createMock(ProjectBuildFileParser.class);
Map<String, Object> retMap1 = new HashMap<>();
retMap1.put("a", "a");
retMap1.put("b", "b");
retMap1.put("c", "c");
retMap1.put("d", "d");
retMap1.put("e", "e");
String[] names = {"a", "b", "c", "d", "e"};
ImmutableMap.Builder<String, ImmutableMap<String, Object>> builder =
ImmutableMap.builderWithExpectedSize(names.length);
for (String name : names) {
builder.put(name, ImmutableMap.copyOf(retMap1));
}
rawTargets = builder.build();
}
public PathReferenceRuleWithMultipleOutputs(
BuildTarget buildTarget,
ProjectFilesystem projectFilesystem,
Path source,
ImmutableMap<OutputLabel, ImmutableSet<Path>> outputLabelsToOutputs) {
super(buildTarget, projectFilesystem, source);
ImmutableMap.Builder<OutputLabel, ImmutableSortedSet<SourcePath>> builder =
ImmutableMap.builderWithExpectedSize(1 + outputLabelsToOutputs.size());
if (!outputLabelsToOutputs.containsKey(OutputLabel.defaultLabel())) {
builder.put(
OutputLabel.defaultLabel(),
source == null
? ImmutableSortedSet.of()
: ImmutableSortedSet.of(ExplicitBuildTargetSourcePath.of(getBuildTarget(), source)));
}
for (Map.Entry<OutputLabel, ImmutableSet<Path>> entry : outputLabelsToOutputs.entrySet()) {
builder.put(
entry.getKey(),
entry.getValue().stream()
.map(path -> ExplicitBuildTargetSourcePath.of(getBuildTarget(), path))
.collect(ImmutableSortedSet.toImmutableSortedSet(Ordering.natural())));
}
outputLabelsToSourcePaths = builder.build();
}
private static <V> ImmutableMap<Artifact, V> transformMap(
ImmutableMap<Artifact, V> data,
Map<OwnerlessArtifactWrapper, Artifact> newArtifactMap,
BiFunction<Artifact, V, V> transform) {
if (data.isEmpty()) {
return data;
}
ImmutableMap.Builder<Artifact, V> result = ImmutableMap.builderWithExpectedSize(data.size());
for (Map.Entry<Artifact, V> entry : data.entrySet()) {
Artifact artifact = entry.getKey();
Artifact newArtifact =
Preconditions.checkNotNull(
newArtifactMap.get(new OwnerlessArtifactWrapper(artifact)),
"Output artifact %s from one shared action not present in another's outputs (%s)",
artifact,
newArtifactMap);
result.put(newArtifact, transform.apply(newArtifact, entry.getValue()));
}
return result.build();
}
public TopLevelConfigurations(
Collection<TargetAndConfiguration> topLevelTargetsAndConfigurations) {
ImmutableMap.Builder<Label, BuildConfiguration> nonNullsBuilder =
ImmutableMap.builderWithExpectedSize(topLevelTargetsAndConfigurations.size());
ImmutableList.Builder<Label> nullsBuilder = new ImmutableList.Builder<>();
for (TargetAndConfiguration targetAndConfiguration : topLevelTargetsAndConfigurations) {
if (targetAndConfiguration.getConfiguration() == null) {
nullsBuilder.add(targetAndConfiguration.getLabel());
} else {
nonNullsBuilder.put(
targetAndConfiguration.getLabel(), targetAndConfiguration.getConfiguration());
}
}
nonNulls = nonNullsBuilder.build();
nonNullConfigs =
ImmutableSortedSet.copyOf(
Comparator.comparing(BuildConfiguration::checksum), nonNulls.values());
nulls = nullsBuilder.build();
}
/**
* Returns a set of paths representing all outputs generated by the given {@code rule}, or null if
* no outputs are available.
*
* <p>For rules that do not provide multiple outputs, the return value is null or a set of one
* output. For rules with multiple outputs, the rule will provide at least the default output
* group, so the return value is a set of zero or more outputs. Note that zero outputs in an
* output group is valid.
*/
@Nullable
private ImmutableMap<OutputLabel, ImmutableSet<Path>> getMultipleOutputPaths(BuildRule rule) {
if (rule instanceof HasMultipleOutputs) {
HasMultipleOutputs multipleOutputsRule = (HasMultipleOutputs) rule;
ProjectFilesystem projectFilesystem = rule.getProjectFilesystem();
ImmutableSet<OutputLabel> outputLabels = multipleOutputsRule.getOutputLabels();
ImmutableMap.Builder<OutputLabel, ImmutableSet<Path>> allPathsBuilder =
ImmutableMap.builderWithExpectedSize(outputLabels.size());
for (OutputLabel outputLabel : outputLabels) {
ImmutableSortedSet<SourcePath> sourcePaths =
multipleOutputsRule.getSourcePathToOutput(outputLabel);
ImmutableSet.Builder<Path> pathBuilderForLabel =
ImmutableSet.builderWithExpectedSize(sourcePaths.size());
for (SourcePath sourcePath : sourcePaths) {
pathBuilderForLabel.add(
relativizeSourcePathToProjectRoot(projectFilesystem, sourcePath).getPath());
}
allPathsBuilder.put(outputLabel, pathBuilderForLabel.build());
}
return allPathsBuilder.build();
}
Path output = getRuleOutputPath(rule);
if (output == null) {
return null;
}
return ImmutableMap.of(OutputLabel.defaultLabel(), ImmutableSet.of(output));
}
@Override
public StringMap injectContextData(List<Property> properties, StringMap reusable) {
ImmutableMap<String, String> current = RequestLoggingContext.get();
if (properties == null || properties.isEmpty()) {
return new JdkMapAdapaterStringMap(current);
}
ImmutableMap.Builder<String, String> injected =
ImmutableMap.builderWithExpectedSize(current.size() + properties.size());
injected.putAll(current);
for (Property prop : properties) {
// This will throw if a property name is already in the context, don't worry about it for now.
injected.put(prop.getName(), prop.getValue());
}
return new JdkMapAdapaterStringMap(injected.build());
}
@Override
protected Object wrapLogEvent(LogEvent event) {
Object wrapped = super.wrapLogEvent(event);
ImmutableMap.Builder<String, String> additional =
ImmutableMap.builderWithExpectedSize(additionalFields.length + 1);
additional.put("severity", stackdriverSeverity(event));
if (wrapped instanceof LogEventWithAdditionalFields) {
additional.putAll(((LogEventWithAdditionalFields) wrapped).getAdditionalFields());
}
return createLogEventWithAdditionalFields(event, additional.build());
}
/**
* Attempt to get a deeply immutable instance of a value passed in from Skylark
*
* <p>Note that if mutable objects are passed in (namely {@link SkylarkList} or {@link
* SkylarkDict}, a copy may be made to get an immutable instance. This may happen in very deeply
* nested structures, so the runtime is variable based on how mutable the objects are. For the
* best performance, only immutable structures should be passed in, as that turns into a simple
* identity function.
*
* @param arg A value from the skylark interpreter. This should only be primitive objects, or
* {@link com.google.devtools.build.lib.skylarkinterface.SkylarkValue} instances
* @return
* <li>The original object if it was already an immutable {@link
* com.google.devtools.build.lib.skylarkinterface.SkylarkValue} or a primitive value
* <li>an immutable {@link SkylarkList} if the original object is a {@link SkylarkList} and
* all values were immutable or could be made immutable. As above, this may be a copy, or
* inner elements may have had to be copied if they were mutable
* <li>An immutable {@link SkylarkDict} if the original object is a {@link SkylarkDict} and
* all keys and values were immutable, or could be made so. Again, note that copies may be
* made in order to make mutable objects immutable
* @throws MutableObjectException If a nested or top level object was mutable, and could not be
* made immutable. This generally only applies to incorrectly implemented native data types
* that are exported to Skylark.
*/
public static Object asDeepImmutable(Object arg) throws MutableObjectException {
// NOTE: For most built in types, this should be reliable. However, if isImmutable is improperly
// implemented on our custom types (that is, it returns "true" when a sub-object is not actually
// immutable), this has the potential to allow an immutable SkylarkList/Dict that contains
// mutable objects. We would rather not pay to iterate and double check for immutability here,
// so for now, just assume things are implemented correctly. The number of user-accessible
// custom objects should be vanishingly small.
// Grab frozen objects and primitives
if (EvalUtils.isImmutable(arg)) {
return arg;
}
if (arg instanceof SkylarkList) {
SkylarkList<?> listArg = ((SkylarkList<?>) arg);
return listArg.stream()
.filter(a -> !EvalUtils.isImmutable(a))
.findFirst()
// if we have a mutable element, like a sub list, try to make it immutable
.map(
mutableElement ->
SkylarkList.createImmutable(
Iterables.transform(listArg, element -> asDeepImmutable(element))))
// Else just copy the list elements into a list with an immutable mutability
// We can't just freeze the list, as it may be mutated elsewhere, but this at least
// elides a copy.
.orElseGet(() -> SkylarkList.createImmutable(listArg));
} else if (arg instanceof SkylarkDict) {
SkylarkDict<?, ?> dictArg = (SkylarkDict<?, ?>) arg;
ImmutableMap.Builder<Object, Object> tempBuilder =
ImmutableMap.builderWithExpectedSize(dictArg.size());
for (Map.Entry<?, ?> entry : dictArg.entrySet()) {
tempBuilder.put(asDeepImmutable(entry.getKey()), asDeepImmutable(entry.getValue()));
}
return SkylarkDict.copyOf(null, tempBuilder.build());
} else {
throw new MutableObjectException(arg);
}
}
public ImmutableMap<String, VersionUniverse> getVersionUniverses(
Optional<TargetConfiguration> targetConfiguration) {
ImmutableSet<String> entries = delegate.getEntriesForSection(UNIVERSES_SECTION).keySet();
ImmutableMap.Builder<String, VersionUniverse> universes =
ImmutableMap.builderWithExpectedSize(entries.size());
for (String name : entries) {
universes.put(name, getVersionUniverse(name, targetConfiguration));
}
return universes.build();
}
@Override
protected Object call(Object[] args, @Nullable FuncallExpression ast, Environment env) {
Verify.verify(isExported, "Tried to call a Provider before exporting it");
ImmutableList<String> fieldNames =
Objects.requireNonNull(getSignature()).getSignature().getNames();
Verify.verify(args.length == fieldNames.size());
ImmutableMap.Builder<String, Object> builder =
ImmutableMap.builderWithExpectedSize(args.length);
for (int i = 0; i < fieldNames.size(); i++) {
builder.put(fieldNames.get(i), args[i]);
}
return new UserDefinedProviderInfo(this, builder.build());
}
private static ImmutableMap<String, String> getClientEnvironment() {
ImmutableMap<String, String> systemEnv = EnvVariablesProvider.getSystemEnv();
ImmutableMap.Builder<String, String> builder =
ImmutableMap.builderWithExpectedSize(systemEnv.size());
systemEnv.entrySet().stream()
.filter(
e ->
!AnsiEnvironmentChecking.NAILGUN_STDOUT_ISTTY_ENV.equals(e.getKey())
&& !AnsiEnvironmentChecking.NAILGUN_STDERR_ISTTY_ENV.equals(e.getKey()))
.forEach(builder::put);
return builder.build();
}
@Override
public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout)
{
SchemaTableName schemaTableName = tableMetadata.getTable();
String schemaName = schemaTableName.getSchemaName();
String tableName = schemaTableName.getTableName();
Schema schema = toIcebergSchema(tableMetadata.getColumns());
PartitionSpec partitionSpec = parsePartitionFields(schema, getPartitioning(tableMetadata.getProperties()));
Database database = metastore.getDatabase(schemaName)
.orElseThrow(() -> new SchemaNotFoundException(schemaName));
HdfsContext hdfsContext = new HdfsContext(session, schemaName, tableName);
HiveIdentity identity = new HiveIdentity(session);
String targetPath = getTableLocation(tableMetadata.getProperties());
if (targetPath == null) {
targetPath = getTableDefaultLocation(database, hdfsContext, hdfsEnvironment, schemaName, tableName).toString();
}
TableOperations operations = new HiveTableOperations(metastore, hdfsEnvironment, hdfsContext, identity, schemaName, tableName, session.getUser(), targetPath);
if (operations.current() != null) {
throw new TableAlreadyExistsException(schemaTableName);
}
ImmutableMap.Builder<String, String> propertiesBuilder = ImmutableMap.builderWithExpectedSize(2);
FileFormat fileFormat = getFileFormat(tableMetadata.getProperties());
propertiesBuilder.put(DEFAULT_FILE_FORMAT, fileFormat.toString());
if (tableMetadata.getComment().isPresent()) {
propertiesBuilder.put(TABLE_COMMENT, tableMetadata.getComment().get());
}
TableMetadata metadata = newTableMetadata(operations, schema, partitionSpec, targetPath, propertiesBuilder.build());
transaction = createTableTransaction(operations, metadata);
return new IcebergWritableTableHandle(
schemaName,
tableName,
SchemaParser.toJson(metadata.schema()),
PartitionSpecParser.toJson(metadata.spec()),
getColumns(metadata.schema(), typeManager),
targetPath,
fileFormat);
}
public GenruleBuildable(
BuildTarget buildTarget,
ProjectFilesystem filesystem,
SandboxExecutionStrategy sandboxExecutionStrategy,
SourceSet srcs,
Optional<Arg> cmd,
Optional<Arg> bash,
Optional<Arg> cmdExe,
Optional<String> type,
Optional<String> out,
Optional<ImmutableMap<OutputLabel, ImmutableSet<String>>> outs,
boolean enableSandboxingInGenrule,
boolean isCacheable,
String environmentExpansionSeparator,
Optional<SandboxProperties> sandboxProperties,
Optional<GenruleAndroidTools> androidTools,
boolean executeRemotely) {
this.buildTarget = buildTarget;
this.sandboxExecutionStrategy = sandboxExecutionStrategy;
this.srcs = srcs;
this.cmd = cmd;
this.bash = bash;
this.cmdExe = cmdExe;
this.type = type;
this.out = out;
this.outputLabelsSupplier = MoreSuppliers.memoize(this::getOutputLabelsSupplier);
this.enableSandboxingInGenrule = enableSandboxingInGenrule;
this.isCacheable = isCacheable;
this.environmentExpansionSeparator = environmentExpansionSeparator;
this.sandboxProperties = sandboxProperties;
this.isWorkerGenrule = isWorkerGenrule();
this.androidTools = androidTools;
this.executeRemotely = executeRemotely;
Preconditions.checkArgument(
out.isPresent() ^ outs.isPresent(), "Genrule unexpectedly has both 'out' and 'outs'.");
if (outs.isPresent()) {
ImmutableMap<OutputLabel, ImmutableSet<String>> outputs = outs.get();
ImmutableMap.Builder<OutputLabel, ImmutableSet<String>> outsBuilder =
ImmutableMap.builderWithExpectedSize(outputs.size() + 1);
ImmutableMap.Builder<OutputLabel, ImmutableSet<OutputPath>> outputPathsBuilder =
ImmutableMap.builderWithExpectedSize(outputs.size() + 1);
for (Map.Entry<OutputLabel, ImmutableSet<String>> outputLabelToOutputs : outputs.entrySet()) {
OutputLabel outputLabel = outputLabelToOutputs.getKey();
outsBuilder.put(outputLabel, outputLabelToOutputs.getValue());
outputPathsBuilder.put(
outputLabel,
outputLabelToOutputs.getValue().stream()
.map(
p -> {
Path path = Paths.get(p);
return new OutputPath(path);
})
.collect(ImmutableSet.toImmutableSet()));
}
if (!outputs.containsKey(OutputLabel.defaultLabel())) {
outsBuilder.put(OutputLabel.defaultLabel(), DEFAULT_OUTS);
outputPathsBuilder.put(OutputLabel.defaultLabel(), DEFAULT_OUTPUTS);
}
this.outs = Optional.of(outsBuilder.build());
this.outputPaths = Optional.of(outputPathsBuilder.build());
this.outputPath = Optional.empty();
} else {
this.outs = Optional.empty();
// Sanity check for the output paths.
this.outputPath = Optional.of(new PublicOutputPath(getLegacyPath(filesystem, out.get())));
this.outputPaths = Optional.empty();
}
}
private ImmutableMap<SkyKey, SkyValue> batchPrefetch(
SkyKey requestor, GroupedList<SkyKey> depKeys, Set<SkyKey> oldDeps, boolean assertDone)
throws InterruptedException, UndonePreviouslyRequestedDeps {
QueryableGraph.PrefetchDepsRequest request = null;
if (PREFETCH_OLD_DEPS) {
request = new QueryableGraph.PrefetchDepsRequest(requestor, oldDeps, depKeys);
evaluatorContext.getGraph().prefetchDeps(request);
}
Map<SkyKey, ? extends NodeEntry> batchMap =
evaluatorContext.getBatchValues(
requestor,
Reason.PREFETCH,
(request != null && request.excludedKeys != null)
? request.excludedKeys
: depKeys.getAllElementsAsIterable());
if (batchMap.size() != depKeys.numElements()) {
Set<SkyKey> difference = Sets.difference(depKeys.toSet(), batchMap.keySet());
evaluatorContext
.getGraphInconsistencyReceiver()
.noteInconsistencyAndMaybeThrow(
requestor, difference, Inconsistency.ALREADY_DECLARED_CHILD_MISSING);
throw new UndonePreviouslyRequestedDeps(ImmutableList.copyOf(difference));
}
ImmutableMap.Builder<SkyKey, SkyValue> depValuesBuilder =
ImmutableMap.builderWithExpectedSize(batchMap.size());
for (Entry<SkyKey, ? extends NodeEntry> entry : batchMap.entrySet()) {
SkyValue valueMaybeWithMetadata = entry.getValue().getValueMaybeWithMetadata();
boolean depDone = valueMaybeWithMetadata != null;
if (assertDone && !depDone) {
// A previously requested dep may have transitioned from done to dirty between when the node
// was read during a previous attempt to build this node and now. Notify the graph
// inconsistency receiver so that we can crash if that's unexpected.
evaluatorContext
.getGraphInconsistencyReceiver()
.noteInconsistencyAndMaybeThrow(
skyKey,
ImmutableList.of(entry.getKey()),
Inconsistency.BUILDING_PARENT_FOUND_UNDONE_CHILD);
throw new UndonePreviouslyRequestedDeps(ImmutableList.of(entry.getKey()));
}
depValuesBuilder.put(entry.getKey(), !depDone ? NULL_MARKER : valueMaybeWithMetadata);
if (depDone) {
maybeUpdateMaxChildVersion(entry.getValue());
}
}
return depValuesBuilder.build();
}
private void injectRemoteArtifact(
Artifact output,
ActionResultMetadata metadata,
Path execRoot,
MetadataInjector metadataInjector,
String actionId)
throws IOException {
if (output.isTreeArtifact()) {
DirectoryMetadata directory =
metadata.directory(execRoot.getRelative(output.getExecPathString()));
if (directory == null) {
// A declared output wasn't created. It might have been an optional output and if not
// SkyFrame will make sure to fail.
return;
}
if (!directory.symlinks().isEmpty()) {
throw new IOException(
"Symlinks in action outputs are not yet supported by "
+ "--experimental_remote_download_outputs=minimal");
}
SpecialArtifact parent = (SpecialArtifact) output;
ImmutableMap.Builder<TreeFileArtifact, FileArtifactValue> childMetadata =
ImmutableMap.builderWithExpectedSize(directory.files.size());
for (FileMetadata file : directory.files()) {
TreeFileArtifact child =
TreeFileArtifact.createTreeOutput(parent, file.path().relativeTo(parent.getPath()));
RemoteFileArtifactValue value =
new RemoteFileArtifactValue(
DigestUtil.toBinaryDigest(file.digest()),
file.digest().getSizeBytes(),
/*locationIndex=*/ 1,
actionId);
childMetadata.put(child, value);
}
metadataInjector.injectDirectory(parent, childMetadata.build());
} else {
FileMetadata outputMetadata = metadata.file(execRoot.getRelative(output.getExecPathString()));
if (outputMetadata == null) {
// A declared output wasn't created. It might have been an optional output and if not
// SkyFrame will make sure to fail.
return;
}
metadataInjector.injectFile(
output,
new RemoteFileArtifactValue(
DigestUtil.toBinaryDigest(outputMetadata.digest()),
outputMetadata.digest().getSizeBytes(),
/*locationIndex=*/ 1,
actionId));
}
}
/** Initialize the parser, starting buck.py. */
private void init() throws IOException {
try (SimplePerfEvent.Scope scope =
SimplePerfEvent.scope(buckEventBus, SimplePerfEvent.PerfEventId.of("ParserInit"))) {
ImmutableMap.Builder<String, String> pythonEnvironmentBuilder =
ImmutableMap.builderWithExpectedSize(environment.size());
// Strip out PYTHONPATH. buck.py manually sets this to include only nailgun. We don't want
// to inject nailgun into the parser's PYTHONPATH, so strip that value out.
// If we wanted to pass on some environmental PYTHONPATH, we would have to do some actual
// merging of this and the BuckConfig's python module search path.
// Also ignore PYTHONHASHSEED environment variable passed by clients since Buck manages it to
// prevent non-determinism.
pythonEnvironmentBuilder.putAll(
Maps.filterKeys(
environment,
k -> !PYTHONPATH_ENV_VAR_NAME.equals(k) && !PYTHON_HASH_SEED_ENV_VAR_NAME.equals(k)));
// set Python hash seed to a fixed number to make parsing reproducible
pythonEnvironmentBuilder.put(PYTHON_HASH_SEED_ENV_VAR_NAME, PYTHON_HASH_SEED_VALUE);
if (options.getPythonModuleSearchPath().isPresent()) {
pythonEnvironmentBuilder.put(
PYTHONPATH_ENV_VAR_NAME, options.getPythonModuleSearchPath().get());
}
ImmutableMap<String, String> pythonEnvironment = pythonEnvironmentBuilder.build();
ProcessExecutorParams params =
ProcessExecutorParams.builder()
.setCommand(buildArgs())
.setEnvironment(pythonEnvironment)
.build();
LOG.debug(
"Starting buck.py command: %s environment: %s",
params.getCommand(), params.getEnvironment());
buckPyProcess = processExecutor.launchProcess(params);
LOG.debug("Started process %s successfully", buckPyProcess);
buckPyProcessInput =
createParserInputStream(
Objects.requireNonNull(buckPyProcess).getStdout(), processedBytes.isPresent());
buckPyProcessJsonGenerator = ObjectMappers.createGenerator(buckPyProcess.getStdin());
// Explicitly use Reader instead of InputStream because in case if InputStream is provided
// Jackson tries to detect encoding be reading first bytes; because the process may not
// output anything at this moment yet this can hang indefinitely.
buckPyProcessJsonParser =
ObjectMappers.createParser(
new InputStreamReader(
Objects.requireNonNull(buckPyProcessInput).getInputStream(), Charsets.UTF_8));
InputStream stderr = buckPyProcess.getStderr();
AtomicInteger numberOfLines = new AtomicInteger(0);
AtomicReference<Path> lastPath = new AtomicReference<Path>();
InputStreamConsumer stderrConsumer =
new InputStreamConsumer(
stderr,
(InputStreamConsumer.Handler)
line -> {
Path path = currentBuildFile.get();
if (!Objects.equals(path, lastPath.get())) {
numberOfLines.set(0);
lastPath.set(path);
}
int count = numberOfLines.getAndIncrement();
if (count == 0) {
buckEventBus.post(
ConsoleEvent.warning("WARNING: Output when parsing %s:", path));
}
buckEventBus.post(ConsoleEvent.warning("| %s", line));
});
stderrConsumerTerminationFuture = new FutureTask<>(stderrConsumer);
stderrConsumerThread =
Threads.namedThread(
PythonDslProjectBuildFileParser.class.getSimpleName(),
stderrConsumerTerminationFuture);
stderrConsumerThread.start();
}
}