org.apache.hadoop.mapred.FileAlreadyExistsException#org.apache.hadoop.yarn.api.records.LocalResource源码实例Demo

static LocalizerHeartbeatResponse createLocalizerHeartbeatResponse() 
    throws URISyntaxException {
  LocalizerHeartbeatResponse ret =
    recordFactory.newRecordInstance(LocalizerHeartbeatResponse.class);
  assertTrue(ret instanceof LocalizerHeartbeatResponsePBImpl);
  ret.setLocalizerAction(LocalizerAction.LIVE);
  LocalResource rsrc = createResource();
  ArrayList<ResourceLocalizationSpec> rsrcs =
    new ArrayList<ResourceLocalizationSpec>();
  ResourceLocalizationSpec resource =
    recordFactory.newRecordInstance(ResourceLocalizationSpec.class);
  resource.setResource(rsrc);
  resource.setDestinationDirectory(ConverterUtils
    .getYarnUrlFromPath(new Path("/tmp" + System.currentTimeMillis())));
  rsrcs.add(resource);
  ret.setResourceSpecs(rsrcs);
  System.out.println(resource);
  return ret;
}
 

/**
 * Process SSLConfig object to set up SSL resources
 *
 * @param sslConfig  SSLConfig object derived from SSL_CONFIG attribute
 * @param fs    HDFS file system object
 * @param appPath    application path for the current application
 * @param localResources  Local resources to modify
 * @throws IOException
 */
private void setupSSLResources(SSLConfig sslConfig, FileSystem fs, Path appPath, Map<String, LocalResource> localResources) throws IOException
{
  if (sslConfig != null) {
    String nodeLocalConfig = sslConfig.getConfigPath();

    if (StringUtils.isNotEmpty(nodeLocalConfig)) {
      // all others should be empty
      if (StringUtils.isNotEmpty(sslConfig.getKeyStorePath()) || StringUtils.isNotEmpty(sslConfig.getKeyStorePassword())
          || StringUtils.isNotEmpty(sslConfig.getKeyStoreKeyPassword())) {
        throw new IllegalArgumentException("Cannot specify both nodeLocalConfigPath and other parameters in " + sslConfig);
      }
      // pass thru: Stram will implement reading the node local SSL config file
    } else {
      // need to package and copy the keyStore file
      String keystorePath = sslConfig.getKeyStorePath();
      String[] sslFileArray = {keystorePath};
      String sslFileNames = copyFromLocal(fs, appPath, sslFileArray);
      LaunchContainerRunnable.addFilesToLocalResources(LocalResourceType.FILE, sslFileNames, localResources, fs);
    }
  }
}
 

/**
 * Uploads and registers a single resource and adds it to <tt>localResources</tt>.
 *
 * @param key
 * 		the key to add the resource under
 * @param fs
 * 		the remote file system to upload to
 * @param appId
 * 		application ID
 * @param localSrcPath
 * 		local path to the file
 * @param localResources
 * 		map of resources
 *
 * @return the remote path to the uploaded resource
 */
private static Path setupSingleLocalResource(
		String key,
		FileSystem fs,
		ApplicationId appId,
		Path localSrcPath,
		Map<String, LocalResource> localResources,
		Path targetHomeDir,
		String relativeTargetPath) throws IOException, URISyntaxException {

	Tuple2<Path, LocalResource> resource = Utils.setupLocalResource(
		fs,
		appId.toString(),
		localSrcPath,
		targetHomeDir,
		relativeTargetPath);

	localResources.put(key, resource.f1);

	return resource.f0;
}
 

static LocalResource createJar(FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  LOG.info("Create jar file " + p);
  File jarFile = new File((files.makeQualified(p)).toUri());
  FileOutputStream stream = new FileOutputStream(jarFile);
  LOG.info("Create jar out stream ");
  JarOutputStream out = new JarOutputStream(stream, new Manifest());
  LOG.info("Done writing jar stream ");
  out.close();
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  FileStatus status = files.getFileStatus(p);
  ret.setSize(status.getLen());
  ret.setTimestamp(status.getModificationTime());
  ret.setType(LocalResourceType.PATTERN);
  ret.setVisibility(vis);
  ret.setPattern("classes/.*");
  return ret;
}
 

/**
 * Creates the JAR file with the global files on the driver and then uploads it to the job folder on
 * (H)DFS.
 *
 * @return the map to be used as the "global" resources when submitting Evaluators.
 * @throws IOException if the creation of the JAR or the upload fails
 */
@Override
public synchronized Map<String, LocalResource> call() throws IOException {
  final Map<String, LocalResource> globalResources = new HashMap<>(1);
  if (!this.isUploaded){
    this.pathToGlobalJar = this.uploader.uploadToJobFolder(makeGlobalJar());
    this.isUploaded = true;
  }

  final LocalResource updatedGlobalJarResource = this.uploader.makeLocalResourceForJarFile(this.pathToGlobalJar);

  if (this.globalJarResource != null
      && this.globalJarResource.getTimestamp() != updatedGlobalJarResource.getTimestamp()) {
    LOG.log(Level.WARNING,
            "The global JAR LocalResource timestamp has been changed from "
            + this.globalJarResource.getTimestamp() + " to " + updatedGlobalJarResource.getTimestamp());
  }

  this.globalJarResource = updatedGlobalJarResource;

  // For now, always rewrite the information due to REEF-348
  globalResources.put(this.fileNames.getGlobalFolderPath(), updatedGlobalJarResource);

  return globalResources;
}
 

@Test
public void testResourceMapWithDefaultValues() {

  Map<String, String> configMap = new HashMap<>();

  configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");

  Config conf = new MapConfig(configMap);

  YarnConfiguration yarnConfiguration = new YarnConfiguration();
  yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

  LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf), yarnConfiguration);
  Map<String, LocalResource> resourceMap = mapper.getResourceMap();

  assertNull("Resource does not exist with a name readme", resourceMap.get("readme"));
  assertNotNull("Resource exists with a name myResource1", resourceMap.get("myResource1"));
  assertEquals("host1.com", resourceMap.get("myResource1").getResource().getHost());
  assertEquals(LocalResourceType.FILE, resourceMap.get("myResource1").getType());
  assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("myResource1").getVisibility());
}
 

static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 

private Map<String, LocalResource> buildResourceMapping() {
  ImmutableMap.Builder<String, LocalResource>  localResourceMapBuilder = ImmutableMap.builder();

  List<String> resourceNames = resourceConfig.getResourceNames();
  for (String resourceName : resourceNames) {
    String resourceLocalName = resourceConfig.getResourceLocalName(resourceName);
    LocalResourceType resourceType = resourceConfig.getResourceLocalType(resourceName);
    LocalResourceVisibility resourceVisibility = resourceConfig.getResourceLocalVisibility(resourceName);
    Path resourcePath = resourceConfig.getResourcePath(resourceName);

    LocalResource localResource = createLocalResource(resourcePath, resourceType, resourceVisibility);

    localResourceMapBuilder.put(resourceLocalName, localResource);
    log.info("preparing local resource: {}", resourceLocalName);
  }

  return localResourceMapBuilder.build();
}
 

private static boolean localResourcesCompatible(Map<String, LocalResource> srcLRs,
    Map<String, LocalResource> reqLRs) {
  Map<String, LocalResource> reqLRsCopy = new HashMap<String, LocalResource>(reqLRs);
  for (Entry<String, LocalResource> srcLREntry : srcLRs.entrySet()) {
    LocalResource requestedLocalResource = reqLRsCopy.remove(srcLREntry.getKey());
    if (requestedLocalResource != null && !srcLREntry.getValue().equals(requestedLocalResource)) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Cannot match container: Attempting to use same target resource name: "
            + srcLREntry.getKey()
            + ", but with different source resources. Already localized: "
            + srcLREntry.getValue() + ", requested: " + requestedLocalResource);
      }
      return false;
    }
  }
  for (Entry<String, LocalResource> additionalLREntry : reqLRsCopy.entrySet()) {
    LocalResource lr = additionalLREntry.getValue();
    if (EnumSet.of(LocalResourceType.ARCHIVE, LocalResourceType.PATTERN).contains(lr.getType())) {
      return false;
    }
  }
  return true;
}
 

private void addAppLocalFiles(String localFilePathList, Optional<Map<String, LocalResource>> resourceMap,
    Path destDir, FileSystem localFs) throws IOException {

  for (String localFilePath : SPLITTER.split(localFilePathList)) {
    Path srcFilePath = new Path(localFilePath);
    Path destFilePath = new Path(destDir, srcFilePath.getName());
    if (localFs.exists(srcFilePath)) {
      this.fs.copyFromLocalFile(srcFilePath, destFilePath);
      if (resourceMap.isPresent()) {
        YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE, resourceMap.get());
      }
    } else {
      LOGGER.warn(String.format("The request file %s doesn't exist", srcFilePath));
    }
  }
}
 

private void addResource(
        FileSystem fs,
        Path destPath,
        Map<String, LocalResource> localResources,
        LocalResourceType resourceType,
        String link) throws IOException {

  FileStatus destStatus = fs.getFileStatus(destPath);
  LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
  amJarRsrc.setType(resourceType);
  amJarRsrc.setVisibility(LocalResourceVisibility.PUBLIC);
  amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(destPath));
  amJarRsrc.setTimestamp(destStatus.getModificationTime());
  amJarRsrc.setSize(destStatus.getLen());
  localResources.put(link, amJarRsrc);
}
 

private AMSchedulerEventTALaunchRequest createLaunchRequestEvent(TezTaskAttemptID taID,
    TaskAttempt ta, Resource capability, String[] hosts, String[] racks, Priority priority,
    String jvmOpts, Map<String, String> environment) {
  return createLaunchRequestEvent(taID, ta, capability, hosts, racks, priority,
      new ContainerContext(new HashMap<String, LocalResource>(), new Credentials(), environment,
          jvmOpts));
}
 

private static LocalResource getMockedResource(Random r, 
    LocalResourceVisibility vis) {
  String name = Long.toHexString(r.nextLong());
  URL url = getPath("/local/PRIVATE/" + name);
  LocalResource rsrc =
      BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
          r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
  return rsrc;
}
 

private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException {
	LocalResource localResource = Records.newRecord(LocalResource.class);
	FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(jarStat.getLen());
	localResource.setTimestamp(jarStat.getModificationTime());
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.APPLICATION);
	return localResource;
}
 

private LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath)
        throws IOException
{
    LocalResource localResource = Records.newRecord(LocalResource.class);
    FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
    localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
    localResource.setSize(jarStat.getLen());
    localResource.setTimestamp(jarStat.getModificationTime());
    localResource.setType(LocalResourceType.FILE);
    localResource.setVisibility(LocalResourceVisibility.APPLICATION);
    return localResource;
}
 

public static void addAdditionalLocalResources(Map<String, LocalResource> additionalLrs,
    Map<String, LocalResource> originalLRs, String logContext) {
  // TODO TEZ-1798. Handle contents of Tez archives for duplicate LocalResource checks
  if (additionalLrs != null && !additionalLrs.isEmpty()) {
    StringBuilder sb = new StringBuilder();
    for (Map.Entry<String, LocalResource> lrEntry : additionalLrs.entrySet()) {
      LocalResource originalLr = originalLRs.get(lrEntry.getKey());
      if (originalLr != null) {
        LocalResource additionalLr = lrEntry.getValue();
        if (originalLr.getSize() != additionalLr.getSize()) {
          throw new TezUncheckedException(
              "Duplicate Resources found with different size for [" + logContext + "]: " + lrEntry.getKey() +
                  " : " + "[" + additionalLr.getResource() + "=" + additionalLr.getSize() +
                  "],[" + originalLr.getResource() + "=" + originalLr.getSize());
        } else {
          if (originalLr.getResource().equals(additionalLr.getResource())) {
            sb.append("[").append(lrEntry.getKey()).append(" : Duplicate]");
          } else {
            sb.append("[").append(lrEntry.getKey()).append(" : DuplicateDifferentPath]");
          }
        }
      }
      // The LR either does not exist, or is an 'equivalent' dupe.
      // Prefer the tez specified LR instead of the equivalent user specified LR for container reuse matching
      originalLRs.put(lrEntry.getKey(), lrEntry.getValue());
    }
    String logString = sb.toString();
    if (!logString.isEmpty()) {
      LOG.warn("Found Resources Duplication in " + logContext + " after including resources from " +
          TezConfiguration.TEZ_LIB_URIS + " and " + TezConfiguration.TEZ_AUX_URIS + ": " +
          logString);
    }
  }
}
 

/**
 * Sets up the LocalResources for a new Evaluator.
 *
 * @param resourceLaunchEvent
 * @return
 * @throws IOException
 */
Map<String, LocalResource> getResources(
    final ResourceLaunchEvent resourceLaunchEvent)
    throws IOException {

  final Map<String, LocalResource> result = new HashMap<>();
  result.putAll(getGlobalResources());

  final File localStagingFolder = this.tempFileCreator.createTempDirectory(this.fileNames.getEvaluatorFolderPrefix());

  // Write the configuration
  final File configurationFile = new File(localStagingFolder, this.fileNames.getEvaluatorConfigurationName());
  this.configurationSerializer.toFile(makeEvaluatorConfiguration(resourceLaunchEvent), configurationFile);

  // Copy files to the staging folder
  JobJarMaker.copy(resourceLaunchEvent.getFileSet(), localStagingFolder);

  // Make a JAR file out of it
  final File localFile = tempFileCreator.createTempFile(
      this.fileNames.getEvaluatorFolderPrefix(), this.fileNames.getJarFileSuffix());
  new JARFileMaker(localFile).addChildren(localStagingFolder).close();

  // Upload the JAR to the job folder
  final Path pathToEvaluatorJar = this.uploader.uploadToJobFolder(localFile);
  result.put(this.fileNames.getLocalFolderPath(), this.uploader.makeLocalResourceForJarFile(pathToEvaluatorJar));

  if (this.deleteTempFiles) {
    LOG.log(Level.FINE, "Marking [{0}] for deletion at the exit of this JVM and deleting [{1}]",
        new Object[]{localFile.getAbsolutePath(), localStagingFolder.getAbsolutePath()});
    localFile.deleteOnExit();
    if (!localStagingFolder.delete()) {
      LOG.log(Level.WARNING, "Failed to delete [{0}]", localStagingFolder.getAbsolutePath());
    }
  } else {
    LOG.log(Level.FINE, "The evaluator staging folder will be kept at [{0}], the JAR at [{1}]",
        new Object[]{localFile.getAbsolutePath(), localStagingFolder.getAbsolutePath()});
  }
  return result;
}
 

@Override
public void setResource(LocalResource resource) {
  maybeInitBuilder();
  if (resource == null)
    builder.clearResource();
  this.resource = resource;
}
 

@Override
public void run() {
  try {
    Map<String, String> env = Utils.setJavaEnv(appMaster.getConfiguration());
    String current = ApplicationConstants.Environment.LD_LIBRARY_PATH.$$();
    env.put("LD_LIBRARY_PATH", current + ":" + "`pwd`");

    Map<String, Path> files = new HashMap<>();
    files.put(Constants.TF_JAR_NAME, new Path(tfJar));
    files.put(Constants.TF_LIB_NAME, new Path(tfLib));

    FileSystem fs = FileSystem.get(appMaster.getConfiguration());
    Map<String, LocalResource> localResources =
        Utils.makeLocalResources(fs, files);

    String command = makeContainerCommand(
        containerMemory, clusterSpec.toBase64EncodedJsonString(),
        taskInfo.jobName, taskInfo.taskIndex);

    LOG.info("Launching a new container."
        + ", containerId=" + container.getId()
        + ", containerNode=" + container.getNodeId().getHost()
        + ":" + container.getNodeId().getPort()
        + ", containerNodeURI=" + container.getNodeHttpAddress()
        + ", containerResourceMemory="
        + container.getResource().getMemorySize()
        + ", containerResourceVirtualCores="
        + container.getResource().getVirtualCores()
        + ", command: " + command);
    ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(
        localResources, env, Lists.newArrayList(command), null, null, null, null);
    appMaster.addContainer(container);
    appMaster.getNMClientAsync().startContainerAsync(container, ctx);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

public ContainerContext(Map<String, LocalResource> localResources,
    Credentials credentials, Map<String, String> environment, String javaOpts,
    @Nullable Vertex vertex) {
  Preconditions.checkNotNull(localResources,
      "localResources should not be null");
  Preconditions.checkNotNull(credentials, "credentials should not be null");
  Preconditions.checkNotNull(environment, "environment should not be null");
  Preconditions.checkNotNull(javaOpts, "javaOpts should not be null");
  this.localResources = localResources;
  this.credentials = credentials;
  this.environment = environment;
  this.javaOpts = javaOpts;
  this.vertex = vertex;
}
 

private void initLocalResources() {
  if (this.localResources != null) {
    return;
  }
  ContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder;
  List<StringLocalResourceMapProto> list = p.getLocalResourcesList();
  this.localResources = new HashMap<String, LocalResource>();

  for (StringLocalResourceMapProto c : list) {
    this.localResources.put(c.getKey(), convertFromProtoFormat(c.getValue()));
  }
}
 

public void addFile(FileStatus fstatus) {
  Path fpath = fstatus.getPath() ;
  LocalResource libJar = Records.newRecord(LocalResource.class);
  libJar.setResource(ConverterUtils.getYarnUrlFromPath(fpath));
  libJar.setSize(fstatus.getLen());
  libJar.setTimestamp(fstatus.getModificationTime());
  libJar.setType(LocalResourceType.FILE);
  libJar.setVisibility(LocalResourceVisibility.PUBLIC);
  put(fpath.getName(), libJar) ;
}
 

/**
 * Creates a YARN resource for the remote object at the given location.
 * @param fs remote filesystem
 * @param remoteRsrcPath resource path to be registered
 * @return YARN resource
 */
private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException {
	FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
	return registerLocalResource(
		remoteRsrcPath,
		jarStat.getLen(),
		jarStat.getModificationTime(),
		LocalResourceVisibility.APPLICATION);
}
 

@Override
public void setLocalResources(
    final Map<String, LocalResource> localResources) {
  if (localResources == null)
    return;
  initLocalResources();
  this.localResources.clear();
  this.localResources.putAll(localResources);
}
 

public static PlanLocalResourcesProto convertFromLocalResources(
  Map<String, LocalResource> localResources) {
  PlanLocalResourcesProto.Builder builder =
    PlanLocalResourcesProto.newBuilder();
  for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) {
    PlanLocalResource plr = convertLocalResourceToPlanLocalResource(
        entry.getKey(), entry.getValue());
    builder.addLocalResources(plr);
  }
  return builder.build();
}
 

public static LocalResource newLocalResource(URL url, LocalResourceType type,
    LocalResourceVisibility visibility, long size, long timestamp,
    boolean shouldBeUploadedToSharedCache) {
  LocalResource resource =
    recordFactory.newRecordInstance(LocalResource.class);
  resource.setResource(url);
  resource.setType(type);
  resource.setVisibility(visibility);
  resource.setSize(size);
  resource.setTimestamp(timestamp);
  resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
  return resource;
}
 

private void addContainerLocalResources(Path destDir, Map<String, LocalResource> resourceMap) throws IOException {
  if (!this.fs.exists(destDir)) {
    LOGGER.warn(String.format("Path %s does not exist so no container LocalResource to add", destDir));
    return;
  }

  FileStatus[] statuses = this.fs.listStatus(destDir);
  if (statuses != null) {
    for (FileStatus status : statuses) {
      YarnHelixUtils.addFileAsLocalResource(this.fs, status.getPath(), LocalResourceType.FILE, resourceMap);
    }
  }
}
 

/**
 *
 */
@Test (timeout=5000)
public void validateSetTezJarLocalResourcesMultipleTarballs() throws Exception {
  FileSystem localFs = FileSystem.getLocal(new Configuration());
  StringBuilder tezLibUris = new StringBuilder();

  // Create 2 files
  Path topDir = new Path(TEST_ROOT_DIR, "validatemultipletarballs");
  if (localFs.exists(topDir)) {
    localFs.delete(topDir, true);
  }
  localFs.mkdirs(topDir);

  Path tarFile1 = new Path(topDir, "f1.tar.gz");
  Path tarFile2 = new Path(topDir, "f2.tar.gz");

  Assert.assertTrue(localFs.createNewFile(tarFile1));
  Assert.assertTrue(localFs.createNewFile(tarFile2));
  tezLibUris.append(localFs.makeQualified(tarFile1).toString()).append("#tar1").append(",");
  tezLibUris.append(localFs.makeQualified(tarFile2).toString()).append("#tar2").append(",");

  TezConfiguration conf = new TezConfiguration();
  conf.set(TezConfiguration.TEZ_LIB_URIS, tezLibUris.toString());
  Credentials credentials = new Credentials();
  Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>();
  TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap);
  Set<String> resourceNames = localizedMap.keySet();
  Assert.assertEquals(2, resourceNames.size());
  Assert.assertTrue(resourceNames.contains("tar1"));
  Assert.assertTrue(resourceNames.contains("tar2"));
  Assert.assertFalse(resourceNames.contains("f1.tar.gz"));
  Assert.assertFalse(resourceNames.contains("f2.tar.gz"));


  Assert.assertTrue(localFs.delete(tarFile1, true));
  Assert.assertTrue(localFs.delete(tarFile2, true));
  Assert.assertTrue(localFs.delete(topDir, true));
}
 

private static Entry<String, LocalResource> getMockRsrc(Random r,
    LocalResourceVisibility vis) {
  String name = Long.toHexString(r.nextLong());
  URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name);
  LocalResource rsrc =
      BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
          r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
  return new SimpleEntry<String, LocalResource>(name, rsrc);
}
 

private LocalResource createApplicationResource(FileContext fs, Path p,
    LocalResourceType type) throws IOException {
  LocalResource rsrc = Records.newRecord(LocalResource.class);
  FileStatus rsrcStat = fs.getFileStatus(p);
  rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
      .getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
  rsrc.setSize(rsrcStat.getLen());
  rsrc.setTimestamp(rsrcStat.getModificationTime());
  rsrc.setType(type);
  rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
  return rsrc;
}