org.apache.hadoop.fs.s3.S3Exception#org.apache.hadoop.io.retry.RetryProxy源码实例Demo

下面列出了org.apache.hadoop.fs.s3.S3Exception#org.apache.hadoop.io.retry.RetryProxy 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。

源代码1 项目： hadoop 文件： RMProxy.java

/**
 * Create a proxy for the specified protocol. For non-HA,
 * this is a direct connection to the ResourceManager address. When HA is
 * enabled, the proxy handles the failover between the ResourceManagers as
 * well.
 */
@Private
protected static <T> T createRMProxy(final Configuration configuration,
    final Class<T> protocol, RMProxy instance) throws IOException {
  YarnConfiguration conf = (configuration instanceof YarnConfiguration)
      ? (YarnConfiguration) configuration
      : new YarnConfiguration(configuration);
  RetryPolicy retryPolicy = createRetryPolicy(conf);
  if (HAUtil.isHAEnabled(conf)) {
    RMFailoverProxyProvider<T> provider =
        instance.createRMFailoverProxyProvider(conf, protocol);
    return (T) RetryProxy.create(protocol, provider, retryPolicy);
  } else {
    InetSocketAddress rmAddress = instance.getRMAddress(conf, protocol);
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress);
    return (T) RetryProxy.create(protocol, proxy, retryPolicy);
  }
}

源代码2 项目： hadoop 文件： NameNodeProxies.java

private static NamenodeProtocol createNNProxyWithNamenodeProtocol(
    InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
    boolean withRetries) throws IOException {
  NamenodeProtocolPB proxy = (NamenodeProtocolPB) createNameNodeProxy(
      address, conf, ugi, NamenodeProtocolPB.class);
  if (withRetries) { // create the proxy with retries
    RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(5, 200,
            TimeUnit.MILLISECONDS);
    Map<String, RetryPolicy> methodNameToPolicyMap
         = new HashMap<String, RetryPolicy>();
    methodNameToPolicyMap.put("getBlocks", timeoutPolicy);
    methodNameToPolicyMap.put("getAccessKeys", timeoutPolicy);
    NamenodeProtocol translatorProxy =
        new NamenodeProtocolTranslatorPB(proxy);
    return (NamenodeProtocol) RetryProxy.create(
        NamenodeProtocol.class, translatorProxy, methodNameToPolicyMap);
  } else {
    return new NamenodeProtocolTranslatorPB(proxy);
  }
}

源代码3 项目： hadoop 文件： NativeS3FileSystem.java

private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  methodNameToPolicyMap.put("rename", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}

源代码4 项目： hadoop 文件： S3FileSystem.java

private static FileSystemStore createDefaultStore(Configuration conf) {
  FileSystemStore store = new Jets3tFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                             conf.getInt("fs.s3.maxRetries", 4),
                                                                             conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                            RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  methodNameToPolicyMap.put("storeBlock", methodPolicy);
  methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
  
  return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
                                             store, methodNameToPolicyMap);
}

源代码5 项目： big-c 文件： RMProxy.java

/**
 * Create a proxy for the specified protocol. For non-HA,
 * this is a direct connection to the ResourceManager address. When HA is
 * enabled, the proxy handles the failover between the ResourceManagers as
 * well.
 */
@Private
protected static <T> T createRMProxy(final Configuration configuration,
    final Class<T> protocol, RMProxy instance) throws IOException {
  YarnConfiguration conf = (configuration instanceof YarnConfiguration)
      ? (YarnConfiguration) configuration
      : new YarnConfiguration(configuration);
  RetryPolicy retryPolicy = createRetryPolicy(conf);
  if (HAUtil.isHAEnabled(conf)) {
    RMFailoverProxyProvider<T> provider =
        instance.createRMFailoverProxyProvider(conf, protocol);
    return (T) RetryProxy.create(protocol, provider, retryPolicy);
  } else {
    InetSocketAddress rmAddress = instance.getRMAddress(conf, protocol);
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress);
    return (T) RetryProxy.create(protocol, proxy, retryPolicy);
  }
}

源代码6 项目： big-c 文件： NameNodeProxies.java

private static NamenodeProtocol createNNProxyWithNamenodeProtocol(
    InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
    boolean withRetries) throws IOException {
  NamenodeProtocolPB proxy = (NamenodeProtocolPB) createNameNodeProxy(
      address, conf, ugi, NamenodeProtocolPB.class);
  if (withRetries) { // create the proxy with retries
    RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(5, 200,
            TimeUnit.MILLISECONDS);
    Map<String, RetryPolicy> methodNameToPolicyMap
         = new HashMap<String, RetryPolicy>();
    methodNameToPolicyMap.put("getBlocks", timeoutPolicy);
    methodNameToPolicyMap.put("getAccessKeys", timeoutPolicy);
    NamenodeProtocol translatorProxy =
        new NamenodeProtocolTranslatorPB(proxy);
    return (NamenodeProtocol) RetryProxy.create(
        NamenodeProtocol.class, translatorProxy, methodNameToPolicyMap);
  } else {
    return new NamenodeProtocolTranslatorPB(proxy);
  }
}

源代码7 项目： big-c 文件： NativeS3FileSystem.java

private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  methodNameToPolicyMap.put("rename", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}

源代码8 项目： big-c 文件： S3FileSystem.java

private static FileSystemStore createDefaultStore(Configuration conf) {
  FileSystemStore store = new Jets3tFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                             conf.getInt("fs.s3.maxRetries", 4),
                                                                             conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                            RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  methodNameToPolicyMap.put("storeBlock", methodPolicy);
  methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
  
  return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
                                             store, methodNameToPolicyMap);
}

源代码9 项目： RDFS 文件： RaidShell.java

private static RaidProtocol createRaidnode(RaidProtocol rpcRaidnode)
  throws IOException {
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();

  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class,
      RetryPolicies.retryByRemoteException(
          RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (RaidProtocol) RetryProxy.create(RaidProtocol.class,
      rpcRaidnode, methodNameToPolicyMap);
}

源代码10 项目： RDFS 文件： SnapshotShell.java

private static SnapshotProtocol createSnapshotNode(SnapshotProtocol rpcSnapshotNode) 
throws IOException {
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                 5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
       new HashMap<Class<? extends Exception>, RetryPolicy>();
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();

  exceptionToPolicyMap.put(RemoteException.class,
RetryPolicies.retryByRemoteException(
    RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
    RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);

  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (SnapshotProtocol) RetryProxy.create(SnapshotProtocol.class, 
                               rpcSnapshotNode, methodNameToPolicyMap);
}

源代码11 项目： RDFS 文件： AvatarZKShell.java

private static AvatarProtocol createAvatarnode(AvatarProtocol rpcAvatarnode)
    throws IOException {
  RetryPolicy createPolicy = RetryPolicies
      .retryUpToMaximumCountWithFixedSleep(5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class, RetryPolicies
      .retryByRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL,
          remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (AvatarProtocol) RetryProxy.create(AvatarProtocol.class,
      rpcAvatarnode, methodNameToPolicyMap);
}

源代码12 项目： RDFS 文件： AvatarShell.java

private static AvatarProtocol createAvatarnode(AvatarProtocol rpcAvatarnode)
    throws IOException {
  RetryPolicy createPolicy = RetryPolicies
      .retryUpToMaximumCountWithFixedSleep(5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class, RetryPolicies
      .retryByRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL,
          remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (AvatarProtocol) RetryProxy.create(AvatarProtocol.class,
      rpcAvatarnode, methodNameToPolicyMap);
}

源代码13 项目： RDFS 文件： NativeS3FileSystem.java

private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}

源代码14 项目： RDFS 文件： S3FileSystem.java

private static FileSystemStore createDefaultStore(Configuration conf) {
  FileSystemStore store = new Jets3tFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                             conf.getInt("fs.s3.maxRetries", 4),
                                                                             conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                            RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  methodNameToPolicyMap.put("storeBlock", methodPolicy);
  methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
  
  return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
                                             store, methodNameToPolicyMap);
}

源代码15 项目： RDFS 文件： DFSClient.java

private static ClientProtocol createNamenode(ClientProtocol rpcNamenode,
    Configuration conf)
  throws IOException {
  long sleepTime = conf.getLong("dfs.client.rpc.retry.sleep",
      LEASE_SOFTLIMIT_PERIOD);
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      5, sleepTime, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class, createPolicy);

  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class,
      RetryPolicies.retryByRemoteException(
          RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (ClientProtocol) RetryProxy.create(ClientProtocol.class,
      rpcNamenode, methodNameToPolicyMap);
}

源代码16 项目： RDFS 文件： HighTideShell.java

private static HighTideProtocol createHighTidenode(HighTideProtocol rpcHighTidenode)
  throws IOException {
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();

  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class,
      RetryPolicies.retryByRemoteException(
          RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (HighTideProtocol) RetryProxy.create(HighTideProtocol.class,
      rpcHighTidenode, methodNameToPolicyMap);
}

源代码17 项目： hadoop-gpu 文件： NativeS3FileSystem.java

private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}

源代码18 项目： hadoop-gpu 文件： S3FileSystem.java

private static FileSystemStore createDefaultStore(Configuration conf) {
  FileSystemStore store = new Jets3tFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                             conf.getInt("fs.s3.maxRetries", 4),
                                                                             conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                            RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  methodNameToPolicyMap.put("storeBlock", methodPolicy);
  methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
  
  return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
                                             store, methodNameToPolicyMap);
}

源代码19 项目： hadoop-gpu 文件： DFSClient.java

private static ClientProtocol createNamenode(ClientProtocol rpcNamenode)
  throws IOException {
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      5, LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
  
  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class, createPolicy);

  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class, 
      RetryPolicies.retryByRemoteException(
          RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  
  methodNameToPolicyMap.put("create", methodPolicy);

  return (ClientProtocol) RetryProxy.create(ClientProtocol.class,
      rpcNamenode, methodNameToPolicyMap);
}

源代码20 项目： TensorFlowOnYARN 文件： TFApplicationRpcClient.java

private TFApplicationRpcClient(String serverAddress, int serverPort) throws IOException {
  InetSocketAddress address = new InetSocketAddress(serverAddress, serverPort);
  Configuration conf = new Configuration();
  RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf, false);
  TensorFlowCluster proxy = RMProxy.createRMProxy(conf, TensorFlowCluster.class, address);
  this.tensorflow = (TensorFlowCluster) RetryProxy.create(
      TensorFlowCluster.class, proxy, retryPolicy);
}

源代码21 项目： hadoop 文件： ServerProxy.java

@SuppressWarnings("unchecked")
protected static <T> T createRetriableProxy(final Configuration conf,
    final Class<T> protocol, final UserGroupInformation user,
    final YarnRPC rpc, final InetSocketAddress serverAddress,
    RetryPolicy retryPolicy) {
  T proxy = user.doAs(new PrivilegedAction<T>() {
    @Override
    public T run() {
      return (T) rpc.getProxy(protocol, serverAddress, conf);
    }
  });
  return (T) RetryProxy.create(protocol, proxy, retryPolicy);
}

源代码22 项目： hadoop 文件： TestNodeStatusUpdater.java

@Override
protected ResourceTracker getRMClient() throws IOException {
  RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf);
  resourceTracker =
      (ResourceTracker) RetryProxy.create(ResourceTracker.class,
        new MyResourceTracker6(rmStartIntervalMS, rmNeverStart),
        retryPolicy);
  return resourceTracker;
}

源代码23 项目： hadoop 文件： NameNodeProxies.java

/**
 * Creates the namenode proxy with the passed protocol. This will handle
 * creation of either HA- or non-HA-enabled proxy objects, depending upon
 * if the provided URI is a configured logical URI.
 *
 * @param conf the configuration containing the required IPC
 *        properties, client failover configurations, etc.
 * @param nameNodeUri the URI pointing either to a specific NameNode
 *        or to a logical nameservice.
 * @param xface the IPC interface which should be created
 * @param fallbackToSimpleAuth set to true or false during calls to indicate if
 *   a secure client falls back to simple auth
 * @return an object containing both the proxy and the associated
 *         delegation token service it corresponds to
 * @throws IOException if there is an error creating the proxy
 **/
@SuppressWarnings("unchecked")
public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
    URI nameNodeUri, Class<T> xface, AtomicBoolean fallbackToSimpleAuth)
    throws IOException {
  AbstractNNFailoverProxyProvider<T> failoverProxyProvider =
      createFailoverProxyProvider(conf, nameNodeUri, xface, true,
        fallbackToSimpleAuth);

  if (failoverProxyProvider == null) {
    // Non-HA case
    return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface,
        UserGroupInformation.getCurrentUser(), true, fallbackToSimpleAuth);
  } else {
    // HA case
    Conf config = new Conf(conf);
    T proxy = (T) RetryProxy.create(xface, failoverProxyProvider,
        RetryPolicies.failoverOnNetworkException(
            RetryPolicies.TRY_ONCE_THEN_FAIL, config.maxFailoverAttempts,
            config.maxRetryAttempts, config.failoverSleepBaseMillis,
            config.failoverSleepMaxMillis));

    Text dtService;
    if (failoverProxyProvider.useLogicalURI()) {
      dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri,
          HdfsConstants.HDFS_URI_SCHEME);
    } else {
      dtService = SecurityUtil.buildTokenService(
          NameNode.getAddress(nameNodeUri));
    }
    return new ProxyAndInfo<T>(proxy, dtService,
        NameNode.getAddress(nameNodeUri));
  }
}

源代码24 项目： hadoop 文件： NameNodeProxies.java

private static ClientProtocol createNNProxyWithClientProtocol(
    InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
    boolean withRetries, AtomicBoolean fallbackToSimpleAuth)
    throws IOException {
  RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);

  final RetryPolicy defaultPolicy = 
      RetryUtils.getDefaultRetryPolicy(
          conf, 
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, 
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT, 
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY,
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
          SafeModeException.class);
  
  final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class);
  ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy(
      ClientNamenodeProtocolPB.class, version, address, ugi, conf,
      NetUtils.getDefaultSocketFactory(conf),
      org.apache.hadoop.ipc.Client.getTimeout(conf), defaultPolicy,
      fallbackToSimpleAuth).getProxy();

  if (withRetries) { // create the proxy with retries

    Map<String, RetryPolicy> methodNameToPolicyMap 
               = new HashMap<String, RetryPolicy>();
  
    ClientProtocol translatorProxy =
      new ClientNamenodeProtocolTranslatorPB(proxy);
    return (ClientProtocol) RetryProxy.create(
        ClientProtocol.class,
        new DefaultFailoverProxyProvider<ClientProtocol>(
            ClientProtocol.class, translatorProxy),
        methodNameToPolicyMap,
        defaultPolicy);
  } else {
    return new ClientNamenodeProtocolTranslatorPB(proxy);
  }
}

源代码25 项目： hadoop 文件： TestIPC.java

/**
 * Test the retry count while used in a retry proxy.
 */
@Test(timeout=60000)
public void testRetryProxy() throws IOException {
  final Client client = new Client(LongWritable.class, conf);
  
  final TestServer server = new TestServer(1, false);
  server.callListener = new Runnable() {
    private int retryCount = 0;
    @Override
    public void run() {
      Assert.assertEquals(retryCount++, Server.getCallRetryCount());
    }
  };

  // try more times, so it is easier to find race condition bug
  // 10000 times runs about 6s on a core i7 machine
  final int totalRetry = 10000;
  DummyProtocol proxy = (DummyProtocol) Proxy.newProxyInstance(
      DummyProtocol.class.getClassLoader(),
      new Class[] { DummyProtocol.class }, new TestInvocationHandler(client,
          server, totalRetry));
  DummyProtocol retryProxy = (DummyProtocol) RetryProxy.create(
      DummyProtocol.class, proxy, RetryPolicies.RETRY_FOREVER);
  
  try {
    server.start();
    retryProxy.dummyRun();
    Assert.assertEquals(TestInvocationHandler.retry, totalRetry + 1);
  } finally {
    Client.setCallIdAndRetryCount(0, 0);
    client.stop();
    server.stop();
  }
}

源代码26 项目： hadoop 文件： TestRPC.java

@Test
public void testWrappedStopProxy() throws IOException {
  StoppedProtocol wrappedProxy = RPC.getProxy(StoppedProtocol.class,
      StoppedProtocol.versionID, null, conf);
  StoppedInvocationHandler invocationHandler = (StoppedInvocationHandler)
      Proxy.getInvocationHandler(wrappedProxy);

  StoppedProtocol proxy = (StoppedProtocol) RetryProxy.create(StoppedProtocol.class,
      wrappedProxy, RetryPolicies.RETRY_FOREVER);

  assertEquals(0, invocationHandler.getCloseCalled());
  RPC.stopProxy(proxy);
  assertEquals(1, invocationHandler.getCloseCalled());
}

源代码27 项目： big-c 文件： ServerProxy.java

@SuppressWarnings("unchecked")
protected static <T> T createRetriableProxy(final Configuration conf,
    final Class<T> protocol, final UserGroupInformation user,
    final YarnRPC rpc, final InetSocketAddress serverAddress,
    RetryPolicy retryPolicy) {
  T proxy = user.doAs(new PrivilegedAction<T>() {
    @Override
    public T run() {
      return (T) rpc.getProxy(protocol, serverAddress, conf);
    }
  });
  return (T) RetryProxy.create(protocol, proxy, retryPolicy);
}

源代码28 项目： big-c 文件： TestNodeStatusUpdater.java

@Override
protected ResourceTracker getRMClient() throws IOException {
  RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf);
  resourceTracker =
      (ResourceTracker) RetryProxy.create(ResourceTracker.class,
        new MyResourceTracker6(rmStartIntervalMS, rmNeverStart),
        retryPolicy);
  return resourceTracker;
}

源代码29 项目： big-c 文件： NameNodeProxies.java

/**
 * Creates the namenode proxy with the passed protocol. This will handle
 * creation of either HA- or non-HA-enabled proxy objects, depending upon
 * if the provided URI is a configured logical URI.
 *
 * @param conf the configuration containing the required IPC
 *        properties, client failover configurations, etc.
 * @param nameNodeUri the URI pointing either to a specific NameNode
 *        or to a logical nameservice.
 * @param xface the IPC interface which should be created
 * @param fallbackToSimpleAuth set to true or false during calls to indicate if
 *   a secure client falls back to simple auth
 * @return an object containing both the proxy and the associated
 *         delegation token service it corresponds to
 * @throws IOException if there is an error creating the proxy
 **/
@SuppressWarnings("unchecked")
public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
    URI nameNodeUri, Class<T> xface, AtomicBoolean fallbackToSimpleAuth)
    throws IOException {
  AbstractNNFailoverProxyProvider<T> failoverProxyProvider =
      createFailoverProxyProvider(conf, nameNodeUri, xface, true,
        fallbackToSimpleAuth);

  if (failoverProxyProvider == null) {
    // Non-HA case
    return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface,
        UserGroupInformation.getCurrentUser(), true, fallbackToSimpleAuth);
  } else {
    // HA case
    Conf config = new Conf(conf);
    T proxy = (T) RetryProxy.create(xface, failoverProxyProvider,
        RetryPolicies.failoverOnNetworkException(
            RetryPolicies.TRY_ONCE_THEN_FAIL, config.maxFailoverAttempts,
            config.maxRetryAttempts, config.failoverSleepBaseMillis,
            config.failoverSleepMaxMillis));

    Text dtService;
    if (failoverProxyProvider.useLogicalURI()) {
      dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri,
          HdfsConstants.HDFS_URI_SCHEME);
    } else {
      dtService = SecurityUtil.buildTokenService(
          NameNode.getAddress(nameNodeUri));
    }
    return new ProxyAndInfo<T>(proxy, dtService,
        NameNode.getAddress(nameNodeUri));
  }
}

源代码30 项目： big-c 文件： NameNodeProxies.java

private static ClientProtocol createNNProxyWithClientProtocol(
    InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
    boolean withRetries, AtomicBoolean fallbackToSimpleAuth)
    throws IOException {
  RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);

  final RetryPolicy defaultPolicy = 
      RetryUtils.getDefaultRetryPolicy(
          conf, 
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, 
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT, 
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY,
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
          SafeModeException.class);
  
  final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class);
  ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy(
      ClientNamenodeProtocolPB.class, version, address, ugi, conf,
      NetUtils.getDefaultSocketFactory(conf),
      org.apache.hadoop.ipc.Client.getTimeout(conf), defaultPolicy,
      fallbackToSimpleAuth).getProxy();

  if (withRetries) { // create the proxy with retries

    Map<String, RetryPolicy> methodNameToPolicyMap 
               = new HashMap<String, RetryPolicy>();
  
    ClientProtocol translatorProxy =
      new ClientNamenodeProtocolTranslatorPB(proxy);
    return (ClientProtocol) RetryProxy.create(
        ClientProtocol.class,
        new DefaultFailoverProxyProvider<ClientProtocol>(
            ClientProtocol.class, translatorProxy),
        methodNameToPolicyMap,
        defaultPolicy);
  } else {
    return new ClientNamenodeProtocolTranslatorPB(proxy);
  }
}

方法所在类

org.apache.hadoop.fs.s3.S3Exception