@Test
public void testRegionServerCoprocessorServiceError() throws Exception {
  final ServerName serverName = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName();
  DummyRegionServerEndpointProtos.DummyRequest request =
      DummyRegionServerEndpointProtos.DummyRequest.getDefaultInstance();
  try {
    admin
        .<DummyRegionServerEndpointProtos.DummyService.Stub,
            DummyRegionServerEndpointProtos.DummyResponse> coprocessorService(
          DummyRegionServerEndpointProtos.DummyService::newStub,
              (s, c, done) -> s.dummyThrow(c, request, done), serverName).get();
    fail("Should have thrown an exception");
  } catch (Exception e) {
    assertTrue(e.getCause() instanceof RetriesExhaustedException);
    assertTrue(e.getCause().getMessage().contains(WHAT_TO_THROW.getClass().getName().trim()));
  }
}
 

/**
 * Do the changes and handle the pool
 * @param tableName table to insert into
 * @param allRows list of actions
 */
private void batch(TableName tableName, Collection<List<Row>> allRows) throws IOException {
  if (allRows.isEmpty()) {
    return;
  }
  AsyncTable<?> table = getConnection().getTable(tableName);
  List<Future<?>> futures = allRows.stream().map(table::batchAll).collect(Collectors.toList());
  for (Future<?> future : futures) {
    try {
      FutureUtils.get(future);
    } catch (RetriesExhaustedException e) {
      if (e.getCause() instanceof TableNotFoundException) {
        throw new TableNotFoundException("'" + tableName + "'");
      }
      throw e;
    }
  }
}
 

private void testRetriesExhaustedFailure(final TableName tableName, final MockRSExecutor executor)
    throws Exception {
  RegionInfo hri = createRegionInfo(tableName, 1);

  // collect AM metrics before test
  collectAssignmentManagerMetrics();

  // Test Assign operation failure
  rsDispatcher.setMockRsExecutor(executor);
  try {
    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
    fail("unexpected assign completion");
  } catch (RetriesExhaustedException e) {
    // expected exception
    LOG.info("expected exception from assign operation: " + e.getMessage(), e);
  }

  // Assign the region (without problems)
  rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
  waitOnFuture(submitProcedure(createAssignProcedure(hri)));
}
 

private void testFailedOpen(final TableName tableName, final MockRSExecutor executor)
    throws Exception {
  final RegionInfo hri = createRegionInfo(tableName, 1);

  // Test Assign operation failure
  rsDispatcher.setMockRsExecutor(executor);
  try {
    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
    fail("unexpected assign completion");
  } catch (RetriesExhaustedException e) {
    // expected exception
    LOG.info("REGION STATE " + am.getRegionStates().getRegionStateNode(hri));
    LOG.info("expected exception from assign operation: " + e.getMessage(), e);
    assertEquals(true, am.getRegionStates().getRegionState(hri).isFailedOpen());
  }
}
 

@Test
public void testSocketClosed() throws IOException, InterruptedException {
  TableName tableName = TableName.valueOf(name.getMethodName());
  UTIL.createTable(tableName, fam1).close();

  Configuration conf = new Configuration(UTIL.getConfiguration());
  conf.set(RpcClientFactory.CUSTOM_RPC_CLIENT_IMPL_CONF_KEY, MyRpcClientImpl.class.getName());
  conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
  try (Connection connection = ConnectionFactory.createConnection(conf);
    Table table = connection.getTable(TableName.valueOf(name.getMethodName()))) {
    MyRpcClientImpl.enableThrowExceptions();
    table.get(new Get(Bytes.toBytes("asd")));
    fail("Should fail because the injected error");
  } catch (RetriesExhaustedException e) {
    // expected
  }
  for (Socket socket : SAVED_SOCKETS) {
    assertTrue("Socket " + socket + " is not closed", socket.isClosed());
  }
}
 

private boolean causeIsPleaseHold(Throwable e) {
    if (e instanceof PleaseHoldException)
        return true;
    if (e instanceof TableNotEnabledException)
        return true;
    if (e instanceof RegionOfflineException)
        return true;
    if (e instanceof RetriesExhaustedException || e instanceof SocketTimeoutException) {
        if (e.getCause() instanceof RemoteException) {
            RemoteException re = (RemoteException) e.getCause();
            if (PleaseHoldException.class.getName().equals(re.getClassName()))
                return true;
        } else if (e.getCause() instanceof MasterNotRunningException) {
            return true;
        }
        return (e.getCause() instanceof IOException && e.getCause().getCause() instanceof CallTimeoutException) ||
               (e.getCause() instanceof RemoteWithExtrasException && e.getMessage().equals(
                       "Table Namespace Manager not fully initialized, try again later"));
    }
    return false;
}
 

int flush() throws IOException {
    try {
        return flush0();
    }
    catch (RetriesExhaustedException x) {
        // when this happens, we need to reconnect or hbase client hangs forever
        HBaseUtil.closeQuietly(this.conn);
        this.conn = null;
        throw x;
    }
}
 

private void assertDisallow(Table table, TableAction action) throws IOException {
  try {
    action.call(table);
    fail("Should not allow the action");
  } catch (DoNotRetryIOException | RetriesExhaustedException e) {
    // expected
    assertThat(e.getMessage(), containsString("STANDBY"));
  }
}
 

@Test
public void testNegativeAuthentication() throws Exception {
  // Validate that we can read that record back out as the user with our custom auth'n
  UserGroupInformation user1 = UserGroupInformation.createUserForTesting("user1", new String[0]);
  user1.addToken(createPasswordToken("user1", "definitely not the password", clusterId));
  user1.doAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run() throws Exception {
      Configuration clientConf = getClientConf();
      clientConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
      // Depending on the registry in use, the following code can throw exceptions at different
      // places. Master registry fails at the createConnection() step because the RPC to the
      // master fails with sasl auth. With ZK registry, connection creation succeeds (since there
      // is no RPC to HBase services involved) but the subsequent get() fails. The root cause
      // should still be a SaslException in both the cases.
      try (Connection conn = ConnectionFactory.createConnection(clientConf);
        Table t = conn.getTable(tableName)) {
        t.get(new Get(Bytes.toBytes("r1")));
        fail("Should not successfully authenticate with HBase");
      } catch (MasterRegistryFetchException mfe) {
        Throwable cause = mfe.getCause();
        assertTrue(cause.getMessage(), cause.getMessage().contains("SaslException"));
      } catch (RetriesExhaustedException re) {
        assertTrue(re.getMessage(), re.getMessage().contains("SaslException"));
      } catch (Exception e) {
        // Any other exception is unexpected.
        fail("Unexpected exception caught, was expecting a authentication error: " +
          Throwables.getStackTraceAsString(e));
      }
      return null;
    }
  });
}
 

/**
 * Tests that a put on a table throws {@link SocketTimeoutException} when the operation takes
 * longer than 'hbase.client.operation.timeout'.
 */
@Test(expected = RetriesExhaustedException.class)
public void testPutTimeout() throws Exception {
  DELAY_MUTATE = 600;

  Put put = new Put(ROW);
  put.addColumn(FAMILY, QUALIFIER, VALUE);
  TABLE.put(put);
}
 

/**
 * Tests that scan on a table throws {@link RetriesExhaustedException} when the operation takes
 * longer than 'hbase.client.scanner.timeout.period'.
 */
@Test(expected = RetriesExhaustedException.class)
public void testScanTimeout() throws Exception {
  DELAY_SCAN = 600;
  ResultScanner scanner = TABLE.getScanner(new Scan());
  scanner.next();
}
 

private void callRefreshRegionHFilesEndPoint() throws IOException {
  try {
    RefreshHFilesClient refreshHFilesClient = new RefreshHFilesClient(CONF);
    refreshHFilesClient.refreshHFiles(TABLE_NAME);
  } catch (RetriesExhaustedException rex) {
    if (rex.getCause() instanceof IOException) {
      throw new IOException();
    }
  } catch (Throwable ex) {
    LOG.error(ex.toString(), ex);
    fail("Couldn't call the RefreshRegionHFilesEndpoint");
  }
}
 

@Override
protected boolean isUnreachableException(Throwable ex) {
    return (super.isUnreachableException(ex) || ex instanceof SocketTimeoutException
            || ex instanceof ConnectException || ex instanceof RetriesExhaustedException);
}
 

@Override
protected boolean isUnreachableException(Throwable ex) {
    return (super.isUnreachableException(ex) || ex instanceof SocketTimeoutException
            || ex instanceof ConnectException || ex instanceof RetriesExhaustedException);
}
 

private Flow confirmOpened(MasterProcedureEnv env, RegionStateNode regionNode)
    throws IOException {
  if (regionNode.isInState(State.OPEN)) {
    retryCounter = null;
    if (lastState == RegionStateTransitionState.REGION_STATE_TRANSITION_CONFIRM_OPENED) {
      // we are the last state, finish
      regionNode.unsetProcedure(this);
      ServerCrashProcedure.updateProgress(env, getParentProcId());
      return Flow.NO_MORE_STATE;
    }
    // It is possible that we arrive here but confirm opened is not the last state, for example,
    // when merging or splitting a region, we unassign the region from a RS and the RS is crashed,
    // then there will be recovered edits for this region, we'd better make the region online
    // again and then unassign it, otherwise we have to fail the merge/split procedure as we may
    // loss data.
    setNextState(RegionStateTransitionState.REGION_STATE_TRANSITION_CLOSE);
    return Flow.HAS_MORE_STATE;
  }

  int retries = env.getAssignmentManager().getRegionStates().addToFailedOpen(regionNode)
      .incrementAndGetRetries();
  int maxAttempts = env.getAssignmentManager().getAssignMaxAttempts();
  LOG.info("Retry={} of max={}; {}; {}", retries, maxAttempts, this, regionNode.toShortString());

  if (retries >= maxAttempts) {
    env.getAssignmentManager().regionFailedOpen(regionNode, true);
    setFailure(getClass().getSimpleName(), new RetriesExhaustedException(
      "Max attempts " + env.getAssignmentManager().getAssignMaxAttempts() + " exceeded"));
    regionNode.unsetProcedure(this);
    return Flow.NO_MORE_STATE;
  }

  env.getAssignmentManager().regionFailedOpen(regionNode, false);
  // we failed to assign the region, force a new plan
  forceNewPlan = true;
  regionNode.setRegionLocation(null);
  setNextState(RegionStateTransitionState.REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE);

  if (retries > env.getAssignmentManager().getAssignRetryImmediatelyMaxAttempts()) {
    // Throw exception to backoff and retry when failed open too many times
    throw new HBaseIOException("Failed confirm OPEN of " + regionNode +
        " (remote log may yield more detail on why).");
  } else {
    // Here we do not throw exception because we want to the region to be online ASAP
    return Flow.HAS_MORE_STATE;
  }
}
 

/**
 * Tests that a get on a table throws {@link SocketTimeoutException} when the operation takes
 * longer than 'hbase.client.operation.timeout'.
 */
@Test(expected = RetriesExhaustedException.class)
public void testGetTimeout() throws Exception {
  DELAY_GET = 600;
  TABLE.get(new Get(ROW));
}
 

protected Response processException(Throwable exp) {
  Throwable curr = exp;
  if(accessDeniedClazz != null) {
    //some access denied exceptions are buried
    while (curr != null) {
      if(accessDeniedClazz.isAssignableFrom(curr.getClass())) {
        throw new WebApplicationException(
            Response.status(Response.Status.FORBIDDEN)
              .type(MIMETYPE_TEXT).entity("Forbidden" + CRLF +
                 StringUtils.stringifyException(exp) + CRLF)
              .build());
      }
      curr = curr.getCause();
    }
  }
  //TableNotFound may also be buried one level deep
  if (exp instanceof TableNotFoundException ||
      exp.getCause() instanceof TableNotFoundException) {
    throw new WebApplicationException(
      Response.status(Response.Status.NOT_FOUND)
        .type(MIMETYPE_TEXT).entity("Not found" + CRLF +
           StringUtils.stringifyException(exp) + CRLF)
        .build());
  }
  if (exp instanceof NoSuchColumnFamilyException){
    throw new WebApplicationException(
      Response.status(Response.Status.NOT_FOUND)
        .type(MIMETYPE_TEXT).entity("Not found" + CRLF +
           StringUtils.stringifyException(exp) + CRLF)
        .build());
  }
  if (exp instanceof RuntimeException) {
    throw new WebApplicationException(
        Response.status(Response.Status.BAD_REQUEST)
          .type(MIMETYPE_TEXT).entity("Bad request" + CRLF +
            StringUtils.stringifyException(exp) + CRLF)
          .build());
  }
  if (exp instanceof RetriesExhaustedException) {
    RetriesExhaustedException retryException = (RetriesExhaustedException) exp;
    processException(retryException.getCause());
  }
  throw new WebApplicationException(
    Response.status(Response.Status.SERVICE_UNAVAILABLE)
      .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF +
        StringUtils.stringifyException(exp) + CRLF)
      .build());
}