org.junit.contrib.java.lang.system.Assertion#org.apache.hadoop.hive.metastore.api.Table源码实例Demo

@Test
public void typical() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY)
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE)
      .build();

  runWaggleDance(runner);
  HiveMetaStoreClient proxy = getWaggleDanceClient();

  // Local table
  Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE);
  Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE);
  assertThat(waggledLocalTable, is(localTable));

  // Remote table
  String waggledRemoteDbName = REMOTE_DATABASE;
  assertTypicalRemoteTable(proxy, waggledRemoteDbName);
}
 

/**
 * {@inheritDoc}
 */
@Override
public List<FieldSchema> get_fields_with_environment_context(
    final String dbName,
    final String tableName,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    return requestWrapper("get_fields_with_environment_context",
        new Object[]{dbName, tableName, environmentContext}, () -> {
            final Table table = get_table(dbName, tableName);

            if (table == null || table.getSd() == null || table.getSd().getCols() == null) {
                throw new MetaException("Unable to get fields for " + dbName + "." + tableName);
            }
            return table.getSd().getCols();
        });
}
 

/**
 * Retrieves the partition values from the partition name. This method also validates the partition keys to that
 * of the table.
 *
 * @param tableQName  table name
 * @param table       table
 * @param partName    partition name
 * @return list of partition values
 */
public static List<String> getPartValuesFromPartName(final QualifiedName tableQName, final Table table,
    final String partName) {
    if (Strings.isNullOrEmpty(partName)) {
        throw new InvalidMetaException(tableQName, partName, null);
    }
    final LinkedHashMap<String, String> partSpec = new LinkedHashMap<>();
    Warehouse.makeSpecFromName(partSpec, new Path(partName));
    final List<String> values = new ArrayList<>();
    for (FieldSchema field : table.getPartitionKeys()) {
        final String key = field.getName();
        final String val = partSpec.get(key);
        if (val == null) {
            throw new InvalidMetaException(tableQName, partName, null);
        }
        values.add(val);
    }
    return values;
}
 

public Partition addTestPartition(Table tbl, List<String> values, int createTime) throws Exception {
  StorageDescriptor partitionSd = new StorageDescriptor();
  if (StringUtils.isNotBlank(tbl.getSd().getLocation())) {
    partitionSd.setLocation(tbl.getSd().getLocation() + values);
  } else {
    partitionSd.setLocation("/tmp/" + tbl.getTableName() + "/part1");
  }

  partitionSd.setSerdeInfo(
      new SerDeInfo("name", "serializationLib", ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy")));
  partitionSd.setCols(tbl.getPartitionKeys());
  Partition partition =
      new Partition(values, tbl.getDbName(), tbl.getTableName(), 1, 1, partitionSd, new HashMap<String, String>());
  partition.setCreateTime(createTime);
  return this.getLocalMetastoreClient().add_partition(partition);

}
 

@Test
public void transformOutboundGetTablesResult() throws Exception {
  Table table = new Table();
  table.setDbName(DB_NAME);
  table.setTableName(TABLE_NAME);
  Table table2 = new Table();
  table2.setDbName(DB_NAME);
  table2.setTableName(TABLE_NAME);
  table2.setViewExpandedText(VIEW_EXPANDED_TEXT);
  table2.setViewOriginalText(VIEW_ORIGINAL_TEXT);
  GetTablesResult result = new GetTablesResult();
  result.setTables(Arrays.asList(table, table2));
  GetTablesResult transformedResult = databaseMapping.transformOutboundGetTablesResult(result);
  assertThat(transformedResult, is(sameInstance(result)));
  assertThat(transformedResult.getTables().size(), is(2));
  assertThat(transformedResult.getTables().get(0), is(sameInstance(result.getTables().get(0))));
  assertThat(transformedResult.getTables().get(0).getDbName(), is(OUT_DB_NAME));
  assertThat(transformedResult.getTables().get(0).getTableName(), is(TABLE_NAME));
  assertThat(transformedResult.getTables().get(0).getViewExpandedText(), nullValue());
  assertThat(transformedResult.getTables().get(0).getViewOriginalText(), nullValue());
  assertThat(transformedResult.getTables().get(1), is(sameInstance(result.getTables().get(1))));
  assertThat(transformedResult.getTables().get(1).getDbName(), is(OUT_DB_NAME));
  assertThat(transformedResult.getTables().get(1).getTableName(), is(TABLE_NAME));
  assertThat(transformedResult.getTables().get(1).getViewExpandedText(), is(VIEW_EXPANDED_TEXT_TRANSFORMED));
  assertThat(transformedResult.getTables().get(1).getViewOriginalText(), is(VIEW_ORIGINAL_TEXT_TRANSFORMED));
}
 

private Table createPartitionedTable(String databaseName, String tableName, HiveServer2Core server) throws Exception {
  Table table = new Table();
  table.setDbName(DATABASE);
  table.setTableName(tableName);
  table.setPartitionKeys(Arrays.asList(new FieldSchema("partcol", "int", null)));
  table.setSd(new StorageDescriptor());
  table.getSd().setCols(Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)));
  table.getSd().setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
  table.getSd().setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
  table.getSd().setSerdeInfo(new SerDeInfo());
  table.getSd().getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
  HiveMetaStoreClient client = server.getCore().newClient();
  client.createTable(table);
  client.close();
  return table;
}
 

private void setParameters(String newMetadataLocation, Table tbl) {
  Map<String, String> parameters = tbl.getParameters();

  if (parameters == null) {
    parameters = new HashMap<>();
  }

  parameters.put(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE.toUpperCase(Locale.ENGLISH));
  parameters.put(METADATA_LOCATION_PROP, newMetadataLocation);

  if (currentMetadataLocation() != null && !currentMetadataLocation().isEmpty()) {
    parameters.put(PREVIOUS_METADATA_LOCATION_PROP, currentMetadataLocation());
  }

  tbl.setParameters(parameters);
}
 

@Override
public Table getTable(final String dbName, final String tableName, boolean ignoreAuthzErrors) throws TException{

  Table table = getTableWithoutTableTypeChecking(dbName, tableName, ignoreAuthzErrors);

  if(table == null){
    return null;
  }

  TableType type = TableType.valueOf(table.getTableType());
  switch (type) {
    case EXTERNAL_TABLE:
    case MANAGED_TABLE:
      return table;

    case VIRTUAL_VIEW:
      throw UserException.unsupportedError().message("Hive views are not supported").build(NOPLogger.NOP_LOGGER);
    default:
      return null;
  }
}
 

/**
 * Copies partitions from oldTable to newTable, partitions copied are modified to take the schema of newTable
 */
public void execute(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws TException {
  int count = 0;
  String databaseName = newTable.getDbName();
  String tableName = newTable.getTableName();
  PartitionIterator partitionIterator = new PartitionIterator(client, oldTable, partitionBatchSize);
  while (partitionIterator.hasNext()) {
    List<Partition> batch = new ArrayList<>();
    for (int i = 0; i < partitionBatchSize && partitionIterator.hasNext(); i++) {
      Partition partition = partitionIterator.next();
      count++;
      Partition copy = new Partition(partition);
      copy.setDbName(databaseName);
      copy.setTableName(tableName);
      StorageDescriptor sd = new StorageDescriptor(partition.getSd());
      sd.setCols(newTable.getSd().getCols());
      copy.setSd(sd);
      batch.add(copy);
    }
    LOG.info("Copying batch of size {} to {}.{}", batch.size(), databaseName, tableName);
    client.add_partitions(batch);
  }
  LOG.info("Copied {} partitions to {}.{}", count, databaseName, tableName);
}
 

@Test
public void testEvolutionDisabledForNewTable() throws IOException {
  boolean isEvolutionEnabled = false;
  Optional<Table> destinationTableMeta = Optional.absent();

  String ddl = HiveAvroORCQueryGenerator
      .generateCreateTableDDL(outputSchema, schemaName, "file:/user/hive/warehouse/" + schemaName,
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<List<String>>absent(),
          Optional.<Map<String, HiveAvroORCQueryGenerator.COLUMN_SORT_ORDER>>absent(), Optional.<Integer>absent(),
          Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(),
          null, isEvolutionEnabled, true, destinationTableMeta,
          new HashMap<String, String>());

  Assert.assertEquals(ddl, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_enabled.ddl"),
      "Generated DDL did not match expected for evolution disabled");

  String dml = HiveAvroORCQueryGenerator
      .generateTableMappingDML(inputSchema, outputSchema, schemaName, schemaName + "_orc", Optional.<String>absent(),
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<Boolean>absent(),
          Optional.<Boolean>absent(), isEvolutionEnabled, destinationTableMeta, rowLimit);

  Assert.assertEquals(dml, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_enabled.dml"),
      "Generated DML did not match expected for evolution disabled");
}
 

@Override
public void rollbackCreateTable(Table tbl) throws MetaException {
    KuduClient client = getKuduClient(tbl.getParameters().get(HiveKuduConstants.MASTER_ADDRESS_NAME));
    String tablename = getKuduTableName(tbl);
    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    try {
        if ( client.tableExists(tablename) && !isExternal) {
            client.deleteTable(tablename);
        }
    } catch (Exception ioe) {
        throw new MetaException("Error dropping table while rollback of create table:" +tablename);
    } finally {
        try {
            client.shutdown();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 

private String getFilePath(Table tbl) throws Exception {

        StorageDescriptor descTable = tbl.getSd();

        InputFormat<?, ?> fformat = HiveDataFragmenter.makeInputFormat(descTable.getInputFormat(), jobConf);

        FileInputFormat.setInputPaths(jobConf, new Path(descTable.getLocation()));

        InputSplit[] splits;
        try {
            splits = fformat.getSplits(jobConf, 1);
        } catch (org.apache.hadoop.mapred.InvalidInputException e) {
            LOG.debug("getSplits failed on " + e.getMessage());
            throw new RuntimeException("Unable to get file path for table.");
        }

        for (InputSplit split : splits) {
            FileSplit fsp = (FileSplit) split;
            String[] hosts = fsp.getLocations();
            String filepath = fsp.getPath().toString();
            return filepath;
        }
        throw new RuntimeException("Unable to get file path for table.");
    }
 

@Test
public void testCheckTableSchemaMappingMissingColumnMapping() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," +
   "col2:dynamo_col2#,hashKey:hashKey,hashMap:hashMap");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("hashMap", "map<string,string>", ""));
  sd.setCols(cols);
  table.setSd(sd);

  exceptionRule.expect(MetaException.class);
  exceptionRule.expectMessage("Could not find column(s) for column mapping(s): ");
  exceptionRule.expectMessage("col2:dynamo_col2#");
  exceptionRule.expectMessage("hashkey:hashKey");
  storageHandler.checkTableSchemaMapping(description, table);
}
 

public void alterTable(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws Exception {
  List<FieldSchema> oldColumns = oldTable.getSd().getCols();
  List<FieldSchema> newColumns = newTable.getSd().getCols();
  if (hasAnyChangedColumns(oldColumns, newColumns)) {
    LOG
        .info("Found columns that have changed type, attempting to recreate target table with the new columns."
            + "Old columns: {}, new columns: {}", oldColumns, newColumns);
    Table tempTable = new Table(newTable);
    String tempName = newTable.getTableName() + "_temp";
    tempTable.setTableName(tempName);
    try {
      client.createTable(tempTable);
      copyPartitionsOperation.execute(client, newTable, tempTable);
      renameTableOperation.execute(client, tempTable, newTable);
    } finally {
      dropTableService.dropTable(client, tempTable.getDbName(), tempName);
    }
  } else {
    client.alter_table(newTable.getDbName(), newTable.getTableName(), newTable);
  }
}
 

@Test
public void testCreate() throws TException {
  // Table should be created in hive metastore
  final Table table = metastoreClient.getTable(DB_NAME, TABLE_NAME);

  // check parameters are in expected state
  final Map<String, String> parameters = table.getParameters();
  Assert.assertNotNull(parameters);
  Assert.assertTrue(ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(parameters.get(TABLE_TYPE_PROP)));
  Assert.assertTrue(ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(table.getTableType()));

  // Ensure the table is pointing to empty location
  Assert.assertEquals(getTableLocation(TABLE_NAME) , table.getSd().getLocation());

  // Ensure it is stored as unpartitioned table in hive.
  Assert.assertEquals(0 , table.getPartitionKeysSize());

  // Only 1 snapshotFile Should exist and no manifests should exist
  Assert.assertEquals(1, metadataVersionFiles(TABLE_NAME).size());
  Assert.assertEquals(0, manifestFiles(TABLE_NAME).size());

  final com.netflix.iceberg.Table icebergTable = new HiveTables(hiveConf).load(DB_NAME, TABLE_NAME);
  // Iceberg schema should match the loaded table
  Assert.assertEquals(schema.asStruct(), icebergTable.schema().asStruct());
}
 

@Override
public synchronized void dropTable(HiveIdentity identity, String databaseName, String tableName, boolean deleteData)
{
    List<String> locations = listAllDataPaths(identity, this, databaseName, tableName);

    SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName);
    Table table = relations.remove(schemaTableName);
    if (table == null) {
        throw new TableNotFoundException(schemaTableName);
    }
    views.remove(schemaTableName);
    partitions.keySet().removeIf(partitionName -> partitionName.matches(databaseName, tableName));

    // remove data
    if (deleteData && table.getTableType().equals(MANAGED_TABLE.name())) {
        for (String location : locations) {
            if (location != null) {
                File directory = new File(new Path(location).toUri());
                checkArgument(isParentDir(directory, baseDirectory), "Table directory must be inside of the metastore base directory");
                deleteDirectory(directory);
            }
        }
    }
}
 

@Test
public void newSourcePartition() throws Exception {
  Table sourceTable = catalog.client().getTable(DATABASE, SOURCE_TABLE);
  File sourcePartition2Location = createPartitionData("part=2", sourceTableUri,
      Arrays.asList("5\troberto", "6\tpedro"));
  Partition sourcePartition2 = newPartition(DATABASE, SOURCE_TABLE, sourceTable.getSd(), Arrays.asList("2"),
      sourcePartition2Location, null, null, false);
  catalog.client().add_partition(sourcePartition2);

  Table replicaTable = catalog.client().getTable(DATABASE, REPLICA_TABLE);

  HiveDifferences
      .builder(diffListener)
      .comparatorRegistry(comparatorRegistry)
      .source(configuration, sourceTable, new PartitionIterator(catalog.client(), sourceTable, PARTITION_BATCH_SIZE))
      .replica(Optional.of(replicaTable),
          Optional.of(new BufferedPartitionFetcher(catalog.client(), replicaTable, PARTITION_BATCH_SIZE)))
      .checksumFunction(checksumFunction)
      .build()
      .run();
  verify(diffListener, never()).onChangedTable(anyList());
  verify(diffListener, times(1))
      .onNewPartition("part=2", catalog.client().getPartition(DATABASE, SOURCE_TABLE, "part=2"));
  verify(diffListener, never()).onChangedPartition(anyString(), any(Partition.class), anyList());
  verify(diffListener, never()).onDataChanged(anyString(), any(Partition.class));
}
 

public SourceLocationManager getLocationManager(
    Table table,
    List<Partition> partitions,
    String eventId,
    Map<String, Object> copierOptions)
  throws IOException {
  if (MetaStoreUtils.isView(table)) {
    return new ViewLocationManager();
  }
  HdfsSnapshotLocationManager hdfsSnapshotLocationManager = new HdfsSnapshotLocationManager(getHiveConf(), eventId,
      table, partitions, snapshotsDisabled, sourceTableLocation, sourceCatalogListener);
  boolean ignoreMissingFolder = MapUtils.getBooleanValue(copierOptions,
      CopierOptions.IGNORE_MISSING_PARTITION_FOLDER_ERRORS, false);
  if (ignoreMissingFolder) {
    return new FilterMissingPartitionsLocationManager(hdfsSnapshotLocationManager, getHiveConf());
  }
  return hdfsSnapshotLocationManager;
}
 

@Override
public void commitDropTable(Table tbl, boolean deleteData)
        throws MetaException {
    KuduClient client = getKuduClient(tbl.getParameters().get(HiveKuduConstants.MASTER_ADDRESS_NAME));
    String tablename = getKuduTableName(tbl);
    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    try {
        if (deleteData && !isExternal) {
            client.deleteTable(tablename);
        }
    } catch (Exception ioe) {
        throw new MetaException("Error dropping table:" +tablename);
    } finally {
        try {
            client.shutdown();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 

public static ConvertibleHiveDataset createTestConvertibleDataset(Config config)
    throws URISyntaxException {
  Table table = getTestTable("db1", "tb1");
  FileSystem mockFs = Mockito.mock(FileSystem.class);
  when(mockFs.getUri()).thenReturn(new URI("test"));
  ConvertibleHiveDataset cd =
      new ConvertibleHiveDataset(mockFs, Mockito.mock(HiveMetastoreClientPool.class), new org.apache.hadoop.hive.ql.metadata.Table(
          table), new Properties(), config);
  return cd;
}
 

@Test
public void testEvolutionDisabledForExistingTable() throws IOException {
  boolean isEvolutionEnabled = false;
  boolean casePreserved = true;
  Optional<Table> destinationTableMeta = createEvolvedDestinationTable(schemaName, "default", "", true);

  String ddl = HiveAvroORCQueryGenerator
      .generateCreateTableDDL(outputSchema, schemaName, "file:/user/hive/warehouse/" + schemaName,
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<List<String>>absent(),
          Optional.<Map<String, HiveAvroORCQueryGenerator.COLUMN_SORT_ORDER>>absent(), Optional.<Integer>absent(),
          Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(),
          null, isEvolutionEnabled, casePreserved, destinationTableMeta,
          new HashMap<String, String>());

  Assert.assertEquals(ddl, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_disabled.ddl"),
      "Generated DDL did not match expected for evolution disabled");

  String dml = HiveAvroORCQueryGenerator
      .generateTableMappingDML(inputSchema, outputSchema, schemaName, schemaName + "_orc", Optional.<String>absent(),
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<Boolean>absent(),
          Optional.<Boolean>absent(), isEvolutionEnabled, destinationTableMeta, rowLimit);

  Assert.assertEquals(dml, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_disabled.dml"),
      "Generated DML did not match expected for evolution disabled");
}
 

@Override
public void preCreateTable(Table tbl) throws MetaException {

    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    if (isExternal) {
        Log.info("Creating External table for Splice...");
    }

    String inputTableName = tbl.getParameters().get(MRConstants.SPLICE_TABLE_NAME);
    if (inputTableName == null)
        throw new MetaException("Wrong param, you are missing " +
        		MRConstants.SPLICE_TABLE_NAME + " ? ");

    // We can choose to support user define column mapping.
    // But currently I don't think it is necessary
    // We map all columns from Splice Table to Hive Table.
    String connStr = tbl.getParameters().get(MRConstants.SPLICE_JDBC_STR);
    if (connStr == null)
        throw new MetaException("Wrong param, did you mean " +
        		MRConstants.SPLICE_JDBC_STR + " ? ");
    if (sqlUtil == null)
        sqlUtil = SMSQLUtil.getInstance(connStr);
    if (inputTableName != null) {
        inputTableName = inputTableName.trim();
        checkTableExists(inputTableName);
    }
}
 

@Test
public void unpartitionedTableReplicateAvroSchema() throws Exception {
  helper.createManagedUnpartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE));
  LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE));

  java.nio.file.Path sourceAvroSchemaPath = Paths.get(sourceWarehouseUri.toString() + "/avro-schema-file.test");
  Files.write(sourceAvroSchemaPath, AVRO_SCHEMA_CONTENT.getBytes());
  String avroSchemaUrl = sourceAvroSchemaPath.toString();

  Table sourceTable = sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE);
  sourceTable.putToParameters("avro.schema.url", avroSchemaUrl);
  sourceCatalog.client().alter_table(sourceTable.getDbName(), sourceTable.getTableName(), sourceTable);

  exit.expectSystemExitWithStatus(0);
  File config = dataFolder.getFile("unpartitioned-single-table-avro-schema.yml");
  CircusTrainRunner runner = CircusTrainRunner
      .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation)
      .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(),
          sourceCatalog.driverClassName())
      .replicaMetaStore(replicaCatalog.getThriftConnectionUri())
      .build();

  exit.checkAssertionAfterwards(new Assertion() {
    @Override
    public void checkAssertion() throws Exception {
      Table replicaHiveTable = replicaCatalog.client().getTable(DATABASE, TARGET_UNPARTITIONED_MANAGED_TABLE);
      String expectedReplicaSchemaUrl = replicaWarehouseUri.toURI().toString() + "ct_database/";
      String transformedAvroUrl = replicaHiveTable.getParameters().get("avro.schema.url");
      assertThat(transformedAvroUrl, startsWith(expectedReplicaSchemaUrl));
      Path copiedSchema = new Path(transformedAvroUrl);
      FileSystem fs = FileSystem.get(replicaCatalog.conf());
      assertTrue(fs.exists(copiedSchema));
      String content = new String(Files.readAllBytes(java.nio.file.Paths.get(copiedSchema.toUri())));
      assertThat(content, is(AVRO_SCHEMA_CONTENT));
    }
  });

  runner.run(config.getAbsolutePath());
}
 

/**
 * COPIED FROM org.apache.hadoop.hive.ql.stats.StatsUtil for backward compatibility
 * 
 * Get basic stats of table
 * @param table
 *          - table
 * @param statType
 *          - type of stats
 * @return value of stats
 */
public static long getBasicStatForTable(org.apache.hadoop.hive.ql.metadata.Table table, String statType) {
    Map<String, String> params = table.getParameters();
    long result = 0;

    if (params != null) {
        try {
            result = Long.parseLong(params.get(statType));
        } catch (NumberFormatException e) {
            result = 0;
        }
    }
    return result;
}
 

private String getPartitionName(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, Table hiveTable) throws PartitionSpecInvalidException {
	List<String> partitionCols = getFieldNames(hiveTable.getPartitionKeys());
	List<String> partitionVals = getOrderedFullPartitionValues(partitionSpec, partitionCols, tablePath);
	List<String> partKVs = new ArrayList<>();
	for (int i = 0; i < partitionCols.size(); i++) {
		partKVs.add(partitionCols.get(i) + "=" + partitionVals.get(i));
	}
	return String.join("/", partKVs);
}
 

private static String partitionName(Table table, Partition partition) {
  try {
    return Warehouse.makePartName(table.getPartitionKeys(), partition.getValues());
  } catch (MetaException e) {
    throw new CircusTrainException("Unable to build partition name for partition values "
        + partition.getValues()
        + " of table "
        + Warehouse.getQualifiedName(table), e);
  }
}
 

@Test
public void alter_table_with_cascade() throws TException {
  Table table = new Table();
  handler.alter_table_with_cascade(DB_P, "table", table, true);
  verify(primaryMapping).checkWritePermissions(DB_P);
  verify(primaryClient).alter_table_with_cascade(DB_P, "table", table, true);
}
 

private void addColumnsToReplica(Table replicaTable) throws Exception {
  List<FieldSchema> columns = replicaTable.getSd().getCols();
  columns.add(new FieldSchema("age", "int", ""));
  columns.add(new FieldSchema("colour", "string", ""));
  replicaTable.getSd().setCols(columns);
  replicaCatalog.client().alter_table(replicaTable.getDbName(), replicaTable.getTableName(), replicaTable);
}
 

/**
 * Create columnStatistics from the given Hive column stats of a hive table.
 */
public static ColumnStatistics createTableColumnStats(
		Table hiveTable,
		Map<String, CatalogColumnStatisticsDataBase> colStats) {
	ColumnStatisticsDesc desc = new ColumnStatisticsDesc(true, hiveTable.getDbName(), hiveTable.getTableName());
	return createHiveColumnStatistics(colStats, hiveTable.getSd(), desc);
}
 

public HiveConvertExtractor(WorkUnitState state, FileSystem fs) throws IOException, TException, HiveException {
  super(state);

  if (Boolean.valueOf(state.getPropAsBoolean(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY))) {
    log.info("Ignoring Watermark workunit for {}", state.getProp(ConfigurationKeys.DATASET_URN_KEY));
    return;
  }

  if (!(this.hiveDataset instanceof ConvertibleHiveDataset)) {
    throw new IllegalStateException("HiveConvertExtractor is only compatible with ConvertibleHiveDataset");
  }

  ConvertibleHiveDataset convertibleHiveDataset = (ConvertibleHiveDataset) this.hiveDataset;

  try (AutoReturnableObject<IMetaStoreClient> client = this.pool.getClient()) {
    Table table = client.get().getTable(this.dbName, this.tableName);

    SchemaAwareHiveTable schemaAwareHiveTable = new SchemaAwareHiveTable(table, AvroSchemaManager.getSchemaFromUrl(this.hiveWorkUnit.getTableSchemaUrl(), fs));

    SchemaAwareHivePartition schemaAwareHivePartition = null;

    if (this.hiveWorkUnit.getPartitionName().isPresent() && this.hiveWorkUnit.getPartitionSchemaUrl().isPresent()) {

      Partition partition = client.get().getPartition(this.dbName, this.tableName, this.hiveWorkUnit.getPartitionName().get());
      schemaAwareHivePartition =
          new SchemaAwareHivePartition(table, partition, AvroSchemaManager.getSchemaFromUrl(this.hiveWorkUnit.getPartitionSchemaUrl().get(), fs));
    }

    QueryBasedHiveConversionEntity entity =
        new QueryBasedHiveConversionEntity(convertibleHiveDataset, schemaAwareHiveTable, Optional.fromNullable(schemaAwareHivePartition));
    this.conversionEntities.add(entity);
  }

}