HBase海量数据存储

#设置JDK的安装目录
export JAVA_HOME=/usr/jdk8/jdk1.8.0_161#true则使用hbase自带的zk服务,false则使用外部的zk服务.
export HBASE_MANAGES_ZK=flase

<!-- 指定HBase日志的存放目录 -->
<property>
<name>hbase.tmp.dir</name>
<value>/usr/hbase/hbase-1.2.8/logs</value>
</property>
<!-- 指定HBase中的数据存储在HDFS中的目录 -->
<property>
<name>hbase.rootdir</name>
<value>hdfs://nameservice:8020/hbase</value>
</property>
<!-- 设置是否是分布式 -->
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<!-- 指定HBase使用的ZK地址 -->
<property>
<name>hbase.zookeeper.quorum</name>
<value>192.168.1.80:2181,192.168.1.81:2181,192.168.1.82:2181</value>
</property>

#查看是否安装了NTP服务
rpm -qa|grep ntp#安装NTP服务
yum install ntp -y#从NTP服务器中获取时间并同步本地
ntpdate 192.168.1.80

#创建表
create 'tableName' , 'columnFamily' , 'columnFamily...'#添加记录
put 'tableName' , 'rowkey' , 'columnFamily:column' , 'value'#查询记录
get 'tableName' , 'rowkey'#统计表的记录数
count 'tableName'#删除记录
deleteall 'tableName' , 'rowkey'#删除记录的某一列
delete 'tableName' , 'rowkey' ,'columnFamily:column'#禁用表
disable 'tableName'#启动表
enable 'tableName'#查看表是否被禁用
is_disabled 'tableName'#删除表
drop 'tableName'#查看表中的所有记录
scan 'tableName'#查看表中指定列的所有记录
scan 'tableName' , {COLUMNS=>'columnFamily:column'}#检查表是否存在
exists 'tableName'#查看当前HBase中的表
list

<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.2.8</version>
</dependency>

Configuration config = HBaseConfiguration.create();
config.set("hbase.zookeeper.quorum","192.168.1.80,192.168.1.81,192.168.1.82");
config.set("hbase.zookeeper.property.clientPort","2181");

Connection conn = ConnectionFactory.createConnection(config);

//判断表是否存在
boolean tableExists(TableName);//遍历HBase中的表定义
HTableDescriptor [] listTables();//遍历HBase中的表名称
TableName [] listTableNames();//根据表名获取表定义
HTableDescriptor getTableDescriptor(TableName);//创建表
void createTable(HTableDescriptor);//删除表
void deleteTable(TableName);//启用表
void enableTable(TableName);//禁用表
void disableTable(TableName);//判断表是否是启用状态
boolean isTableEnabled(TableName);//判断表是否是禁用状态
boolean isTableDisabled(TableName);//为表添加列簇
void addColumn(TableName,HColumnDescriptor);//删除表中的列簇
void deleteColumn(TableName,byte);//修改表中的列簇
void modifyColumn(TableName,HColumnDescriptor);

//判断指定RowKey的数据是否存在
boolean exists(Get get);//根据RowKey获取数据
Result get(Get get);//根据多个RowKey获取数据
Result [] get(List<Get>);//获取表的扫描器
ResultScanner getScanner(Scan);//添加数据
void put(Put);//批量添加数据
void put(List<Put>);//删除数据
void delete(Delete);//批量删除数据
void delete(List<Delete>)

/**
* @Auther: ZHUANGHAOTANG
* @Date: 2018/11/26 11:40
* @Description:
*/
public class HBaseUtils { private static final Logger logger = LoggerFactory.getLogger(HBaseUtils.class); /**
* ZK集群地址
*/
private static final String ZK_CLUSTER_HOSTS = "192.168.1.80,192.168.1.81,192.168.1.82"; /**
* ZK端口
*/
private static final String ZK_CLUSTER_PORT = "2181"; /**
* HBase全局连接
*/
private static Connection connection; static {
//默认加载classpath下hbase-site.xml文件
Configuration configuration = HBaseConfiguration.create();
configuration.set("hbase.zookeeper.quorum", ZK_CLUSTER_HOSTS);
configuration.set("hbase.zookeeper.property.clientPort", ZK_CLUSTER_PORT);
try {
connection = ConnectionFactory.createConnection(configuration);
} catch (Exception e) {
logger.info("初始化HBase连接失败：", e);
}
} /**
* 返回连接
*/
public static Connection getConnection() {
return connection;
} /**
* 创建表
*/
public static void createTable(String tableName, String... families) throws Exception {
Admin admin = connection.getAdmin();
if (admin.tableExists(TableName.valueOf(tableName))) {
throw new UnsupportedOperationException("tableName " + tableName + " is already exists");
}
HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(tableName));
for (String family : families)
descriptor.addFamily(new HColumnDescriptor(family));
admin.createTable(descriptor);
} /**
* 删除表
*/
public static void deleteTable(String tableName) throws Exception {
Admin admin = connection.getAdmin();
if (admin.tableExists(TableName.valueOf(tableName))) {
admin.disableTable(TableName.valueOf(tableName));
admin.deleteTable(TableName.valueOf(tableName));
}
} /**
* 获取所有表名称
*/
public static TableName[] getTableNameList() throws Exception {
Admin admin = connection.getAdmin();
return admin.listTableNames();
} /**
* 获取所有表定义
*/
public static HTableDescriptor[] getTableDescriptorList() throws Exception {
Admin admin = connection.getAdmin();
return admin.listTables();
} /**
* 为表添加列簇
*/
public static void addFamily(String tableName, String family) throws Exception {
Admin admin = connection.getAdmin();
if (!admin.tableExists(TableName.valueOf(tableName))) {
throw new UnsupportedOperationException("tableName " + tableName + " is not exists");
}
admin.addColumn(TableName.valueOf(tableName), new HColumnDescriptor(family));
} /**
* 删除表中指定的列簇
*/
public static void deleteFamily(String tableName, String family) throws Exception {
Admin admin = connection.getAdmin();
admin.deleteColumn(TableName.valueOf(tableName), Bytes.toBytes(family));
} /**
* 为表添加一条数据
*/
public static void put(String tableName, String rowKey, String family, Map<String, String> values) throws Exception {
Table table = connection.getTable(TableName.valueOf(tableName));
Put put = new Put(Bytes.toBytes(rowKey));
for (Map.Entry<String, String> entry : values.entrySet())
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(entry.getKey()), Bytes.toBytes(entry.getValue()));
table.put(put);
} /**
* 批量为表添加数据
*/
public static void batchPut(String tableName, String family, Map<String, Map<String, String>> values) throws Exception {
Table table = connection.getTable(TableName.valueOf(tableName));
List<Put> puts = new ArrayList<>();
for (Map.Entry<String, Map<String, String>> entry : values.entrySet()) {
Put put = new Put(Bytes.toBytes(entry.getKey()));
for (Map.Entry<String, String> subEntry : entry.getValue().entrySet())
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(subEntry.getKey()), Bytes.toBytes(subEntry.getValue()));
puts.add(put);
}
table.put(puts);
} /**
* 删除RowKey中的某列
*/
public static void deleteColumn(String tableName, String rowKey, String family, String qualifier) throws Exception {
Table table = connection.getTable(TableName.valueOf(tableName));
Delete delete = new Delete(Bytes.toBytes(rowKey));
delete.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
table.delete(delete);
} /**
* 删除RowKey
*/
public static void delete(String tableName, String rowKey) throws Exception {
Table table = connection.getTable(TableName.valueOf(tableName));
table.delete(new Delete(Bytes.toBytes(rowKey)));
} /**
* 批量删除RowKey
*/
public static void batchDelete(String tableName, String... rowKeys) throws Exception {
Table table = connection.getTable(TableName.valueOf(tableName));
List<Delete> deletes = new ArrayList<>();
for (String rowKey : rowKeys)
deletes.add(new Delete(Bytes.toBytes(rowKey)));
table.delete(deletes);
} /**
* 根据RowKey获取数据
*/
public static Map<String, String> get(String tableName, String rowKey) throws Exception {
Table table = connection.getTable(TableName.valueOf(tableName));
Result result = table.get(new Get(Bytes.toBytes(rowKey)));
List<Cell> cells = result.listCells();
Map<String, String> cellsMap = new HashMap<>();
for (Cell cell : cells) {
cellsMap.put(Bytes.toString(CellUtil.cloneQualifier(cell)), Bytes.toString(CellUtil.cloneValue(cell)));
}
return cellsMap;
} /**
* 获取全表数据
*/
public static Map<String, Map<String, String>> scan(String tableName) throws Exception {
Table table = connection.getTable(TableName.valueOf(tableName));
ResultScanner resultScanner = table.getScanner(new Scan());
return getResult(resultScanner);
} /**
* 获取某列数据
*/
public static Map<String, Map<String, String>> scan(String tableName, String family, String qualifier) throws Exception {
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
ResultScanner resultScanner = table.getScanner(scan);
return getResult(resultScanner);
} private static Map<String, Map<String, String>> getResult(ResultScanner resultScanner) {
Map<String, Map<String, String>> resultMap = new HashMap<>();
for (Result result : resultScanner) {
List<Cell> cells = result.listCells();
Map<String, String> cellsMap = new HashMap<>();
for (Cell cell : cells)
cellsMap.put(Bytes.toString(CellUtil.cloneQualifier(cell)), Bytes.toString(CellUtil.cloneValue(cell)));
resultMap.put(Bytes.toString(result.getRow()), cellsMap);
}
return resultMap;
}}