Flink操作Hbase
现在有这样一个场景，我们需要将hbase做成一个数据流，而不是数据集。根据Flink自带的Flink-Hbase只能帮我们做到数据集，所以这个时候选择了重写Hbase的数据源。
package com.yjp.flink.demo11;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.table.shaded.org.joda.time.DateTime;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* 以Hbase为数据源
* 从Hbase中获取数据，然后以流的形式发射
* Date : 9:50 2018/3/12
*/
public class HbaseSource implements SourceFunction<String> {
private static Logger loggerFactory = LoggerFactory.getLogger(HbaseSource.class);
private static final long serialVersionUID = 1;
private volatile boolean isRunning = true;
/**
* 开始的时间戳
*/
private long startTime;
/**
* 每次查询多长时间的数据
*/
private long interval;
/**
* 需要查询的列名
*/
private ArrayList<String> columns;
/**
* 需要查询的表名
*/
private String tableName;
public HbaseSource(long startTime, long interval, ArrayList<String> columns, String tableName) {
this.startTime = startTime;
this.interval = interval;
this.columns = columns;
this.tableName = tableName;
}
public HbaseSource() {
}
@Override
public void run(SourceContext<String> out) {
if (isRunning) {
long endTime = DateTime.now().getMillis() - interval;
ResultScanner rs = new HbaseSource().getHbaseData(tableName, startTime, endTime - startTime, columns);
new HbaseSource().transmitData(rs, out);
startTime = endTime;
}
while (isRunning) {
ResultScanner rs = new HbaseSource().getHbaseData(tableName, startTime, interval, columns);
new HbaseSource().transmitData(rs, out);
startTime += interval;
try {
Thread.sleep(interval);
} catch (InterruptedException e) {
throw new RuntimeException("休眠异常", e);
}
}
}
@Override
public void cancel() {
}
/**
* 获取数据集
*
* @param startTime 时间戳开始的时间
* @param interval 间隔时间
* @return 对应的结果集
*/
private ResultScanner getHbaseData(String tableName, long startTime, long interval, List<String> columns) {
Configuration conf = HBaseConfiguration.create();
HTable table;
Scan scan;
try {
table = new HTable(conf, tableName);
scan = new Scan();
scan.setTimeRange(startTime, startTime + interval);
for (String column : columns) {
String[] columnName = column.split(":");
scan.addColumn(Bytes.toBytes(columnName[0]), Bytes.toBytes(columnName[1]));
}
return table.getScanner(scan);
} catch (IOException e) {
throw new RuntimeException("读取数据异常", e);
}
}
private void transmitData(ResultScanner rs, SourceContext<String> out) {
Result result;
try {
while ((result = rs.next()) != null && isRunning) {
KeyValue[] kvs = result.raw();
for (KeyValue kv : kvs) {
String value = new String(kv.getValue());
out.collect(value);
}
}
} catch (IOException e) {
throw new RuntimeException("结果集遍历异常", e);
}
}
}
然后将数据结果加工和处理存入Hbase中
package com.yjp.flink.hbase;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.java.StreamTableEnvironment;
import org.apache.flink.util.Collector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
public class HbaseToHbase {
public static Logger logger = LoggerFactory.getLogger(HbaseToHbase.class);
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tEnv = StreamTableEnvironment.getTableEnvironment(sEnv);
sEnv.getConfig().disableSysoutLogging();
List<String> getColumns = new ArrayList<String>(3);
getColumns.add("cf1_name");
getColumns.add("cf2_amount");
getColumns.add("cf3_groupId");
List<String> columnFamily = new ArrayList<>(3);
columnFamily.add("cf1");
columnFamily.add("cf2");
columnFamily.add("cf3");
List<String> setColumns = new ArrayList<>(3);
setColumns.add("cf2:result");
DataStreamSource<Orders>
orderDataStream = sEnv.addSource(new
HbaseStreamDataSource("Orders", 0L, 2000L, getColumns, Orders.class));
DataStream<Tuple3<String, Double, Integer>> dataStream = orderDataStream.flatMap(
new FlatMapFunction<Orders, Tuple3<String, Double, Integer>>() {
@Override
public void flatMap(Orders value, Collector<Tuple3<String, Double, Integer>> out) throws Exception {
out.collect(new Tuple3<String, Double, Integer>(value.getCf1_name(),
value.getCf2_amount(), value.getCf3_groupId()));
}
});
dataStream.keyBy(2).sum(1).addSink(
new SinkHbase<Tuple3<String, Double, Integer>>(
"OrderResult", columnFamily, setColumns, "result"));
sEnv.execute("test Hbase");
}
}
package com.yjp.flink.hbase;
import org.apache.flink.api.java.tuple.*;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Table;
import java.lang.reflect.Method;
import java.util.*;
/**
* 自定义Sink
*
* Date : 17:23 2018/3/12
*/
public class SinkHbase<T> extends RichSinkFunction<T> {
private static final long serialVersionUID = 1L;
/**
* 表名
*/
private String tableName;
/**
* 列族名
*/
private List<String> columnFails;
/**
* 列名 以 family:column的形式传入 column与tuple中的值一一对应
*/
private List<String> columns;
/**
* 行名
*/
private String rowKey;
/**
* @param tableName 表名
* @param columnFamily 列族名 当表存在时不用输入
* @param columns 储存的列名 列族:列名
* @param rowKey 传入的行名
*/
public SinkHbase(String tableName, List<String> columnFamily, List<String> columns, String rowKey) {
this.tableName = tableName;
this.columnFails = columnFamily;
this.columns = columns;
this.rowKey = rowKey;
}
/**
* @param tableName 表名
* @param columns 储存的列名 列族:列名
* @param rowKey 传入的行名
*/
public SinkHbase(String tableName, List<String> columns, String rowKey) {
this.tableName = tableName;
this.columns = columns;
this.rowKey = rowKey;
}
public SinkHbase() {
}
/**
* 初始化完成连接 当表不存在的时候 新建表和family列
*
* @param parameters 调用父类的方法
* @throws Exception 创建连接失败
*/
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
Admin admin = FactoryConnect.getConnection().getAdmin();
final TableName tableName1 = TableName.valueOf(tableName);
if (!admin.tableExists(tableName1)) {
HTableDescriptor hTableDescriptor = new HTableDescriptor(tableName1);
for (String columnFamily : columnFails) {
hTableDescriptor.addFamily(new HColumnDescriptor(columnFamily));
}
admin.createTable(hTableDescriptor);
}
}
/**
* 执行方法 将数据存入hbase
*
* @param value 传入的结果
*/
@Override
public void invoke(T value, Context context) throws Exception {
Map<Class, Method> map = new HashMap<>(25);
new SinkHbase<T>().initMap(map);
Table table = FactoryConnect.getConnection().getTable(TableName.valueOf(tableName));
Set<Class> keys = map.keySet();
for (Class key : keys) {
if (value.getClass() == key) {
map.get(key).invoke(new AssignmentTuple(), value, rowKey, columns, table);
return;
}
}
}
private void initMap(Map<Class, Method> map) {
try {
map.put(Tuple1.class, AssignmentTuple.class.getMethod("setTuple1", Tuple1.class, String.class, ArrayList.class, Table.class));
map.put(Tuple2.class, AssignmentTuple.class.getMethod("setTuple2", Tuple2.class, String.class, ArrayList.class, Table.class));
map.put(Tuple3.class, AssignmentTuple.class.getMethod("setTuple3", Tuple3.class, String.class, ArrayList.class, Table.class));
map.put(Tuple4.class, AssignmentTuple.class.getMethod("setTuple4", Tuple4.class, String.class, ArrayList.class, Table.class));
map.put(Tuple5.class, AssignmentTuple.class.getMethod("setTuple5", Tuple5.class, String.class, ArrayList.class, Table.class));
map.put(Tuple6.class, AssignmentTuple.class.getMethod("setTuple6", Tuple6.class, String.class, ArrayList.class, Table.class));
map.put(Tuple7.class, AssignmentTuple.class.getMethod("setTuple7", Tuple7.class, String.class, ArrayList.class, Table.class));
map.put(Tuple8.class, AssignmentTuple.class.getMethod("setTuple8", Tuple8.class, String.class, ArrayList.class, Table.class));
map.put(Tuple9.class, AssignmentTuple.class.getMethod("setTuple9", Tuple9.class, String.class, ArrayList.class, Table.class));
map.put(Tuple10.class, AssignmentTuple.class.getMethod("setTuple10", Tuple10.class, String.class, ArrayList.class, Table.class));
map.put(Tuple11.class, AssignmentTuple.class.getMethod("setTuple11", Tuple11.class, String.class, ArrayList.class, Table.class));
map.put(Tuple12.class, AssignmentTuple.class.getMethod("setTuple12", Tuple12.class, String.class, ArrayList.class, Table.class));
map.put(Tuple13.class, AssignmentTuple.class.getMethod("setTuple13", Tuple13.class, String.class, ArrayList.class, Table.class));
map.put(Tuple14.class, AssignmentTuple.class.getMethod("setTuple14", Tuple14.class, String.class, ArrayList.class, Table.class));
map.put(Tuple15.class, AssignmentTuple.class.getMethod("setTuple15", Tuple15.class, String.class, ArrayList.class, Table.class));
map.put(Tuple16.class, AssignmentTuple.class.getMethod("setTuple16", Tuple16.class, String.class, ArrayList.class, Table.class));
map.put(Tuple17.class, AssignmentTuple.class.getMethod("setTuple17", Tuple17.class, String.class, ArrayList.class, Table.class));
map.put(Tuple18.class, AssignmentTuple.class.getMethod("setTuple18", Tuple18.class, String.class, ArrayList.class, Table.class));
map.put(Tuple19.class, AssignmentTuple.class.getMethod("setTuple19", Tuple19.class, String.class, ArrayList.class, Table.class));
map.put(Tuple20.class, AssignmentTuple.class.getMethod("setTuple20", Tuple20.class, String.class, ArrayList.class, Table.class));
map.put(Tuple21.class, AssignmentTuple.class.getMethod("setTuple21", Tuple21.class, String.class, ArrayList.class, Table.class));
map.put(Tuple22.class, AssignmentTuple.class.getMethod("setTuple22", Tuple22.class, String.class, ArrayList.class, Table.class));
map.put(Tuple23.class, AssignmentTuple.class.getMethod("setTuple23", Tuple23.class, String.class, ArrayList.class, Table.class));
map.put(Tuple24.class, AssignmentTuple.class.getMethod("setTuple24", Tuple24.class, String.class, ArrayList.class, Table.class));
map.put(Tuple25.class, AssignmentTuple.class.getMethod("setTuple25", Tuple25.class, String.class, ArrayList.class, Table.class));
} catch (NoSuchMethodException e) {
throw new RuntimeException("反射失败", e);
}
}
}
package com.yjp.flink.hbase;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import java.io.IOException;
import java.io.Serializable;
/**
* 单例模式 安全的拿到连接
*
* Date : 16:45 2018/3/16
*/
public class FactoryConnect implements Serializable {
private static volatile Connection connection;
private FactoryConnect() {
}
public static Connection getConnection() throws IOException {
if (null == connection) {
synchronized (FactoryConnect.class) {
try {
if (null == connection) {
org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create();
connection = ConnectionFactory.createConnection(conf);
}
} catch (Exception e) {
System.err.println("读取配置文件异常");
}
}
}
return connection;
}
}
package com.yjp.flink.hbase;
import org.apache.flink.api.java.tuple.*;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
/**
* 将tuple中的存放在Hbase中
*
* Date : 16:49 2018/3/12
*/
public class AssignmentTuple {
/**
* tuple 为1
*
* @param tuple1 传入tuple的值
* @param rowKey 传入的rowkey的值
* @param columns 需要赋值的列
* @param table put的table对象
*/
public void setTuple1(Tuple1<Object> tuple1, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple1, rowKey, columns, table);
}
public void setTuple2(Tuple2<Object, Object> tuple2, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple2, rowKey, columns, table);
}
public void setTuple3(Tuple3<Object, Object, Object> tuple3, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple3, rowKey, columns, table);
}
public void setTuple4(Tuple4<Object, Object, Object, Object> tuple4, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple4, rowKey, columns, table);
}
public void setTuple5(Tuple5<Object, Object, Object, Object, Object> tuple5, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple5, rowKey, columns, table);
}
public void setTuple6(Tuple6 tuple6, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple6, rowKey, columns, table);
}
public void setTuple7(Tuple7 tuple7, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple7, rowKey, columns, table);
}
public void setTuple8(Tuple8 tuple8, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple8, rowKey, columns, table);
}
public void setTuple9(Tuple9 tuple9, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple9, rowKey, columns, table);
}
public void setTuple10(Tuple10 tuple10, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple10, rowKey, columns, table);
}
public void setTuple11(Tuple11 tuple11, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple11, rowKey, columns, table);
}
public void setTuple12(Tuple12 tuple12, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple12, rowKey, columns, table);
}
public void setTuple13(Tuple13 tuple13, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple13, rowKey, columns, table);
}
public void setTuple14(Tuple14 tuple14, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple14, rowKey, columns, table);
}
public void setTuple15(Tuple15 tuple15, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple15, rowKey, columns, table);
}
public void setTuple16(Tuple16 tuple16, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple16, rowKey, columns, table);
}
public void setTuple17(Tuple17 tuple17, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple17, rowKey, columns, table);
}
public void setTuple18(Tuple18 tuple18, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple18, rowKey, columns, table);
}
public void setTuple19(Tuple19 tuple19, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple19, rowKey, columns, table);
}
public void setTuple20(Tuple20 tuple20, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple20, rowKey, columns, table);
}
public void setTuple21(Tuple21 tuple21, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple21, rowKey, columns, table);
}
public void setTuple22(Tuple22 tuple22, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple22, rowKey, columns, table);
}
public void setTuple23(Tuple23 tuple23, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple23, rowKey, columns, table);
}
public void setTuple24(Tuple24 tuple24, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple24, rowKey, columns, table);
}
public void setTuple25(Tuple25 tuple25, String rowKey, ArrayList<String> columns, Table table) {
new AssignmentTuple().putData(tuple25, rowKey, columns, table);
}
/**
* 将tuple中的数据一一对应的赋值给列
*
* @param tuple tuple中的数据
* @param rowKey 设置的行值
* @param columns 对应的列名
* @param table 对应的table对象
*/
public void putData(Tuple tuple, String rowKey, List<String> columns, Table table) {
Put put = new Put(Bytes.toBytes(rowKey));
Long timeStamp = Instant.now().toEpochMilli();
for (int i = 0; i < columns.size(); i++) {
String[] split = columns.get(i).split(":");
put.addColumn(Bytes.toBytes(split[0]), Bytes.toBytes(split[1]), timeStamp, Bytes.toBytes(tuple.getField(i).toString()));
}
try {
table.put(put);
} catch (IOException e) {
throw new RuntimeException("存放失败", e);
}
}
}
为了做到一个通用的数据源和数据存储，于是采用了反射的方法。
