Java连接kafka,并将数据写入hdfs

原创
2017/04/27 16:17
阅读数 1K
 public static void main(String[] args) throws IOException {
        Properties props = new Properties();
        //set brokers
        Gson gson = new Gson();
        props.put("bootstrap.servers", "139.198.XXX.XXX:9092,139.198.XXX.XXX:9093,139.198.XXX.XXX:9094");
        props.put("group.id", "bitautoHDFS");
        props.put("enable.auto.commit", "true");
        props.put("auto.commit.interval.ms", "1000");
        props.put("session.timeout.ms", "30000");
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
        consumer.subscribe(Arrays.asList(topicName));
        BitautoEntity bitautoEntity = null;
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            for (ConsumerRecord<String, String> record : records) {
                try {
//                    写为entity
                    bitautoEntity = gson.fromJson(record.value(), BitautoEntity.class);
//                    将Entity转换为json
                    String result = gson.toJson(bitautoEntity);
//                    写入大数据平台
                    WriteContent.write2hdfs(result, hdfs_path);
                    System.out.println("推送数据成功!");
                } catch (Exception ex) {
                    System.out.print(ex.getMessage());
                }
            }
        }
    }

 

展开阅读全文
打赏
0
0 收藏
分享
加载中
更多评论
打赏
0 评论
0 收藏
0
分享
返回顶部
顶部