java sqoop api 导mysql数据到hdfs

package com.example.demo.controller;import org.apache.hadoop.conf.Configuration;
import org.apache.sqoop.Sqoop;
import org.apache.sqoop.tool.SqoopTool;
import org.apache.commons.lang3.StringUtils;
/*** @创建人 **** @创建时间 2018/12/20*/
public class sqooptest {public int sqoopt(){
//public static void main(String[] args) {System.out.println(" begin test sqoop");String[] argument = new String[]{"--connect", "jdbc:mysql://192.168.1.111:3306/test?useSSL=false","--driver","com.mysql.jdbc.Driver","--username", "root","--password", "123","--table", "psn","--columns", "id,name,msg,age","-m","1","--target-dir","/user/psn"};com.cloudera.sqoop.tool.SqoopTool sqoopTool=(com.cloudera.sqoop.tool.SqoopTool)SqoopTool.getTool("import");Configuration conf= new Configuration();conf.set("fs.default.name","hdfs://192.168.1.111:8020");Sqoop sqoop=new Sqoop(sqoopTool,SqoopTool.loadPlugins(conf));int res = Sqoop.runSqoop(sqoop,argument);System.out.println("--------"+res);System.out.println("执行sqoop结束");return res;}
}

pom.xml文件

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"><modelVersion>4.0.0</modelVersion><parent><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-parent</artifactId><version>2.1.1.RELEASE</version><relativePath/> <!-- lookup parent from repository --></parent><groupId>com.example</groupId><artifactId>demo</artifactId><version>0.0.1-SNAPSHOT</version><!-- war包 --><packaging>war</packaging><name>demo</name><description>Demo project for Spring Boot</description><properties><java.version>1.8</java.version></properties><dependencies><!--因配置外部TOMCAT 而配置--><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-tomcat</artifactId><scope>provided</scope></dependency><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-devtools</artifactId><optional>true</optional> <!-- 这个需要为 true 热部署才有效 --></dependency><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-web</artifactId></dependency><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-test</artifactId><scope>test</scope></dependency><dependency><groupId>mysql</groupId><artifactId>mysql-connector-java</artifactId><scope>runtime</scope></dependency><!--<dependency>--><!--<groupId>sqoop</groupId>--><!--<artifactId>sqoop</artifactId>--><!--<version>1.4.6</version>--><!--<scope>system</scope>--><!--<systemPath>${project.basedir}/src/main/webapp/lib/</systemPath>--><!--</dependency>--><dependency><groupId>org.apache.sqoop</groupId><artifactId>sqoop</artifactId><version>1.4.6</version></dependency><!--<dependency>--><!--<groupId>ojdbc14</groupId>--><!--<artifactId>ojdbc14</artifactId>--><!--<version>1.0</version>--><!--<scope>system</scope>--><!--<systemPath>${project.basedir}/src/main/webapp/lib/ojdbc6-4.0.jar</systemPath>--><!--</dependency>--><!--<dependency>--><!--<groupId>mysql</groupId>--><!--<artifactId>mysql</artifactId>--><!--<version>5.1.31</version>--><!--<scope>system</scope>--><!--<systemPath>${project.basedir}/src/main/webapp/lib/mysql-connector-java-5.1.31-bin.jar</systemPath>--><!--</dependency>--><dependency><groupId>org.apache.commons</groupId><artifactId>commons-lang3</artifactId><version>3.0</version></dependency><!--hadoop--><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-common</artifactId><version>2.7.4</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-hdfs</artifactId><version>2.7.4</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-mapreduce-client-core</artifactId><version>2.7.4</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-mapreduce-client-common</artifactId><version>2.7.4</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-mapreduce-client-jobclient</artifactId><version>2.7.4</version><scope>test</scope></dependency><dependency><groupId>org.apache.avro</groupId><artifactId>avro-mapred</artifactId><version>1.8.1</version></dependency><dependency><groupId>org.apache.hive</groupId><artifactId>hive-common</artifactId><version>2.3.2</version></dependency><dependency><groupId>org.apache.avro</groupId><artifactId>avro</artifactId><version>1.8.1</version></dependency></dependencies><build><plugins><plugin><groupId>org.springframework.boot</groupId><artifactId>spring-boot-maven-plugin</artifactId></plugin></plugins><finalName>sqoop</finalName><resources><resource><directory>src/main/resources/lib</directory><targetPath>BOOT-INF/lib/</targetPath><includes><include>**/*.jar</include></includes></resource><resource><directory>src/main/webapp/lib</directory><targetPath>BOOT-INF/lib/</targetPath></resource><!--<resource>--><!--<directory>src/main/webapp/lib1</directory>--><!--<targetPath>BOOT-INF/lib/</targetPath>--><!--</resource>--></resources></build></project>

参照：https://www.cnblogs.com/claren/p/7240735.html

java sqoop api 导mysql数据到hdfs相关推荐

Sqoop在导入MySQL数据时遇到Timestamp列为0000-00-00 00:00:00报错
为什么80%的码农都做不了架构师?>>> Sqoop在导入MySQL数据时遇到Timestamp列为'0000-00-00 00:00:00'时报错,解决方法是:在JDBC连接 ...
如何完美解决Sqoop导入导出MySQL数据错位问题
我发现小伙伴们在使用Sqoop把数据从MySQL导入到Hive的过程中经常会遇到数据错位的问题,虽然最后都是通过添加参数的方法来解决这个问题,但是我认为这并不是一个完美的解决方案,所以花了一点时间研究 ...
【kettle抽取Orecle/Mysql数据至HDFS】诸如‘\u0001’等特殊分隔符表示法
[kettle抽取Orecle/Mysql数据至HDFS]诸如'\u0001'等特殊分隔符表示法前言设计问题分析解决办法后记前言由于需要设计一个每天多批次的定时作业,由于单端逻辑不能抽取 ...
sqoop增量导入hive_Sqoop 增量导MySQL数据至Hive
通过Sqoop 增量导数据到Hive, 命令如下: ./sqoop-job --meta-connect jdbc:hsqldb:hsql://127.0.0.1:16000/sqoop --crea ...
使用 Sqoop 将 30W+ MySQL 数据导入 Hive
本实验完成的是,使用 Sqoop 从 MySQL 导出数据到 Hive. 整体步骤分为: 初始化 MySQL 的 30W+ 数据安装配置 Sqoop 在 Hive 中初始化目标表 Sqoop 脚本实 ...
使用Java/Scala API对Iceberg数据湖进行snapshot快照expire过期处理
目录 1. expire过期处理的作用 2. 查看表snapshot-log 3. expire过期程序 3.1 pom.xml 3.2 放置hdfs-site.xml 3.3 flink_test. ...
数据同步Sqoop用法之mysql与Hive导入导出
目录一. Sqoop介绍二. Mysql 数据导入到 Hive 三. Hive数据导入到Mysql 四. mysql数据增量导入hive 1). 新建一张表 2). 初始化hive表: 1). 创 ...
sqoop动态分区导入mysql,使用sqoop import从mysql往hive含分区表中导入数据的一些注意事项...
先看下面这条语句,它实现的功能是将特定日期的数据从mysql表中直接导入hive $ sqoop import \ --connect jdbc:mysql://192.168.xx.xx:3306/ ...
【Sqoop】从MySQL同步数据到Hive的操作参数解析
参数解析: 必选参数 --connect :连接的mysql数据库的jdbc信息 --username:mysql数据库的登录账户 --password:mysql数据库的登录密码 --table: ...

java sqoop api 导mysql数据到hdfs

java sqoop api 导mysql数据到hdfs相关推荐

最新文章

热门文章