4、Intelij+Maven搭建Hadoop项目

4、Intelij+Maven搭建Hadoop项目

1创建工程

2通过maven导入项目依赖jar包

pom.xml配置文件如下：

4.0.0 com.kaikeba.hadoop com.kaikeba.hadoop 1.0-SNAPSHOT jar 2.7.3 commons-cli commons-cli 1.2 commons-logging commons-logging 1.1.3 org.apache.hadoop hadoop-mapreduce-client-jobclient ${hadoop.version} org.apache.hadoop hadoop-common ${hadoop.version} org.apache.hadoop hadoop-hdfs 2.7.3 org.apache.hadoop hadoop-hdfs ${hadoop.version} org.apache.hadoop hadoop-mapreduce-client-app ${hadoop.version} org.apache.hadoop hadoop-mapreduce-client-hs ${hadoop.version}

3编写HDFS读写程序

****把本地文件传输到HDFS**** package com.kaikeba.hadoop.hdfs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import java.io.*; import java.net.URI; /** * 将本地文件系统的文件通过java-API写入到HDFS文件 */ public class FileCopyFromLocal { public static void main(String[] args) { String source = "E:\\aa.mp4"; //获取/data的m目录存在（根据自己的环境更改） String destination = "hdfs://122.51.241.109:9000/data/hdfs01.mp4"; InputStream in = null; try { in = new BufferedInputStream(new FileInputStream(source)); //HDFS读写的配置文件 Configuration conf = new Configuration(); //生成一个文件系统对象 FileSystem fs = FileSystem.get(URI.create(destination),conf); //生成一个输出流 OutputStream out = fs.create(new Path(destination)); IOUtils.copyBytes(in, out, 4096, true); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } } }

**把HDFS中的文件传输到本地** package com.kaikeba.hadoop.hdfs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import java.io.BufferedOutputStream; import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; /** * 从HDFS读取文件 * 打包运行jar包 [bruce@node-01 Desktop]$ hadoop jar com.kaikeba.hadoop-1.0-SNAPSHOT.jar com.kaikeba.hadoop.hdfs.FileReadFromHdfs */ public class FileReadFromHdfs { public static void main(String[] args) { try { // String srcFile = "hdfs://122.51.241.109:9000/data/hdfs01.mp4"; Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(srcFile),conf); FSDataInputStream hdfsInStream = fs.open(new Path(srcFile)); BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream("/opt/hdfs01.mp4")); IOUtils.copyBytes(hdfsInStream, outputStream, 4096, true); } catch (IOException e) { e.printStackTrace(); } } }

4通过运行jar包的方式验证

版权声明：本文内容由网络用户投稿，版权归原作者所有，本站不拥有其著作权，亦不承担相应法律责任。如果您发现本站中有涉嫌抄袭或描述失实的内容，请联系我们jiasou666@gmail.com 处理，核实后本网站将在24小时内删除侵权内容。