1、pom.xml配置
<!--配置--> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.target>1.8</maven.compiler.target> <hadoop.version>3.1.3</hadoop.version> </properties> <!--依赖库--> <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies>
2、创建与删除
//导包 import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.io.IOException; public static void main( String[] args ){ //初始化hadoop文件系统的configration对象 Configuration conf = new Configuration(); //将hadoop的configration信息传入 conf.set("fs.defaultFS","hdfs://192.168.50.102:9000"); //初始化Hadoop文件系统的句柄 FileSystem fs=null; try { //配置Hadoop的文件句柄信息 fs=FileSystem.get(conf); //定义Hadoop的文件路径 final String PATH="/test/kb16/hadoop/ratings.csv"; //初始化Hadoop的路径信息 Path path = new Path(PATH); //如果文件路径存在就删除 if (fs.exists(path)) { System.out.println("DELETE "+fs.delete(path, true)); }else{ //如果文件路径不存在就创建 System.out.println("CREATE "+fs.create(path)); } } catch (IOException e) { e.printStackTrace(); }finally { //结束的时候,句柄还没有释放就进行释放 if (fs!=null) { try { fs.close() ; }catch (IOException e) { e.printStackTrace(); } } } }
3、文件上传
//导包 import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.io.File; import java.io.IOException; public static void main(String[] args) { //定义本地上传文件路径 final String formPath="E:\\ratings.csv"; //本地文件不存在就报错,并强制让程序终止 if (!new File(formPath).exists()) { System.out.println(formPath +"doesn't exits"); return; } //初始化hadoop文件系统的configration对象 Configuration conf = new Configuration(); //将hadoop的configration信息传入 conf.set("fs.defaultFS","hdfs://192.168.50.102:9000"); //初始化Hadoop文件系统的句柄 FileSystem fs=null; try { //将config信息传入 fs=FileSystem.get(conf); //定义上传到HDFS的路径 final String toPath="/test/kb16/hive"; //初始化路径 Path to =new Path(toPath); //如果文件路径存在不执行,如果文件路径不存在就尝试创建,如果创建失败就跳过 if (!fs.exists(to)&& !fs.mkdirs(to)) { System.out.println(toPath +"doesn't exit and can't be created"); return; } //初始化上传文件路径 Path from=new Path(formPath); //利用方法将本地文件复制到HDFS中 fs.copyFromLocalFile(from, to); System.out.println("succeed in copying from "+formPath+" to "+toPath); } catch (IOException e) { e.printStackTrace(); System.out.println("FAILURE"); }finally{ //如果结束Hadoop文件系统句柄没有关闭,利用方法进行句柄释放 if (null!=fs) { try { fs.close(); } catch (IOException e) { e.printStackTrace(); } } } }
4、文件下载
//导包 import com.google.inject.internal.cglib.core.$LocalVariablesSorter; import com.google.inject.internal.cglib.proxy.$Factory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.io.File; import java.io.IOException; public class Download { public static void main(String[] args) { //定义文件下载路径 final String toPath = "C:\\Users\\Jialin\\Desktop"; //获取路径 File to = new File(toPath); //如果路存在或者文件路径不存在但是创建成功就不执行if方法 if (!to.exists()&&!to.mkdirs()) { System.err.println(toPath + "doesn't exist and can't be created"); return; } //初始化hadoop文件系统的configration对象 Configuration config = new Configuration(); //将hadoop的configration信息传入 config.set("fs.defaultFS", "hdfs://192.168.50.102:9000"); //初始化Hadoop文件系统的句柄 FileSystem fs = null; try { //将config信息传入 fs = FileSystem.get(config); //定义下载文件路径 final String fromPath = "/test/kb16/hive/ratings.csv"; //获取路径信息 Path from = new Path(fromPath); //如果指定下载文件不存在就退出 if (!fs.exists(from)) { System.err.println(toPath + "doesn't exist "); return; } //获取文件下载路径信息 Path _to = new Path(toPath); //利用方法将Hadoop文件下载到本地 fs.copyToLocalFile(from,_to); System.out.println("succeed in downloading from "+fromPath+" to"+toPath); } catch (IOException e) { e.printStackTrace(); System.out.println("FAILURE"); } finally { //如果结束Hadoop文件系统句柄没有关闭,利用方法进行句柄释放 if (null != fs) try { fs.close(); } catch (IOException e) { e.printStackTrace(); } } } }
The above is the detailed content of How to implement HDFS file upload and download in Java. For more information, please follow other related articles on the PHP Chinese website!

本篇文章给大家带来了关于java的相关知识,其中主要介绍了关于结构化数据处理开源库SPL的相关问题,下面就一起来看一下java下理想的结构化数据处理类库,希望对大家有帮助。

本篇文章给大家带来了关于java的相关知识,其中主要介绍了关于PriorityQueue优先级队列的相关知识,Java集合框架中提供了PriorityQueue和PriorityBlockingQueue两种类型的优先级队列,PriorityQueue是线程不安全的,PriorityBlockingQueue是线程安全的,下面一起来看一下,希望对大家有帮助。

本篇文章给大家带来了关于java的相关知识,其中主要介绍了关于java锁的相关问题,包括了独占锁、悲观锁、乐观锁、共享锁等等内容,下面一起来看一下,希望对大家有帮助。

本篇文章给大家带来了关于java的相关知识,其中主要介绍了关于多线程的相关问题,包括了线程安装、线程加锁与线程不安全的原因、线程安全的标准类等等内容,希望对大家有帮助。

本篇文章给大家带来了关于Java的相关知识,其中主要介绍了关于关键字中this和super的相关问题,以及他们的一些区别,下面一起来看一下,希望对大家有帮助。

本篇文章给大家带来了关于java的相关知识,其中主要介绍了关于枚举的相关问题,包括了枚举的基本操作、集合类对枚举的支持等等内容,下面一起来看一下,希望对大家有帮助。

封装是一种信息隐藏技术,是指一种将抽象性函式接口的实现细节部分包装、隐藏起来的方法;封装可以被认为是一个保护屏障,防止指定类的代码和数据被外部类定义的代码随机访问。封装可以通过关键字private,protected和public实现。

本篇文章给大家带来了关于java的相关知识,其中主要介绍了关于设计模式的相关问题,主要将装饰器模式的相关内容,指在不改变现有对象结构的情况下,动态地给该对象增加一些职责的模式,希望对大家有帮助。


Hot AI Tools

Undresser.AI Undress
AI-powered app for creating realistic nude photos

AI Clothes Remover
Online AI tool for removing clothes from photos.

Undress AI Tool
Undress images for free

Clothoff.io
AI clothes remover

AI Hentai Generator
Generate AI Hentai for free.

Hot Article

Hot Tools

Dreamweaver CS6
Visual web development tools

SecLists
SecLists is the ultimate security tester's companion. It is a collection of various types of lists that are frequently used during security assessments, all in one place. SecLists helps make security testing more efficient and productive by conveniently providing all the lists a security tester might need. List types include usernames, passwords, URLs, fuzzing payloads, sensitive data patterns, web shells, and more. The tester can simply pull this repository onto a new test machine and he will have access to every type of list he needs.

MantisBT
Mantis is an easy-to-deploy web-based defect tracking tool designed to aid in product defect tracking. It requires PHP, MySQL and a web server. Check out our demo and hosting services.

mPDF
mPDF is a PHP library that can generate PDF files from UTF-8 encoded HTML. The original author, Ian Back, wrote mPDF to output PDF files "on the fly" from his website and handle different languages. It is slower than original scripts like HTML2FPDF and produces larger files when using Unicode fonts, but supports CSS styles etc. and has a lot of enhancements. Supports almost all languages, including RTL (Arabic and Hebrew) and CJK (Chinese, Japanese and Korean). Supports nested block-level elements (such as P, DIV),

ZendStudio 13.5.1 Mac
Powerful PHP integrated development environment
