1、pom.xml配置
<!--配置--> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.target>1.8</maven.compiler.target> <hadoop.version>3.1.3</hadoop.version> </properties> <!--依赖库--> <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies>
2、创建与删除
//导包 import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.io.IOException; public static void main( String[] args ){ //初始化hadoop文件系统的configration对象 Configuration conf = new Configuration(); //将hadoop的configration信息传入 conf.set("fs.defaultFS","hdfs://192.168.50.102:9000"); //初始化Hadoop文件系统的句柄 FileSystem fs=null; try { //配置Hadoop的文件句柄信息 fs=FileSystem.get(conf); //定义Hadoop的文件路径 final String PATH="/test/kb16/hadoop/ratings.csv"; //初始化Hadoop的路径信息 Path path = new Path(PATH); //如果文件路径存在就删除 if (fs.exists(path)) { System.out.println("DELETE "+fs.delete(path, true)); }else{ //如果文件路径不存在就创建 System.out.println("CREATE "+fs.create(path)); } } catch (IOException e) { e.printStackTrace(); }finally { //结束的时候,句柄还没有释放就进行释放 if (fs!=null) { try { fs.close() ; }catch (IOException e) { e.printStackTrace(); } } } }
3、文件上传
//导包 import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.io.File; import java.io.IOException; public static void main(String[] args) { //定义本地上传文件路径 final String formPath="E:\\ratings.csv"; //本地文件不存在就报错,并强制让程序终止 if (!new File(formPath).exists()) { System.out.println(formPath +"doesn't exits"); return; } //初始化hadoop文件系统的configration对象 Configuration conf = new Configuration(); //将hadoop的configration信息传入 conf.set("fs.defaultFS","hdfs://192.168.50.102:9000"); //初始化Hadoop文件系统的句柄 FileSystem fs=null; try { //将config信息传入 fs=FileSystem.get(conf); //定义上传到HDFS的路径 final String toPath="/test/kb16/hive"; //初始化路径 Path to =new Path(toPath); //如果文件路径存在不执行,如果文件路径不存在就尝试创建,如果创建失败就跳过 if (!fs.exists(to)&& !fs.mkdirs(to)) { System.out.println(toPath +"doesn't exit and can't be created"); return; } //初始化上传文件路径 Path from=new Path(formPath); //利用方法将本地文件复制到HDFS中 fs.copyFromLocalFile(from, to); System.out.println("succeed in copying from "+formPath+" to "+toPath); } catch (IOException e) { e.printStackTrace(); System.out.println("FAILURE"); }finally{ //如果结束Hadoop文件系统句柄没有关闭,利用方法进行句柄释放 if (null!=fs) { try { fs.close(); } catch (IOException e) { e.printStackTrace(); } } } }
4、文件下载
//导包 import com.google.inject.internal.cglib.core.$LocalVariablesSorter; import com.google.inject.internal.cglib.proxy.$Factory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.io.File; import java.io.IOException; public class Download { public static void main(String[] args) { //定义文件下载路径 final String toPath = "C:\\Users\\Jialin\\Desktop"; //获取路径 File to = new File(toPath); //如果路存在或者文件路径不存在但是创建成功就不执行if方法 if (!to.exists()&&!to.mkdirs()) { System.err.println(toPath + "doesn't exist and can't be created"); return; } //初始化hadoop文件系统的configration对象 Configuration config = new Configuration(); //将hadoop的configration信息传入 config.set("fs.defaultFS", "hdfs://192.168.50.102:9000"); //初始化Hadoop文件系统的句柄 FileSystem fs = null; try { //将config信息传入 fs = FileSystem.get(config); //定义下载文件路径 final String fromPath = "/test/kb16/hive/ratings.csv"; //获取路径信息 Path from = new Path(fromPath); //如果指定下载文件不存在就退出 if (!fs.exists(from)) { System.err.println(toPath + "doesn't exist "); return; } //获取文件下载路径信息 Path _to = new Path(toPath); //利用方法将Hadoop文件下载到本地 fs.copyToLocalFile(from,_to); System.out.println("succeed in downloading from "+fromPath+" to"+toPath); } catch (IOException e) { e.printStackTrace(); System.out.println("FAILURE"); } finally { //如果结束Hadoop文件系统句柄没有关闭,利用方法进行句柄释放 if (null != fs) try { fs.close(); } catch (IOException e) { e.printStackTrace(); } } } }
The above is the detailed content of How to implement HDFS file upload and download in Java. For more information, please follow other related articles on the PHP Chinese website!

JVM handles operating system API differences through JavaNativeInterface (JNI) and Java standard library: 1. JNI allows Java code to call local code and directly interact with the operating system API. 2. The Java standard library provides a unified API, which is internally mapped to different operating system APIs to ensure that the code runs across platforms.

modularitydoesnotdirectlyaffectJava'splatformindependence.Java'splatformindependenceismaintainedbytheJVM,butmodularityinfluencesapplicationstructureandmanagement,indirectlyimpactingplatformindependence.1)Deploymentanddistributionbecomemoreefficientwi

BytecodeinJavaistheintermediaterepresentationthatenablesplatformindependence.1)Javacodeiscompiledintobytecodestoredin.classfiles.2)TheJVMinterpretsorcompilesthisbytecodeintomachinecodeatruntime,allowingthesamebytecodetorunonanydevicewithaJVM,thusfulf

JavaachievesplatformindependencethroughtheJavaVirtualMachine(JVM),whichexecutesbytecodeonanydevicewithaJVM.1)Javacodeiscompiledintobytecode.2)TheJVMinterpretsandexecutesthisbytecodeintomachine-specificinstructions,allowingthesamecodetorunondifferentp

Platform independence in JavaGUI development faces challenges, but can be dealt with by using Swing, JavaFX, unifying appearance, performance optimization, third-party libraries and cross-platform testing. JavaGUI development relies on AWT and Swing, which aims to provide cross-platform consistency, but the actual effect varies from operating system to operating system. Solutions include: 1) using Swing and JavaFX as GUI toolkits; 2) Unify the appearance through UIManager.setLookAndFeel(); 3) Optimize performance to suit different platforms; 4) using third-party libraries such as ApachePivot or SWT; 5) conduct cross-platform testing to ensure consistency.

Javadevelopmentisnotentirelyplatform-independentduetoseveralfactors.1)JVMvariationsaffectperformanceandbehavioracrossdifferentOS.2)NativelibrariesviaJNIintroduceplatform-specificissues.3)Filepathsandsystempropertiesdifferbetweenplatforms.4)GUIapplica

Java code will have performance differences when running on different platforms. 1) The implementation and optimization strategies of JVM are different, such as OracleJDK and OpenJDK. 2) The characteristics of the operating system, such as memory management and thread scheduling, will also affect performance. 3) Performance can be improved by selecting the appropriate JVM, adjusting JVM parameters and code optimization.

Java'splatformindependencehaslimitationsincludingperformanceoverhead,versioncompatibilityissues,challengeswithnativelibraryintegration,platform-specificfeatures,andJVMinstallation/maintenance.Thesefactorscomplicatethe"writeonce,runanywhere"


Hot AI Tools

Undresser.AI Undress
AI-powered app for creating realistic nude photos

AI Clothes Remover
Online AI tool for removing clothes from photos.

Undress AI Tool
Undress images for free

Clothoff.io
AI clothes remover

Video Face Swap
Swap faces in any video effortlessly with our completely free AI face swap tool!

Hot Article

Hot Tools

Safe Exam Browser
Safe Exam Browser is a secure browser environment for taking online exams securely. This software turns any computer into a secure workstation. It controls access to any utility and prevents students from using unauthorized resources.

Dreamweaver Mac version
Visual web development tools

WebStorm Mac version
Useful JavaScript development tools

EditPlus Chinese cracked version
Small size, syntax highlighting, does not support code prompt function

mPDF
mPDF is a PHP library that can generate PDF files from UTF-8 encoded HTML. The original author, Ian Back, wrote mPDF to output PDF files "on the fly" from his website and handle different languages. It is slower than original scripts like HTML2FPDF and produces larger files when using Unicode fonts, but supports CSS styles etc. and has a lot of enhancements. Supports almost all languages, including RTL (Arabic and Hebrew) and CJK (Chinese, Japanese and Korean). Supports nested block-level elements (such as P, DIV),
