java 读取 hadoop 的文件 demo

叁歲伎倆 2022-05-17 10:38 80阅读 0赞

是什么？

使用java 读取 hadoop 中的一个文件。

为什么？

用于初步接触hadoop开发。

怎么做？

一、依赖jar

<?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0"
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <modelVersion>4.0.0</modelVersion>
    
        <groupId>myHadoop</groupId>
        <artifactId>myHadoop</artifactId>
        <version>1.0-SNAPSHOT</version>
    
        <properties>
            <hadoop.hdfs.version>2.4.0</hadoop.hdfs.version>
            <hadoop.mapreduce.client.core.version>2.4.0</hadoop.mapreduce.client.core.version>
            <hadoop.common.version>2.4.0</hadoop.common.version>
            <hadoop.mapreduce.client.common.version>2.4.0</hadoop.mapreduce.client.common.version>
            <hadoop.mapreduce.client.jobclient.version>2.4.0</hadoop.mapreduce.client.jobclient.version>
            <commons.lang.version>3.3.2</commons.lang.version>
            <commons.io.version>2.4</commons.io.version>
        </properties>
    
        <dependencies>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-hdfs</artifactId>
                <version>${hadoop.hdfs.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-mapreduce-client-core</artifactId>
                <version>${hadoop.mapreduce.client.core.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-common</artifactId>
                <version>${hadoop.common.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-mapreduce-client-common</artifactId>
                <version>${hadoop.mapreduce.client.common.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
                <version>${hadoop.mapreduce.client.jobclient.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.commons</groupId>
                <artifactId>commons-lang3</artifactId>
                <version>${commons.lang.version}</version>
            </dependency>
            <dependency>
                <groupId>commons-io</groupId>
                <artifactId>commons-io</artifactId>
                <version>${commons.io.version}</version>
            </dependency>
        </dependencies>
    
        <build>
            <finalName>gQuery-${version}</finalName>
            
            <plugins>
                <plugin>
                    <groupId>org.apache.maven.plugins</groupId>
                    <artifactId>maven-compiler-plugin</artifactId>
                    <version>2.3.1</version>
                    <configuration>
                        <source>1.7</source>
                        <target>1.7</target>
                        <encoding>utf8</encoding>
                    </configuration>
                </plugin>
                
                <plugin>
                    <groupId>org.apache.maven.plugins</groupId>
                    <artifactId>maven-jar-plugin</artifactId>
                    <version>2.4</version>
                    
                    <configuration>
                        
                        <archive>
                            
                            <addMavenDescriptor>false</addMavenDescriptor>
    
                            
                            <manifest>
                                
                                <addClasspath>true</addClasspath>
                                
                                <classpathPrefix>lib/</classpathPrefix>
                                
                                <mainClass>com.geotmt.hadoop.App</mainClass>
                            </manifest>
                            
                            <manifestEntries>
                                <Class-Path>.</Class-Path>
                            </manifestEntries>
                        </archive>
                        
                        <excludes>
                            <exclude>**/*.properties</exclude>
                            <exclude>**/*.xml</exclude>
                        </excludes>
                    </configuration>
                </plugin>
    
                <plugin>
                    <groupId>org.apache.maven.plugins</groupId>
                    <artifactId>maven-resources-plugin</artifactId>
                    <version>3.1.0</version>
                    <executions>
                        <execution>
                            <id>copy-resources</id>
                            <phase>package</phase>
                            <goals>
                                <goal>copy-resources</goal>
                            </goals>
                            <configuration>
                                <encoding>UTF-8</encoding>
                                
                                <outputDirectory>${project.build.directory}/etc</outputDirectory>
                                <resources>
                                    
                                    <resource>
                                        
                                        <directory>src/main/resources/</directory>
                                        
                                        <includes>
                                            <exclude>**/*.properties</exclude>
                                            <exclude>**/*.xml</exclude>
                                        </includes>
                                        
                                        <filtering>true</filtering>
                                    </resource>
                                </resources>
                            </configuration>
                        </execution>
                    </executions>
                </plugin>
    
                
                <plugin>
                    <groupId>org.apache.maven.plugins</groupId>
                    <artifactId>maven-dependency-plugin</artifactId>
                    
                    <executions>
                        
                        <execution>
                            
                            <id>copy-dependencies</id>
                            
                            <phase>package</phase>
                            
                            <goals>
                                <goal>copy-dependencies</goal>
                            </goals>
                            <configuration>
                                
                                <outputDirectory>
                                    ${project.build.directory}/etc/lib
                                </outputDirectory>
                                <overWriteReleases>false</overWriteReleases>
                                <overWriteSnapshots>false</overWriteSnapshots>
                                <overWriteIfNewer>true</overWriteIfNewer>
                                <excludeArtifactIds>junit</excludeArtifactIds>
                            </configuration>
                        </execution>
                    </executions>
                </plugin>
            </plugins>
        </build>
    </project>

java代码

package com.geotmt.hadoop.hdfs;
    
    import org.apache.commons.io.IOUtils;
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    
    import java.io.InputStream;
    import java.net.URI;
    
    /**
     * 从Hdfs读取一个文件
     *
     * Created by c.z on 2018/8/10. */
    public class ReadFileHandle {
    
        public static void main(String[] args) throws Exception {
            ReadFileHandle readFileHandle = new ReadFileHandle();
            readFileHandle.readFile();
        }
        
        public void readFile() throws Exception{
            String uri = "hdfs://10.111.32.142:8020/user/zhaochao/20180727/sms.1532681090912.log.tmp";
            Configuration conf = new Configuration();
            FileSystem fs = FileSystem.get(URI.create(uri),conf);
            try(InputStream in = fs.open(new Path(uri)))
            {
                IOUtils.copy(in,System.out);
            }
        }
    }