CODEWITHSUNDEEP
javahadoophdfs
user3025127
user3025127

Reputation: 687

Hadoop HDFS PutMerge doesn't work

I try to run the PutMerge in Hadoop in Action on my computer. But it doesn't work. I create several files on my local machine Following is the code

package org.apache.hadoop.examples;

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class PutMerge {

public static void main(String[] args) throws IOException {

    Configuration conf = new Configuration();
    FileSystem hdfs = FileSystem.get(conf);
    FileSystem local = FileSystem.getLocal(conf);

    Path inputDir = new Path("/home/yehang/Documents/example");
    Path hdfsFile = new Path("hdfs://localhost:54310:/user/yehang");

    //System.out.println(inputDir);
    //System.out.println(hdfsFile);

    try {
        FileStatus[] inputFiles = local.listStatus(inputDir);
        //System.out.println(inputFiles);
        FSDataOutputStream out = hdfs.create(hdfsFile);
        //System.out.println(out);

        for (int i = 0; i < inputFiles.length; i++) {
            System.out.println(inputFiles[i].getPath().getName());
            FSDataInputStream in = local.open(inputFiles[i].getPath());
            byte buffer[] = new byte[256];
            int bytesRead = 0;
            while ((bytesRead = in.read(buffer)) > 0) {
                out.write(buffer, 0, bytesRead);
            }
            in.close();
        }
        out.close();
    } catch (IOException e) {
        e.printStackTrace();

    }

}

}

And when I run it on hadoop, it came out following error:

  Exception in thread "main" java.lang.IllegalArgumentException: Wrong FS:        hdfs://localhost:54310:/user, expected: file:///
at org.apache.hadoop.fs.FileSystem.checkPath(FileSystem.java:381)
at org.apache.hadoop.fs.RawLocalFileSystem.pathToFile(RawLocalFileSystem.java:55)
at org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:335)
at org.apache.hadoop.fs.ChecksumFileSystem.mkdirs(ChecksumFileSystem.java:492)
at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:377)
at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:364)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:555)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:536)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:443)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:435)
at org.apache.hadoop.examples.PutMerge.main(PutMerge.java:28)

Who can help me?

Upvotes: 1

Views: 679

Answers (1)

SachinJose
SachinJose

Reputation: 8522

Hadoop Distributed FileSystem(HDFS) related class files are defined inside the project hadoop-hdfs. So you got to import the jar hadoop-hdfs-*.jar in your Java classpath inorder to resolve this issue.

Upvotes: 1

Related Questions