<property>
<name>dfs.permissions</name>
<value>false</value>
</property>

<configuration>
<property>
<name>mapred.job.tracker</name>
<value>localhost:9001</value>
</property>
</configuration>

<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://localhost:9000</value>
</property>
</configuration>

package org.liky.sina.save;

//這里用到了JSoup開發(fā)包，該包可以很簡單的提取到HTML中的有效信息
import java.io.File;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class SinaNewsData {

  private static Configuration conf = new Configuration();

  private static FileSystem fs;

  private static Path path;

  private static int count = 0;

  public static void main(String[] args) {
    parseAllFile(new File(
        "E:/heritrix-1.12.1/jobs/sina_news_job_02-20170814013255352/mirror/"));
  }

  public static void parseAllFile(File file) {
    // 判斷類型
    if (file.isDirectory()) {
      // 文件夾
      File[] allFile = file.listFiles();
      if (allFile != null) {
        for (File f : allFile) {
          parseAllFile(f);
        }
      }
    } else {
      // 文件
      if (file.getName().endsWith(".html")
          || file.getName().endsWith(".shtml")) {
        parseContent(file.getAbsolutePath());
      }
    }
  }

  public static void parseContent(String filePath) {
    try {
        //用jsoup的方法讀取文件路徑
      Document doc = Jsoup.parse(new File(filePath), "utf-8");
      //讀取標(biāo)題
      String title = doc.title();
      Elements descElem = doc.getElementsByAttributeValue("name",
          "description");
      Element descE = descElem.first();
      
      // 讀取內(nèi)容
      String content = descE.attr("content");

      if (title != null && content != null) {
        //通過Path來保存數(shù)據(jù)到HDFS中
        path = new Path("hdfs://localhost:9000/input/"
            + System.currentTimeMillis() + ".txt");

        fs = path.getFileSystem(conf);

        // 建立輸出流對象
        FSDataOutputStream os = fs.create(path);
        // 使用os完成輸出
        os.writeChars(title + "\r\n" + content);

        os.close();

        count++;

        System.out.println("已經(jīng)完成" + count + " 個!");
      }

    } catch (Exception e) {
      e.printStackTrace();
    }
  }
}