Hadoop实战演练：搜索数据分析----数据去重（1）
网友 CSDN博客

一、环境与数据


1、本地开发环境 

Windows7 + Eclipse Luna

hadoop版本:2.6.2

JDK版本:1.8


2、数据来源:

搜狗实验室

http://www.sogou.com/labs/resource/q.php


3、数据格式

用户ID\t[查询词]\t该URL在返回结果中的排名\t用户点击的顺序号\t用户点击的URL
其中,用户ID是根据用户使用浏览器访问搜索引擎时的Cookie信息自动赋值,即同一次使用浏览器输入的不同查询对应同一个用户ID

样例:

blob.png

3、数据去重

每一人的同一个搜索关键词能可会有多个点击条数。这里要去除同一个人同一个关键词重复的条数

二、编程实现

  1. package com.lin.keyword;  


  3. import java.io.IOException;  

  4. import java.util.StringTokenizer;  


  6. import org.apache.hadoop.conf.Configuration;  

  7. import org.apache.hadoop.fs.Path;  

  8. import org.apache.hadoop.io.Text;  

  9. import org.apache.hadoop.mapreduce.Job;  

  10. import org.apache.hadoop.mapreduce.Mapper;  

  11. import org.apache.hadoop.mapreduce.Reducer;  

  12. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  

  13. import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;  

  14. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  

  15. import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;  

  16. import org.apache.hadoop.util.GenericOptionsParser;  


  18. /** 

  19.  * 功能概要:数据去重 

  20.  *  

  21.  * @author linbingwen 

  22.  * @since 2016年7月31日 

  23.  */

  24. publicclass CleanSameData {  

  25.     // map将输入中的value复制到输出数据的key上,并直接输出

  26.     publicstaticclass Map extends Mapper<Object, Text, Text, Text> {  


  28.         // 实现map函数

  29.         @Override

  30.         publicvoid map(Object key, Text value, Context context) throws IOException, InterruptedException {  

  31.             // 将输入的纯文本文件的数据转化成String

  32.             String line = value.toString();  

  33.             // 将输入的数据首先按行进行分割

  34.             StringTokenizer tokenizerArticle = new StringTokenizer(line, "\n");  

  35.             // 分别对每一行进行处理

  36.             while (tokenizerArticle.hasMoreElements()) {  

  37.                 // 每行按空格划分

  38.                 StringTokenizer tokenizerLine = new StringTokenizer(tokenizerArticle.nextToken());  

  39.                 String c1 = tokenizerLine.nextToken();//

  40.                 String c2 = tokenizerLine.nextToken();// 关键词

  41.                 c2 = c2.substring(1, c2.length() - 1);  

  42.                 Text newline = new Text(c1 + "    " + c2);  

  43.                 context.write(newline, new Text(""));  

  44.             }  


  46.         }  


  48.     }  


  50.     // reduce将输入中的key复制到输出数据的key上,并直接输出

  51.     publicstaticclass Reduce extends Reducer<Text, Text, Text, Text> {  

  52.         // 实现reduce函数

  53.         @Override

  54.         publicvoid reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {  

  55.             context.write(key, new Text(""));  

  56.         }  

  57.     }  


  59.     publicstaticvoid main(String[] args) throws Exception {  


  61.         Configuration conf = new Configuration();  

  62.         // 设置hadoop的机器、端口

  63.         conf.set("mapred.job.tracker", "10.75.201.125:9000");  

  64.         // 设置输入输出文件目录

  65.         String[] ioArgs = new String[] { "hdfs://hmaster:9000/data_in", "hdfs://hmaster:9000/clean_same_out" };  

  66.         String[] otherArgs = new GenericOptionsParser(conf, ioArgs).getRemainingArgs();  

  67.         if (otherArgs.length != 2) {  

  68.             System.err.println("Usage:  <in> <out>");  

  69.             System.exit(2);  

  70.         }  

  71.         // 设置一个job

  72.         Job job = Job.getInstance(conf, "clean same data");  

  73.         job.setJarByClass(CleanSameData.class);  


  75.         // 设置Map、Combine和Reduce处理类

  76.         job.setMapperClass(Map.class);  

  77.         job.setCombinerClass(Reduce.class);  

  78.         job.setReducerClass(Reduce.class);  


  80.         // 设置输出类型

  81.         job.setOutputKeyClass(Text.class);  

  82.         job.setOutputValueClass(Text.class);  


  84.         // 将输入的数据集分割成小数据块splites,提供一个RecordReder的实现

  85.         job.setInputFormatClass(TextInputFormat.class);  


  87.         // 提供一个RecordWriter的实现,负责数据输出

  88.         job.setOutputFormatClass(TextOutputFormat.class);  


  90.         // 设置输入和输出目录

  91.         FileInputFormat.addInputPath(job, new Path(otherArgs[0]));  

  92.         FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));  

  93.         System.exit(job.waitForCompletion(true) ? 0 : 1);  


  95.     }  


  97. }  


maven文件:

  1. <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

  2.     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">  

  3.     <modelVersion>4.0.0</modelVersion>  

  4.     <groupId>com.lin</groupId>  

  5.     <artifactId>zdy-hadoop</artifactId>  

  6.     <version>0.0.1-SNAPSHOT</version>  

  7.     <properties>  

  8.         <!-- spring版本号 -->  

  9.         <spring.version>4.3.0.RELEASE</spring.version>  

  10.         <!-- log4j日志文件管理包版本 -->  

  11.         <slf4j.version>1.6.6</slf4j.version>  

  12.         <log4j.version>1.2.12</log4j.version>  

  13.         <!-- junit版本号 -->  

  14.         <junit.version>4.12</junit.version>  

  15.         <hadoop.version>2.6.0</hadoop.version>  

  16.         <!-- mybatis版本号 -->  

  17.         <mybatis.version>3.2.1</mybatis.version>  

  18.         <mysql.connect.version>6.0.3</mysql.connect.version>  

  19.     </properties>  


  21.     <dependencies>          

  22.         <!--单元测试依赖 -->  

  23.         <dependency>  

  24.             <groupId>junit</groupId>  

  25.             <artifactId>junit</artifactId>  

  26.             <version>${junit.version}</version>  

  27.             <scope>test</scope>  

  28.         </dependency>  

  29.         <!-- logback start -->  

  30.         <dependency>  

  31.             <groupId>log4j</groupId>  

  32.             <artifactId>log4j</artifactId>  

  33.             <version>${log4j.version}</version>  

  34.         </dependency>  

  35.         <dependency>  

  36.             <groupId>org.slf4j</groupId>  

  37.             <artifactId>slf4j-api</artifactId>  

  38.             <version>${slf4j.version}</version>  

  39.         </dependency>  

  40.         <dependency>  

  41.             <groupId>ch.qos.logback</groupId>  

  42.             <artifactId>logback-classic</artifactId>  

  43.             <version>1.1.2</version>  

  44.         </dependency>  

  45.         <dependency>  

  46.             <groupId>ch.qos.logback</groupId>  

  47.             <artifactId>logback-core</artifactId>  

  48.             <version>1.1.2</version>  

  49.         </dependency>  

  50.         <dependency>  

  51.             <groupId>org.logback-extensions</groupId>  

  52.             <artifactId>logback-ext-spring</artifactId>  

  53.             <version>0.1.1</version>  

  54.         </dependency>  

  55.         <dependency>  

  56.             <groupId>org.apache.hadoop</groupId>  

  57.             <artifactId>hadoop-common</artifactId>  

  58.             <version>2.6.0</version>  

  59.         </dependency>  

  60.         <dependency>  

  61.             <groupId>org.apache.hadoop</groupId>  

  62.             <artifactId>hadoop-hdfs</artifactId>  

  63.             <version>2.6.0</version>  

  64.         </dependency>  

  65.         <dependency>  

  66.             <groupId>org.apache.hadoop</groupId>  

  67.             <artifactId>hadoop-client</artifactId>  

  68.             <version>2.6.0</version>  

  69.         </dependency>  

  70.         <dependency>  

  71.             <groupId>jdk.tools</groupId>  

  72.             <artifactId>jdk.tools</artifactId>  

  73.             <version>1.8</version>  

  74.             <scope>system</scope>  

  75.             <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>  

  76.         </dependency>  

  77.          <!-- mysql驱动包 -->  

  78.         <dependency>  

  79.             <groupId>mysql</groupId>  

  80.             <artifactId>mysql-connector-java</artifactId>  

  81.             <version>${mysql.connect.version}</version>  

  82.         </dependency>  

  83.     </dependencies>  


  85.     <build>  

  86.         <finalName>zdy-hadoop</finalName>  

  87.         <filters>  

  88.             <!-- 表示打包哪个环境下的properties文件 -->  

  89.             <filter>src/main/resources/env/${env}.properties</filter>  

  90.         </filters>  

  91.         <!-- 配置将properties时的变量用pom.xml里的变量赋值替换 -->  

  92.         <resources>  

  93.             <resource>  

  94.                 <directory>src/main/resources</directory>  

  95.                 <targetPath>${basedir}/target/classes</targetPath>  

  96.                 <includes>  

  97.                     <include>**/*.properties</include> 

  98.                     <include>**/*.xml</include>  

  99.                 </includes>  

  100.                 <filtering>true</filtering>  

  101.             </resource>  

  102.             <resource>  

  103.                 <directory>src/main/resources</directory>  

  104.                 <targetPath>${basedir}/target/resources</targetPath>  

  105.                 <includes>  

  106.                     <include>**/*.properties</include> 

  107.                     <include>**/*.xml</include>  

  108.                 </includes>  

  109.                 <filtering>true</filtering>  

  110.             </resource>  

  111.         </resources>  


  113.         <!-- </plugins> -->  

  114.         <plugins>  

  115.             <plugin>  

  116.                 <groupId>org.apache.maven.plugins</groupId>  

  117.                 <artifactId>maven-compiler-plugin</artifactId>  

  118.                 <version>2.0.2</version>  

  119.                 <configuration>  

  120.                     <source>1.8</source>  

  121.                     <target>1.8</target>  

  122.                     <encoding>UTF-8</encoding>  

  123.                 </configuration>  

  124.             </plugin>  

  125.             <plugin>  

  126.                 <groupId>org.apache.maven.plugins</groupId>  

  127.                 <artifactId>maven-jar-plugin</artifactId>  

  128.                 <executions>  

  129.                     <!-- 定义在prepare-package时将classes/com打jar -->  

  130.                     <execution>  

  131.                         <phase>prepare-package</phase>  

  132.                         <goals>  

  133.                             <goal>jar</goal>  

  134.                         </goals>  

  135.                         <configuration>  

  136.                             <includes>  

  137.                                 <include>com/**</include>  

  138.                             </includes>  

  139.                         </configuration>  

  140.                     </execution>  

  141.                 </executions>  

  142.             </plugin>  

  143.             <plugin>  

  144.                 <artifactId>maven-assembly-plugin</artifactId>  

  145.                 <configuration>  

  146.                     <!-- not append assembly id in release file name -->  

  147.                     <appendAssemblyId>false</appendAssemblyId>  

  148.                     <finalName>${project.artifactId}</finalName>  

  149.                     <descriptors>  

  150.                         <descriptor>src/main/assemble/assembly.xml</descriptor>  

  151.                     </descriptors>  

  152.                 </configuration>  

  153.                 <executions>  

  154.                     <execution>  

  155.                         <id>make-assembly</id>  

  156.                         <phase>package</phase>  

  157.                         <goals>  

  158.                             <goal>single</goal>  

  159.                         </goals>  

  160.                     </execution>  

  161.                 </executions>  

  162.             </plugin>  

  163.     </plugins>  

  164.     </build>  

  165.     <profiles>  

  166.         <!-- 开发环境,配置文件放在src/main/resources/env下 -->  

  167.         <profile>  

  168.             <id>dev</id>  

  169.             <activation>  

  170.                 <activeByDefault>true</activeByDefault>  

  171.             </activation>  

  172.             <properties>  

  173.                 <env>dev</env>  

  174.             </properties>  

  175.         </profile>  

  176.     </profiles>  

  177. </project>  



输出结果:

blob.png

CIO之家 www.ciozj.com 公众号:imciow
关联的文档
也许您喜欢