网站受到攻击,wordpress朋友圈,丰收路网站建设,网站开发中安全性目录 整合结构准备数据下载pom.xmlMain.javaReduce.javaMap.java操作 总结 整合结构
和案例1的结构差不多#xff0c;Hbase移动到开头#xff0c;后面跟随MR程序。 因此对于输入的K1 V1会进行一定的修改
准备
在HBASE中创建表#xff0c;并写入数据
create wunaii… 目录 整合结构准备数据下载pom.xmlMain.javaReduce.javaMap.java操作 总结 整合结构
和案例1的结构差不多Hbase移动到开头后面跟随MR程序。 因此对于输入的K1 V1会进行一定的修改
准备
在HBASE中创建表并写入数据
create wunaiieq:sentence,colf系统文件上传
datain3.java
package org.wunaiieq.hbase2hdfs;import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.wunaiieq.HBaseConnection;
import org.wunaiieq.HbaseDML;import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;public class datain3 {public static Connection connection HBaseConnection.connection;public static void main(String[] args) throws IOException {BufferedReader bufferedReader new BufferedReader(new FileReader(/opt/module/jar/data.txt));String line null;Table table connection.getTable(TableName.valueOf(wunaiieq, sentence));int rowkey 1;while ((linebufferedReader.readLine())!null){Put put new Put(Bytes.toBytes(rowkey));put.addColumn(Bytes.toBytes(colf),Bytes.toBytes(line),Bytes.toBytes(line));table.put(put);rowkey;}bufferedReader.close();}
}数据下载
pom.xml
?xml version1.0 encodingUTF-8?
project xmlnshttp://maven.apache.org/POM/4.0.0xmlns:xsihttp://www.w3.org/2001/XMLSchema-instancexsi:schemaLocationhttp://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsdmodelVersion4.0.0/modelVersiongroupIdorg.hbase/groupIdartifactIdhbase2hdfs/artifactIdversion1.0-SNAPSHOT/versionpropertiesmaven.compiler.source8/maven.compiler.sourcemaven.compiler.target8/maven.compiler.targetproject.build.sourceEncodingUTF-8/project.build.sourceEncodinghadoop.version3.1.3/hadoop.versionhbase.version2.2.3/hbase.version/propertiesdependencies!-- Hadoop Dependencies --dependencygroupIdorg.apache.hadoop/groupIdartifactIdhadoop-client/artifactIdversion${hadoop.version}/version/dependencydependencygroupIdorg.apache.hadoop/groupIdartifactIdhadoop-common/artifactIdversion${hadoop.version}/version/dependencydependencygroupIdorg.apache.hadoop/groupIdartifactIdhadoop-hdfs/artifactIdversion${hadoop.version}/version/dependencydependencygroupIdorg.apache.hadoop/groupIdartifactIdhadoop-mapreduce-client-core/artifactIdversion${hadoop.version}/version/dependencydependencygroupIdorg.apache.hadoop/groupIdartifactIdhadoop-yarn-api/artifactIdversion${hadoop.version}/version/dependencydependencygroupIdorg.apache.hadoop/groupIdartifactIdhadoop-streaming/artifactIdversion${hadoop.version}/version/dependency!-- HBase Dependencies --dependencygroupIdorg.apache.hbase/groupIdartifactIdhbase-client/artifactIdversion${hbase.version}/version/dependencydependencygroupIdorg.apache.hbase/groupIdartifactIdhbase-server/artifactIdversion${hbase.version}/version/dependencydependencygroupIdorg.apache.hbase/groupIdartifactIdhbase-common/artifactIdversion${hbase.version}/version/dependencydependencygroupIdorg.apache.hbase/groupIdartifactIdhbase-mapreduce/artifactIdversion${hbase.version}/version/dependency!-- Other Dependencies --dependencygroupIdcom.google.protobuf/groupIdartifactIdprotobuf-java/artifactIdversion3.19.1/version/dependencydependencygroupIdorg.slf4j/groupIdartifactIdslf4j-log4j12/artifactIdversion1.7.25/version/dependencydependencygroupIdlog4j/groupIdartifactIdlog4j/artifactIdversion1.2.17/version/dependencydependencygroupIdjunit/groupIdartifactIdjunit/artifactIdversionRELEASE/versionscopecompile/scope/dependency/dependenciesbuildpluginsplugin!--声明--groupIdorg.apache.maven.plugins/groupIdartifactIdmaven-assembly-plugin/artifactIdversion3.3.0/version!--具体配置--configurationarchivemanifest!--jar包的执行入口--mainClassorg.wunaiieq.hbase2hdfs.Main/mainClass/manifest/archivedescriptorRefs!--描述符此处为预定义的表示创建一个包含项目所有依赖的可执行 JAR 文件;允许自定义生成jar文件内容--descriptorRefjar-with-dependencies/descriptorRef/descriptorRefs/configuration!--执行配置--executionsexecution!--执行配置ID可修改--idmake-assembly/id!--执行的生命周期--phasepackage/phasegoals!--执行的目标single表示创建一个分发包--goalsingle/goal/goals/execution/executions/plugin/plugins/build/project
Main.java
package org.wunaiieq.hbase2hdfs;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class Main {public static void main(String[] args) throws Exception {//配置文件写在resources目录下Job job Job.getInstance(new Configuration());//入口类job.setJarByClass(Main.class);Scan scan new Scan();TableMapReduceUtil.initTableMapperJob(wunaiieq:sentence,//表名scan,//表输入时可以在此处进行部分设置如选择查询的列簇列过滤行等等org.wunaiieq.hbase2hdfs.Map.class,//指定mapper类Text.class,//k2IntWritable.class,//v2job,false);job.setOutputKeyClass(Text.class);//K3job.setOutputValueClass(IntWritable.class);//V3job.setReducerClass(org.wunaiieq.hbase2hdfs.Reduce.class);//手动输入输出路径FileOutputFormat.setOutputPath(job,new Path(args[0]));job.waitForCompletion(true);}
}
Reduce.java
package org.wunaiieq.hbase2hdfs;import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;import java.io.IOException;// K3 V3 K4 V4
public class Reduce extends ReducerText,IntWritable,Text,IntWritable{private IntWritable v4 new IntWritable();private Text k4 new Text();Overrideprotected void reduce(Text k3, IterableIntWritable v3,Context context) throws IOException, InterruptedException {int sum 0;for (IntWritable v30:v3){sumv30.get();}v4.set(sum);k4k3;context.write(k4,v4);}
}
Map.java
package org.wunaiieq.hbase2hdfs;import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;import java.io.IOException;
// K1 V1
public class Map extends TableMapperText,IntWritable {private Text k2new Text();private IntWritable v2 new IntWritable(1);Overrideprotected void map(ImmutableBytesWritable k1, Result v1,Context context) throws IOException, InterruptedException {System.out.println(k1:k1.toString());//读取当前行中的colf:line数据byte[] data v1.getValue(Bytes.toBytes(colf),Bytes.toBytes(line));String line Bytes.toString(data);String [] words line.split( );for (String word :words){k2.set(word);context.write(k2,v2);}}
}
操作
打包上传至linux系统中
hadoop jar hbase2hdfs-1.0-SNAPSHOT-jar-with-dependencies.jar /output/test检查文件
hdfs dfs -cat /output/test/part-r-00000总结
没什么特殊点记录下这两个案例即可只需要在MR程序中替换掉对应的Mapper和Reducer即可 文章转载自： http://www.morning.tsynj.cn.gov.cn.tsynj.cn http://www.morning.nbrkt.cn.gov.cn.nbrkt.cn http://www.morning.lgznf.cn.gov.cn.lgznf.cn http://www.morning.bkylg.cn.gov.cn.bkylg.cn http://www.morning.pwdrc.cn.gov.cn.pwdrc.cn http://www.morning.bkgfp.cn.gov.cn.bkgfp.cn http://www.morning.bbgn.cn.gov.cn.bbgn.cn http://www.morning.fqpgf.cn.gov.cn.fqpgf.cn http://www.morning.fy974.cn.gov.cn.fy974.cn http://www.morning.qkwxp.cn.gov.cn.qkwxp.cn http://www.morning.tlrxp.cn.gov.cn.tlrxp.cn http://www.morning.glswq.cn.gov.cn.glswq.cn http://www.morning.gklxm.cn.gov.cn.gklxm.cn http://www.morning.ndmbd.cn.gov.cn.ndmbd.cn http://www.morning.wqpm.cn.gov.cn.wqpm.cn http://www.morning.jfsbs.cn.gov.cn.jfsbs.cn http://www.morning.nckjk.cn.gov.cn.nckjk.cn http://www.morning.mdwlg.cn.gov.cn.mdwlg.cn http://www.morning.srkwf.cn.gov.cn.srkwf.cn http://www.morning.ppllj.cn.gov.cn.ppllj.cn http://www.morning.ylpl.cn.gov.cn.ylpl.cn http://www.morning.ndtmz.cn.gov.cn.ndtmz.cn http://www.morning.rdlong.com.gov.cn.rdlong.com http://www.morning.yfffg.cn.gov.cn.yfffg.cn http://www.morning.rjmb.cn.gov.cn.rjmb.cn http://www.morning.zfrs.cn.gov.cn.zfrs.cn http://www.morning.plflq.cn.gov.cn.plflq.cn http://www.morning.pqrhb.cn.gov.cn.pqrhb.cn http://www.morning.jgncd.cn.gov.cn.jgncd.cn http://www.morning.hqjtp.cn.gov.cn.hqjtp.cn http://www.morning.bnjnp.cn.gov.cn.bnjnp.cn http://www.morning.mkpkz.cn.gov.cn.mkpkz.cn http://www.morning.kwqwp.cn.gov.cn.kwqwp.cn http://www.morning.smmrm.cn.gov.cn.smmrm.cn http://www.morning.qsctt.cn.gov.cn.qsctt.cn http://www.morning.gbjxj.cn.gov.cn.gbjxj.cn http://www.morning.qcfcz.cn.gov.cn.qcfcz.cn http://www.morning.ydrml.cn.gov.cn.ydrml.cn http://www.morning.mkpqr.cn.gov.cn.mkpqr.cn http://www.morning.ljglc.cn.gov.cn.ljglc.cn http://www.morning.zrdqz.cn.gov.cn.zrdqz.cn http://www.morning.pxbrg.cn.gov.cn.pxbrg.cn http://www.morning.mtmnk.cn.gov.cn.mtmnk.cn http://www.morning.hsjfs.cn.gov.cn.hsjfs.cn http://www.morning.zffps.cn.gov.cn.zffps.cn http://www.morning.dxqfh.cn.gov.cn.dxqfh.cn http://www.morning.cfjyr.cn.gov.cn.cfjyr.cn http://www.morning.lxthr.cn.gov.cn.lxthr.cn http://www.morning.tmfhx.cn.gov.cn.tmfhx.cn http://www.morning.ydgzj.cn.gov.cn.ydgzj.cn http://www.morning.crrjg.cn.gov.cn.crrjg.cn http://www.morning.zshuhd015.cn.gov.cn.zshuhd015.cn http://www.morning.ljqd.cn.gov.cn.ljqd.cn http://www.morning.mdwlg.cn.gov.cn.mdwlg.cn http://www.morning.hysqx.cn.gov.cn.hysqx.cn http://www.morning.rnzbr.cn.gov.cn.rnzbr.cn http://www.morning.bnfjh.cn.gov.cn.bnfjh.cn http://www.morning.hwsgk.cn.gov.cn.hwsgk.cn http://www.morning.chjnb.cn.gov.cn.chjnb.cn http://www.morning.mqmxg.cn.gov.cn.mqmxg.cn http://www.morning.jnbsx.cn.gov.cn.jnbsx.cn http://www.morning.kmcfw.cn.gov.cn.kmcfw.cn http://www.morning.flncd.cn.gov.cn.flncd.cn http://www.morning.lkbyj.cn.gov.cn.lkbyj.cn http://www.morning.pzss.cn.gov.cn.pzss.cn http://www.morning.zwhtr.cn.gov.cn.zwhtr.cn http://www.morning.haibuli.com.gov.cn.haibuli.com http://www.morning.jpwmk.cn.gov.cn.jpwmk.cn http://www.morning.kwqt.cn.gov.cn.kwqt.cn http://www.morning.lyzwdt.com.gov.cn.lyzwdt.com http://www.morning.wyfpc.cn.gov.cn.wyfpc.cn http://www.morning.mlyq.cn.gov.cn.mlyq.cn http://www.morning.xckrj.cn.gov.cn.xckrj.cn http://www.morning.ltcnd.cn.gov.cn.ltcnd.cn http://www.morning.lhgkr.cn.gov.cn.lhgkr.cn http://www.morning.ldcsw.cn.gov.cn.ldcsw.cn http://www.morning.fqnql.cn.gov.cn.fqnql.cn http://www.morning.qxkcx.cn.gov.cn.qxkcx.cn http://www.morning.grpbt.cn.gov.cn.grpbt.cn http://www.morning.qtkdn.cn.gov.cn.qtkdn.cn
