C# Hadoop学习笔记（一）
一、安装环境1，前期准备：官网下载“NuGet Package Manager”，按自己已有的VS环境下载对应版本；
2，利用NuGet下载Hadoop For .NET SDK,地址“http://hadoopsdk.codeplex.com/”
3，安装。
4，通过HDInsight，安装Windows Azure，目前是预览版本。
5，参照网址“http://blogs.msdn.com/b/data_otaku/archive/2013/08/14/hadoop-for-net-developers.aspx” 系统学习API
二、测试DEMO
view plain copy
[*]using System;
[*]using System.Collections.Generic;
[*]using System.Linq;
[*]using System.Text;
[*]using System.Threading.Tasks;
[*]using Microsoft.Hadoop;
[*]using Microsoft.Hadoop.MapReduce;
[*]using Microsoft.Hadoop.WebClient.WebHCatClient;
[*]using System.Diagnostics;
[*]using System.IO;
[*]using System.IO.MemoryMappedFiles;
[*]
[*]
[*]namespace HadoopConsol
[*]{
[*] class Program
[*] {
[*] static void Main(string[] args)
[*] {
[*] Stopwatch sw = new Stopwatch();
[*] long hadoopTime=0;
[*] long normalTime=0;
[*]
[*] sw.Start();
[*]
[*] //start hadoop
[*] Console.WriteLine(" Hadoop Process Strating ....");
[*]
[*] #region Hadoop time
[*]
[*] #region hadoopconnet
[*]
[*] Console.WriteLine(" Hadoop Connect Strating ....");
[*]
[*] //establish job configuration
[*]
[*] HadoopJobConfiguration myConfig = new HadoopJobConfiguration();
[*]
[*] myConfig.InputPath = "/demo/simple/in";
[*]
[*] myConfig.OutputFolder = "/demo/simple/out";
[*]
[*]
[*]
[*] //connect to cluster
[*]
[*] Uri myUri = new Uri("http://localhost");
[*]
[*] string userName = "hadoop";
[*]
[*] string passWord = null;
[*]
[*] IHadoop myCluster = Hadoop.Connect(myUri, userName, passWord);
[*]
[*] hadoopTime += sw.ElapsedMilliseconds;
[*]
[*] Console.WriteLine(" Hadoop Connect End.");
[*]
[*] Console.WriteLine(" Hadoop Connect time:" + sw.ElapsedMilliseconds);
[*]
[*] #endregion
[*]
[*]
[*] #region hadoopmapreduce
[*]
[*] sw.Reset();
[*] sw.Start();
[*]
[*] Console.WriteLine(" Hadoop MapReduce Strating ....");
[*]
[*] //execute mapreduce job
[*]
[*] MapReduceResult jobResult =
[*]
[*] myCluster.MapReduceJob.Execute<MySimpleMapper, MySimpleReducer>(myConfig);
[*]
[*] hadoopTime += sw.ElapsedMilliseconds;
[*]
[*] Console.WriteLine(" Hadoop MapReduce End.");
[*] Console.WriteLine(" Hadoop MapReduce Time:"+sw.ElapsedMilliseconds);
[*]
[*] #endregion
[*]
[*] #region Hadoop End
[*]
[*] sw.Reset();
[*] sw.Start();
[*]
[*] Console.WriteLine(" Hadoop Endprocess Strating ....");
[*]
[*] //write job result to console
[*]
[*] int exitCode = jobResult.Info.ExitCode;
[*]
[*]
[*]
[*] string exitStatus = "Failure";
[*]
[*] if (exitCode == 0) exitStatus = "Success";
[*]
[*] exitStatus = exitCode + " (" + exitStatus + ")";
[*]
[*] Console.WriteLine();
[*]
[*] Console.Write("Exit Code = " + exitStatus);
[*]
[*] Console.WriteLine(" Hadoop Endprocess End.");
[*] hadoopTime += sw.ElapsedMilliseconds;
[*] Console.WriteLine(" Hadoop Exit Time:" + sw.ElapsedMilliseconds);
[*] Console.WriteLine(" Hadoop Process All Time:" + hadoopTime);
[*] #endregion
[*]
[*] #endregion
[*]
[*]
[*] #region Normal time
[*] //start Normal
[*] Console.WriteLine(" Normal Process Strating ....");
[*]
[*] sw.Reset();
[*] sw.Start();
[*]
[*] //normal process
[*] #region Normal Process
[*]
[*] int myevenCount = 0;
[*] int myeventSum = 0;
[*]
[*] int myoddCount = 0;
[*] int myoddSum = 0;
[*]
[*] StreamReader fs = new StreamReader(@"c:\TEMP\integers.txt");
[*]
[*] while (fs.Peek() >= 0)
[*] {
[*] string strTemp = fs.ReadLine();
[*] if (Int32.Parse(strTemp) % 2 == 0)
[*] {
[*] myevenCount++;
[*] myeventSum += Int32.Parse(strTemp);
[*] }
[*] else
[*] {
[*] myoddCount++;
[*] myoddSum += Int32.Parse(strTemp);
[*] }
[*] }
[*] //MemoryMappedFile m = MemoryMappedFile.
[*] Console.WriteLine("even:" + "\t" + myevenCount + "\t" + myeventSum);
[*] Console.WriteLine("odd:" + "\t" + myoddCount + "\t" + myoddSum);
[*]
[*] #endregion
[*]
[*] Console.WriteLine(" Normal Process End.");
[*]
[*] normalTime += sw.ElapsedMilliseconds;
[*] Console.WriteLine(" Normal Exit Time:" + sw.ElapsedMilliseconds);
[*] Console.WriteLine(" Normal Process All Time:" + normalTime);
[*]
[*] #endregion
[*]
[*] sw.Stop();
[*]
[*] Console.Read();
[*]
[*] }
[*]
[*] }
[*]
[*] public class MySimpleMapper : MapperBase
[*] {
[*]
[*] public override void Map(string inputLine, MapperContext context)
[*] {
[*]
[*] //interpret the incoming line as an integer value
[*]
[*] int value = int.Parse(inputLine);
[*]
[*] //determine whether value is even or odd
[*]
[*] string key = (value % 2 == 0) ? "even" : "odd";
[*]
[*] //output key assignment with value
[*]
[*] context.EmitKeyValue(key, value.ToString());
[*]
[*] }
[*]
[*] }
[*]
[*] public class MySimpleReducer : ReducerCombinerBase
[*] {
[*]
[*] public override void Reduce(
[*]
[*] string key, IEnumerable<string> values, ReducerCombinerContext context
[*]
[*] )
[*] {
[*]
[*] //initialize counters
[*]
[*] int myCount = 0;
[*]
[*] int mySum = 0;
[*]
[*]
[*]
[*] //count and sum incoming values
[*]
[*] foreach (string value in values)
[*] {
[*]
[*] mySum += int.Parse(value);
[*]
[*] myCount++;
[*]
[*] }
[*]
[*]
[*]
[*] //output results
[*]
[*] context.EmitKeyValue(key, myCount + "\t" + mySum);
[*]
[*] }
[*]
[*]
[*] }
[*]}
三、测试结果
页:
[1]