使用RawComparator加速Hadoop程序

小木木

　　http://yoyzhou.github.io/blog/2013/05/13/hadoop-write-ur-own-rawcomparator/

static class MyComparator extends WritableComparator {
static {
WritableComparator.define(MyWritable.class, new MyComparator());
}
protected MyComparator() {
super(MyWritable.class);
}
public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
int cmp = 1;
//determine how many bytes the first VLong takes
int n1 = WritableUtils.decodeVIntSize(b1[s1]);
int n2 = WritableUtils.decodeVIntSize(b2[s2]);
try {
//read value from VLongWritable byte array
long l11 = readVLong(b1, s1);
long l21 = readVLong(b2, s2);
cmp = l11 > l21 ? 1 : (l11 == l21 ? 0 : -1);
if (cmp != 0) {
return cmp;
} else {
long l12 = readVLong(b1, s1 + n1);
long l22 = readVLong(b2, s2 + n2);
return cmp = l12 > l22 ? 1 : (l12 == l22 ? 0 : -1);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
static class MyWritable extends BinaryComparable implements WritableComparable<BinaryComparable> {
@Override
public void write(DataOutput out) throws IOException {
out.write("hello".getBytes());
}
@Override
public void readFields(DataInput in) throws IOException {
String str = WritableUtils.readString(in);
System.out.println(str);
}
@Override
public int getLength() {
return 0;
}
@Override
public byte[] getBytes() {
return null;
}
}