cascading.flow.hadoop.util
Class HadoopUtil

java.lang.Object
  extended by cascading.flow.hadoop.util.HadoopUtil

public class HadoopUtil
extends java.lang.Object


Constructor Summary
HadoopUtil()
           
 
Method Summary
static JobConf createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties, JobConf defaultJobconf)
           
static java.util.Map<java.lang.Object,java.lang.Object> createProperties(Configuration jobConf)
           
static java.lang.Object deserializeBase64(java.lang.String string)
          This method deserializes the Base64 encoded String into an Object instance.
static java.lang.Object deserializeBase64(java.lang.String string, boolean decompress)
           
static java.util.List<java.util.Map<java.lang.String,java.lang.String>> deserializeListMapBase64(java.lang.String string, boolean decompress)
           
static java.util.Map<java.lang.String,java.lang.String> deserializeMapBase64(java.lang.String string, boolean decompress)
           
static java.lang.Class findMainClass(java.lang.Class defaultType)
           
static java.util.Map<java.lang.String,java.lang.String> getConfig(JobConf defaultConf, JobConf updatedConf)
           
static java.lang.Thread getHDFSShutdownHook()
           
static JobConf[] getJobConfs(JobConf job, java.util.List<java.util.Map<java.lang.String,java.lang.String>> configs)
           
static void initLog4j(JobConf jobConf)
           
static JobConf mergeConf(JobConf job, java.util.Map<java.lang.String,java.lang.String> config, boolean directly)
           
static java.lang.String readStateFromDistCache(JobConf jobConf, java.lang.String id)
           
static JobConf removePropertiesFrom(JobConf jobConf, java.lang.String... keys)
           
static boolean removeStateFromDistCache(JobConf conf, java.lang.String path)
           
static java.lang.String serializeBase64(java.lang.Object object)
          This method serializes the given Object instance and retunrs a String Base64 representation.
static java.lang.String serializeBase64(java.lang.Object object, boolean compress)
           
static java.lang.String serializeListMapBase64(java.util.List<java.util.Map<java.lang.String,java.lang.String>> listMap, boolean compress)
           
static java.lang.String serializeMapBase64(java.util.Map<java.lang.String,java.lang.String> map, boolean compress)
           
static java.lang.String writeStateToDistCache(JobConf conf, java.lang.String id, java.lang.String stepState)
           
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 

Constructor Detail

HadoopUtil

public HadoopUtil()
Method Detail

initLog4j

public static void initLog4j(JobConf jobConf)

createJobConf

public static JobConf createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties,
                                    JobConf defaultJobconf)

createProperties

public static java.util.Map<java.lang.Object,java.lang.Object> createProperties(Configuration jobConf)

getHDFSShutdownHook

public static java.lang.Thread getHDFSShutdownHook()

serializeMapBase64

public static java.lang.String serializeMapBase64(java.util.Map<java.lang.String,java.lang.String> map,
                                                  boolean compress)
                                           throws java.io.IOException
Throws:
java.io.IOException

deserializeMapBase64

public static java.util.Map<java.lang.String,java.lang.String> deserializeMapBase64(java.lang.String string,
                                                                                    boolean decompress)
                                                                             throws java.io.IOException
Throws:
java.io.IOException

serializeListMapBase64

public static java.lang.String serializeListMapBase64(java.util.List<java.util.Map<java.lang.String,java.lang.String>> listMap,
                                                      boolean compress)
                                               throws java.io.IOException
Throws:
java.io.IOException

deserializeListMapBase64

public static java.util.List<java.util.Map<java.lang.String,java.lang.String>> deserializeListMapBase64(java.lang.String string,
                                                                                                        boolean decompress)
                                                                                                 throws java.io.IOException
Throws:
java.io.IOException

serializeBase64

public static java.lang.String serializeBase64(java.lang.Object object)
                                        throws java.io.IOException
This method serializes the given Object instance and retunrs a String Base64 representation.

Parameters:
object - to be serialized
Returns:
String
Throws:
java.io.IOException

serializeBase64

public static java.lang.String serializeBase64(java.lang.Object object,
                                               boolean compress)
                                        throws java.io.IOException
Throws:
java.io.IOException

deserializeBase64

public static java.lang.Object deserializeBase64(java.lang.String string)
                                          throws java.io.IOException
This method deserializes the Base64 encoded String into an Object instance.

Parameters:
string -
Returns:
an Object
Throws:
java.io.IOException

deserializeBase64

public static java.lang.Object deserializeBase64(java.lang.String string,
                                                 boolean decompress)
                                          throws java.io.IOException
Throws:
java.io.IOException

findMainClass

public static java.lang.Class findMainClass(java.lang.Class defaultType)

getConfig

public static java.util.Map<java.lang.String,java.lang.String> getConfig(JobConf defaultConf,
                                                                         JobConf updatedConf)

getJobConfs

public static JobConf[] getJobConfs(JobConf job,
                                    java.util.List<java.util.Map<java.lang.String,java.lang.String>> configs)

mergeConf

public static JobConf mergeConf(JobConf job,
                                java.util.Map<java.lang.String,java.lang.String> config,
                                boolean directly)

removePropertiesFrom

public static JobConf removePropertiesFrom(JobConf jobConf,
                                           java.lang.String... keys)

removeStateFromDistCache

public static boolean removeStateFromDistCache(JobConf conf,
                                               java.lang.String path)
                                        throws java.io.IOException
Throws:
java.io.IOException

writeStateToDistCache

public static java.lang.String writeStateToDistCache(JobConf conf,
                                                     java.lang.String id,
                                                     java.lang.String stepState)

readStateFromDistCache

public static java.lang.String readStateFromDistCache(JobConf jobConf,
                                                      java.lang.String id)
                                               throws java.io.IOException
Throws:
java.io.IOException


Copyright © 2007-2011 Concurrent, Inc. All Rights Reserved.