WeightedPageRankFromMatrix

A weighted PageRank implementation using the Scalding Matrix API. This assumes that all rows and columns are of type Int and values or egde weights are Double. If you want an unweighted PageRank, simply set the weights on the edges to 1.

Input arguments:

d -- damping factor n -- number of nodes in the graph currentIteration -- start with 0 probably maxIterations -- stop after n iterations convergenceThreshold -- using the sum of the absolute difference between iteration solutions, iterating stops once we reach this threshold rootDir -- the root directory holding all starting, intermediate and final data/output

The expected structure of the rootDir is:

rootDir |- iterations | |- 0 <-- a TSV of (row, value) of size n, value can be 1/n (generate this) | |- n <-- holds future iterations/solutions |- edges <-- a TSV of (row, column, value) for edges in the graph |- onesVector <-- a TSV of (row, 1) of size n (generate this) |- diff <-- a single line representing the difference between the last iterations |- constants <-- built at iteration 0, these are constant for any given matrix/graph |- M_hat |- priorVector

Don't forget to set the number of reducers for this job: -D mapred.reduce.tasks=n

Linear Supertypes

Job, Serializable, FieldConversions, LowPriorityFieldConversions, AnyRef, Any

Instance Constructors

new WeightedPageRankFromMatrix(args: Args)

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def M_hat: Matrix[Int, Int, Double]

Load or generate on first iteration the matrix M^ given A.
implicit def _implicitJobArgs: Args

Attributes
protected
Definition Classes
Job
def anyToFieldArg(f: Any): Comparable[_]

Attributes
protected
Definition Classes
LowPriorityFieldConversions
val args: Args

Definition Classes
Job
final def asInstanceOf[T0]: T0

Definition Classes
Any
def asList(f: Fields): List[Comparable[_]]

Definition Classes
FieldConversions
def asSet(f: Fields): Set[Comparable[_]]

Definition Classes
FieldConversions
def buildFlow: Flow[_]

Definition Classes
Job
def classIdentifier: String

Definition Classes
Job
def clear(): Unit

Definition Classes
Job
def clone(nextargs: Args): Job

Definition Classes
Job
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def colVectorFromTsv(input: String): ColVector[Int, Double]
def config: Map[AnyRef, AnyRef]

Definition Classes
Job
val convergenceThreshold: Double
val currentIteration: Int
val d: Double
implicit def dateParser: DateParser

Definition Classes
Job
def defaultComparator: Option[Class[_ <: Comparator[_]]]

Definition Classes
Job
def defaultMode(fromFields: Fields, toFields: Fields): Fields

Definition Classes
FieldConversions
def defaultSpillThreshold: Int

Definition Classes
Job
val diffLoc: String
val edgesLoc: String
final def ensureUniqueFields(left: Fields, right: Fields, rightPipe: Pipe): (Fields, Pipe)

Definition Classes
FieldConversions
implicit def enumValueToFields(x: Value): Fields

Definition Classes
FieldConversions
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
implicit def fieldFields[T <: TraversableOnce[Field[_]]](f: T): RichFields

Definition Classes
FieldConversions
implicit def fieldToFields(f: Field[_]): RichFields

Definition Classes
FieldConversions
implicit def fields[T <: TraversableOnce[Symbol]](f: T): Fields

Definition Classes
FieldConversions
implicit def fieldsToRichFields(fields: Fields): RichFields

Definition Classes
FieldConversions
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
implicit val flowDef: FlowDef

Attributes
protected
Definition Classes
Job
implicit def fromEnum[T <: Enumeration](enumeration: T): Fields

Definition Classes
FieldConversions
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getField(f: Fields, idx: Int): Fields

Definition Classes
FieldConversions
def handleStats(statsData: CascadingStats): Unit

Attributes
protected
Definition Classes
Job
def hasInts(f: Fields): Boolean

Definition Classes
FieldConversions
def hashCode(): Int

Definition Classes
AnyRef → Any
implicit def intFields[T <: TraversableOnce[Int]](f: T): Fields

Definition Classes
FieldConversions
implicit def intToFields(x: Int): Fields

Definition Classes
FieldConversions
implicit def integerToFields(x: Integer): Fields

Definition Classes
FieldConversions
def ioSerializations: List[Class[_ <: Serialization[_]]]

Definition Classes
Job
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
implicit def iterableToRichPipe[T](iter: Iterable[T])(implicit set: TupleSetter[T], conv: TupleConverter[T]): RichPipe

Definition Classes
Job
val iterationsDir: String
def keepAlive(): Unit

Definition Classes
Job
def listeners: List[FlowListener]

Definition Classes
Job
def matrixFromTsv(input: String): Matrix[Int, Int, Double]
val maxIterations: Int
def measureConvergenceAndStore(): Unit

Measure convergence by calculating the total of the absolute difference between the previous and next vectors.
Measure convergence by calculating the total of the absolute difference between the previous and next vectors. This stores the result after calculation.
implicit def mode: Mode

Definition Classes
Job
val n: Int
def name: String

Definition Classes
Job
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def newSymbol(avoid: Set[Symbol], guess: Symbol, trial: Int): Symbol

Definition Classes
FieldConversions
Annotations
@tailrec()
def next: Option[Job]

Recurse and iterate again iff we are under the max number of iterations and vector has not converged.
Recurse and iterate again iff we are under the max number of iterations and vector has not converged.

Definition Classes
WeightedPageRankFromMatrix → Job
val nextVector: ColVector[Int, Double]
val nextVectorLoc: String
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val onesVectorLoc: String
implicit def parseAnySeqToFields[T <: TraversableOnce[Any]](anyf: T): Fields

Definition Classes
FieldConversions
implicit def pipeToRichPipe(pipe: Pipe): RichPipe

Definition Classes
Job
val previousVector: ColVector[Int, Double]
val previousVectorLoc: String
def priorVector: ColVector[Int, Double]

Load or generate on first iteration the prior vector given d and n.
implicit def productToFields(f: Product): Fields

Definition Classes
LowPriorityFieldConversions
implicit def read(src: Source): Pipe

Definition Classes
Job
val rootDir: String
def run(): Boolean

Definition Classes
Job
implicit def scaldingConfig: Config

Attributes
protected
Definition Classes
Job
def skipStrategy: Option[FlowSkipStrategy]

Definition Classes
Job
implicit def sourceToRichPipe(src: Source): RichPipe

Definition Classes
Job
def stepListeners: List[FlowStepListener]

Definition Classes
Job
def stepStrategy: Option[FlowStepStrategy[_]]

Definition Classes
Job
implicit def strFields[T <: TraversableOnce[String]](f: T): Fields

Definition Classes
FieldConversions
implicit def stringToFields(x: String): Fields

Definition Classes
FieldConversions
implicit def symbolToFields(x: Symbol): Fields

Definition Classes
FieldConversions
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def timeout[T](timeout: AbsoluteDuration)(t: ⇒ T): Option[T]

Definition Classes
Job
implicit def toPipe[T](iter: Iterable[T])(implicit set: TupleSetter[T], conv: TupleConverter[T]): Pipe

Definition Classes
Job
def toString(): String

Definition Classes
AnyRef → Any
implicit def tuple2ToFieldsPair[T, U](pair: (T, U))(implicit tf: (T) ⇒ Fields, uf: (U) ⇒ Fields): (Fields, Fields)

Definition Classes
FieldConversions
implicit def unitToFields(u: Unit): Fields

Definition Classes
FieldConversions
def validate(): Unit

Definition Classes
Job
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def write(pipe: Pipe, src: Source): Unit

Definition Classes
Job

Related Doc: package examples

class WeightedPageRankFromMatrix extends Job

Instance Constructors

new WeightedPageRankFromMatrix(args: Args)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def M_hat: Matrix[Int, Int, Double]

implicit def _implicitJobArgs: Args

def anyToFieldArg(f: Any): Comparable[_]

val args: Args

final def asInstanceOf[T0]: T0

def asList(f: Fields): List[Comparable[_]]

def asSet(f: Fields): Set[Comparable[_]]

def buildFlow: Flow[_]

def classIdentifier: String

def clear(): Unit

def clone(nextargs: Args): Job

def clone(): AnyRef

def colVectorFromTsv(input: String): ColVector[Int, Double]

def config: Map[AnyRef, AnyRef]

val convergenceThreshold: Double

val currentIteration: Int

val d: Double

implicit def dateParser: DateParser

def defaultComparator: Option[Class[_ <: Comparator[_]]]

def defaultMode(fromFields: Fields, toFields: Fields): Fields

def defaultSpillThreshold: Int

val diffLoc: String

val edgesLoc: String

final def ensureUniqueFields(left: Fields, right: Fields, rightPipe: Pipe): (Fields, Pipe)

implicit def enumValueToFields(x: Value): Fields

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

implicit def fieldFields[T <: TraversableOnce[Field[_]]](f: T): RichFields

implicit def fieldToFields(f: Field[_]): RichFields

implicit def fields[T <: TraversableOnce[Symbol]](f: T): Fields

implicit def fieldsToRichFields(fields: Fields): RichFields

def finalize(): Unit

implicit val flowDef: FlowDef

implicit def fromEnum[T <: Enumeration](enumeration: T): Fields

final def getClass(): Class[_]

def getField(f: Fields, idx: Int): Fields

def handleStats(statsData: CascadingStats): Unit

def hasInts(f: Fields): Boolean

def hashCode(): Int

implicit def intFields[T <: TraversableOnce[Int]](f: T): Fields

implicit def intToFields(x: Int): Fields

implicit def integerToFields(x: Integer): Fields

def ioSerializations: List[Class[_ <: Serialization[_]]]

final def isInstanceOf[T0]: Boolean

implicit def iterableToRichPipe[T](iter: Iterable[T])(implicit set: TupleSetter[T], conv: TupleConverter[T]): RichPipe

val iterationsDir: String

def keepAlive(): Unit

def listeners: List[FlowListener]

def matrixFromTsv(input: String): Matrix[Int, Int, Double]

val maxIterations: Int

def measureConvergenceAndStore(): Unit

implicit def mode: Mode

val n: Int

def name: String

final def ne(arg0: AnyRef): Boolean

final def newSymbol(avoid: Set[Symbol], guess: Symbol, trial: Int): Symbol

def next: Option[Job]

val nextVector: ColVector[Int, Double]

val nextVectorLoc: String

final def notify(): Unit

final def notifyAll(): Unit

val onesVectorLoc: String

implicit def parseAnySeqToFields[T <: TraversableOnce[Any]](anyf: T): Fields

implicit def pipeToRichPipe(pipe: Pipe): RichPipe

val previousVector: ColVector[Int, Double]

val previousVectorLoc: String

def priorVector: ColVector[Int, Double]

implicit def productToFields(f: Product): Fields

implicit def read(src: Source): Pipe

val rootDir: String

def run(): Boolean