YakeModel

Instance Constructors

new YakeModel()

Annotator reference id.
Annotator reference id. Used to identify elements in metadata or to refer to this annotator type
new YakeModel(uid: String)

Type Members

type AnnotationContent = Seq[Row]

Attributes
protected
Definition Classes
AnnotatorModel
type AnnotatorType = String

Definition Classes
HasOutputAnnotatorType

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def $[T](param: Param[T]): T

Attributes
protected
Definition Classes
Params
def $$[T](feature: StructFeature[T]): T

Attributes
protected
Definition Classes
HasFeatures
def $$[K, V](feature: MapFeature[K, V]): Map[K, V]

Attributes
protected
Definition Classes
HasFeatures
def $$[T](feature: SetFeature[T]): Set[T]

Attributes
protected
Definition Classes
HasFeatures
def $$[T](feature: ArrayFeature[T]): Array[T]

Attributes
protected
Definition Classes
HasFeatures
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def _transform(dataset: Dataset[_], recursivePipeline: Option[PipelineModel]): DataFrame

Attributes
protected
Definition Classes
AnnotatorModel
def afterAnnotate(dataset: DataFrame): DataFrame

Attributes
protected
Definition Classes
AnnotatorModel
def annotate(annotations: Seq[Annotation]): Seq[Annotation]

Definition Classes
YakeModel → AnnotatorModel
final def asInstanceOf[T0]: T0

Definition Classes
Any
def assignTags(resultFlattenIndexed: Array[(String, Int)]): Array[(String, Int, Int, String)]
def beforeAnnotate(dataset: Dataset[_]): Dataset[_]

Attributes
protected
Definition Classes
AnnotatorModel
def calculateTokenScores(basicStats: Array[(String, Int)], coOccurLeftAggregate: Map[String, Map[String, Int]], coOccurRightAggregate: Map[String, Map[String, Int]]): Iterable[Token]

Calculate token scores given statistics
Calculate token scores given statistics
Refer YAKE Paper
T Position = ln ( ln ( 3 + Median(Sentence Index)) T Case = max(TF(U(t)) , TF(A(t))) / ln(TF(t)) TF Norm =TF(t) / (MeanTF + 1 ∗ σ) T Rel = 1 + ( DL + DR ) * TF(t)/MaxTF T Sentence = SF(t)/# Sentences TS = ( TPos ∗ TRel ) / ( TCase + (( TFNorm + TSent ) / TRel ))
basicStats
Basic stats
coOccurLeftAggregate
Left Co Occurrence
coOccurRightAggregate
Right Co Occurrence
final def checkSchema(schema: StructType, inputAnnotatorType: String): Boolean

Attributes
protected
Definition Classes
HasInputAnnotationCols
final def clear(param: Param[_]): YakeModel.this.type

Definition Classes
Params
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def copy(extra: ParamMap): YakeModel

Definition Classes
RawAnnotator → Model → Transformer → PipelineStage → Params
def copyValues[T <: Params](to: T, extra: ParamMap): T

Attributes
protected
Definition Classes
Params
final def defaultCopy[T <: Params](extra: ParamMap): T

Attributes
protected
Definition Classes
Params
def dfAnnotate: UserDefinedFunction

Attributes
protected
Definition Classes
AnnotatorModel
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def explainParam(param: Param[_]): String

Definition Classes
Params
def explainParams(): String

Definition Classes
Params
def extraValidate(structType: StructType): Boolean

Attributes
protected
Definition Classes
RawAnnotator
def extraValidateMsg: String

Attributes
protected
Definition Classes
RawAnnotator
final def extractParamMap(): ParamMap

Definition Classes
Params
final def extractParamMap(extra: ParamMap): ParamMap

Definition Classes
Params
val features: ArrayBuffer[Feature[_, _, _]]

Definition Classes
HasFeatures
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def get[T](feature: StructFeature[T]): Option[T]

Attributes
protected
Definition Classes
HasFeatures
def get[K, V](feature: MapFeature[K, V]): Option[Map[K, V]]

Attributes
protected
Definition Classes
HasFeatures
def get[T](feature: SetFeature[T]): Option[Set[T]]

Attributes
protected
Definition Classes
HasFeatures
def get[T](feature: ArrayFeature[T]): Option[Array[T]]

Attributes
protected
Definition Classes
HasFeatures
final def get[T](param: Param[T]): Option[T]

Definition Classes
Params
def getBasicStats(result: Array[Annotation]): Array[(String, Int)]

Calculates basic statistics like total Sentences in the document and assign a tag for each token
Calculates basic statistics like total Sentences in the document and assign a tag for each token
result
Document to annotate as array of tokens with sentence metadata
returns
Dataframe with columns SentenceID, token, totalSentences, tag
def getCandidateKeywords(sentences: Array[(String, Int, Int, String)]): Map[String, Int]

Generate candidate keywords
Generate candidate keywords
sentences
sentences as a list
returns
candidate keywords
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getCoOccurrence(sentences: ListBuffer[ListBuffer[String]], left: Boolean): Map[String, Map[String, Int]]

Calculate Co Occurrence for left to right given a window size
Calculate Co Occurrence for left to right given a window size
sentences
DataFrame with tokens
returns
Co Occurrence for token x from left to right as a Map
final def getDefault[T](param: Param[T]): Option[T]

Definition Classes
Params
def getInputCols: Array[String]

Definition Classes
HasInputAnnotationCols
def getKeywords(candidate: Map[String, Int], tokens: Iterable[Token]): ListMap[String, Double]

Extract keywords
Extract keywords
candidate
candidate keywords
tokens
tokens with scores
returns
keywords
def getLazyAnnotator: Boolean

Definition Classes
CanBeLazy
final def getOrDefault[T](param: Param[T]): T

Definition Classes
Params
final def getOutputCol: String

Definition Classes
HasOutputAnnotationCol
def getParam(paramName: String): Param[Any]

Definition Classes
Params
def getSentences(tokenizedArray: Array[Annotation]): ListBuffer[ListBuffer[String]]

Separate sentences given tokens with sentence metadata
Separate sentences given tokens with sentence metadata
tokenizedArray
Tokens with sentence metadata
returns
separated sentences
def getStopWords: Array[String]

Definition Classes
YakeParams
final def hasDefault[T](param: Param[T]): Boolean

Definition Classes
Params
def hasParam(paramName: String): Boolean

Definition Classes
Params
def hasParent: Boolean

Definition Classes
Model
def hashCode(): Int

Definition Classes
AnyRef → Any
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

Attributes
protected
Definition Classes
Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
val inputAnnotatorTypes: Array[AnnotatorType]

Definition Classes
YakeModel → HasInputAnnotationCols
final val inputCols: StringArrayParam

Attributes
protected
Definition Classes
HasInputAnnotationCols
final def isDefined(param: Param[_]): Boolean

Definition Classes
Params
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def isSet(param: Param[_]): Boolean

Definition Classes
Params
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
val lazyAnnotator: BooleanParam

Definition Classes
CanBeLazy
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
val maxNGrams: IntParam

Definition Classes
YakeParams
val minNGrams: IntParam

Definition Classes
YakeParams
def msgHelper(schema: StructType): String

Attributes
protected
Definition Classes
HasInputAnnotationCols
val nKeywords: IntParam

Definition Classes
YakeParams
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def onWrite(path: String, spark: SparkSession): Unit

Attributes
protected
Definition Classes
ParamsAndFeaturesWritable
val outputAnnotatorType: AnnotatorType

Definition Classes
YakeModel → HasOutputAnnotatorType
final val outputCol: Param[String]

Attributes
protected
Definition Classes
HasOutputAnnotationCol
lazy val params: Array[Param[_]]

Definition Classes
Params
var parent: Estimator[YakeModel]

Definition Classes
Model
def processSentences(annotations: Seq[Annotation]): Seq[Annotation]

Execute the YAKE algorithm for each sentence
Execute the YAKE algorithm for each sentence
annotations
token array to annotate
returns
annotated token array
def save(path: String): Unit

Definition Classes
MLWritable
Annotations
@Since( "1.6.0" ) @throws( ... )
def set[T](feature: StructFeature[T], value: T): YakeModel.this.type

Attributes
protected
Definition Classes
HasFeatures
def set[K, V](feature: MapFeature[K, V], value: Map[K, V]): YakeModel.this.type

Attributes
protected
Definition Classes
HasFeatures
def set[T](feature: SetFeature[T], value: Set[T]): YakeModel.this.type

Attributes
protected
Definition Classes
HasFeatures
def set[T](feature: ArrayFeature[T], value: Array[T]): YakeModel.this.type

Attributes
protected
Definition Classes
HasFeatures
final def set(paramPair: ParamPair[_]): YakeModel.this.type

Attributes
protected
Definition Classes
Params
final def set(param: String, value: Any): YakeModel.this.type

Attributes
protected
Definition Classes
Params
final def set[T](param: Param[T], value: T): YakeModel.this.type

Definition Classes
Params
def setDefault[T](feature: StructFeature[T], value: () ⇒ T): YakeModel.this.type

Attributes
protected
Definition Classes
HasFeatures
def setDefault[K, V](feature: MapFeature[K, V], value: () ⇒ Map[K, V]): YakeModel.this.type

Attributes
protected
Definition Classes
HasFeatures
def setDefault[T](feature: SetFeature[T], value: () ⇒ Set[T]): YakeModel.this.type

Attributes
protected
Definition Classes
HasFeatures
def setDefault[T](feature: ArrayFeature[T], value: () ⇒ Array[T]): YakeModel.this.type

Attributes
protected
Definition Classes
HasFeatures
final def setDefault(paramPairs: ParamPair[_]*): YakeModel.this.type

Attributes
protected
Definition Classes
Params
final def setDefault[T](param: Param[T], value: T): YakeModel.this.type

Attributes
protected
Definition Classes
Params
final def setInputCols(value: String*): YakeModel.this.type

Definition Classes
HasInputAnnotationCols
final def setInputCols(value: Array[String]): YakeModel.this.type

Definition Classes
HasInputAnnotationCols
def setLazyAnnotator(value: Boolean): YakeModel.this.type

Definition Classes
CanBeLazy
def setMaxNGrams(value: Int): YakeModel.this.type

Definition Classes
YakeParams
def setMinNGrams(value: Int): YakeModel.this.type

Definition Classes
YakeParams
def setNKeywords(value: Int): YakeModel.this.type

Definition Classes
YakeParams
final def setOutputCol(value: String): YakeModel.this.type

Definition Classes
HasOutputAnnotationCol
def setParent(parent: Estimator[YakeModel]): YakeModel

Definition Classes
Model
def setStopWords(value: Array[String]): YakeModel.this.type

Definition Classes
YakeParams
def setThreshold(value: Float): YakeModel.this.type

Definition Classes
YakeParams
def setWindowSize(value: Int): YakeModel.this.type

Definition Classes
YakeParams
val stopWords: StringArrayParam

Definition Classes
YakeParams
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
val threshold: FloatParam

Definition Classes
YakeParams
def toString(): String

Definition Classes
Identifiable → AnyRef → Any
final def transform(dataset: Dataset[_]): DataFrame

Definition Classes
AnnotatorModel → Transformer
def transform(dataset: Dataset[_], paramMap: ParamMap): DataFrame

Definition Classes
Transformer
Annotations
@Since( "2.0.0" )
def transform(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): DataFrame

Definition Classes
Transformer
Annotations
@Since( "2.0.0" ) @varargs()
final def transformSchema(schema: StructType): StructType

Definition Classes
RawAnnotator → PipelineStage
def transformSchema(schema: StructType, logging: Boolean): StructType

Attributes
protected
Definition Classes
PipelineStage
Annotations
@DeveloperApi()
val uid: String

Definition Classes
YakeModel → Identifiable
def validate(schema: StructType): Boolean

Attributes
protected
Definition Classes
RawAnnotator
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
val windowSize: IntParam

Definition Classes
YakeParams
def wrapColumnMetadata(col: Column): Column

Attributes
protected
Definition Classes
RawAnnotator
def write: MLWriter

Definition Classes
ParamsAndFeaturesWritable → DefaultParamsWritable → MLWritable

Related Docs: object YakeModel | package yake

class YakeModel extends AnnotatorModel[YakeModel] with YakeParams

Instance Constructors

new YakeModel()

new YakeModel(uid: String)

Type Members

type AnnotationContent = Seq[Row]

type AnnotatorType = String

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def $[T](param: Param[T]): T

def $$[T](feature: StructFeature[T]): T

def $$[K, V](feature: MapFeature[K, V]): Map[K, V]

def $$[T](feature: SetFeature[T]): Set[T]

def $$[T](feature: ArrayFeature[T]): Array[T]

final def ==(arg0: Any): Boolean

def _transform(dataset: Dataset[_], recursivePipeline: Option[PipelineModel]): DataFrame

def afterAnnotate(dataset: DataFrame): DataFrame

def annotate(annotations: Seq[Annotation]): Seq[Annotation]

final def asInstanceOf[T0]: T0

def assignTags(resultFlattenIndexed: Array[(String, Int)]): Array[(String, Int, Int, String)]

def beforeAnnotate(dataset: Dataset[_]): Dataset[_]

def calculateTokenScores(basicStats: Array[(String, Int)], coOccurLeftAggregate: Map[String, Map[String, Int]], coOccurRightAggregate: Map[String, Map[String, Int]]): Iterable[Token]

final def checkSchema(schema: StructType, inputAnnotatorType: String): Boolean

final def clear(param: Param[_]): YakeModel.this.type

def clone(): AnyRef

def copy(extra: ParamMap): YakeModel

def copyValues[T <: Params](to: T, extra: ParamMap): T

final def defaultCopy[T <: Params](extra: ParamMap): T

def dfAnnotate: UserDefinedFunction

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def explainParam(param: Param[_]): String

def explainParams(): String

def extraValidate(structType: StructType): Boolean

def extraValidateMsg: String

final def extractParamMap(): ParamMap

final def extractParamMap(extra: ParamMap): ParamMap

val features: ArrayBuffer[Feature[_, _, _]]

def finalize(): Unit

def get[T](feature: StructFeature[T]): Option[T]

def get[K, V](feature: MapFeature[K, V]): Option[Map[K, V]]

def get[T](feature: SetFeature[T]): Option[Set[T]]

def get[T](feature: ArrayFeature[T]): Option[Array[T]]

final def get[T](param: Param[T]): Option[T]

def getBasicStats(result: Array[Annotation]): Array[(String, Int)]

def getCandidateKeywords(sentences: Array[(String, Int, Int, String)]): Map[String, Int]

final def getClass(): Class[_]

def getCoOccurrence(sentences: ListBuffer[ListBuffer[String]], left: Boolean): Map[String, Map[String, Int]]

final def getDefault[T](param: Param[T]): Option[T]

def getInputCols: Array[String]

def getKeywords(candidate: Map[String, Int], tokens: Iterable[Token]): ListMap[String, Double]

def getLazyAnnotator: Boolean

final def getOrDefault[T](param: Param[T]): T

final def getOutputCol: String

def getParam(paramName: String): Param[Any]

def getSentences(tokenizedArray: Array[Annotation]): ListBuffer[ListBuffer[String]]

def getStopWords: Array[String]

final def hasDefault[T](param: Param[T]): Boolean

def hasParam(paramName: String): Boolean

def hasParent: Boolean

def hashCode(): Int

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

val inputAnnotatorTypes: Array[AnnotatorType]

final val inputCols: StringArrayParam

final def isDefined(param: Param[_]): Boolean

final def isInstanceOf[T0]: Boolean

final def isSet(param: Param[_]): Boolean

def isTraceEnabled(): Boolean

val lazyAnnotator: BooleanParam

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit