com.johnsnowlabs.finance.sequence_generation

FinanceSummarizer

Companion object FinanceSummarizer

class FinanceSummarizer extends MedicalSummarizer

Linear Supertypes

MedicalSummarizer, MedicalEncoderDecoder, CheckLicense, WriteSentencePieceModel, HasEngine, WriteOnnxModel, WriteTensorflowModel, HasBatchedAnnotate[MedicalEncoderDecoder], HasCaseSensitiveProperties, AnnotatorModel[MedicalEncoderDecoder], CanBeLazy, RawAnnotator[MedicalEncoderDecoder], HasOutputAnnotationCol, HasInputAnnotationCols, HasOutputAnnotatorType, ParamsAndFeaturesWritable, HasFeatures, DefaultParamsWritable, MLWritable, Model[MedicalEncoderDecoder], Transformer, PipelineStage, Logging, Params, Serializable, Serializable, Identifiable, AnyRef, Any

Ordering

Grouped
Alphabetic
By Inheritance

Inherited

FinanceSummarizer
MedicalSummarizer
MedicalEncoderDecoder
CheckLicense
WriteSentencePieceModel
HasEngine
WriteOnnxModel
WriteTensorflowModel
HasBatchedAnnotate
HasCaseSensitiveProperties
AnnotatorModel
CanBeLazy
RawAnnotator
HasOutputAnnotationCol
HasInputAnnotationCols
HasOutputAnnotatorType
ParamsAndFeaturesWritable
HasFeatures
DefaultParamsWritable
MLWritable
Model
Transformer
PipelineStage
Logging
Params
Serializable
Serializable
Identifiable
AnyRef
Any

Hide All
Show All

Visibility

Public
All

Instance Constructors

new FinanceSummarizer()
new FinanceSummarizer(uid: String)

Type Members

type AnnotationContent = Seq[Row]

Attributes
protected
Definition Classes
AnnotatorModel
type AnnotatorType = String

Definition Classes
HasOutputAnnotatorType

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def $[T](param: Param[T]): T

Attributes
protected
Definition Classes
Params
def $$[T](feature: StructFeature[T]): T

Attributes
protected
Definition Classes
HasFeatures
def $$[K, V](feature: MapFeature[K, V]): Map[K, V]

Attributes
protected
Definition Classes
HasFeatures
def $$[T](feature: SetFeature[T]): Set[T]

Attributes
protected
Definition Classes
HasFeatures
def $$[T](feature: ArrayFeature[T]): Array[T]

Attributes
protected
Definition Classes
HasFeatures
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def _transform(dataset: Dataset[_], recursivePipeline: Option[PipelineModel]): DataFrame

Attributes
protected
Definition Classes
AnnotatorModel
def afterAnnotate(dataset: DataFrame): DataFrame

Attributes
protected
Definition Classes
AnnotatorModel
final def asInstanceOf[T0]: T0

Definition Classes
Any
def batchAnnotate(batchedAnnotations: Seq[Array[Annotation]]): Seq[Seq[Annotation]]

Definition Classes
MedicalEncoderDecoder → HasBatchedAnnotate
def batchProcess(rows: Iterator[_]): Iterator[Row]

Definition Classes
HasBatchedAnnotate
val batchSize: IntParam

Definition Classes
HasBatchedAnnotate
def beforeAnnotate(dataset: Dataset[_]): Dataset[_]

Attributes
protected
Definition Classes
AnnotatorModel
val caseSensitive: BooleanParam

Definition Classes
HasCaseSensitiveProperties
final def checkSchema(schema: StructType, inputAnnotatorType: String): Boolean

Attributes
protected
Definition Classes
HasInputAnnotationCols
def checkValidEnvironment(spark: Option[SparkSession], scopes: Seq[String]): Unit

Definition Classes
CheckLicense
def checkValidScope(scope: String): Unit

Definition Classes
CheckLicense
def checkValidScopeAndEnvironment(scope: String, spark: Option[SparkSession], checkLp: Boolean): Unit

Definition Classes
CheckLicense
def checkValidScopesAndEnvironment(scopes: Seq[String], spark: Option[SparkSession], checkLp: Boolean): Unit

Definition Classes
CheckLicense
final def clear(param: Param[_]): FinanceSummarizer.this.type

Definition Classes
Params
def clone(): AnyRef

Attributes
protected[lang]
Definition Classes
AnyRef
Annotations
@throws( ... ) @native()
val configProtoBytes: IntArrayParam
ConfigProto from tensorflow, serialized into byte array.
ConfigProto from tensorflow, serialized into byte array. Get with config_proto.SerializeToString()

Definition Classes
MedicalEncoderDecoder
def copy(extra: ParamMap): MedicalEncoderDecoder

Definition Classes
RawAnnotator → Model → Transformer → PipelineStage → Params
def copyValues[T <: Params](to: T, extra: ParamMap): T

Attributes
protected
Definition Classes
Params
final def defaultCopy[T <: Params](extra: ParamMap): T

Attributes
protected
Definition Classes
Params
val doSample: BooleanParam
Whether or not to use sampling, use greedy decoding otherwise (Default: false)
Whether or not to use sampling, use greedy decoding otherwise (Default: false)

Definition Classes
MedicalEncoderDecoder
val engine: Param[String]

Definition Classes
HasEngine
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def explainParam(param: Param[_]): String

Definition Classes
Params
def explainParams(): String

Definition Classes
Params
def extraValidate(structType: StructType): Boolean

Attributes
protected
Definition Classes
RawAnnotator
def extraValidateMsg: String

Attributes
protected
Definition Classes
RawAnnotator
final def extractParamMap(): ParamMap

Definition Classes
Params
final def extractParamMap(extra: ParamMap): ParamMap

Definition Classes
Params
val features: ArrayBuffer[Feature[_, _, _]]

Definition Classes
HasFeatures
def finalize(): Unit

Attributes
protected[lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def get[T](feature: StructFeature[T]): Option[T]

Attributes
protected
Definition Classes
HasFeatures
def get[K, V](feature: MapFeature[K, V]): Option[Map[K, V]]

Attributes
protected
Definition Classes
HasFeatures
def get[T](feature: SetFeature[T]): Option[Set[T]]

Attributes
protected
Definition Classes
HasFeatures
def get[T](feature: ArrayFeature[T]): Option[Array[T]]

Attributes
protected
Definition Classes
HasFeatures
final def get[T](param: Param[T]): Option[T]

Definition Classes
Params
def getBatchSize: Int

Definition Classes
HasBatchedAnnotate
def getCaseSensitive: Boolean

Definition Classes
HasCaseSensitiveProperties
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
Annotations
@native()
def getConfigProtoBytes: Option[Array[Byte]]

Definition Classes
MedicalEncoderDecoder
final def getDefault[T](param: Param[T]): Option[T]

Definition Classes
Params
def getDoSample: Boolean

Definition Classes
MedicalEncoderDecoder
def getEngine: String

Definition Classes
HasEngine
def getIgnoreTokenIds: Array[Int]

Definition Classes
MedicalEncoderDecoder
def getInputCols: Array[String]

Definition Classes
HasInputAnnotationCols
def getLazyAnnotator: Boolean

Definition Classes
CanBeLazy
def getMaxNewTokens: Int

Definition Classes
MedicalEncoderDecoder
def getMaxTextLength: Int

Definition Classes
MedicalEncoderDecoder
def getMlFrameworkType: String
Get ML framework type
Get ML framework type

Definition Classes
MedicalEncoderDecoder
def getModelIfNotSet: MedicalEncoderDecoderModel

Definition Classes
MedicalEncoderDecoder
def getNoRepeatNgramSize: Int

Definition Classes
MedicalEncoderDecoder
final def getOrDefault[T](param: Param[T]): T

Definition Classes
Params
final def getOutputCol: String

Definition Classes
HasOutputAnnotationCol
def getParam(paramName: String): Param[Any]

Definition Classes
Params
def getRandomSeed: Option[Int]

Definition Classes
MedicalEncoderDecoder
def getRefineChunkSize: Int

Definition Classes
MedicalEncoderDecoder
def getRefineMaxAttempts: Int

Definition Classes
MedicalEncoderDecoder
def getRefineSummary: Boolean

Definition Classes
MedicalEncoderDecoder
def getRefineSummaryTargetLength: Int

Definition Classes
MedicalEncoderDecoder
def getSignatures: Option[Map[String, String]]

Definition Classes
MedicalEncoderDecoder
def getStopAtEos: Boolean
Checks whether text generation stops when the end-of-sentence token is encountered.
Checks whether text generation stops when the end-of-sentence token is encountered.

Definition Classes
MedicalSummarizer → MedicalEncoderDecoder
def getTask: String

Attributes
protected
Definition Classes
MedicalEncoderDecoder
def getTopK: Int

Attributes
protected
Definition Classes
MedicalEncoderDecoder
def getUseCache: Boolean

Definition Classes
MedicalEncoderDecoder
final def hasDefault[T](param: Param[T]): Boolean

Definition Classes
Params
def hasParam(paramName: String): Boolean

Definition Classes
Params
def hasParent: Boolean

Definition Classes
Model
def hashCode(): Int

Definition Classes
AnyRef → Any
Annotations
@native()
var ignoreTokenIds: IntArrayParam
A list of token ids which are ignored in the decoder's output (Default: Array())
A list of token ids which are ignored in the decoder's output (Default: Array())

Definition Classes
MedicalEncoderDecoder
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

Attributes
protected
Definition Classes
Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
val inputAnnotatorTypes: Array[AnnotatorType]
Input annotator type : DOCUMENT
Input annotator type : DOCUMENT

Definition Classes
MedicalEncoderDecoder → HasInputAnnotationCols
final val inputCols: StringArrayParam

Attributes
protected
Definition Classes
HasInputAnnotationCols
final def isDefined(param: Param[_]): Boolean

Definition Classes
Params
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def isSet(param: Param[_]): Boolean

Definition Classes
Params
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
val lazyAnnotator: BooleanParam

Definition Classes
CanBeLazy
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
val maxNewTokens: IntParam
Maximum number of new tokens to be generated (Default: 30)
Maximum number of new tokens to be generated (Default: 30)

Definition Classes
MedicalEncoderDecoder
val maxTextLength: IntParam
Maximum length of context text.
Maximum length of context text. (Default: 1024)

Definition Classes
MedicalEncoderDecoder
val mlFrameworkType: Param[String]
ML framework type
ML framework type

Definition Classes
MedicalEncoderDecoder
def msgHelper(schema: StructType): String

Attributes
protected
Definition Classes
HasInputAnnotationCols
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
val noRepeatNgramSize: IntParam
If set to int > 0, all ngrams of that size can only occur once (Default: 0)
If set to int > 0, all ngrams of that size can only occur once (Default: 0)

Definition Classes
MedicalEncoderDecoder
final def notify(): Unit

Definition Classes
AnyRef
Annotations
@native()
final def notifyAll(): Unit

Definition Classes
AnyRef
Annotations
@native()
def onWrite(path: String, spark: SparkSession): Unit

Definition Classes
MedicalSummarizer → MedicalEncoderDecoder → ParamsAndFeaturesWritable
def onWriteSkip(path: String, spark: SparkSession): Unit

Attributes
protected
Definition Classes
MedicalEncoderDecoder
val optionalInputAnnotatorTypes: Array[String]

Definition Classes
HasInputAnnotationCols
val outputAnnotatorType: String
Output annotator type : DOCUMENT
Output annotator type : DOCUMENT

Definition Classes
MedicalEncoderDecoder → HasOutputAnnotatorType
final val outputCol: Param[String]

Attributes
protected
Definition Classes
HasOutputAnnotationCol
lazy val params: Array[Param[_]]

Definition Classes
Params
var parent: Estimator[MedicalEncoderDecoder]

Definition Classes
Model
var randomSeed: Option[Int]
Optional Random seed for the model.
Optional Random seed for the model. Needs to be of type Long.

Definition Classes
MedicalEncoderDecoder
val refineChunkSize: IntParam
How large should refined chunks Be.
How large should refined chunks Be. Should be equal to LLM context window size in tokens. Takes only effect when refineSummary=True

Definition Classes
MedicalEncoderDecoder
val refineMaxAttempts: IntParam
How many times should chunks be re-summarized while they are above SummaryTargetLength before stopping.
How many times should chunks be re-summarized while they are above SummaryTargetLength before stopping. Takes only effect when refineSummary=True

Definition Classes
MedicalEncoderDecoder
val refineSummary: BooleanParam
Set true to perform refined summarization at increased computation cost.
Set true to perform refined summarization at increased computation cost.

Definition Classes
MedicalEncoderDecoder
val refineSummaryTargetLength: IntParam
Target length for refined summary.
Target length for refined summary. Text will be chunked and re-summarized until it is below this length or maximum attempts are used up.

Definition Classes
MedicalEncoderDecoder
def save(path: String): Unit

Definition Classes
MLWritable
Annotations
@Since( "1.6.0" ) @throws( ... )
def set[T](feature: StructFeature[T], value: T): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def set[K, V](feature: MapFeature[K, V], value: Map[K, V]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def set[T](feature: SetFeature[T], value: Set[T]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def set[T](feature: ArrayFeature[T], value: Array[T]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
final def set(paramPair: ParamPair[_]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
Params
final def set(param: String, value: Any): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
Params
final def set[T](param: Param[T], value: T): FinanceSummarizer.this.type

Definition Classes
Params
def setBatchSize(size: Int): FinanceSummarizer.this.type

Definition Classes
HasBatchedAnnotate
def setCaseSensitive(value: Boolean): FinanceSummarizer.this.type

Definition Classes
HasCaseSensitiveProperties
def setConfigProtoBytes(bytes: Array[Int]): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setDefault[T](feature: StructFeature[T], value: () ⇒ T): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def setDefault[K, V](feature: MapFeature[K, V], value: () ⇒ Map[K, V]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def setDefault[T](feature: SetFeature[T], value: () ⇒ Set[T]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def setDefault[T](feature: ArrayFeature[T], value: () ⇒ Array[T]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
final def setDefault(paramPairs: ParamPair[_]*): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
Params
final def setDefault[T](param: Param[T], value: T): FinanceSummarizer.this.type

Attributes
protected[org.apache.spark.ml]
Definition Classes
Params
def setDoSample(value: Boolean): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setIgnoreTokenIds(tokenIds: Array[Int]): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
final def setInputCols(value: String*): FinanceSummarizer.this.type

Definition Classes
HasInputAnnotationCols
def setInputCols(value: Array[String]): FinanceSummarizer.this.type

Definition Classes
HasInputAnnotationCols
def setLazyAnnotator(value: Boolean): FinanceSummarizer.this.type

Definition Classes
CanBeLazy
def setMaxNewTokens(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setMaxTextLength(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setMlFrameworkType(value: String): FinanceSummarizer.this.type
Set ML framework type
Set ML framework type

Definition Classes
MedicalEncoderDecoder
def setModelIfNotSet(spark: SparkSession, encoder: OnnxWrapper, decoder: OnnxWrapper, spp: SentencePieceWrapper): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setModelIfNotSet(spark: SparkSession, tfWrapper: TensorflowWrapper, spp: SentencePieceWrapper, useCache: Boolean): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setNoRepeatNgramSize(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
final def setOutputCol(value: String): FinanceSummarizer.this.type

Definition Classes
HasOutputAnnotationCol
def setParent(parent: Estimator[MedicalEncoderDecoder]): MedicalEncoderDecoder

Definition Classes
Model
def setRandomSeed(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setRefineChunkSize(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setRefineMaxAttempts(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setRefineSummary(value: Boolean): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setRefineSummaryTargetLength(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setSignatures(value: Map[String, String]): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setStopAtEos(value: Boolean): FinanceSummarizer.this.type
Determines whether text generation stops when the end-of-sentence token is encountered.
Determines whether text generation stops when the end-of-sentence token is encountered.

Definition Classes
MedicalEncoderDecoder
def setTask(value: String): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
MedicalEncoderDecoder
def setTopK(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setUseCache(value: Boolean): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
MedicalEncoderDecoder
val signatures: MapFeature[String, String]

Definition Classes
MedicalEncoderDecoder
val stopAtEos: BooleanParam

Definition Classes
MedicalEncoderDecoder
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
val task: Param[String]
Set transformer task, e.g.
Set transformer task, e.g. "summarize:" (Default: ""). The T5 task needs to be in the format "task:".

Definition Classes
MedicalEncoderDecoder
def toString(): String

Definition Classes
Identifiable → AnyRef → Any
val topK: IntParam
The number of highest probability vocabulary tokens to keep for top-k-filtering (Default: 50)
The number of highest probability vocabulary tokens to keep for top-k-filtering (Default: 50)

Definition Classes
MedicalEncoderDecoder
final def transform(dataset: Dataset[_]): DataFrame

Definition Classes
AnnotatorModel → Transformer
def transform(dataset: Dataset[_], paramMap: ParamMap): DataFrame

Definition Classes
Transformer
Annotations
@Since( "2.0.0" )
def transform(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): DataFrame

Definition Classes
Transformer
Annotations
@Since( "2.0.0" ) @varargs()
final def transformSchema(schema: StructType): StructType

Definition Classes
RawAnnotator → PipelineStage
def transformSchema(schema: StructType, logging: Boolean): StructType

Attributes
protected
Definition Classes
PipelineStage
Annotations
@DeveloperApi()
val uid: String

Definition Classes
FinanceSummarizer → MedicalSummarizer → MedicalEncoderDecoder → Identifiable
val useCache: BooleanParam
Cache internal state of the model to improve performance
Cache internal state of the model to improve performance

Definition Classes
MedicalEncoderDecoder
def validate(schema: StructType): Boolean

Attributes
protected
Definition Classes
RawAnnotator
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... ) @native()
def wrapColumnMetadata(col: Column): Column

Attributes
protected
Definition Classes
RawAnnotator
def write: MLWriter

Definition Classes
ParamsAndFeaturesWritable → DefaultParamsWritable → MLWritable
def writeOnnxModel(path: String, spark: SparkSession, onnxWrapper: OnnxWrapper, suffix: String, fileName: String): Unit

Definition Classes
WriteOnnxModel
def writeOnnxModels(path: String, spark: SparkSession, onnxWrappersWithNames: Seq[(OnnxWrapper, String)], suffix: String): Unit

Definition Classes
WriteOnnxModel
def writeSentencePieceModel(path: String, spark: SparkSession, spp: SentencePieceWrapper, suffix: String, filename: String): Unit

Definition Classes
WriteSentencePieceModel
def writeTensorflowHub(path: String, tfPath: String, spark: SparkSession, suffix: String): Unit

Definition Classes
WriteTensorflowModel
def writeTensorflowModel(path: String, spark: SparkSession, tensorflow: TensorflowWrapper, suffix: String, filename: String, configProtoBytes: Option[Array[Byte]]): Unit

Definition Classes
WriteTensorflowModel
def writeTensorflowModelV2(path: String, spark: SparkSession, tensorflow: TensorflowWrapper, suffix: String, filename: String, configProtoBytes: Option[Array[Byte]], savedSignatures: Option[Map[String, String]]): Unit

Definition Classes
WriteTensorflowModel

Inherited from MedicalSummarizer

Value Members

def getStopAtEos: Boolean
Checks whether text generation stops when the end-of-sentence token is encountered.
Checks whether text generation stops when the end-of-sentence token is encountered.

Definition Classes
MedicalSummarizer → MedicalEncoderDecoder
def onWrite(path: String, spark: SparkSession): Unit

Definition Classes
MedicalSummarizer → MedicalEncoderDecoder → ParamsAndFeaturesWritable

Inherited from MedicalEncoderDecoder

Value Members

def batchAnnotate(batchedAnnotations: Seq[Array[Annotation]]): Seq[Seq[Annotation]]

Definition Classes
MedicalEncoderDecoder → HasBatchedAnnotate
val configProtoBytes: IntArrayParam
ConfigProto from tensorflow, serialized into byte array.
ConfigProto from tensorflow, serialized into byte array. Get with config_proto.SerializeToString()

Definition Classes
MedicalEncoderDecoder
val doSample: BooleanParam
Whether or not to use sampling, use greedy decoding otherwise (Default: false)
Whether or not to use sampling, use greedy decoding otherwise (Default: false)

Definition Classes
MedicalEncoderDecoder
def getConfigProtoBytes: Option[Array[Byte]]

Definition Classes
MedicalEncoderDecoder
def getDoSample: Boolean

Definition Classes
MedicalEncoderDecoder
def getIgnoreTokenIds: Array[Int]

Definition Classes
MedicalEncoderDecoder
def getMaxNewTokens: Int

Definition Classes
MedicalEncoderDecoder
def getMaxTextLength: Int

Definition Classes
MedicalEncoderDecoder
def getMlFrameworkType: String
Get ML framework type
Get ML framework type

Definition Classes
MedicalEncoderDecoder
def getModelIfNotSet: MedicalEncoderDecoderModel

Definition Classes
MedicalEncoderDecoder
def getNoRepeatNgramSize: Int

Definition Classes
MedicalEncoderDecoder
def getRandomSeed: Option[Int]

Definition Classes
MedicalEncoderDecoder
def getRefineChunkSize: Int

Definition Classes
MedicalEncoderDecoder
def getRefineMaxAttempts: Int

Definition Classes
MedicalEncoderDecoder
def getRefineSummary: Boolean

Definition Classes
MedicalEncoderDecoder
def getRefineSummaryTargetLength: Int

Definition Classes
MedicalEncoderDecoder
def getSignatures: Option[Map[String, String]]

Definition Classes
MedicalEncoderDecoder
def getTask: String

Attributes
protected
Definition Classes
MedicalEncoderDecoder
def getTopK: Int

Attributes
protected
Definition Classes
MedicalEncoderDecoder
def getUseCache: Boolean

Definition Classes
MedicalEncoderDecoder
var ignoreTokenIds: IntArrayParam
A list of token ids which are ignored in the decoder's output (Default: Array())
A list of token ids which are ignored in the decoder's output (Default: Array())

Definition Classes
MedicalEncoderDecoder
val inputAnnotatorTypes: Array[AnnotatorType]
Input annotator type : DOCUMENT
Input annotator type : DOCUMENT

Definition Classes
MedicalEncoderDecoder → HasInputAnnotationCols
val maxNewTokens: IntParam
Maximum number of new tokens to be generated (Default: 30)
Maximum number of new tokens to be generated (Default: 30)

Definition Classes
MedicalEncoderDecoder
val maxTextLength: IntParam
Maximum length of context text.
Maximum length of context text. (Default: 1024)

Definition Classes
MedicalEncoderDecoder
val mlFrameworkType: Param[String]
ML framework type
ML framework type

Definition Classes
MedicalEncoderDecoder
val noRepeatNgramSize: IntParam
If set to int > 0, all ngrams of that size can only occur once (Default: 0)
If set to int > 0, all ngrams of that size can only occur once (Default: 0)

Definition Classes
MedicalEncoderDecoder
def onWriteSkip(path: String, spark: SparkSession): Unit

Attributes
protected
Definition Classes
MedicalEncoderDecoder
val outputAnnotatorType: String
Output annotator type : DOCUMENT
Output annotator type : DOCUMENT

Definition Classes
MedicalEncoderDecoder → HasOutputAnnotatorType
var randomSeed: Option[Int]
Optional Random seed for the model.
Optional Random seed for the model. Needs to be of type Long.

Definition Classes
MedicalEncoderDecoder
val refineChunkSize: IntParam
How large should refined chunks Be.
How large should refined chunks Be. Should be equal to LLM context window size in tokens. Takes only effect when refineSummary=True

Definition Classes
MedicalEncoderDecoder
val refineMaxAttempts: IntParam
How many times should chunks be re-summarized while they are above SummaryTargetLength before stopping.
How many times should chunks be re-summarized while they are above SummaryTargetLength before stopping. Takes only effect when refineSummary=True

Definition Classes
MedicalEncoderDecoder
val refineSummary: BooleanParam
Set true to perform refined summarization at increased computation cost.
Set true to perform refined summarization at increased computation cost.

Definition Classes
MedicalEncoderDecoder
val refineSummaryTargetLength: IntParam
Target length for refined summary.
Target length for refined summary. Text will be chunked and re-summarized until it is below this length or maximum attempts are used up.

Definition Classes
MedicalEncoderDecoder
def setConfigProtoBytes(bytes: Array[Int]): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setDoSample(value: Boolean): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setIgnoreTokenIds(tokenIds: Array[Int]): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setMaxNewTokens(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setMaxTextLength(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setMlFrameworkType(value: String): FinanceSummarizer.this.type
Set ML framework type
Set ML framework type

Definition Classes
MedicalEncoderDecoder
def setModelIfNotSet(spark: SparkSession, encoder: OnnxWrapper, decoder: OnnxWrapper, spp: SentencePieceWrapper): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setModelIfNotSet(spark: SparkSession, tfWrapper: TensorflowWrapper, spp: SentencePieceWrapper, useCache: Boolean): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setNoRepeatNgramSize(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setRandomSeed(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setRefineChunkSize(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setRefineMaxAttempts(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setRefineSummary(value: Boolean): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setRefineSummaryTargetLength(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setSignatures(value: Map[String, String]): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setStopAtEos(value: Boolean): FinanceSummarizer.this.type
Determines whether text generation stops when the end-of-sentence token is encountered.
Determines whether text generation stops when the end-of-sentence token is encountered.

Definition Classes
MedicalEncoderDecoder
def setTask(value: String): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
MedicalEncoderDecoder
def setTopK(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setUseCache(value: Boolean): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
MedicalEncoderDecoder
val signatures: MapFeature[String, String]

Definition Classes
MedicalEncoderDecoder
val stopAtEos: BooleanParam

Definition Classes
MedicalEncoderDecoder
val task: Param[String]
Set transformer task, e.g.
Set transformer task, e.g. "summarize:" (Default: ""). The T5 task needs to be in the format "task:".

Definition Classes
MedicalEncoderDecoder
val topK: IntParam
The number of highest probability vocabulary tokens to keep for top-k-filtering (Default: 50)
The number of highest probability vocabulary tokens to keep for top-k-filtering (Default: 50)

Definition Classes
MedicalEncoderDecoder
val useCache: BooleanParam
Cache internal state of the model to improve performance
Cache internal state of the model to improve performance

Definition Classes
MedicalEncoderDecoder

Inherited from CheckLicense

Value Members

def checkValidEnvironment(spark: Option[SparkSession], scopes: Seq[String]): Unit

Definition Classes
CheckLicense
def checkValidScope(scope: String): Unit

Definition Classes
CheckLicense
def checkValidScopeAndEnvironment(scope: String, spark: Option[SparkSession], checkLp: Boolean): Unit

Definition Classes
CheckLicense
def checkValidScopesAndEnvironment(scopes: Seq[String], spark: Option[SparkSession], checkLp: Boolean): Unit

Definition Classes
CheckLicense

Inherited from WriteSentencePieceModel

Value Members

def writeSentencePieceModel(path: String, spark: SparkSession, spp: SentencePieceWrapper, suffix: String, filename: String): Unit

Definition Classes
WriteSentencePieceModel

Inherited from HasEngine

Value Members

val engine: Param[String]

Definition Classes
HasEngine
def getEngine: String

Definition Classes
HasEngine

Inherited from WriteOnnxModel

Value Members

def writeOnnxModel(path: String, spark: SparkSession, onnxWrapper: OnnxWrapper, suffix: String, fileName: String): Unit

Definition Classes
WriteOnnxModel
def writeOnnxModels(path: String, spark: SparkSession, onnxWrappersWithNames: Seq[(OnnxWrapper, String)], suffix: String): Unit

Definition Classes
WriteOnnxModel

Inherited from WriteTensorflowModel

Value Members

def writeTensorflowHub(path: String, tfPath: String, spark: SparkSession, suffix: String): Unit

Definition Classes
WriteTensorflowModel
def writeTensorflowModel(path: String, spark: SparkSession, tensorflow: TensorflowWrapper, suffix: String, filename: String, configProtoBytes: Option[Array[Byte]]): Unit

Definition Classes
WriteTensorflowModel
def writeTensorflowModelV2(path: String, spark: SparkSession, tensorflow: TensorflowWrapper, suffix: String, filename: String, configProtoBytes: Option[Array[Byte]], savedSignatures: Option[Map[String, String]]): Unit

Definition Classes
WriteTensorflowModel

Inherited from HasBatchedAnnotate[MedicalEncoderDecoder]

Value Members

def batchProcess(rows: Iterator[_]): Iterator[Row]

Definition Classes
HasBatchedAnnotate
val batchSize: IntParam

Definition Classes
HasBatchedAnnotate
def getBatchSize: Int

Definition Classes
HasBatchedAnnotate
def setBatchSize(size: Int): FinanceSummarizer.this.type

Definition Classes
HasBatchedAnnotate

Inherited from HasCaseSensitiveProperties

Value Members

val caseSensitive: BooleanParam

Definition Classes
HasCaseSensitiveProperties
def getCaseSensitive: Boolean

Definition Classes
HasCaseSensitiveProperties
def setCaseSensitive(value: Boolean): FinanceSummarizer.this.type

Definition Classes
HasCaseSensitiveProperties

Inherited from AnnotatorModel[MedicalEncoderDecoder]

Value Members

def _transform(dataset: Dataset[_], recursivePipeline: Option[PipelineModel]): DataFrame

Attributes
protected
Definition Classes
AnnotatorModel
def afterAnnotate(dataset: DataFrame): DataFrame

Attributes
protected
Definition Classes
AnnotatorModel
def beforeAnnotate(dataset: Dataset[_]): Dataset[_]

Attributes
protected
Definition Classes
AnnotatorModel
final def transform(dataset: Dataset[_]): DataFrame

Definition Classes
AnnotatorModel → Transformer

Inherited from CanBeLazy

Value Members

def getLazyAnnotator: Boolean

Definition Classes
CanBeLazy
val lazyAnnotator: BooleanParam

Definition Classes
CanBeLazy
def setLazyAnnotator(value: Boolean): FinanceSummarizer.this.type

Definition Classes
CanBeLazy

Inherited from RawAnnotator[MedicalEncoderDecoder]

Value Members

def copy(extra: ParamMap): MedicalEncoderDecoder

Definition Classes
RawAnnotator → Model → Transformer → PipelineStage → Params
def extraValidate(structType: StructType): Boolean

Attributes
protected
Definition Classes
RawAnnotator
def extraValidateMsg: String

Attributes
protected
Definition Classes
RawAnnotator
final def transformSchema(schema: StructType): StructType

Definition Classes
RawAnnotator → PipelineStage
def validate(schema: StructType): Boolean

Attributes
protected
Definition Classes
RawAnnotator
def wrapColumnMetadata(col: Column): Column

Attributes
protected
Definition Classes
RawAnnotator

Inherited from HasOutputAnnotationCol

Value Members

final def getOutputCol: String

Definition Classes
HasOutputAnnotationCol
final val outputCol: Param[String]

Attributes
protected
Definition Classes
HasOutputAnnotationCol
final def setOutputCol(value: String): FinanceSummarizer.this.type

Definition Classes
HasOutputAnnotationCol

Inherited from HasInputAnnotationCols

Value Members

final def checkSchema(schema: StructType, inputAnnotatorType: String): Boolean

Attributes
protected
Definition Classes
HasInputAnnotationCols
def getInputCols: Array[String]

Definition Classes
HasInputAnnotationCols
final val inputCols: StringArrayParam

Attributes
protected
Definition Classes
HasInputAnnotationCols
def msgHelper(schema: StructType): String

Attributes
protected
Definition Classes
HasInputAnnotationCols
val optionalInputAnnotatorTypes: Array[String]

Definition Classes
HasInputAnnotationCols
final def setInputCols(value: String*): FinanceSummarizer.this.type

Definition Classes
HasInputAnnotationCols
def setInputCols(value: Array[String]): FinanceSummarizer.this.type

Definition Classes
HasInputAnnotationCols

Inherited from ParamsAndFeaturesWritable

Value Members

def write: MLWriter

Definition Classes
ParamsAndFeaturesWritable → DefaultParamsWritable → MLWritable

Inherited from HasFeatures

Value Members

def $$[T](feature: StructFeature[T]): T

Attributes
protected
Definition Classes
HasFeatures
def $$[K, V](feature: MapFeature[K, V]): Map[K, V]

Attributes
protected
Definition Classes
HasFeatures
def $$[T](feature: SetFeature[T]): Set[T]

Attributes
protected
Definition Classes
HasFeatures
def $$[T](feature: ArrayFeature[T]): Array[T]

Attributes
protected
Definition Classes
HasFeatures
val features: ArrayBuffer[Feature[_, _, _]]

Definition Classes
HasFeatures
def get[T](feature: StructFeature[T]): Option[T]

Attributes
protected
Definition Classes
HasFeatures
def get[K, V](feature: MapFeature[K, V]): Option[Map[K, V]]

Attributes
protected
Definition Classes
HasFeatures
def get[T](feature: SetFeature[T]): Option[Set[T]]

Attributes
protected
Definition Classes
HasFeatures
def get[T](feature: ArrayFeature[T]): Option[Array[T]]

Attributes
protected
Definition Classes
HasFeatures
def set[T](feature: StructFeature[T], value: T): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def set[K, V](feature: MapFeature[K, V], value: Map[K, V]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def set[T](feature: SetFeature[T], value: Set[T]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def set[T](feature: ArrayFeature[T], value: Array[T]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def setDefault[T](feature: StructFeature[T], value: () ⇒ T): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def setDefault[K, V](feature: MapFeature[K, V], value: () ⇒ Map[K, V]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def setDefault[T](feature: SetFeature[T], value: () ⇒ Set[T]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def setDefault[T](feature: ArrayFeature[T], value: () ⇒ Array[T]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures

Inherited from MLWritable

Value Members

def save(path: String): Unit

Definition Classes
MLWritable
Annotations
@Since( "1.6.0" ) @throws( ... )

Inherited from Model[MedicalEncoderDecoder]

Value Members

def hasParent: Boolean

Definition Classes
Model
var parent: Estimator[MedicalEncoderDecoder]

Definition Classes
Model
def setParent(parent: Estimator[MedicalEncoderDecoder]): MedicalEncoderDecoder

Definition Classes
Model

Inherited from Transformer

Value Members

def transform(dataset: Dataset[_], paramMap: ParamMap): DataFrame

Definition Classes
Transformer
Annotations
@Since( "2.0.0" )
def transform(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): DataFrame

Definition Classes
Transformer
Annotations
@Since( "2.0.0" ) @varargs()

Inherited from PipelineStage

Value Members

def transformSchema(schema: StructType, logging: Boolean): StructType

Attributes
protected
Definition Classes
PipelineStage
Annotations
@DeveloperApi()

Inherited from Logging

Value Members

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

Attributes
protected
Definition Classes
Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging

Inherited from Params

Value Members

final def $[T](param: Param[T]): T

Attributes
protected
Definition Classes
Params
final def clear(param: Param[_]): FinanceSummarizer.this.type

Definition Classes
Params
def copyValues[T <: Params](to: T, extra: ParamMap): T

Attributes
protected
Definition Classes
Params
final def defaultCopy[T <: Params](extra: ParamMap): T

Attributes
protected
Definition Classes
Params
def explainParam(param: Param[_]): String

Definition Classes
Params
def explainParams(): String

Definition Classes
Params
final def extractParamMap(): ParamMap

Definition Classes
Params
final def extractParamMap(extra: ParamMap): ParamMap

Definition Classes
Params
final def get[T](param: Param[T]): Option[T]

Definition Classes
Params
final def getDefault[T](param: Param[T]): Option[T]

Definition Classes
Params
final def getOrDefault[T](param: Param[T]): T

Definition Classes
Params
def getParam(paramName: String): Param[Any]

Definition Classes
Params
final def hasDefault[T](param: Param[T]): Boolean

Definition Classes
Params
def hasParam(paramName: String): Boolean

Definition Classes
Params
final def isDefined(param: Param[_]): Boolean

Definition Classes
Params
final def isSet(param: Param[_]): Boolean

Definition Classes
Params
lazy val params: Array[Param[_]]

Definition Classes
Params
final def set(paramPair: ParamPair[_]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
Params
final def set(param: String, value: Any): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
Params
final def set[T](param: Param[T], value: T): FinanceSummarizer.this.type

Definition Classes
Params
final def setDefault(paramPairs: ParamPair[_]*): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
Params
final def setDefault[T](param: Param[T], value: T): FinanceSummarizer.this.type

Attributes
protected[org.apache.spark.ml]
Definition Classes
Params

Inherited from Identifiable

Value Members

def toString(): String

Definition Classes
Identifiable → AnyRef → Any

Inherited from AnyRef

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def clone(): AnyRef

Attributes
protected[lang]
Definition Classes
AnyRef
Annotations
@throws( ... ) @native()
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
Annotations
@native()
def hashCode(): Int

Definition Classes
AnyRef → Any
Annotations
@native()
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
Annotations
@native()
final def notifyAll(): Unit

Definition Classes
AnyRef
Annotations
@native()
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... ) @native()

Inherited from Any

Value Members

final def asInstanceOf[T0]: T0

Definition Classes
Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any

getParam

def getConfigProtoBytes: Option[Array[Byte]]

Definition Classes
MedicalEncoderDecoder
def getDoSample: Boolean

Definition Classes
MedicalEncoderDecoder
def getIgnoreTokenIds: Array[Int]

Definition Classes
MedicalEncoderDecoder
def getMaxNewTokens: Int

Definition Classes
MedicalEncoderDecoder
def getMaxTextLength: Int

Definition Classes
MedicalEncoderDecoder
def getMlFrameworkType: String
Get ML framework type
Get ML framework type

Definition Classes
MedicalEncoderDecoder
def getNoRepeatNgramSize: Int

Definition Classes
MedicalEncoderDecoder
def getRandomSeed: Option[Int]

Definition Classes
MedicalEncoderDecoder
def getRefineChunkSize: Int

Definition Classes
MedicalEncoderDecoder
def getRefineMaxAttempts: Int

Definition Classes
MedicalEncoderDecoder
def getRefineSummaryTargetLength: Int

Definition Classes
MedicalEncoderDecoder
def getStopAtEos: Boolean
Checks whether text generation stops when the end-of-sentence token is encountered.
Checks whether text generation stops when the end-of-sentence token is encountered.

Definition Classes
MedicalSummarizer → MedicalEncoderDecoder
def getTask: String

Attributes
protected
Definition Classes
MedicalEncoderDecoder
def getTopK: Int

Attributes
protected
Definition Classes
MedicalEncoderDecoder

param

val configProtoBytes: IntArrayParam
ConfigProto from tensorflow, serialized into byte array.
ConfigProto from tensorflow, serialized into byte array. Get with config_proto.SerializeToString()

Definition Classes
MedicalEncoderDecoder
val doSample: BooleanParam
Whether or not to use sampling, use greedy decoding otherwise (Default: false)
Whether or not to use sampling, use greedy decoding otherwise (Default: false)

Definition Classes
MedicalEncoderDecoder
var ignoreTokenIds: IntArrayParam
A list of token ids which are ignored in the decoder's output (Default: Array())
A list of token ids which are ignored in the decoder's output (Default: Array())

Definition Classes
MedicalEncoderDecoder
val inputAnnotatorTypes: Array[AnnotatorType]
Input annotator type : DOCUMENT
Input annotator type : DOCUMENT

Definition Classes
MedicalEncoderDecoder → HasInputAnnotationCols
val maxNewTokens: IntParam
Maximum number of new tokens to be generated (Default: 30)
Maximum number of new tokens to be generated (Default: 30)

Definition Classes
MedicalEncoderDecoder
val maxTextLength: IntParam
Maximum length of context text.
Maximum length of context text. (Default: 1024)

Definition Classes
MedicalEncoderDecoder
val mlFrameworkType: Param[String]
ML framework type
ML framework type

Definition Classes
MedicalEncoderDecoder
val noRepeatNgramSize: IntParam
If set to int > 0, all ngrams of that size can only occur once (Default: 0)
If set to int > 0, all ngrams of that size can only occur once (Default: 0)

Definition Classes
MedicalEncoderDecoder
val outputAnnotatorType: String
Output annotator type : DOCUMENT
Output annotator type : DOCUMENT

Definition Classes
MedicalEncoderDecoder → HasOutputAnnotatorType
var randomSeed: Option[Int]
Optional Random seed for the model.
Optional Random seed for the model. Needs to be of type Long.

Definition Classes
MedicalEncoderDecoder
val refineChunkSize: IntParam
How large should refined chunks Be.
How large should refined chunks Be. Should be equal to LLM context window size in tokens. Takes only effect when refineSummary=True

Definition Classes
MedicalEncoderDecoder
val refineMaxAttempts: IntParam
How many times should chunks be re-summarized while they are above SummaryTargetLength before stopping.
How many times should chunks be re-summarized while they are above SummaryTargetLength before stopping. Takes only effect when refineSummary=True

Definition Classes
MedicalEncoderDecoder
val refineSummary: BooleanParam
Set true to perform refined summarization at increased computation cost.
Set true to perform refined summarization at increased computation cost.

Definition Classes
MedicalEncoderDecoder
val refineSummaryTargetLength: IntParam
Target length for refined summary.
Target length for refined summary. Text will be chunked and re-summarized until it is below this length or maximum attempts are used up.

Definition Classes
MedicalEncoderDecoder
val task: Param[String]
Set transformer task, e.g.
Set transformer task, e.g. "summarize:" (Default: ""). The T5 task needs to be in the format "task:".

Definition Classes
MedicalEncoderDecoder
val topK: IntParam
The number of highest probability vocabulary tokens to keep for top-k-filtering (Default: 50)
The number of highest probability vocabulary tokens to keep for top-k-filtering (Default: 50)

Definition Classes
MedicalEncoderDecoder
val useCache: BooleanParam
Cache internal state of the model to improve performance
Cache internal state of the model to improve performance

Definition Classes
MedicalEncoderDecoder

setParam

def setConfigProtoBytes(bytes: Array[Int]): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setDoSample(value: Boolean): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setIgnoreTokenIds(tokenIds: Array[Int]): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setMaxNewTokens(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setMaxTextLength(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setMlFrameworkType(value: String): FinanceSummarizer.this.type
Set ML framework type
Set ML framework type

Definition Classes
MedicalEncoderDecoder
def setModelIfNotSet(spark: SparkSession, tfWrapper: TensorflowWrapper, spp: SentencePieceWrapper, useCache: Boolean): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setNoRepeatNgramSize(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setRandomSeed(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setRefineChunkSize(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setRefineMaxAttempts(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setRefineSummaryTargetLength(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setSignatures(value: Map[String, String]): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setStopAtEos(value: Boolean): FinanceSummarizer.this.type
Determines whether text generation stops when the end-of-sentence token is encountered.
Determines whether text generation stops when the end-of-sentence token is encountered.

Definition Classes
MedicalEncoderDecoder
def setTask(value: String): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
MedicalEncoderDecoder
def setTopK(value: Int): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder

Ungrouped

type AnnotationContent = Seq[Row]

Attributes
protected
Definition Classes
AnnotatorModel
type AnnotatorType = String

Definition Classes
HasOutputAnnotatorType

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def $[T](param: Param[T]): T

Attributes
protected
Definition Classes
Params
def $$[T](feature: StructFeature[T]): T

Attributes
protected
Definition Classes
HasFeatures
def $$[K, V](feature: MapFeature[K, V]): Map[K, V]

Attributes
protected
Definition Classes
HasFeatures
def $$[T](feature: SetFeature[T]): Set[T]

Attributes
protected
Definition Classes
HasFeatures
def $$[T](feature: ArrayFeature[T]): Array[T]

Attributes
protected
Definition Classes
HasFeatures
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def _transform(dataset: Dataset[_], recursivePipeline: Option[PipelineModel]): DataFrame

Attributes
protected
Definition Classes
AnnotatorModel
def afterAnnotate(dataset: DataFrame): DataFrame

Attributes
protected
Definition Classes
AnnotatorModel
final def asInstanceOf[T0]: T0

Definition Classes
Any
def batchAnnotate(batchedAnnotations: Seq[Array[Annotation]]): Seq[Seq[Annotation]]

Definition Classes
MedicalEncoderDecoder → HasBatchedAnnotate
def batchProcess(rows: Iterator[_]): Iterator[Row]

Definition Classes
HasBatchedAnnotate
val batchSize: IntParam

Definition Classes
HasBatchedAnnotate
def beforeAnnotate(dataset: Dataset[_]): Dataset[_]

Attributes
protected
Definition Classes
AnnotatorModel
val caseSensitive: BooleanParam

Definition Classes
HasCaseSensitiveProperties
final def checkSchema(schema: StructType, inputAnnotatorType: String): Boolean

Attributes
protected
Definition Classes
HasInputAnnotationCols
def checkValidEnvironment(spark: Option[SparkSession], scopes: Seq[String]): Unit

Definition Classes
CheckLicense
def checkValidScope(scope: String): Unit

Definition Classes
CheckLicense
def checkValidScopeAndEnvironment(scope: String, spark: Option[SparkSession], checkLp: Boolean): Unit

Definition Classes
CheckLicense
def checkValidScopesAndEnvironment(scopes: Seq[String], spark: Option[SparkSession], checkLp: Boolean): Unit

Definition Classes
CheckLicense
final def clear(param: Param[_]): FinanceSummarizer.this.type

Definition Classes
Params
def clone(): AnyRef

Attributes
protected[lang]
Definition Classes
AnyRef
Annotations
@throws( ... ) @native()
def copy(extra: ParamMap): MedicalEncoderDecoder

Definition Classes
RawAnnotator → Model → Transformer → PipelineStage → Params
def copyValues[T <: Params](to: T, extra: ParamMap): T

Attributes
protected
Definition Classes
Params
final def defaultCopy[T <: Params](extra: ParamMap): T

Attributes
protected
Definition Classes
Params
val engine: Param[String]

Definition Classes
HasEngine
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def explainParam(param: Param[_]): String

Definition Classes
Params
def explainParams(): String

Definition Classes
Params
def extraValidate(structType: StructType): Boolean

Attributes
protected
Definition Classes
RawAnnotator
def extraValidateMsg: String

Attributes
protected
Definition Classes
RawAnnotator
final def extractParamMap(): ParamMap

Definition Classes
Params
final def extractParamMap(extra: ParamMap): ParamMap

Definition Classes
Params
val features: ArrayBuffer[Feature[_, _, _]]

Definition Classes
HasFeatures
def finalize(): Unit

Attributes
protected[lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def get[T](feature: StructFeature[T]): Option[T]

Attributes
protected
Definition Classes
HasFeatures
def get[K, V](feature: MapFeature[K, V]): Option[Map[K, V]]

Attributes
protected
Definition Classes
HasFeatures
def get[T](feature: SetFeature[T]): Option[Set[T]]

Attributes
protected
Definition Classes
HasFeatures
def get[T](feature: ArrayFeature[T]): Option[Array[T]]

Attributes
protected
Definition Classes
HasFeatures
final def get[T](param: Param[T]): Option[T]

Definition Classes
Params
def getBatchSize: Int

Definition Classes
HasBatchedAnnotate
def getCaseSensitive: Boolean

Definition Classes
HasCaseSensitiveProperties
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
Annotations
@native()
final def getDefault[T](param: Param[T]): Option[T]

Definition Classes
Params
def getEngine: String

Definition Classes
HasEngine
def getInputCols: Array[String]

Definition Classes
HasInputAnnotationCols
def getLazyAnnotator: Boolean

Definition Classes
CanBeLazy
def getModelIfNotSet: MedicalEncoderDecoderModel

Definition Classes
MedicalEncoderDecoder
final def getOrDefault[T](param: Param[T]): T

Definition Classes
Params
final def getOutputCol: String

Definition Classes
HasOutputAnnotationCol
def getParam(paramName: String): Param[Any]

Definition Classes
Params
def getRefineSummary: Boolean

Definition Classes
MedicalEncoderDecoder
def getSignatures: Option[Map[String, String]]

Definition Classes
MedicalEncoderDecoder
def getUseCache: Boolean

Definition Classes
MedicalEncoderDecoder
final def hasDefault[T](param: Param[T]): Boolean

Definition Classes
Params
def hasParam(paramName: String): Boolean

Definition Classes
Params
def hasParent: Boolean

Definition Classes
Model
def hashCode(): Int

Definition Classes
AnyRef → Any
Annotations
@native()
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

Attributes
protected
Definition Classes
Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
final val inputCols: StringArrayParam

Attributes
protected
Definition Classes
HasInputAnnotationCols
final def isDefined(param: Param[_]): Boolean

Definition Classes
Params
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def isSet(param: Param[_]): Boolean

Definition Classes
Params
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
val lazyAnnotator: BooleanParam

Definition Classes
CanBeLazy
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def msgHelper(schema: StructType): String

Attributes
protected
Definition Classes
HasInputAnnotationCols
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
Annotations
@native()
final def notifyAll(): Unit

Definition Classes
AnyRef
Annotations
@native()
def onWrite(path: String, spark: SparkSession): Unit

Definition Classes
MedicalSummarizer → MedicalEncoderDecoder → ParamsAndFeaturesWritable
def onWriteSkip(path: String, spark: SparkSession): Unit

Attributes
protected
Definition Classes
MedicalEncoderDecoder
val optionalInputAnnotatorTypes: Array[String]

Definition Classes
HasInputAnnotationCols
final val outputCol: Param[String]

Attributes
protected
Definition Classes
HasOutputAnnotationCol
lazy val params: Array[Param[_]]

Definition Classes
Params
var parent: Estimator[MedicalEncoderDecoder]

Definition Classes
Model
def save(path: String): Unit

Definition Classes
MLWritable
Annotations
@Since( "1.6.0" ) @throws( ... )
def set[T](feature: StructFeature[T], value: T): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def set[K, V](feature: MapFeature[K, V], value: Map[K, V]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def set[T](feature: SetFeature[T], value: Set[T]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def set[T](feature: ArrayFeature[T], value: Array[T]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
final def set(paramPair: ParamPair[_]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
Params
final def set(param: String, value: Any): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
Params
final def set[T](param: Param[T], value: T): FinanceSummarizer.this.type

Definition Classes
Params
def setBatchSize(size: Int): FinanceSummarizer.this.type

Definition Classes
HasBatchedAnnotate
def setCaseSensitive(value: Boolean): FinanceSummarizer.this.type

Definition Classes
HasCaseSensitiveProperties
def setDefault[T](feature: StructFeature[T], value: () ⇒ T): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def setDefault[K, V](feature: MapFeature[K, V], value: () ⇒ Map[K, V]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def setDefault[T](feature: SetFeature[T], value: () ⇒ Set[T]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
def setDefault[T](feature: ArrayFeature[T], value: () ⇒ Array[T]): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
HasFeatures
final def setDefault(paramPairs: ParamPair[_]*): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
Params
final def setDefault[T](param: Param[T], value: T): FinanceSummarizer.this.type

Attributes
protected[org.apache.spark.ml]
Definition Classes
Params
final def setInputCols(value: String*): FinanceSummarizer.this.type

Definition Classes
HasInputAnnotationCols
def setInputCols(value: Array[String]): FinanceSummarizer.this.type

Definition Classes
HasInputAnnotationCols
def setLazyAnnotator(value: Boolean): FinanceSummarizer.this.type

Definition Classes
CanBeLazy
def setModelIfNotSet(spark: SparkSession, encoder: OnnxWrapper, decoder: OnnxWrapper, spp: SentencePieceWrapper): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
final def setOutputCol(value: String): FinanceSummarizer.this.type

Definition Classes
HasOutputAnnotationCol
def setParent(parent: Estimator[MedicalEncoderDecoder]): MedicalEncoderDecoder

Definition Classes
Model
def setRefineSummary(value: Boolean): FinanceSummarizer.this.type

Definition Classes
MedicalEncoderDecoder
def setUseCache(value: Boolean): FinanceSummarizer.this.type

Attributes
protected
Definition Classes
MedicalEncoderDecoder
val signatures: MapFeature[String, String]

Definition Classes
MedicalEncoderDecoder
val stopAtEos: BooleanParam

Definition Classes
MedicalEncoderDecoder
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
Identifiable → AnyRef → Any
final def transform(dataset: Dataset[_]): DataFrame

Definition Classes
AnnotatorModel → Transformer
def transform(dataset: Dataset[_], paramMap: ParamMap): DataFrame

Definition Classes
Transformer
Annotations
@Since( "2.0.0" )
def transform(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): DataFrame

Definition Classes
Transformer
Annotations
@Since( "2.0.0" ) @varargs()
final def transformSchema(schema: StructType): StructType

Definition Classes
RawAnnotator → PipelineStage
def transformSchema(schema: StructType, logging: Boolean): StructType

Attributes
protected
Definition Classes
PipelineStage
Annotations
@DeveloperApi()
val uid: String

Definition Classes
FinanceSummarizer → MedicalSummarizer → MedicalEncoderDecoder → Identifiable
def validate(schema: StructType): Boolean

Attributes
protected
Definition Classes
RawAnnotator
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... ) @native()
def wrapColumnMetadata(col: Column): Column

Attributes
protected
Definition Classes
RawAnnotator
def write: MLWriter

Definition Classes
ParamsAndFeaturesWritable → DefaultParamsWritable → MLWritable
def writeOnnxModel(path: String, spark: SparkSession, onnxWrapper: OnnxWrapper, suffix: String, fileName: String): Unit

Definition Classes
WriteOnnxModel
def writeOnnxModels(path: String, spark: SparkSession, onnxWrappersWithNames: Seq[(OnnxWrapper, String)], suffix: String): Unit

Definition Classes
WriteOnnxModel
def writeSentencePieceModel(path: String, spark: SparkSession, spp: SentencePieceWrapper, suffix: String, filename: String): Unit

Definition Classes
WriteSentencePieceModel
def writeTensorflowHub(path: String, tfPath: String, spark: SparkSession, suffix: String): Unit

Definition Classes
WriteTensorflowModel
def writeTensorflowModel(path: String, spark: SparkSession, tensorflow: TensorflowWrapper, suffix: String, filename: String, configProtoBytes: Option[Array[Byte]]): Unit

Definition Classes
WriteTensorflowModel
def writeTensorflowModelV2(path: String, spark: SparkSession, tensorflow: TensorflowWrapper, suffix: String, filename: String, configProtoBytes: Option[Array[Byte]], savedSignatures: Option[Map[String, String]]): Unit

Definition Classes
WriteTensorflowModel

Packages

FinanceSummarizer 

Companion object FinanceSummarizer

class FinanceSummarizer extends MedicalSummarizer

Instance Constructors

Type Members

Value Members

Inherited from MedicalSummarizer

Value Members

Inherited from MedicalEncoderDecoder

Value Members

Inherited from CheckLicense

Value Members

Inherited from WriteSentencePieceModel

Value Members

Inherited from HasEngine

Value Members

Inherited from WriteOnnxModel

Value Members

Inherited from WriteTensorflowModel

Value Members

Inherited from HasBatchedAnnotate[MedicalEncoderDecoder]

Value Members

Inherited from HasCaseSensitiveProperties

Value Members

Inherited from AnnotatorModel[MedicalEncoderDecoder]

Value Members

Inherited from CanBeLazy

Value Members

Inherited from RawAnnotator[MedicalEncoderDecoder]

Value Members

Inherited from HasOutputAnnotationCol

Value Members

Inherited from HasInputAnnotationCols

Value Members

Inherited from ParamsAndFeaturesWritable

Value Members

Inherited from HasFeatures

Value Members

Inherited from MLWritable

Value Members

Inherited from Model[MedicalEncoderDecoder]

Value Members

Inherited from Transformer

Value Members

Inherited from PipelineStage

Value Members

Inherited from Logging

Value Members

Inherited from Params

Value Members

Inherited from Identifiable

Value Members

Inherited from AnyRef

Value Members

Inherited from Any

Value Members

getParam

param

setParam

Ungrouped

FinanceSummarizer