Packages

class FinanceTextGenerator extends MedicalTextGenerator

Linear Supertypes
MedicalTextGenerator, CheckLicense, HasEngine, WriteSentencePieceModel, WriteOnnxModel, WriteTensorflowModel, HasBatchedAnnotate[MedicalQuestionAnswering], HasCaseSensitiveProperties, GPTGenerationParams, AnnotatorModel[MedicalQuestionAnswering], CanBeLazy, RawAnnotator[MedicalQuestionAnswering], HasOutputAnnotationCol, HasInputAnnotationCols, HasOutputAnnotatorType, ParamsAndFeaturesWritable, HasFeatures, DefaultParamsWritable, MLWritable, Model[MedicalQuestionAnswering], Transformer, PipelineStage, Logging, Params, Serializable, Serializable, Identifiable, AnyRef, Any
Ordering
  1. Grouped
  2. Alphabetic
  3. By Inheritance
Inherited
  1. FinanceTextGenerator
  2. MedicalTextGenerator
  3. CheckLicense
  4. HasEngine
  5. WriteSentencePieceModel
  6. WriteOnnxModel
  7. WriteTensorflowModel
  8. HasBatchedAnnotate
  9. HasCaseSensitiveProperties
  10. GPTGenerationParams
  11. AnnotatorModel
  12. CanBeLazy
  13. RawAnnotator
  14. HasOutputAnnotationCol
  15. HasInputAnnotationCols
  16. HasOutputAnnotatorType
  17. ParamsAndFeaturesWritable
  18. HasFeatures
  19. DefaultParamsWritable
  20. MLWritable
  21. Model
  22. Transformer
  23. PipelineStage
  24. Logging
  25. Params
  26. Serializable
  27. Serializable
  28. Identifiable
  29. AnyRef
  30. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

getParam

  1. def getAdditionalTokens: Map[Int, String]

    Get additional tokens

    Get additional tokens

    Definition Classes
    MedicalTextGenerator
  2. def getAdditionalTokensStr: String

    Get additional tokens in string format

    Get additional tokens in string format

    Definition Classes
    MedicalTextGenerator
  3. def getConfigProtoBytes: Option[Array[Byte]]

    Definition Classes
    MedicalTextGenerator
  4. def getDoSample: Boolean

    Definition Classes
    GPTGenerationParams
  5. def getIgnoreTokenIds: Array[Int]

    Definition Classes
    GPTGenerationParams
  6. def getMaxContextLength: Int

    Definition Classes
    GPTGenerationParams
  7. def getMaxNewTokens: Int

    Definition Classes
    GPTGenerationParams
  8. def getMaxTextLength: Int

    Definition Classes
    MedicalTextGenerator
  9. def getMlFrameworkType: String

    Get ML framework type

    Get ML framework type

    Definition Classes
    MedicalTextGenerator
  10. def getModelIfNotSet: MedicalEncoderDecoderModel

    Definition Classes
    MedicalTextGenerator
  11. def getModelType: String

    Get model type

    Get model type

    Definition Classes
    MedicalTextGenerator
  12. def getNoRepeatNgramSize: Int

    Definition Classes
    GPTGenerationParams
  13. def getRandomSeed: Option[Int]

    Definition Classes
    GPTGenerationParams
  14. def getStopAtEos: Boolean

    Checks whether text generation stops when the end-of-sentence token is encountered.

    Checks whether text generation stops when the end-of-sentence token is encountered.

    Definition Classes
    MedicalTextGenerator
  15. def getTopK: Int

    Definition Classes
    GPTGenerationParams

param

  1. val additionalTokens: MapFeature[Int, String]

    Additional tokens

    Additional tokens

    Definition Classes
    MedicalTextGenerator
  2. val configProtoBytes: IntArrayParam

    ConfigProto from tensorflow, serialized into byte array.

    ConfigProto from tensorflow, serialized into byte array. Get with config_proto.SerializeToString()

    Definition Classes
    MedicalTextGenerator
  3. val customPrompt: Param[String]

    Custom model prompt

    Custom model prompt

    Definition Classes
    MedicalTextGenerator
  4. val doSample: BooleanParam

    Whether or not to use sampling, use greedy decoding otherwise (Default: false)

    Whether or not to use sampling, use greedy decoding otherwise (Default: false)

    Definition Classes
    GPTGenerationParams
  5. val ignoreTokenIds: IntArrayParam

    A list of token ids which are ignored in the decoder's output (Default: Array())

    A list of token ids which are ignored in the decoder's output (Default: Array())

    Definition Classes
    GPTGenerationParams
  6. val inputAnnotatorTypes: Array[String]

    Input annotator type : DOCUMENT, DOCUMENT

    Input annotator type : DOCUMENT, DOCUMENT

    Definition Classes
    MedicalTextGenerator → HasInputAnnotationCols
  7. val maxContextLength: IntParam

    Maximum length of context text.

    Maximum length of context text. (Default: 1024)

    Definition Classes
    GPTGenerationParams
  8. val maxNewTokens: IntParam

    Maximum number of new tokens to be generated (Default: 30)

    Maximum number of new tokens to be generated (Default: 30)

    Definition Classes
    GPTGenerationParams
  9. val maxTextLength: IntParam

    Maximum length of context text.

    Maximum length of context text. (Default: 1024)

    Definition Classes
    MedicalTextGenerator
  10. val merges: MapFeature[(String, String), Int]

    Holding merges.txt coming from RoBERTa model

    Holding merges.txt coming from RoBERTa model

    Definition Classes
    MedicalTextGenerator
  11. val mlFrameworkType: Param[String]

    ML framework type

    ML framework type

    Definition Classes
    MedicalTextGenerator
  12. val modelType: Param[String]

    Model type

    Model type

    Definition Classes
    MedicalTextGenerator
  13. val noRepeatNgramSize: IntParam

    If set to int > 0, all ngrams of that size can only occur once (Default: 0)

    If set to int > 0, all ngrams of that size can only occur once (Default: 0)

    Definition Classes
    GPTGenerationParams
  14. val outputAnnotatorType: String

    Output annotator type : DOCUMENT

    Output annotator type : DOCUMENT

    Definition Classes
    MedicalTextGenerator → HasOutputAnnotatorType
  15. val randomSeed: Option[Int]

    Optional Random seed for the model.

    Optional Random seed for the model. Needs to be of type Long.

    Definition Classes
    GPTGenerationParams
  16. val signatures: MapFeature[String, String]

    It contains TF model signatures for the laded saved model

    It contains TF model signatures for the laded saved model

    Definition Classes
    MedicalTextGenerator
  17. val stopAtEos: BooleanParam

    Stop text generation when the end-of-sentence token is encountered.

    Stop text generation when the end-of-sentence token is encountered.

    Definition Classes
    MedicalTextGenerator
  18. val topK: IntParam

    The number of highest probability vocabulary tokens to consider

    The number of highest probability vocabulary tokens to consider

    Definition Classes
    GPTGenerationParams
  19. val useCache: BooleanParam

    Cache internal state of the model to improve performance

    Cache internal state of the model to improve performance

    Definition Classes
    MedicalTextGenerator
  20. val vocabulary: MapFeature[String, Int]

    Vocabulary used to encode the words to ids with bpeTokenizer.encode

    Vocabulary used to encode the words to ids with bpeTokenizer.encode

    Definition Classes
    MedicalTextGenerator

setParam

  1. def setAdditionalTokens(values: HashMap[Int, String]): FinanceTextGenerator.this.type

    Set additional tokens

    Set additional tokens

    Definition Classes
    MedicalTextGenerator
  2. def setAdditionalTokens(value: Map[Int, String]): FinanceTextGenerator.this.type

    Set additional tokens

    Set additional tokens

    Definition Classes
    MedicalTextGenerator
  3. def setConfigProtoBytes(bytes: Array[Int]): FinanceTextGenerator.this.type

    Definition Classes
    MedicalTextGenerator
  4. def setCustomPrompt(value: String): FinanceTextGenerator.this.type

    Set custom model prompt

    Set custom model prompt

    Definition Classes
    MedicalTextGenerator
  5. def setDoSample(value: Boolean): FinanceTextGenerator.this.type

    Definition Classes
    GPTGenerationParams
  6. def setIgnoreTokenIds(tokenIds: Array[Int]): FinanceTextGenerator.this.type

    Definition Classes
    GPTGenerationParams
  7. def setMaxContextLength(value: Int): FinanceTextGenerator.this.type

    Definition Classes
    GPTGenerationParams
  8. def setMaxNewTokens(value: Int): FinanceTextGenerator.this.type

    Definition Classes
    GPTGenerationParams
  9. def setMaxTextLength(value: Int): FinanceTextGenerator.this.type

    Definition Classes
    MedicalTextGenerator
  10. def setMerges(value: Map[(String, String), Int]): FinanceTextGenerator.this.type

    Definition Classes
    MedicalTextGenerator
  11. def setMlFrameworkType(value: String): FinanceTextGenerator.this.type

    Set ML framework type

    Set ML framework type

    Definition Classes
    MedicalTextGenerator
  12. def setModelIfNotSet(spark: SparkSession, tfWrapper: TensorflowWrapper): FinanceTextGenerator.this.type

    Definition Classes
    MedicalTextGenerator
  13. def setModelType(value: String): FinanceTextGenerator.this.type

    Set model type

    Set model type

    Definition Classes
    MedicalTextGenerator
  14. def setNoRepeatNgramSize(value: Int): FinanceTextGenerator.this.type

    Definition Classes
    GPTGenerationParams
  15. def setRandomSeed(value: Int): FinanceTextGenerator.this.type

    Definition Classes
    GPTGenerationParams
  16. def setSignatures(value: Map[String, String]): FinanceTextGenerator.this.type

    Definition Classes
    MedicalTextGenerator
  17. def setStopAtEos(value: Boolean): FinanceTextGenerator.this.type

    Determines whether text generation stops when the end-of-sentence token is encountered.

    Determines whether text generation stops when the end-of-sentence token is encountered.

    Definition Classes
    MedicalTextGenerator
  18. def setTopK(value: Int): FinanceTextGenerator.this.type

    Definition Classes
    GPTGenerationParams
  19. def setVocabulary(value: Map[String, Int]): FinanceTextGenerator.this.type

    Definition Classes
    MedicalTextGenerator

Ungrouped

  1. type AnnotatorType = String
    Definition Classes
    HasOutputAnnotatorType
  1. def batchAnnotate(batchedAnnotations: Seq[Array[Annotation]]): Seq[Seq[Annotation]]

    takes a document and annotations and produces new annotations of this annotator's annotation type

    takes a document and annotations and produces new annotations of this annotator's annotation type

    batchedAnnotations

    Annotations that correspond to inputAnnotationCols generated by previous annotators if any

    returns

    any number of annotations processed for every input annotation. Not necessary one to one relationship

    Definition Classes
    FinanceTextGenerator → MedicalTextGenerator → HasBatchedAnnotate
  2. def batchProcess(rows: Iterator[_]): Iterator[Row]
    Definition Classes
    HasBatchedAnnotate
  3. val batchSize: IntParam
    Definition Classes
    HasBatchedAnnotate
  4. def batchedAnnotateWithoutPromptTemplate(batchedAnnotations: Seq[Array[Annotation]]): Seq[Seq[Annotation]]
    Definition Classes
    MedicalTextGenerator
  5. val caseSensitive: BooleanParam
    Definition Classes
    HasCaseSensitiveProperties
  6. def checkValidEnvironment(spark: Option[SparkSession], scopes: Seq[String]): Unit
    Definition Classes
    CheckLicense
  7. def checkValidScope(scope: String): Unit
    Definition Classes
    CheckLicense
  8. def checkValidScopeAndEnvironment(scope: String, spark: Option[SparkSession], checkLp: Boolean): Unit
    Definition Classes
    CheckLicense
  9. def checkValidScopesAndEnvironment(scopes: Seq[String], spark: Option[SparkSession], checkLp: Boolean): Unit
    Definition Classes
    CheckLicense
  10. final def clear(param: Param[_]): FinanceTextGenerator.this.type
    Definition Classes
    Params
  11. def copy(extra: ParamMap): MedicalQuestionAnswering
    Definition Classes
    RawAnnotator → Model → Transformer → PipelineStage → Params
  12. val engine: Param[String]
    Definition Classes
    HasEngine
  13. def explainParam(param: Param[_]): String
    Definition Classes
    Params
  14. def explainParams(): String
    Definition Classes
    Params
  15. final def extractParamMap(): ParamMap
    Definition Classes
    Params
  16. final def extractParamMap(extra: ParamMap): ParamMap
    Definition Classes
    Params
  17. val features: ArrayBuffer[Feature[_, _, _]]
    Definition Classes
    HasFeatures
  18. final def get[T](param: Param[T]): Option[T]
    Definition Classes
    Params
  19. def getBatchSize: Int
    Definition Classes
    HasBatchedAnnotate
  20. def getCaseSensitive: Boolean
    Definition Classes
    HasCaseSensitiveProperties
  21. final def getDefault[T](param: Param[T]): Option[T]
    Definition Classes
    Params
  22. def getEngine: String
    Definition Classes
    HasEngine
  23. def getInputCols: Array[String]
    Definition Classes
    HasInputAnnotationCols
  24. def getLazyAnnotator: Boolean
    Definition Classes
    CanBeLazy
  25. final def getOrDefault[T](param: Param[T]): T
    Definition Classes
    Params
  26. final def getOutputCol: String
    Definition Classes
    HasOutputAnnotationCol
  27. def getParam(paramName: String): Param[Any]
    Definition Classes
    Params
  28. def getSignatures: Option[Map[String, String]]
    Definition Classes
    MedicalTextGenerator
  29. def getUseCache: Boolean
    Definition Classes
    MedicalTextGenerator
  30. final def hasDefault[T](param: Param[T]): Boolean
    Definition Classes
    Params
  31. def hasParam(paramName: String): Boolean
    Definition Classes
    Params
  32. def hasParent: Boolean
    Definition Classes
    Model
  33. final def isDefined(param: Param[_]): Boolean
    Definition Classes
    Params
  34. final def isSet(param: Param[_]): Boolean
    Definition Classes
    Params
  35. val lazyAnnotator: BooleanParam
    Definition Classes
    CanBeLazy
  36. def onWrite(path: String, spark: SparkSession): Unit
    Definition Classes
    MedicalTextGenerator → ParamsAndFeaturesWritable
  37. val optionalInputAnnotatorTypes: Array[String]
    Definition Classes
    HasInputAnnotationCols
  38. lazy val params: Array[Param[_]]
    Definition Classes
    Params
  39. var parent: Estimator[MedicalQuestionAnswering]
    Definition Classes
    Model
  40. def save(path: String): Unit
    Definition Classes
    MLWritable
    Annotations
    @Since( "1.6.0" ) @throws( ... )
  41. final def set[T](param: Param[T], value: T): FinanceTextGenerator.this.type
    Definition Classes
    Params
  42. def setBatchSize(size: Int): FinanceTextGenerator.this.type
    Definition Classes
    HasBatchedAnnotate
  43. def setCaseSensitive(value: Boolean): FinanceTextGenerator.this.type
    Definition Classes
    HasCaseSensitiveProperties
  44. final def setInputCols(value: String*): FinanceTextGenerator.this.type
    Definition Classes
    HasInputAnnotationCols
  45. def setInputCols(value: Array[String]): FinanceTextGenerator.this.type
    Definition Classes
    HasInputAnnotationCols
  46. def setLazyAnnotator(value: Boolean): FinanceTextGenerator.this.type
    Definition Classes
    CanBeLazy
  47. def setModelIfNotSet(spark: SparkSession, model: MedicalEncoderDecoderModel): FinanceTextGenerator.this.type
    Definition Classes
    MedicalTextGenerator
  48. def setModelIfNotSet(spark: SparkSession, encoder: OnnxWrapper, decoder: OnnxWrapper, spp: SentencePieceWrapper): FinanceTextGenerator.this.type
    Definition Classes
    MedicalTextGenerator
  49. def setModelIfNotSet(spark: SparkSession, tfWrapper: TensorflowWrapper, spp: SentencePieceWrapper, useCache: Boolean): FinanceTextGenerator.this.type
    Definition Classes
    MedicalTextGenerator
  50. final def setOutputCol(value: String): FinanceTextGenerator.this.type
    Definition Classes
    HasOutputAnnotationCol
  51. def setParent(parent: Estimator[MedicalQuestionAnswering]): MedicalQuestionAnswering
    Definition Classes
    Model
  52. def setUseCache(value: Boolean): FinanceTextGenerator.this.type
    Definition Classes
    MedicalTextGenerator
  53. def toString(): String
    Definition Classes
    Identifiable → AnyRef → Any
  54. final def transform(dataset: Dataset[_]): DataFrame
    Definition Classes
    AnnotatorModel → Transformer
  55. def transform(dataset: Dataset[_], paramMap: ParamMap): DataFrame
    Definition Classes
    Transformer
    Annotations
    @Since( "2.0.0" )
  56. def transform(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): DataFrame
    Definition Classes
    Transformer
    Annotations
    @Since( "2.0.0" ) @varargs()
  57. final def transformSchema(schema: StructType): StructType
    Definition Classes
    RawAnnotator → PipelineStage
  58. val uid: String
    Definition Classes
    FinanceTextGenerator → MedicalTextGenerator → Identifiable
  59. def write: MLWriter
    Definition Classes
    ParamsAndFeaturesWritable → DefaultParamsWritable → MLWritable
  60. def writeOnnxModel(path: String, spark: SparkSession, onnxWrapper: OnnxWrapper, suffix: String, fileName: String): Unit
    Definition Classes
    WriteOnnxModel
  61. def writeOnnxModels(path: String, spark: SparkSession, onnxWrappersWithNames: Seq[(OnnxWrapper, String)], suffix: String): Unit
    Definition Classes
    WriteOnnxModel
  62. def writeSentencePieceModel(path: String, spark: SparkSession, spp: SentencePieceWrapper, suffix: String, filename: String): Unit
    Definition Classes
    WriteSentencePieceModel
  63. def writeTensorflowHub(path: String, tfPath: String, spark: SparkSession, suffix: String): Unit
    Definition Classes
    WriteTensorflowModel
  64. def writeTensorflowModel(path: String, spark: SparkSession, tensorflow: TensorflowWrapper, suffix: String, filename: String, configProtoBytes: Option[Array[Byte]]): Unit
    Definition Classes
    WriteTensorflowModel
  65. def writeTensorflowModelV2(path: String, spark: SparkSession, tensorflow: TensorflowWrapper, suffix: String, filename: String, configProtoBytes: Option[Array[Byte]], savedSignatures: Option[Map[String, String]]): Unit
    Definition Classes
    WriteTensorflowModel