class MedicalEncoderDecoder extends AnnotatorModel[MedicalEncoderDecoder] with HasCaseSensitiveProperties with HasBatchedAnnotate[MedicalEncoderDecoder] with ParamsAndFeaturesWritable with WriteTensorflowModel with WriteOnnxModel with HasEngine with WriteSentencePieceModel with CheckLicense

Linear Supertypes
CheckLicense, WriteSentencePieceModel, HasEngine, WriteOnnxModel, WriteTensorflowModel, HasBatchedAnnotate[MedicalEncoderDecoder], HasCaseSensitiveProperties, AnnotatorModel[MedicalEncoderDecoder], CanBeLazy, RawAnnotator[MedicalEncoderDecoder], HasOutputAnnotationCol, HasInputAnnotationCols, HasOutputAnnotatorType, ParamsAndFeaturesWritable, HasFeatures, DefaultParamsWritable, MLWritable, Model[MedicalEncoderDecoder], Transformer, PipelineStage, Logging, Params, Serializable, Serializable, Identifiable, AnyRef, Any
Known Subclasses
FinanceEncoderDecoder, FinanceSummarizer, LegalEncoderDecoder, LegalSummarizer, MedicalSummarizer
Ordering
  1. Grouped
  2. Alphabetic
  3. By Inheritance
Inherited
  1. MedicalEncoderDecoder
  2. CheckLicense
  3. WriteSentencePieceModel
  4. HasEngine
  5. WriteOnnxModel
  6. WriteTensorflowModel
  7. HasBatchedAnnotate
  8. HasCaseSensitiveProperties
  9. AnnotatorModel
  10. CanBeLazy
  11. RawAnnotator
  12. HasOutputAnnotationCol
  13. HasInputAnnotationCols
  14. HasOutputAnnotatorType
  15. ParamsAndFeaturesWritable
  16. HasFeatures
  17. DefaultParamsWritable
  18. MLWritable
  19. Model
  20. Transformer
  21. PipelineStage
  22. Logging
  23. Params
  24. Serializable
  25. Serializable
  26. Identifiable
  27. AnyRef
  28. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new MedicalEncoderDecoder()
  2. new MedicalEncoderDecoder(uid: String)

Type Members

  1. type AnnotationContent = Seq[Row]
    Attributes
    protected
    Definition Classes
    AnnotatorModel
  2. type AnnotatorType = String
    Definition Classes
    HasOutputAnnotatorType

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def $[T](param: Param[T]): T
    Attributes
    protected
    Definition Classes
    Params
  4. def $$[T](feature: StructFeature[T]): T
    Attributes
    protected
    Definition Classes
    HasFeatures
  5. def $$[K, V](feature: MapFeature[K, V]): Map[K, V]
    Attributes
    protected
    Definition Classes
    HasFeatures
  6. def $$[T](feature: SetFeature[T]): Set[T]
    Attributes
    protected
    Definition Classes
    HasFeatures
  7. def $$[T](feature: ArrayFeature[T]): Array[T]
    Attributes
    protected
    Definition Classes
    HasFeatures
  8. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  9. def _transform(dataset: Dataset[_], recursivePipeline: Option[PipelineModel]): DataFrame
    Attributes
    protected
    Definition Classes
    AnnotatorModel
  10. def afterAnnotate(dataset: DataFrame): DataFrame
    Attributes
    protected
    Definition Classes
    AnnotatorModel
  11. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  12. def batchAnnotate(batchedAnnotations: Seq[Array[Annotation]]): Seq[Seq[Annotation]]
    Definition Classes
    MedicalEncoderDecoder → HasBatchedAnnotate
  13. def batchProcess(rows: Iterator[_]): Iterator[Row]
    Definition Classes
    HasBatchedAnnotate
  14. val batchSize: IntParam
    Definition Classes
    HasBatchedAnnotate
  15. def beforeAnnotate(dataset: Dataset[_]): Dataset[_]
    Attributes
    protected
    Definition Classes
    AnnotatorModel
  16. val caseSensitive: BooleanParam
    Definition Classes
    HasCaseSensitiveProperties
  17. final def checkSchema(schema: StructType, inputAnnotatorType: String): Boolean
    Attributes
    protected
    Definition Classes
    HasInputAnnotationCols
  18. def checkValidEnvironment(spark: Option[SparkSession], scopes: Seq[String]): Unit
    Definition Classes
    CheckLicense
  19. def checkValidScope(scope: String): Unit
    Definition Classes
    CheckLicense
  20. def checkValidScopeAndEnvironment(scope: String, spark: Option[SparkSession], checkLp: Boolean): Unit
    Definition Classes
    CheckLicense
  21. def checkValidScopesAndEnvironment(scopes: Seq[String], spark: Option[SparkSession], checkLp: Boolean): Unit
    Definition Classes
    CheckLicense
  22. final def clear(param: Param[_]): MedicalEncoderDecoder.this.type
    Definition Classes
    Params
  23. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  24. val configProtoBytes: IntArrayParam

    ConfigProto from tensorflow, serialized into byte array.

    ConfigProto from tensorflow, serialized into byte array. Get with config_proto.SerializeToString()

  25. def copy(extra: ParamMap): MedicalEncoderDecoder
    Definition Classes
    RawAnnotator → Model → Transformer → PipelineStage → Params
  26. def copyValues[T <: Params](to: T, extra: ParamMap): T
    Attributes
    protected
    Definition Classes
    Params
  27. final def defaultCopy[T <: Params](extra: ParamMap): T
    Attributes
    protected
    Definition Classes
    Params
  28. val doSample: BooleanParam

    Whether or not to use sampling, use greedy decoding otherwise (Default: false)

  29. val engine: Param[String]
    Definition Classes
    HasEngine
  30. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  31. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  32. def explainParam(param: Param[_]): String
    Definition Classes
    Params
  33. def explainParams(): String
    Definition Classes
    Params
  34. def extraValidate(structType: StructType): Boolean
    Attributes
    protected
    Definition Classes
    RawAnnotator
  35. def extraValidateMsg: String
    Attributes
    protected
    Definition Classes
    RawAnnotator
  36. final def extractParamMap(): ParamMap
    Definition Classes
    Params
  37. final def extractParamMap(extra: ParamMap): ParamMap
    Definition Classes
    Params
  38. val features: ArrayBuffer[Feature[_, _, _]]
    Definition Classes
    HasFeatures
  39. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  40. def get[T](feature: StructFeature[T]): Option[T]
    Attributes
    protected
    Definition Classes
    HasFeatures
  41. def get[K, V](feature: MapFeature[K, V]): Option[Map[K, V]]
    Attributes
    protected
    Definition Classes
    HasFeatures
  42. def get[T](feature: SetFeature[T]): Option[Set[T]]
    Attributes
    protected
    Definition Classes
    HasFeatures
  43. def get[T](feature: ArrayFeature[T]): Option[Array[T]]
    Attributes
    protected
    Definition Classes
    HasFeatures
  44. final def get[T](param: Param[T]): Option[T]
    Definition Classes
    Params
  45. def getBatchSize: Int
    Definition Classes
    HasBatchedAnnotate
  46. def getCaseSensitive: Boolean
    Definition Classes
    HasCaseSensitiveProperties
  47. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  48. def getConfigProtoBytes: Option[Array[Byte]]

  49. final def getDefault[T](param: Param[T]): Option[T]
    Definition Classes
    Params
  50. def getDoSample: Boolean

  51. def getEngine: String
    Definition Classes
    HasEngine
  52. def getIgnoreTokenIds: Array[Int]

  53. def getInputCols: Array[String]
    Definition Classes
    HasInputAnnotationCols
  54. def getLazyAnnotator: Boolean
    Definition Classes
    CanBeLazy
  55. def getMaxNewTokens: Int

  56. def getMaxTextLength: Int

  57. def getMlFrameworkType: String

    Get ML framework type

  58. def getModelIfNotSet: MedicalEncoderDecoderModel
  59. def getNoRepeatNgramSize: Int

  60. final def getOrDefault[T](param: Param[T]): T
    Definition Classes
    Params
  61. final def getOutputCol: String
    Definition Classes
    HasOutputAnnotationCol
  62. def getParam(paramName: String): Param[Any]
    Definition Classes
    Params
  63. def getRandomSeed: Option[Int]

  64. def getRefineChunkSize: Int

  65. def getRefineMaxAttempts: Int

  66. def getRefineSummary: Boolean
  67. def getRefineSummaryTargetLength: Int

  68. def getSignatures: Option[Map[String, String]]
  69. def getStopAtEos: Boolean

    Checks whether text generation stops when the end-of-sentence token is encountered.

  70. def getTask: String

    Attributes
    protected
  71. def getTopK: Int

    Attributes
    protected
  72. def getUseCache: Boolean
  73. final def hasDefault[T](param: Param[T]): Boolean
    Definition Classes
    Params
  74. def hasParam(paramName: String): Boolean
    Definition Classes
    Params
  75. def hasParent: Boolean
    Definition Classes
    Model
  76. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  77. var ignoreTokenIds: IntArrayParam

    A list of token ids which are ignored in the decoder's output (Default: Array())

  78. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  79. def initializeLogIfNecessary(isInterpreter: Boolean): Unit
    Attributes
    protected
    Definition Classes
    Logging
  80. val inputAnnotatorTypes: Array[AnnotatorType]

    Input annotator type : DOCUMENT

    Input annotator type : DOCUMENT

    Definition Classes
    MedicalEncoderDecoder → HasInputAnnotationCols
  81. final val inputCols: StringArrayParam
    Attributes
    protected
    Definition Classes
    HasInputAnnotationCols
  82. final def isDefined(param: Param[_]): Boolean
    Definition Classes
    Params
  83. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  84. final def isSet(param: Param[_]): Boolean
    Definition Classes
    Params
  85. def isTraceEnabled(): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  86. val lazyAnnotator: BooleanParam
    Definition Classes
    CanBeLazy
  87. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  88. def logDebug(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  89. def logDebug(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  90. def logError(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  91. def logError(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  92. def logInfo(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  93. def logInfo(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  94. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  95. def logTrace(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  96. def logTrace(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  97. def logWarning(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  98. def logWarning(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  99. val maxNewTokens: IntParam

    Maximum number of new tokens to be generated (Default: 30)

  100. val maxTextLength: IntParam

    Maximum length of context text.

    Maximum length of context text. (Default: 1024)

  101. val mlFrameworkType: Param[String]

    ML framework type

  102. def msgHelper(schema: StructType): String
    Attributes
    protected
    Definition Classes
    HasInputAnnotationCols
  103. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  104. val noRepeatNgramSize: IntParam

    If set to int > 0, all ngrams of that size can only occur once (Default: 0)

  105. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  106. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  107. def onWrite(path: String, spark: SparkSession): Unit
    Definition Classes
    MedicalEncoderDecoder → ParamsAndFeaturesWritable
  108. def onWriteSkip(path: String, spark: SparkSession): Unit
    Attributes
    protected
  109. val optionalInputAnnotatorTypes: Array[String]
    Definition Classes
    HasInputAnnotationCols
  110. val outputAnnotatorType: String

    Output annotator type : DOCUMENT

    Output annotator type : DOCUMENT

    Definition Classes
    MedicalEncoderDecoder → HasOutputAnnotatorType
  111. final val outputCol: Param[String]
    Attributes
    protected
    Definition Classes
    HasOutputAnnotationCol
  112. lazy val params: Array[Param[_]]
    Definition Classes
    Params
  113. var parent: Estimator[MedicalEncoderDecoder]
    Definition Classes
    Model
  114. var randomSeed: Option[Int]

    Optional Random seed for the model.

    Optional Random seed for the model. Needs to be of type Long.

  115. val refineChunkSize: IntParam

    How large should refined chunks Be.

    How large should refined chunks Be. Should be equal to LLM context window size in tokens. Takes only effect when refineSummary=True

  116. val refineMaxAttempts: IntParam

    How many times should chunks be re-summarized while they are above SummaryTargetLength before stopping.

    How many times should chunks be re-summarized while they are above SummaryTargetLength before stopping. Takes only effect when refineSummary=True

  117. val refineSummary: BooleanParam

    Set true to perform refined summarization at increased computation cost.

  118. val refineSummaryTargetLength: IntParam

    Target length for refined summary.

    Target length for refined summary. Text will be chunked and re-summarized until it is below this length or maximum attempts are used up.

  119. def save(path: String): Unit
    Definition Classes
    MLWritable
    Annotations
    @Since( "1.6.0" ) @throws( ... )
  120. def set[T](feature: StructFeature[T], value: T): MedicalEncoderDecoder.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  121. def set[K, V](feature: MapFeature[K, V], value: Map[K, V]): MedicalEncoderDecoder.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  122. def set[T](feature: SetFeature[T], value: Set[T]): MedicalEncoderDecoder.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  123. def set[T](feature: ArrayFeature[T], value: Array[T]): MedicalEncoderDecoder.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  124. final def set(paramPair: ParamPair[_]): MedicalEncoderDecoder.this.type
    Attributes
    protected
    Definition Classes
    Params
  125. final def set(param: String, value: Any): MedicalEncoderDecoder.this.type
    Attributes
    protected
    Definition Classes
    Params
  126. final def set[T](param: Param[T], value: T): MedicalEncoderDecoder.this.type
    Definition Classes
    Params
  127. def setBatchSize(size: Int): MedicalEncoderDecoder.this.type
    Definition Classes
    HasBatchedAnnotate
  128. def setCaseSensitive(value: Boolean): MedicalEncoderDecoder.this.type
    Definition Classes
    HasCaseSensitiveProperties
  129. def setConfigProtoBytes(bytes: Array[Int]): MedicalEncoderDecoder.this.type

  130. def setDefault[T](feature: StructFeature[T], value: () ⇒ T): MedicalEncoderDecoder.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  131. def setDefault[K, V](feature: MapFeature[K, V], value: () ⇒ Map[K, V]): MedicalEncoderDecoder.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  132. def setDefault[T](feature: SetFeature[T], value: () ⇒ Set[T]): MedicalEncoderDecoder.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  133. def setDefault[T](feature: ArrayFeature[T], value: () ⇒ Array[T]): MedicalEncoderDecoder.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  134. final def setDefault(paramPairs: ParamPair[_]*): MedicalEncoderDecoder.this.type
    Attributes
    protected
    Definition Classes
    Params
  135. final def setDefault[T](param: Param[T], value: T): MedicalEncoderDecoder.this.type
    Attributes
    protected[org.apache.spark.ml]
    Definition Classes
    Params
  136. def setDoSample(value: Boolean): MedicalEncoderDecoder.this.type

  137. def setIgnoreTokenIds(tokenIds: Array[Int]): MedicalEncoderDecoder.this.type

  138. final def setInputCols(value: String*): MedicalEncoderDecoder.this.type
    Definition Classes
    HasInputAnnotationCols
  139. def setInputCols(value: Array[String]): MedicalEncoderDecoder.this.type
    Definition Classes
    HasInputAnnotationCols
  140. def setLazyAnnotator(value: Boolean): MedicalEncoderDecoder.this.type
    Definition Classes
    CanBeLazy
  141. def setMaxNewTokens(value: Int): MedicalEncoderDecoder.this.type

  142. def setMaxTextLength(value: Int): MedicalEncoderDecoder.this.type

  143. def setMlFrameworkType(value: String): MedicalEncoderDecoder.this.type

    Set ML framework type

  144. def setModelIfNotSet(spark: SparkSession, encoder: OnnxWrapper, decoder: OnnxWrapper, spp: SentencePieceWrapper): MedicalEncoderDecoder.this.type
  145. def setModelIfNotSet(spark: SparkSession, tfWrapper: TensorflowWrapper, spp: SentencePieceWrapper, useCache: Boolean): MedicalEncoderDecoder.this.type

  146. def setNoRepeatNgramSize(value: Int): MedicalEncoderDecoder.this.type

  147. final def setOutputCol(value: String): MedicalEncoderDecoder.this.type
    Definition Classes
    HasOutputAnnotationCol
  148. def setParent(parent: Estimator[MedicalEncoderDecoder]): MedicalEncoderDecoder
    Definition Classes
    Model
  149. def setRandomSeed(value: Int): MedicalEncoderDecoder.this.type

  150. def setRefineChunkSize(value: Int): MedicalEncoderDecoder.this.type

  151. def setRefineMaxAttempts(value: Int): MedicalEncoderDecoder.this.type

  152. def setRefineSummary(value: Boolean): MedicalEncoderDecoder.this.type
  153. def setRefineSummaryTargetLength(value: Int): MedicalEncoderDecoder.this.type

  154. def setSignatures(value: Map[String, String]): MedicalEncoderDecoder.this.type

  155. def setStopAtEos(value: Boolean): MedicalEncoderDecoder.this.type

    Determines whether text generation stops when the end-of-sentence token is encountered.

  156. def setTask(value: String): MedicalEncoderDecoder.this.type

    Attributes
    protected
  157. def setTopK(value: Int): MedicalEncoderDecoder.this.type

  158. def setUseCache(value: Boolean): MedicalEncoderDecoder.this.type
    Attributes
    protected
  159. val signatures: MapFeature[String, String]
  160. val stopAtEos: BooleanParam
  161. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  162. val task: Param[String]

    Set transformer task, e.g.

    Set transformer task, e.g. "summarize:" (Default: ""). The T5 task needs to be in the format "task:".

  163. def toString(): String
    Definition Classes
    Identifiable → AnyRef → Any
  164. val topK: IntParam

    The number of highest probability vocabulary tokens to keep for top-k-filtering (Default: 50)

  165. final def transform(dataset: Dataset[_]): DataFrame
    Definition Classes
    AnnotatorModel → Transformer
  166. def transform(dataset: Dataset[_], paramMap: ParamMap): DataFrame
    Definition Classes
    Transformer
    Annotations
    @Since( "2.0.0" )
  167. def transform(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): DataFrame
    Definition Classes
    Transformer
    Annotations
    @Since( "2.0.0" ) @varargs()
  168. final def transformSchema(schema: StructType): StructType
    Definition Classes
    RawAnnotator → PipelineStage
  169. def transformSchema(schema: StructType, logging: Boolean): StructType
    Attributes
    protected
    Definition Classes
    PipelineStage
    Annotations
    @DeveloperApi()
  170. val uid: String
    Definition Classes
    MedicalEncoderDecoder → Identifiable
  171. val useCache: BooleanParam

    Cache internal state of the model to improve performance

  172. def validate(schema: StructType): Boolean
    Attributes
    protected
    Definition Classes
    RawAnnotator
  173. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  174. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  175. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  176. def wrapColumnMetadata(col: Column): Column
    Attributes
    protected
    Definition Classes
    RawAnnotator
  177. def write: MLWriter
    Definition Classes
    ParamsAndFeaturesWritable → DefaultParamsWritable → MLWritable
  178. def writeOnnxModel(path: String, spark: SparkSession, onnxWrapper: OnnxWrapper, suffix: String, fileName: String): Unit
    Definition Classes
    WriteOnnxModel
  179. def writeOnnxModels(path: String, spark: SparkSession, onnxWrappersWithNames: Seq[(OnnxWrapper, String)], suffix: String): Unit
    Definition Classes
    WriteOnnxModel
  180. def writeSentencePieceModel(path: String, spark: SparkSession, spp: SentencePieceWrapper, suffix: String, filename: String): Unit
    Definition Classes
    WriteSentencePieceModel
  181. def writeTensorflowHub(path: String, tfPath: String, spark: SparkSession, suffix: String): Unit
    Definition Classes
    WriteTensorflowModel
  182. def writeTensorflowModel(path: String, spark: SparkSession, tensorflow: TensorflowWrapper, suffix: String, filename: String, configProtoBytes: Option[Array[Byte]]): Unit
    Definition Classes
    WriteTensorflowModel
  183. def writeTensorflowModelV2(path: String, spark: SparkSession, tensorflow: TensorflowWrapper, suffix: String, filename: String, configProtoBytes: Option[Array[Byte]], savedSignatures: Option[Map[String, String]]): Unit
    Definition Classes
    WriteTensorflowModel

Inherited from CheckLicense

Inherited from WriteSentencePieceModel

Inherited from HasEngine

Inherited from WriteOnnxModel

Inherited from WriteTensorflowModel

Inherited from HasBatchedAnnotate[MedicalEncoderDecoder]

Inherited from HasCaseSensitiveProperties

Inherited from AnnotatorModel[MedicalEncoderDecoder]

Inherited from CanBeLazy

Inherited from RawAnnotator[MedicalEncoderDecoder]

Inherited from HasOutputAnnotationCol

Inherited from HasInputAnnotationCols

Inherited from HasOutputAnnotatorType

Inherited from ParamsAndFeaturesWritable

Inherited from HasFeatures

Inherited from DefaultParamsWritable

Inherited from MLWritable

Inherited from Model[MedicalEncoderDecoder]

Inherited from Transformer

Inherited from PipelineStage

Inherited from Logging

Inherited from Params

Inherited from Serializable

Inherited from Serializable

Inherited from Identifiable

Inherited from AnyRef

Inherited from Any

getParam

param

setParam

Ungrouped