com.johnsnowlabs.nlp.annotators.assertion.logreg
AssertionLogRegModel
Companion object AssertionLogRegModel
class AssertionLogRegModel extends Model[AssertionLogRegModel] with RawAnnotator[AssertionLogRegModel] with Windowing with HasStorageRef with CanBeLazy with Licensed
This is a main class in AssertionLogReg family. It contains all the parameters and main point of interest method transform() which projects the input dataset into DataFrame. Unlike DL Model, this class does not extend AnnotatorModel. Instead it extends the RawAnnotator, that's why the main point of interest is method transform()
- Alphabetic
- By Inheritance
- AssertionLogRegModel
- Licensed
- CanBeLazy
- HasStorageRef
- Windowing
- RawAnnotator
- HasOutputAnnotationCol
- HasInputAnnotationCols
- HasOutputAnnotatorType
- ParamsAndFeaturesWritable
- HasFeatures
- DefaultParamsWritable
- MLWritable
- Model
- Transformer
- PipelineStage
- Logging
- Params
- Serializable
- Serializable
- Identifiable
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Instance Constructors
Type Members
-
type
AnnotatorType = String
- Definition Classes
- HasOutputAnnotatorType
-
case class
VectorizedChunk(vector: Vector, begin: Int, end: Int, sentenceId: Int, chunkId: Int) extends Product with Serializable
- Attributes
- protected
- Definition Classes
- Windowing
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
$[T](param: Param[T]): T
- Attributes
- protected
- Definition Classes
- Params
-
def
$$[T](feature: StructFeature[T]): T
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
$$[K, V](feature: MapFeature[K, V]): Map[K, V]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
$$[T](feature: SetFeature[T]): Set[T]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
$$[T](feature: ArrayFeature[T]): Array[T]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
lazy val
after: Int
- Definition Classes
- AssertionLogRegModel → Windowing
- val afterParam: IntParam
-
def
applyWindow(tokenizedSentence: WordpieceEmbeddingsSentence, s: Int, e: Int, embeddingsDim: Int): Array[Double]
- Definition Classes
- Windowing
-
def
applyWindowContext(tokenizedSentence: WordpieceEmbeddingsSentence, s: Int, e: Int, embeddingsDim: Int): (Array[Array[Float]], Array[Array[Float]], Array[Array[Float]])
- Definition Classes
- Windowing
-
def
applyWindowUdf(embeddingsDim: Int): UserDefinedFunction
- Definition Classes
- Windowing
-
def
applyWindowUdfChunk(embeddingsDim: Int): UserDefinedFunction
- Definition Classes
- Windowing
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
lazy val
before: Int
- Definition Classes
- AssertionLogRegModel → Windowing
- val beforeParam: IntParam
-
final
def
checkSchema(schema: StructType, inputAnnotatorType: String): Boolean
- Attributes
- protected
- Definition Classes
- HasInputAnnotationCols
-
final
def
clear(param: Param[_]): AssertionLogRegModel.this.type
- Definition Classes
- Params
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
copy(extra: ParamMap): AssertionLogRegModel
requirement for annotators copies
requirement for annotators copies
- Definition Classes
- AssertionLogRegModel → RawAnnotator → Model → Transformer → PipelineStage → Params
-
def
copyValues[T <: Params](to: T, extra: ParamMap): T
- Attributes
- protected
- Definition Classes
- Params
-
def
createDatabaseConnection(database: Name): RocksDBConnection
- Definition Classes
- HasStorageRef
-
final
def
defaultCopy[T <: Params](extra: ParamMap): T
- Attributes
- protected
- Definition Classes
- Params
- val endCol: Param[String]
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
equals(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
explainParam(param: Param[_]): String
- Definition Classes
- Params
-
def
explainParams(): String
- Definition Classes
- Params
-
def
extraValidate(structType: StructType): Boolean
- Attributes
- protected
- Definition Classes
- RawAnnotator
-
def
extraValidateMsg: String
- Attributes
- protected
- Definition Classes
- RawAnnotator
-
final
def
extractParamMap(): ParamMap
- Definition Classes
- Params
-
final
def
extractParamMap(extra: ParamMap): ParamMap
- Definition Classes
- Params
-
val
features: ArrayBuffer[Feature[_, _, _]]
- Definition Classes
- HasFeatures
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
def
get[T](feature: StructFeature[T]): Option[T]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
get[K, V](feature: MapFeature[K, V]): Option[Map[K, V]]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
get[T](feature: SetFeature[T]): Option[Set[T]]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
get[T](feature: ArrayFeature[T]): Option[Array[T]]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
final
def
get[T](param: Param[T]): Option[T]
- Definition Classes
- Params
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
final
def
getDefault[T](param: Param[T]): Option[T]
- Definition Classes
- Params
-
def
getInputCols: Array[String]
- Definition Classes
- HasInputAnnotationCols
-
def
getLazyAnnotator: Boolean
- Definition Classes
- CanBeLazy
-
final
def
getOrDefault[T](param: Param[T]): T
- Definition Classes
- Params
-
final
def
getOutputCol: String
- Definition Classes
- HasOutputAnnotationCol
-
def
getParam(paramName: String): Param[Any]
- Definition Classes
- Params
-
def
getStorageRef: String
- Definition Classes
- HasStorageRef
-
final
def
hasDefault[T](param: Param[T]): Boolean
- Definition Classes
- Params
-
def
hasParam(paramName: String): Boolean
- Definition Classes
- Params
-
def
hasParent: Boolean
- Definition Classes
- Model
-
def
hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
initializeLogIfNecessary(isInterpreter: Boolean): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
val
inputAnnotatorTypes: Array[String]
- Definition Classes
- AssertionLogRegModel → HasInputAnnotationCols
-
final
val
inputCols: StringArrayParam
- Attributes
- protected
- Definition Classes
- HasInputAnnotationCols
-
final
def
isDefined(param: Param[_]): Boolean
- Definition Classes
- Params
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
final
def
isSet(param: Param[_]): Boolean
- Definition Classes
- Params
-
def
isTraceEnabled(): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
l2norm(xs: Array[Double]): Double
- Definition Classes
- Windowing
- var labelMap: MapFeature[Double, String]
-
val
lazyAnnotator: BooleanParam
- Definition Classes
- CanBeLazy
-
def
log: Logger
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logName: String
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- var model: StructFeature[LogisticRegressionModel]
-
def
msgHelper(schema: StructType): String
- Attributes
- protected
- Definition Classes
- HasInputAnnotationCols
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
normalize(vec: Array[Double]): Array[Double]
- Definition Classes
- Windowing
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
def
onWrite(path: String, spark: SparkSession): Unit
- Attributes
- protected
- Definition Classes
- ParamsAndFeaturesWritable
-
val
outputAnnotatorType: AnnotatorType
- Definition Classes
- AssertionLogRegModel → HasOutputAnnotatorType
-
final
val
outputCol: Param[String]
- Attributes
- protected
- Definition Classes
- HasOutputAnnotationCol
-
lazy val
params: Array[Param[_]]
- Definition Classes
- Params
-
var
parent: Estimator[AssertionLogRegModel]
- Definition Classes
- Model
-
def
save(path: String): Unit
- Definition Classes
- MLWritable
- Annotations
- @Since( "1.6.0" ) @throws( ... )
-
def
set[T](feature: StructFeature[T], value: T): AssertionLogRegModel.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
set[K, V](feature: MapFeature[K, V], value: Map[K, V]): AssertionLogRegModel.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
set[T](feature: SetFeature[T], value: Set[T]): AssertionLogRegModel.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
set[T](feature: ArrayFeature[T], value: Array[T]): AssertionLogRegModel.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
final
def
set(paramPair: ParamPair[_]): AssertionLogRegModel.this.type
- Attributes
- protected
- Definition Classes
- Params
-
final
def
set(param: String, value: Any): AssertionLogRegModel.this.type
- Attributes
- protected
- Definition Classes
- Params
-
final
def
set[T](param: Param[T], value: T): AssertionLogRegModel.this.type
- Definition Classes
- Params
- def setAfter(after: Int): AssertionLogRegModel.this.type
- def setBefore(before: Int): AssertionLogRegModel.this.type
-
def
setDefault[T](feature: StructFeature[T], value: () ⇒ T): AssertionLogRegModel.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
setDefault[K, V](feature: MapFeature[K, V], value: () ⇒ Map[K, V]): AssertionLogRegModel.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
setDefault[T](feature: SetFeature[T], value: () ⇒ Set[T]): AssertionLogRegModel.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
setDefault[T](feature: ArrayFeature[T], value: () ⇒ Array[T]): AssertionLogRegModel.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
final
def
setDefault(paramPairs: ParamPair[_]*): AssertionLogRegModel.this.type
- Attributes
- protected
- Definition Classes
- Params
-
final
def
setDefault[T](param: Param[T], value: T): AssertionLogRegModel.this.type
- Attributes
- protected
- Definition Classes
- Params
- def setEndCol(end: String): AssertionLogRegModel.this.type
-
final
def
setInputCols(value: String*): AssertionLogRegModel.this.type
- Definition Classes
- HasInputAnnotationCols
-
final
def
setInputCols(value: Array[String]): AssertionLogRegModel.this.type
- Definition Classes
- HasInputAnnotationCols
- def setLabelMap(labelMappings: Map[String, Double]): AssertionLogRegModel.this.type
-
def
setLazyAnnotator(value: Boolean): AssertionLogRegModel.this.type
- Definition Classes
- CanBeLazy
- def setModel(m: LogisticRegressionModel): AssertionLogRegModel.this.type
-
final
def
setOutputCol(value: String): AssertionLogRegModel.this.type
- Definition Classes
- HasOutputAnnotationCol
-
def
setParent(parent: Estimator[AssertionLogRegModel]): AssertionLogRegModel
- Definition Classes
- Model
- def setStartCol(start: String): AssertionLogRegModel.this.type
-
def
setStorageRef(value: String): AssertionLogRegModel.this.type
- Definition Classes
- HasStorageRef
- val startCol: Param[String]
-
val
storageRef: Param[String]
- Definition Classes
- HasStorageRef
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
toString(): String
- Definition Classes
- Identifiable → AnyRef → Any
-
def
tokenIndexToChunkIndex(doc: Array[TokenPieceEmbeddings], start: Int, end: Int): (Int, Int)
- Definition Classes
- Windowing
-
final
def
transform(dataset: Dataset[_]): DataFrame
This is main point of interest.
This is main point of interest. It gets the input dataset transformed into the dataframe with predictions and probabilities This method utilises chunk groups, word piece embeddings and conversion to chunk indexes
- dataset
an input dataset
- returns
a DataFrame with probabilities of predictions
- Definition Classes
- AssertionLogRegModel → Transformer
-
def
transform(dataset: Dataset[_], paramMap: ParamMap): DataFrame
- Definition Classes
- Transformer
- Annotations
- @Since( "2.0.0" )
-
def
transform(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): DataFrame
- Definition Classes
- Transformer
- Annotations
- @Since( "2.0.0" ) @varargs()
-
final
def
transformSchema(schema: StructType): StructType
- Definition Classes
- RawAnnotator → PipelineStage
-
def
transformSchema(schema: StructType, logging: Boolean): StructType
- Attributes
- protected
- Definition Classes
- PipelineStage
- Annotations
- @DeveloperApi()
-
val
uid: String
- Definition Classes
- AssertionLogRegModel → Identifiable
-
def
validate(schema: StructType): Boolean
- Attributes
- protected
- Definition Classes
- RawAnnotator
-
def
validateStorageRef(dataset: Dataset[_], inputCols: Array[String], annotatorType: String): Unit
- Definition Classes
- HasStorageRef
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
wrapColumnMetadata(col: Column): Column
- Attributes
- protected
- Definition Classes
- RawAnnotator
-
def
write: MLWriter
- Definition Classes
- ParamsAndFeaturesWritable → DefaultParamsWritable → MLWritable