Packages

class Router extends AnnotatorModel[Router] with HasSimpleAnnotate[Router] with ParamsAndFeaturesWritable with CheckLicense

This class allows to filter any annotation based on the medatata fields.

val testData = ResourceHelper.spark.createDataFrame(Seq(
   (1, "\"Jesus live in Leon. Madrid is the capital of Spain")
 )).toDF("id", "text")

 val document = new DocumentAssembler()
   .setInputCol("text")
   .setOutputCol("document")

 val sentence = new SentenceDetector()
   .setInputCols("document")
   .setOutputCol("sentence")

 val regexMatcher = new RegexMatcher()
   .setExternalRules(ExternalResource("src/test/resources/regex-matcher/rules2.txt", ReadAs.TEXT, Map("delimiter" -> ",")))
   .setInputCols(Array("sentence"))
   .setOutputCol("regex")
   .setStrategy("MATCH_ALL")

 val chunk2Doc = new Chunk2Doc().setInputCols("regex").setOutputCol("doc_chunk")

 val embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L2_128")
   .setInputCols("doc_chunk")
   .setOutputCol("bert")
   .setCaseSensitive(false)
   .setMaxSentenceLength(32)

 val routerName = new Router()
   .setInputType("sentence_embeddings")
   .setInputCols(Array("bert"))
   .setMetadataField("identifier")
   .setFilterFieldsElements(Array("name"))
   .setOutputCol("names")
 val routerCity = new Router()
   .setInputType("sentence_embeddings")
   .setInputCols(Array("bert"))
   .setMetadataField("identifier")
   .setFilterFieldsElements(Array("city"))
   .setOutputCol("cities")
Linear Supertypes
CheckLicense, HasSimpleAnnotate[Router], AnnotatorModel[Router], CanBeLazy, RawAnnotator[Router], HasOutputAnnotationCol, HasInputAnnotationCols, HasOutputAnnotatorType, ParamsAndFeaturesWritable, HasFeatures, DefaultParamsWritable, MLWritable, Model[Router], Transformer, PipelineStage, Logging, Params, Serializable, Serializable, Identifiable, AnyRef, Any
Ordering
  1. Grouped
  2. Alphabetic
  3. By Inheritance
Inherited
  1. Router
  2. CheckLicense
  3. HasSimpleAnnotate
  4. AnnotatorModel
  5. CanBeLazy
  6. RawAnnotator
  7. HasOutputAnnotationCol
  8. HasInputAnnotationCols
  9. HasOutputAnnotatorType
  10. ParamsAndFeaturesWritable
  11. HasFeatures
  12. DefaultParamsWritable
  13. MLWritable
  14. Model
  15. Transformer
  16. PipelineStage
  17. Logging
  18. Params
  19. Serializable
  20. Serializable
  21. Identifiable
  22. AnyRef
  23. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Parameters

  1. val filterFieldsElements: StringArrayParam

    The filterfieldsElements are the allowed values for the metadata field that is being used

  2. val inputType: Param[String]

    The type of the entity that you want to filter by default sentence_embeddings.Possible values document|token|wordpiece|word_embeddings| sentence_embeddings|category|date|sentiment|pos|chunk|named_entity|regex|dependency|labeled_dependency|language|keyword

  3. val metadataField: Param[String]

    The metadataField is the field that in metadata that you want to filter (by default 'entity')

Members

  1. type AnnotatorType = String
    Definition Classes
    HasOutputAnnotatorType
  1. def afterAnnotate(dataset: DataFrame): DataFrame
    Definition Classes
    Router → AnnotatorModel
  2. def annotate(annotations: Seq[Annotation]): Seq[Annotation]
    Definition Classes
    Router → HasSimpleAnnotate
  3. def beforeAnnotate(dataset: Dataset[_]): Dataset[_]
    Definition Classes
    Router → AnnotatorModel
  4. def checkValidEnvironment(spark: Option[SparkSession]): Unit
    Definition Classes
    CheckLicense
  5. def checkValidScope(scope: String): Unit
    Definition Classes
    CheckLicense
  6. def checkValidScopeAndEnvironment(scope: String, spark: Option[SparkSession], checkLp: Boolean): Unit
    Definition Classes
    CheckLicense
  7. def checkValidScopesAndEnvironment(scopes: Seq[String], spark: Option[SparkSession], checkLp: Boolean): Unit
    Definition Classes
    CheckLicense
  8. final def clear(param: Param[_]): Router.this.type
    Definition Classes
    Params
  9. def copy(extra: ParamMap): Router
    Definition Classes
    RawAnnotator → Model → Transformer → PipelineStage → Params
  10. def dfAnnotate: UserDefinedFunction
    Definition Classes
    HasSimpleAnnotate
  11. def explainParam(param: Param[_]): String
    Definition Classes
    Params
  12. def explainParams(): String
    Definition Classes
    Params
  13. final def extractParamMap(): ParamMap
    Definition Classes
    Params
  14. final def extractParamMap(extra: ParamMap): ParamMap
    Definition Classes
    Params
  15. val features: ArrayBuffer[Feature[_, _, _]]
    Definition Classes
    HasFeatures
  16. final def get[T](param: Param[T]): Option[T]
    Definition Classes
    Params
  17. final def getDefault[T](param: Param[T]): Option[T]
    Definition Classes
    Params
  18. def getFilterFieldsElements(): Array[String]
  19. def getInputCols: Array[String]
    Definition Classes
    HasInputAnnotationCols
  20. def getInputType(): String
  21. def getLazyAnnotator: Boolean
    Definition Classes
    CanBeLazy
  22. def getMetadataField(): String
  23. final def getOrDefault[T](param: Param[T]): T
    Definition Classes
    Params
  24. final def getOutputCol: String
    Definition Classes
    HasOutputAnnotationCol
  25. def getParam(paramName: String): Param[Any]
    Definition Classes
    Params
  26. final def hasDefault[T](param: Param[T]): Boolean
    Definition Classes
    Params
  27. def hasParam(paramName: String): Boolean
    Definition Classes
    Params
  28. def hasParent: Boolean
    Definition Classes
    Model
  29. lazy val inputAnnotatorTypes: Array[String]
    Definition Classes
    Router → HasInputAnnotationCols
  30. final def isDefined(param: Param[_]): Boolean
    Definition Classes
    Params
  31. final def isSet(param: Param[_]): Boolean
    Definition Classes
    Params
  32. val lazyAnnotator: BooleanParam
    Definition Classes
    CanBeLazy
  33. val optionalInputAnnotatorTypes: Array[String]
    Definition Classes
    HasInputAnnotationCols
  34. lazy val outputAnnotatorType: AnnotatorType
    Definition Classes
    Router → HasOutputAnnotatorType
  35. lazy val params: Array[Param[_]]
    Definition Classes
    Params
  36. var parent: Estimator[Router]
    Definition Classes
    Model
  37. def save(path: String): Unit
    Definition Classes
    MLWritable
    Annotations
    @Since( "1.6.0" ) @throws( ... )
  38. final def set[T](param: Param[T], value: T): Router.this.type
    Definition Classes
    Params
  39. def setFilterFieldsElements(value: Array[String]): Router
  40. final def setInputCols(value: String*): Router.this.type
    Definition Classes
    HasInputAnnotationCols
  41. def setInputCols(value: Array[String]): Router.this.type
    Definition Classes
    HasInputAnnotationCols
  42. def setInputType(value: String): Router
  43. def setLazyAnnotator(value: Boolean): Router.this.type
    Definition Classes
    CanBeLazy
  44. def setMetadataField(value: String): Router
  45. final def setOutputCol(value: String): Router.this.type
    Definition Classes
    HasOutputAnnotationCol
  46. def setParent(parent: Estimator[Router]): Router
    Definition Classes
    Model
  47. val storageRef: Param[String]
  48. def toString(): String
    Definition Classes
    Identifiable → AnyRef → Any
  49. final def transform(dataset: Dataset[_]): DataFrame
    Definition Classes
    AnnotatorModel → Transformer
  50. def transform(dataset: Dataset[_], paramMap: ParamMap): DataFrame
    Definition Classes
    Transformer
    Annotations
    @Since( "2.0.0" )
  51. def transform(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): DataFrame
    Definition Classes
    Transformer
    Annotations
    @Since( "2.0.0" ) @varargs()
  52. final def transformSchema(schema: StructType): StructType
    Definition Classes
    RawAnnotator → PipelineStage
  53. val uid: String
    Definition Classes
    Router → Identifiable
  54. def write: MLWriter
    Definition Classes
    ParamsAndFeaturesWritable → DefaultParamsWritable → MLWritable