Sentence Entity Resolver for SNOMED CT (Clinical Findings) (sbiobert_base_cased_mli_onnx embeddings)

Description

This model maps extracted clinical NER entities to SNOMED CT concepts using sbiobert_base_cased_mli_onnx embeddings.

It is trained on SNOMED CT US Edition 20260901 release.

Live Demo Open in Colab Copy S3 URI

How to use

documentAssembler = DocumentAssembler()\
    .setInputCol("text")\
    .setOutputCol("document")

sentenceDetectorDL = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare", "en", "clinical/models")\
    .setInputCols(["document"])\
    .setOutputCol("sentence")

tokenizer = Tokenizer()\
    .setInputCols(["sentence"])\
    .setOutputCol("token")

word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical","en","clinical/models")\
    .setInputCols(["sentence","token"])\
    .setOutputCol("embeddings")

ner_model = MedicalNerModel.pretrained("ner_jsl","en","clinical/models")\
    .setInputCols(["sentence","token","embeddings"])\
    .setOutputCol("ner_tags")

ner_converter = NerConverterInternal()\
    .setInputCols(["sentence","token","ner_tags"])\
    .setOutputCol("ner_chunk")\
    .setWhiteList(["Kidney_Disease", "Cerebrovascular_Disease", "Heart_Disease", "Disease_Syndrome_Disorder", "ImagingFindings", "Symptom", "VS_Finding", "EKG_Findings", "Communicable_Disease"])

chunk2doc = Chunk2Doc()\
    .setInputCols(["ner_chunk"])\
    .setOutputCol("ner_chunk_doc")

embedder = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli_onnx", "en", "clinical/models")\
    .setInputCols(["ner_chunk_doc"])\
    .setOutputCol("sbert_embeddings")\
    .setCaseSensitive(False)

resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_snomed_findings_20260901","en","clinical/models")\
    .setInputCols(["sbert_embeddings"])\
    .setOutputCol("snomed_code")\
    .setDistanceFunction("EUCLIDEAN")\
    .setThreshold(1000)

pipeline = Pipeline(stages=[\
    documentAssembler, sentenceDetectorDL, tokenizer, word_embeddings, ner_model, ner_converter, chunk2doc, embedder, resolver\
])

data = spark.createDataFrame([["The patient presented with recurrent fevers. Clinically she appeared cachectic with hepatosplenomegaly. Laboratory results confirmed pancytopenia."]]).toDF("text")
result = pipeline.fit(data).transform(data)
documentAssembler = nlp.DocumentAssembler()\
    .setInputCol("text")\
    .setOutputCol("document")

sentenceDetectorDL = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare", "en", "clinical/models")\
    .setInputCols(["document"])\
    .setOutputCol("sentence")

tokenizer = nlp.Tokenizer()\
    .setInputCols(["sentence"])\
    .setOutputCol("token")

word_embeddings = nlp.WordEmbeddingsModel.pretrained("embeddings_clinical","en","clinical/models")\
    .setInputCols(["sentence","token"])\
    .setOutputCol("embeddings")

ner_model = medical.NerModel.pretrained("ner_jsl","en","clinical/models")\
    .setInputCols(["sentence","token","embeddings"])\
    .setOutputCol("ner_tags")

ner_converter = medical.NerConverterInternal()\
    .setInputCols(["sentence","token","ner_tags"])\
    .setOutputCol("ner_chunk")\
    .setWhiteList(["Kidney_Disease", "Cerebrovascular_Disease", "Heart_Disease", "Disease_Syndrome_Disorder", "ImagingFindings", "Symptom", "VS_Finding", "EKG_Findings", "Communicable_Disease"])

chunk2doc = nlp.Chunk2Doc()\
    .setInputCols(["ner_chunk"])\
    .setOutputCol("ner_chunk_doc")

embedder = nlp.BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli_onnx", "en", "clinical/models")\
    .setInputCols(["ner_chunk_doc"])\
    .setOutputCol("sbert_embeddings")\
    .setCaseSensitive(False)

resolver = medical.SentenceEntityResolverModel.pretrained("sbiobertresolve_snomed_findings_20260901","en","clinical/models")\
    .setInputCols(["sbert_embeddings"])\
    .setOutputCol("snomed_code")\
    .setDistanceFunction("EUCLIDEAN")\
    .setThreshold(1000)

pipeline = nlp.Pipeline(stages=[\
    documentAssembler, sentenceDetectorDL, tokenizer, word_embeddings, ner_model, ner_converter, chunk2doc, embedder, resolver\
])

data = spark.createDataFrame([["The patient presented with recurrent fevers. Clinically she appeared cachectic with hepatosplenomegaly. Laboratory results confirmed pancytopenia."]]).toDF("text")
result = pipeline.fit(data).transform(data)

val documentAssembler = new DocumentAssembler()
    .setInputCol("text")
    .setOutputCol("document")

val sentenceDetectorDL = SentenceDetectorDLModel
    .pretrained("sentence_detector_dl_healthcare", "en", "clinical/models")
    .setInputCols(Array("document"))
    .setOutputCol("sentence")

val tokenizer = new Tokenizer()
    .setInputCols("sentence")
    .setOutputCol("token")

val word_embeddings = WordEmbeddingsModel
    .pretrained("embeddings_clinical", "en", "clinical/models")
    .setInputCols(Array("sentence", "token"))
    .setOutputCol("embeddings")

val ner_model = MedicalNerModel
    .pretrained("ner_jsl", "en", "clinical/models")
    .setInputCols(Array("sentence", "token", "embeddings"))
    .setOutputCol("ner_tags")

val ner_converter = new NerConverterInternal()
    .setInputCols(Array("sentence", "token", "ner_tags"))
    .setOutputCol("ner_chunk")
    .setWhiteList(Array("Kidney_Disease", "Cerebrovascular_Disease", "Heart_Disease", "Disease_Syndrome_Disorder", "ImagingFindings", "Symptom", "VS_Finding", "EKG_Findings", "Communicable_Disease"))

val chunk2doc = new Chunk2Doc()
    .setInputCols(Array("ner_chunk"))
    .setOutputCol("ner_chunk_doc")

val embedder = BertSentenceEmbeddings
    .pretrained("sbiobert_base_cased_mli_onnx", "en", "clinical/models")
    .setInputCols(Array("ner_chunk_doc"))
    .setOutputCol("sbert_embeddings")
    .setCaseSensitive(false)

val resolver = SentenceEntityResolverModel
    .pretrained("sbiobertresolve_snomed_findings_20260901", "en", "clinical/models")
    .setInputCols(Array("sbert_embeddings"))
    .setOutputCol("snomed_code")
    .setDistanceFunction("EUCLIDEAN")
    .setThreshold(1000)

val pipeline = new Pipeline().setStages(Array(
    documentAssembler, sentenceDetectorDL, tokenizer, word_embeddings, ner_model, ner_converter, chunk2doc, embedder, resolver
))

val data = Seq("The patient presented with recurrent fevers. Clinically she appeared cachectic with hepatosplenomegaly. Laboratory results confirmed pancytopenia.").toDF("text")
val res = pipeline.fit(data).transform(data)

Results

| chunk              | label      |   snomed_code | resolution         | all_codes                                                                           | all_resolutions                                                                     |
|:-------------------|:-----------|--------------:|:-------------------|:------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|
| fevers             | VS_Finding |     386661006 | fever              | 386661006:::77957000:::271751000:::248435007:::271754008:::186694006:::704425001... | fever:::intermittent fever:::sustained fever:::prolonged fever:::crisis of fever... |
| cachectic          | Symptom    |     238108007 | cachectic          | 238108007:::422003001:::284529003:::788876001:::240128005:::288517002:::89476005... | cachectic:::cachexia associated with aids:::cardiac cachexia:::malignant cachexi... |
| hepatosplenomegaly | Symptom    |      36760000 | hepatosplenomegaly | 36760000:::16294009:::19058002:::191382009:::80378000:::240630008:::190794006:::... | hepatosplenomegaly:::splenomegaly:::congestive splenomegaly:::chronic congestive... |
| pancytopenia       | Symptom    |     127034005 | pancytopenia       | 127034005:::736024007:::5876000:::124961001:::417672002:::302215000:::38970002::... | pancytopenia:::drug induced pancytopenia:::pancytopenia - acquired:::reticulocyt... |

Model Information

Model Name: sbiobertresolve_snomed_findings_20260901
Compatibility: Healthcare NLP 6.4.1+
License: Licensed
Edition: Official
Input Labels: [sbert_embeddings]
Output Labels: [snomed_code]
Language: en
Size: 687.4 MB
Case sensitive: false