Skip to content

Commit

Permalink
version 2.1.0 updated libraries
Browse files Browse the repository at this point in the history
  • Loading branch information
heitorbarbieri committed Apr 18, 2023
1 parent 0cf3b7c commit 4f043e2
Show file tree
Hide file tree
Showing 54 changed files with 38 additions and 33 deletions.
29 changes: 15 additions & 14 deletions build.sbt
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
lazy val commonSettings = Seq(
organization := "org.bireme",
version := "1.0.0",
scalaVersion := "2.13.3" //"2.12.8" // org.scala-lang.modules:scala-xml _2.13, _2.12
scalaVersion := "2.13.10" //"2.12.8" // org.scala-lang.modules:scala-xml _2.13, _2.12
)

lazy val root = (project in file(".")).
Expand All @@ -10,14 +10,14 @@ lazy val root = (project in file(".")).
name := "BVS-InfoButton"
)

val akkaVersion = "10.1.12" //"10.1.9"
val playJsonVersion = "2.9.0" //"2.8.1"
val scalaXmlVersion = "1.3.0" //"1.2.0"
val dom4jVersion = "2.1.3" //"2.1.1"
val scalaLoggingVersion = "3.9.2" //"3.9.0"
val logbackVersion = "1.2.3"
val scalaTestVersion = "3.2.0" //"3.1.2"
val luceneVersion = "8.6.0" //"8.5.1"
val akkaVersion = "10.5.0" //"10.1.12"
val playJsonVersion = "2.9.4" //"2.9.0"
val scalaXmlVersion = "2.1.0" //"1.3.0"
val dom4jVersion = "2.1.4" //"2.1.3"
val scalaLoggingVersion = "3.9.5" //"3.9.2"
val logbackVersion = "1.4.6" //"1.2.3"
val scalaTestVersion = "3.2.15" //"3.2.0"
val luceneVersion = "9.5.0" //"8.6.0"
val scalajHttpVersion = "2.4.2" //"2.4.1"

libraryDependencies ++= Seq(
Expand All @@ -31,24 +31,25 @@ libraryDependencies ++= Seq(
"org.scalactic" %% "scalactic" % scalaTestVersion,
"org.scalatest" %% "scalatest" % scalaTestVersion % "test",
"org.apache.lucene" % "lucene-core" % luceneVersion,
"org.apache.lucene" % "lucene-analyzers-common" % luceneVersion,
"org.apache.lucene" % "lucene-analysis-common" % luceneVersion,
//"org.apache.lucene" % "lucene-analyzers-common" % luceneVersion,
"org.apache.lucene" % "lucene-queryparser" % luceneVersion,
"org.apache.lucene" % "lucene-queries" % luceneVersion,
"org.apache.lucene" % "lucene-backward-codecs" % luceneVersion,
"org.scalaj" %% "scalaj-http" % scalajHttpVersion
)

logBuffered in Test := false
Test / logBuffered := false
trapExit := false // To allow System.exit() without an exception (TestIndex.scala)

scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Ywarn-unused")

assemblyMergeStrategy in assembly := {
assembly / assemblyMergeStrategy := {
case "module-info.class" => MergeStrategy.first //MergeStrategy.discard
case x =>
val oldStrategy = (assemblyMergeStrategy in assembly).value
val oldStrategy = (assembly / assemblyMergeStrategy).value
oldStrategy(x)
}

test in assembly := {}
assembly / test := {}

17 changes: 10 additions & 7 deletions src/main/scala/org/bireme/infob/MeshConverter.scala
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import scala.util.{Either, Left, Right}
*/
class MeshConverter(indexes: String) {
//"ICD9CM", "ICD10CM", "IDC10", "SNOMED-CT", "RXNORM"
val thesauri: Map[String, String] = Map(
private val thesauri: Map[String, String] = Map(
"ICD9-CM" -> "ICD9-CM",
"ICD10-CM" -> "ICD10-CM",
"ICD10" -> "ICD10",
Expand All @@ -34,13 +34,13 @@ class MeshConverter(indexes: String) {
)

// Names of Lucene indexes to convert one thesaurus into another
val thes2thes: Map[String, String] = Map(
private val thes2thes: Map[String, String] = Map(
"DeCS" -> s"$indexes/DeCS",
"UMLS" -> s"$indexes/UMLS"
)

// Lucene index searchers to convert one system into another
val thes2thesSearchers: Map[String, IndexSearcher] = thes2thes.map {
private val thes2thesSearchers: Map[String, IndexSearcher] = thes2thes.map {
case (k, v) =>
val directory: FSDirectory = FSDirectory.open(new File(v).toPath)
require(DirectoryReader.indexExists(directory))
Expand All @@ -49,7 +49,7 @@ class MeshConverter(indexes: String) {
(k, new IndexSearcher(ireader))
}

val analyzer: KeywordAnalyzer = new KeywordAnalyzer()
private val analyzer: KeywordAnalyzer = new KeywordAnalyzer()

def close(): Unit = {
thes2thesSearchers.foreach {
Expand Down Expand Up @@ -131,12 +131,14 @@ class MeshConverter(indexes: String) {
*/
private def getCode(docIds: Array[Int],
searcher: IndexSearcher): Either[Option[Set[String]], String] = {
docIds.map(id => searcher.doc(id)).find(doc1 => doc1.get("meshCode") != null) match {
docIds.map(id => searcher.storedFields().document(id)).find(doc1 => doc1.get("meshCode") != null) match {
//docIds.map(id => searcher.storedFields().document(id)).find(doc1 => doc1.get("meshCode") != null) match {
case Some(doc) => Right(doc.get("meshCode"))
case None =>
val labelSet: Set[String] = docIds.foldLeft(Set[String]()) {
case (set, docId) =>
searcher.doc(docId).getValues("termLabel").foldLeft(set) {
searcher.storedFields().document(docId).getValues("termLabel").foldLeft(set) {
//searcher.doc(docId).getValues("termLabel").foldLeft(set) {
case (set1, value) => set1 + value.trim.toLowerCase
}
}
Expand Down Expand Up @@ -164,7 +166,8 @@ class MeshConverter(indexes: String) {
if (topDocs.totalHits.value == 0) {
None
} else {
val doc = searcher.doc(topDocs.scoreDocs.head.doc)
val doc = searcher.storedFields().document(topDocs.scoreDocs.head.doc)
//val doc = searcher.doc(topDocs.scoreDocs.head.doc)
val eDescr = doc.get("ENGLISH_DESCR")
if (eDescr == null) {
val sDescr = doc.get("SPANISH_DESCR")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import scala.jdk.CollectionConverters._
*/
object MeSH_DeCS_Index extends App {
val DEF_DECS_INDEX = "web/BVSInfoButton/indexes/DeCS"
val DEF_DECS_MASTER = "/usr/local/bireme/tabs/decs"
val DEF_DECS_MASTER = "isis/decs" //"/usr/local/bireme/tabs/decs"

val MESH_ID_TAG = 480
val HIERARCHICAL_CODE_TAG = 20
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ object UMLS_Index extends App {

val DEF_UMLS_INDEX = "web/BVSInfoButton/indexes/UMLS"
val DEF_UMLS_YEAR = 2020
val DEF_UMLS_MASTER = s"/bases/umls/${DEF_UMLS_YEAR}AA/decsnamecuisty"
val DEF_UMLS_MASTER = "isis/decsnamecuisty" //s"/bases/umls/${DEF_UMLS_YEAR}AA/decsnamecuisty"

val thesauri = HashSet("ICD9CM", "ICD10CM", "ICD10", "SNOMEDCT_US", "RXNORM",
"NDC", "LOINC")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,22 +27,22 @@ import scala.io.Source
* [-index=<UMLSIndex>] - path to Lucene index to be updated"
*/
object UMLS_UpdateIndex extends App {
val DEF_LUCENE_INDEX = "web/BVSInfoButton/indexes/UMLS"
private val DEF_LUCENE_INDEX = "web/BVSInfoButton/indexes/UMLS"
//val DEF_FILES = "ICD10:cid10/ver2010/all.txt,ICD10:cid10/espanhol/all.txt,ICD10:cid10/datasus/ver2008/all.txt,SNOMED-CT:Snomed-CT/all.txt"
//val DEF_FILES = "ICD10:cid10/one.txt"
val DEF_FILES = "ICD10:cid10/ver2010/all.txt,ICD10:cid10/espanhol/all.txt,ICD10:cid10/datasus/ver2008/all.txt"
private val DEF_FILES = "ICD10:cid10/ver2010/all.txt,ICD10:cid10/espanhol/all.txt,ICD10:cid10/datasus/ver2008/all.txt"

val parameters = args.foldLeft[Map[String, String]](Map()) {
private val parameters = args.foldLeft[Map[String, String]](Map()) {
case (map, par) =>
val split = par.split(" *= *", 2)
map + ((split(0).substring(1), split(1)))
}
val files: Set[(String, String)] = parameters.getOrElse("files", DEF_FILES).split(" *, *")
private val files: Set[(String, String)] = parameters.getOrElse("files", DEF_FILES).split(" *, *")
.map { elem =>
val split = elem.split(" *: *")
(split(0), split(1))
}.toSet // Set((thesaurus_name,path)...)
val index = parameters.getOrElse("index", DEF_LUCENE_INDEX)
private val index = parameters.getOrElse("index", DEF_LUCENE_INDEX)

updateIndex(index, files)

Expand Down Expand Up @@ -99,7 +99,8 @@ object UMLS_UpdateIndex extends App {
doc.add(new StringField("termLabelNorm", Tools.uniformString(termLabel), Field.Store.YES))
writer.addDocument(doc)
} else {
val doc1: Document = searcher2.doc(topDocs.scoreDocs.head.doc)
//val doc1: Document = searcher2.doc(topDocs.scoreDocs.head.doc)
val doc1: Document = searcher2.storedFields().document(topDocs.scoreDocs.head.doc)
val labels = getLabels(doc1) + termLabel
labels.foreach {
label =>
Expand Down
8 changes: 4 additions & 4 deletions src/main/scala/org/bireme/infob/parameters/Age.scala
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,13 @@ import scala.util.{Try, Success, Failure}

class Age(value: Option[String],
unit: Option[String]) extends SearchParameter {
val value2: Int = value.getOrElse("0").toInt
private val value2: Int = value.getOrElse("0").toInt
require(value2 > 0)
val unit2: String = unit.getOrElse("").toLowerCase
private val unit2: String = unit.getOrElse("").toLowerCase
require(unit2.equals("min") || unit2.equals("h") || unit2.equals("d") ||
unit2.equals("wk") || unit2.equals("mo") || unit2.equals("a"))

val agroup: Option[String] = convertToMonth(value.get, unit.get) match {
private val agroup: Option[String] = convertToMonth(value.get, unit.get) match {
case Some(x) if (x >= 0) && (x <= 1) => Some("infant, newborn")
case Some(x) if (x > 1) && (x < 24) => Some("infant")
case Some(x) if (x >= 24) && (x < 72) => Some("child, preschool")
Expand All @@ -29,7 +29,7 @@ class Age(value: Option[String],
case Some(x) if (x >= 540) && (x < 780) => Some("middle aged")
case Some(x) if (x >= 780) && (x < 960) => Some("aged")
case Some(x) if x >= 960 => Some("aged, 80 and older")
case None => None
case _ => None
}

private def convertToMonth(value: String, unit: String): Option[Int] = {
Expand Down
Binary file removed web/BVSInfoButton/indexes/DeCS/_4.cfe
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/DeCS/_4.cfs
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/DeCS/_4.si
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/DeCS/_5.fdm
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/DeCS/_5.fdt
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/DeCS/_5.fdx
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/DeCS/_5.fnm
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/DeCS/_5.si
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/DeCS/_5_Lucene90_0.doc
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/DeCS/_5_Lucene90_0.tim
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/DeCS/_5_Lucene90_0.tip
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/DeCS/_5_Lucene90_0.tmd
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/DeCS/segments_4
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/DeCS/segments_5
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/ICD10/_3.cfe
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/ICD10/_3.cfs
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/ICD10/_3.si
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/ICD10/_4.fdm
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/ICD10/_4.fdt
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/ICD10/_4.fdx
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/ICD10/_4.fnm
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/ICD10/_4.nvd
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/ICD10/_4.nvm
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/ICD10/_4.si
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/ICD10/_4_Lucene90_0.doc
Binary file not shown.
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/ICD10/_4_Lucene90_0.tim
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/ICD10/_4_Lucene90_0.tip
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/ICD10/_4_Lucene90_0.tmd
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/ICD10/segments_4
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/ICD10/segments_5
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/UMLS/_p0b7.fdm
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/UMLS/_p0b7.fdt
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/UMLS/_p0b7.fdx
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/UMLS/_p0b7.fnm
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/UMLS/_p0b7.si
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/UMLS/_p0b7_Lucene84_0.tmd
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/UMLS/_p0b8.fdm
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/UMLS/_p0b8.fdt
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/UMLS/_p0b8.fdx
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/UMLS/_p0b8.fnm
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/UMLS/_p0b8.si
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/UMLS/_p0b8_Lucene90_0.tmd
Binary file not shown.
Binary file removed web/BVSInfoButton/indexes/UMLS/segments_d
Binary file not shown.
Binary file added web/BVSInfoButton/indexes/UMLS/segments_e
Binary file not shown.

0 comments on commit 4f043e2

Please sign in to comment.