Upload folder using huggingface_hub

CRF/java/.am

@@ -1,27 +1,27 @@
-TARGET=MeCab
-JAVAC=javac
-JAVA=java
-JAR=jar
-CXX=c++
-INCLUDE=/usr/lib/jvm/java-6-openjdk/include
-
-PACKAGE=org/chasen/mecab
-
-LIBS=`mecab-config --libs`
-INC=`mecab-config --cflags` -I$(INCLUDE) -I$(INCLUDE)/linux
-
-all:
-	$(CXX) -O3 -c -fpic $(TARGET)_wrap.cxx  $(INC)
-	$(CXX) -shared  $(TARGET)_wrap.o -o lib$(TARGET).so $(LIBS)
-	$(JAVAC) $(PACKAGE)/*.java
-	$(JAVAC) test.java
-	$(JAR) cfv $(TARGET).jar $(PACKAGE)/*.class
-
-test:
-	env LD_LIBRARY_PATH=. $(JAVA) test
-
-clean:
-	rm -fr *.jar *.o *.so *.class $(PACKAGE)/*.class
-	
-cleanall:
-	rm -fr $(TARGET).java *.cxx
+TARGET=MeCab
+JAVAC=javac
+JAVA=java
+JAR=jar
+CXX=c++
+INCLUDE=/usr/lib/jvm/java-6-openjdk/include
+
+PACKAGE=org/chasen/mecab
+
+LIBS=`mecab-config --libs`
+INC=`mecab-config --cflags` -I$(INCLUDE) -I$(INCLUDE)/linux
+
+all:
+	$(CXX) -O3 -c -fpic $(TARGET)_wrap.cxx  $(INC)
+	$(CXX) -shared  $(TARGET)_wrap.o -o lib$(TARGET).so $(LIBS)
+	$(JAVAC) $(PACKAGE)/*.java
+	$(JAVAC) test.java
+	$(JAR) cfv $(TARGET).jar $(PACKAGE)/*.class
+
+test:
+	env LD_LIBRARY_PATH=. $(JAVA) test
+
+clean:
+	rm -fr *.jar *.o *.so *.class $(PACKAGE)/*.class
+	
+cleanall:
+	rm -fr $(TARGET).java *.cxx

CRF/java/org/chasen/crfpp/Model.java

@@ -1,51 +1,51 @@
-/* ----------------------------------------------------------------------------
- * This file was automatically generated by SWIG (http://www.swig.org).
- * Version 1.3.40
- *
- * Do not make changes to this file unless you know what you are doing--modify
- * the SWIG interface file instead.
- * ----------------------------------------------------------------------------- */
-
-package org.chasen.crfpp;
-
-public class Model {
-  private long swigCPtr;
-  protected boolean swigCMemOwn;
-
-  protected Model(long cPtr, boolean cMemoryOwn) {
-    swigCMemOwn = cMemoryOwn;
-    swigCPtr = cPtr;
-  }
-
-  protected static long getCPtr(Model obj) {
-    return (obj == null) ? 0 : obj.swigCPtr;
-  }
-
-  protected void finalize() {
-    delete();
-  }
-
-  public synchronized void delete() {
-    if (swigCPtr != 0) {
-      if (swigCMemOwn) {
-        swigCMemOwn = false;
-        CRFPPJNI.delete_Model(swigCPtr);
-      }
-      swigCPtr = 0;
-    }
-  }
-
-  public Tagger createTagger() {
-    long cPtr = CRFPPJNI.Model_createTagger(swigCPtr, this);
-    return (cPtr == 0) ? null : new Tagger(cPtr, false);
-  }
-
-  public String what() {
-    return CRFPPJNI.Model_what(swigCPtr, this);
-  }
-
-  public Model(String arg) {
-    this(CRFPPJNI.new_Model(arg), true);
-  }
-
-}
+/* ----------------------------------------------------------------------------
+ * This file was automatically generated by SWIG (http://www.swig.org).
+ * Version 1.3.40
+ *
+ * Do not make changes to this file unless you know what you are doing--modify
+ * the SWIG interface file instead.
+ * ----------------------------------------------------------------------------- */
+
+package org.chasen.crfpp;
+
+public class Model {
+  private long swigCPtr;
+  protected boolean swigCMemOwn;
+
+  protected Model(long cPtr, boolean cMemoryOwn) {
+    swigCMemOwn = cMemoryOwn;
+    swigCPtr = cPtr;
+  }
+
+  protected static long getCPtr(Model obj) {
+    return (obj == null) ? 0 : obj.swigCPtr;
+  }
+
+  protected void finalize() {
+    delete();
+  }
+
+  public synchronized void delete() {
+    if (swigCPtr != 0) {
+      if (swigCMemOwn) {
+        swigCMemOwn = false;
+        CRFPPJNI.delete_Model(swigCPtr);
+      }
+      swigCPtr = 0;
+    }
+  }
+
+  public Tagger createTagger() {
+    long cPtr = CRFPPJNI.Model_createTagger(swigCPtr, this);
+    return (cPtr == 0) ? null : new Tagger(cPtr, false);
+  }
+
+  public String what() {
+    return CRFPPJNI.Model_what(swigCPtr, this);
+  }
+
+  public Model(String arg) {
+    this(CRFPPJNI.new_Model(arg), true);
+  }
+
+}

CRF/perl/Makefile.old

@@ -1,931 +1,931 @@
-# This Makefile is for the CRFPP extension to perl.
-#
-# It was generated automatically by MakeMaker version
-# 6.56 (Revision: 65600) from the contents of
-# Makefile.PL. Don't edit this file, edit Makefile.PL instead.
-#
-#       ANY CHANGES MADE HERE WILL BE LOST!
-#
-#   MakeMaker ARGV: ()
-#
-
-#   MakeMaker Parameters:
-
-#     BUILD_REQUIRES => {  }
-#     CC => q[c++]
-#     INC => q[]
-#     LD => q[c++]
-#     LIBS => q[-lpthread -lcrfpp]
-#     NAME => q[CRFPP]
-#     OBJECT => q[CRFPP_wrap.o]
-#     PREREQ_PM => {  }
-
-# --- MakeMaker post_initialize section:
-
-
-# --- MakeMaker const_config section:
-
-# These definitions are from config.sh (via /usr/lib/perl/5.12/Config.pm).
-# They may have been overridden via Makefile.PL or on the command line.
-AR = ar
-CC = c++
-CCCDLFLAGS = -fPIC
-CCDLFLAGS = -Wl,-E
-DLEXT = so
-DLSRC = dl_dlopen.xs
-EXE_EXT = 
-FULL_AR = /usr/bin/ar
-LD = c++
-LDDLFLAGS = -shared -O2 -g -L/usr/local/lib -fstack-protector
-LDFLAGS =  -fstack-protector -L/usr/local/lib
-LIBC = 
-LIB_EXT = .a
-OBJ_EXT = .o
-OSNAME = linux
-OSVERS = 2.6.24-28-server
-RANLIB = :
-SITELIBEXP = /usr/local/share/perl/5.12.4
-SITEARCHEXP = /usr/local/lib/perl/5.12.4
-SO = so
-VENDORARCHEXP = /usr/lib/perl5
-VENDORLIBEXP = /usr/share/perl5
-
-
-# --- MakeMaker constants section:
-AR_STATIC_ARGS = cr
-DIRFILESEP = /
-DFSEP = $(DIRFILESEP)
-NAME = CRFPP
-NAME_SYM = CRFPP
-VERSION = 
-VERSION_MACRO = VERSION
-VERSION_SYM = 
-DEFINE_VERSION = -D$(VERSION_MACRO)=\"$(VERSION)\"
-XS_VERSION = 
-XS_VERSION_MACRO = XS_VERSION
-XS_DEFINE_VERSION = -D$(XS_VERSION_MACRO)=\"$(XS_VERSION)\"
-INST_ARCHLIB = blib/arch
-INST_SCRIPT = blib/script
-INST_BIN = blib/bin
-INST_LIB = blib/lib
-INST_MAN1DIR = blib/man1
-INST_MAN3DIR = blib/man3
-MAN1EXT = 1p
-MAN3EXT = 3pm
-INSTALLDIRS = site
-DESTDIR = 
-PREFIX = /usr
-PERLPREFIX = $(PREFIX)
-SITEPREFIX = $(PREFIX)/local
-VENDORPREFIX = $(PREFIX)
-INSTALLPRIVLIB = $(PERLPREFIX)/share/perl/5.12
-DESTINSTALLPRIVLIB = $(DESTDIR)$(INSTALLPRIVLIB)
-INSTALLSITELIB = $(SITEPREFIX)/share/perl/5.12.4
-DESTINSTALLSITELIB = $(DESTDIR)$(INSTALLSITELIB)
-INSTALLVENDORLIB = $(VENDORPREFIX)/share/perl5
-DESTINSTALLVENDORLIB = $(DESTDIR)$(INSTALLVENDORLIB)
-INSTALLARCHLIB = $(PERLPREFIX)/lib/perl/5.12
-DESTINSTALLARCHLIB = $(DESTDIR)$(INSTALLARCHLIB)
-INSTALLSITEARCH = $(SITEPREFIX)/lib/perl/5.12.4
-DESTINSTALLSITEARCH = $(DESTDIR)$(INSTALLSITEARCH)
-INSTALLVENDORARCH = $(VENDORPREFIX)/lib/perl5
-DESTINSTALLVENDORARCH = $(DESTDIR)$(INSTALLVENDORARCH)
-INSTALLBIN = $(PERLPREFIX)/bin
-DESTINSTALLBIN = $(DESTDIR)$(INSTALLBIN)
-INSTALLSITEBIN = $(SITEPREFIX)/bin
-DESTINSTALLSITEBIN = $(DESTDIR)$(INSTALLSITEBIN)
-INSTALLVENDORBIN = $(VENDORPREFIX)/bin
-DESTINSTALLVENDORBIN = $(DESTDIR)$(INSTALLVENDORBIN)
-INSTALLSCRIPT = $(PERLPREFIX)/bin
-DESTINSTALLSCRIPT = $(DESTDIR)$(INSTALLSCRIPT)
-INSTALLSITESCRIPT = $(SITEPREFIX)/bin
-DESTINSTALLSITESCRIPT = $(DESTDIR)$(INSTALLSITESCRIPT)
-INSTALLVENDORSCRIPT = $(VENDORPREFIX)/bin
-DESTINSTALLVENDORSCRIPT = $(DESTDIR)$(INSTALLVENDORSCRIPT)
-INSTALLMAN1DIR = $(PERLPREFIX)/share/man/man1
-DESTINSTALLMAN1DIR = $(DESTDIR)$(INSTALLMAN1DIR)
-INSTALLSITEMAN1DIR = $(SITEPREFIX)/man/man1
-DESTINSTALLSITEMAN1DIR = $(DESTDIR)$(INSTALLSITEMAN1DIR)
-INSTALLVENDORMAN1DIR = $(VENDORPREFIX)/share/man/man1
-DESTINSTALLVENDORMAN1DIR = $(DESTDIR)$(INSTALLVENDORMAN1DIR)
-INSTALLMAN3DIR = $(PERLPREFIX)/share/man/man3
-DESTINSTALLMAN3DIR = $(DESTDIR)$(INSTALLMAN3DIR)
-INSTALLSITEMAN3DIR = $(SITEPREFIX)/man/man3
-DESTINSTALLSITEMAN3DIR = $(DESTDIR)$(INSTALLSITEMAN3DIR)
-INSTALLVENDORMAN3DIR = $(VENDORPREFIX)/share/man/man3
-DESTINSTALLVENDORMAN3DIR = $(DESTDIR)$(INSTALLVENDORMAN3DIR)
-PERL_LIB = /usr/share/perl/5.12
-PERL_ARCHLIB = /usr/lib/perl/5.12
-LIBPERL_A = libperl.a
-FIRST_MAKEFILE = Makefile
-MAKEFILE_OLD = Makefile.old
-MAKE_APERL_FILE = Makefile.aperl
-PERLMAINCC = $(CC)
-PERL_INC = /usr/lib/perl/5.12/CORE
-PERL = /usr/bin/perl
-FULLPERL = /usr/bin/perl
-ABSPERL = $(PERL)
-PERLRUN = $(PERL)
-FULLPERLRUN = $(FULLPERL)
-ABSPERLRUN = $(ABSPERL)
-PERLRUNINST = $(PERLRUN) "-I$(INST_ARCHLIB)" "-I$(INST_LIB)"
-FULLPERLRUNINST = $(FULLPERLRUN) "-I$(INST_ARCHLIB)" "-I$(INST_LIB)"
-ABSPERLRUNINST = $(ABSPERLRUN) "-I$(INST_ARCHLIB)" "-I$(INST_LIB)"
-PERL_CORE = 0
-PERM_DIR = 755
-PERM_RW = 644
-PERM_RWX = 755
-
-MAKEMAKER   = /usr/share/perl/5.12/ExtUtils/MakeMaker.pm
-MM_VERSION  = 6.56
-MM_REVISION = 65600
-
-# FULLEXT = Pathname for extension directory (eg Foo/Bar/Oracle).
-# BASEEXT = Basename part of FULLEXT. May be just equal FULLEXT. (eg Oracle)
-# PARENT_NAME = NAME without BASEEXT and no trailing :: (eg Foo::Bar)
-# DLBASE  = Basename part of dynamic library. May be just equal BASEEXT.
-MAKE = make
-FULLEXT = CRFPP
-BASEEXT = CRFPP
-PARENT_NAME = 
-DLBASE = $(BASEEXT)
-VERSION_FROM = 
-INC = 
-OBJECT = CRFPP_wrap$(OBJ_EXT)
-LDFROM = $(OBJECT)
-LINKTYPE = dynamic
-BOOTDEP = 
-
-# Handy lists of source code files:
-XS_FILES = 
-C_FILES  = CRFPP_wrap.cxx
-O_FILES  = CRFPP_wrap.o
-H_FILES  = 
-MAN1PODS = 
-MAN3PODS = 
-
-# Where is the Config information that we are using/depend on
-CONFIGDEP = $(PERL_ARCHLIB)$(DFSEP)Config.pm $(PERL_INC)$(DFSEP)config.h
-
-# Where to build things
-INST_LIBDIR      = $(INST_LIB)
-INST_ARCHLIBDIR  = $(INST_ARCHLIB)
-
-INST_AUTODIR     = $(INST_LIB)/auto/$(FULLEXT)
-INST_ARCHAUTODIR = $(INST_ARCHLIB)/auto/$(FULLEXT)
-
-INST_STATIC      = $(INST_ARCHAUTODIR)/$(BASEEXT)$(LIB_EXT)
-INST_DYNAMIC     = $(INST_ARCHAUTODIR)/$(DLBASE).$(DLEXT)
-INST_BOOT        = $(INST_ARCHAUTODIR)/$(BASEEXT).bs
-
-# Extra linker info
-EXPORT_LIST        = 
-PERL_ARCHIVE       = 
-PERL_ARCHIVE_AFTER = 
-
-
-TO_INST_PM = CRFPP.pm
-
-PM_TO_BLIB = CRFPP.pm \
-	$(INST_LIB)/CRFPP.pm
-
-
-# --- MakeMaker platform_constants section:
-MM_Unix_VERSION = 6.56
-PERL_MALLOC_DEF = -DPERL_EXTMALLOC_DEF -Dmalloc=Perl_malloc -Dfree=Perl_mfree -Drealloc=Perl_realloc -Dcalloc=Perl_calloc
-
-
-# --- MakeMaker tool_autosplit section:
-# Usage: $(AUTOSPLITFILE) FileToSplit AutoDirToSplitInto
-AUTOSPLITFILE = $(ABSPERLRUN)  -e 'use AutoSplit;  autosplit($$ARGV[0], $$ARGV[1], 0, 1, 1)' --
-
-
-
-# --- MakeMaker tool_xsubpp section:
-
-XSUBPPDIR = /usr/share/perl/5.12/ExtUtils
-XSUBPP = $(XSUBPPDIR)$(DFSEP)xsubpp
-XSUBPPRUN = $(PERLRUN) $(XSUBPP)
-XSPROTOARG = 
-XSUBPPDEPS = /usr/share/perl/5.12/ExtUtils/typemap $(XSUBPP)
-XSUBPPARGS = -typemap /usr/share/perl/5.12/ExtUtils/typemap
-XSUBPP_EXTRA_ARGS = 
-
-
-# --- MakeMaker tools_other section:
-SHELL = /bin/sh
-CHMOD = chmod
-CP = cp
-MV = mv
-NOOP = $(TRUE)
-NOECHO = @
-RM_F = rm -f
-RM_RF = rm -rf
-TEST_F = test -f
-TOUCH = touch
-UMASK_NULL = umask 0
-DEV_NULL = > /dev/null 2>&1
-MKPATH = $(ABSPERLRUN) -MExtUtils::Command -e 'mkpath' --
-EQUALIZE_TIMESTAMP = $(ABSPERLRUN) -MExtUtils::Command -e 'eqtime' --
-FALSE = false
-TRUE = true
-ECHO = echo
-ECHO_N = echo -n
-UNINST = 0
-VERBINST = 0
-MOD_INSTALL = $(ABSPERLRUN) -MExtUtils::Install -e 'install([ from_to => {@ARGV}, verbose => '\''$(VERBINST)'\'', uninstall_shadows => '\''$(UNINST)'\'', dir_mode => '\''$(PERM_DIR)'\'' ]);' --
-DOC_INSTALL = $(ABSPERLRUN) -MExtUtils::Command::MM -e 'perllocal_install' --
-UNINSTALL = $(ABSPERLRUN) -MExtUtils::Command::MM -e 'uninstall' --
-WARN_IF_OLD_PACKLIST = $(ABSPERLRUN) -MExtUtils::Command::MM -e 'warn_if_old_packlist' --
-MACROSTART = 
-MACROEND = 
-USEMAKEFILE = -f
-FIXIN = $(ABSPERLRUN) -MExtUtils::MY -e 'MY->fixin(shift)' --
-
-
-# --- MakeMaker makemakerdflt section:
-makemakerdflt : all
-	$(NOECHO) $(NOOP)
-
-
-# --- MakeMaker dist section:
-TAR = tar
-TARFLAGS = cvf
-ZIP = zip
-ZIPFLAGS = -r
-COMPRESS = gzip --best
-SUFFIX = .gz
-SHAR = shar
-PREOP = $(NOECHO) $(NOOP)
-POSTOP = $(NOECHO) $(NOOP)
-TO_UNIX = $(NOECHO) $(NOOP)
-CI = ci -u
-RCS_LABEL = rcs -Nv$(VERSION_SYM): -q
-DIST_CP = best
-DIST_DEFAULT = tardist
-DISTNAME = CRFPP
-DISTVNAME = CRFPP-
-
-
-# --- MakeMaker macro section:
-
-
-# --- MakeMaker depend section:
-
-
-# --- MakeMaker cflags section:
-
-CCFLAGS = -D_REENTRANT -D_GNU_SOURCE -DDEBIAN -fno-strict-aliasing -pipe -fstack-protector -I/usr/local/include -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
-OPTIMIZE = -O2 -g
-PERLTYPE = 
-MPOLLUTE = 
-
-
-# --- MakeMaker const_loadlibs section:
-
-# CRFPP might depend on some other libraries:
-# See ExtUtils::Liblist for details
-#
-EXTRALIBS = -lcrfpp
-LDLOADLIBS = -lpthread -lcrfpp
-BSLOADLIBS = 
-
-
-# --- MakeMaker const_cccmd section:
-CCCMD = $(CC) -c $(PASTHRU_INC) $(INC) \
-	$(CCFLAGS) $(OPTIMIZE) \
-	$(PERLTYPE) $(MPOLLUTE) $(DEFINE_VERSION) \
-	$(XS_DEFINE_VERSION)
-
-# --- MakeMaker post_constants section:
-
-
-# --- MakeMaker pasthru section:
-
-PASTHRU = LIBPERL_A="$(LIBPERL_A)"\
-	LINKTYPE="$(LINKTYPE)"\
-	OPTIMIZE="$(OPTIMIZE)"\
-	PREFIX="$(PREFIX)"\
-	PASTHRU_INC="$(PASTHRU_INC)"
-
-
-# --- MakeMaker special_targets section:
-.SUFFIXES : .xs .c .C .cpp .i .s .cxx .cc $(OBJ_EXT)
-
-.PHONY: all config static dynamic test linkext manifest blibdirs clean realclean disttest distdir
-
-
-
-# --- MakeMaker c_o section:
-
-.c.i:
-	cc -E -c $(PASTHRU_INC) $(INC) \
-	$(CCFLAGS) $(OPTIMIZE) \
-	$(PERLTYPE) $(MPOLLUTE) $(DEFINE_VERSION) \
-	$(XS_DEFINE_VERSION) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.c > $*.i
-
-.c.s:
-	$(CCCMD) -S $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.c
-
-.c$(OBJ_EXT):
-	$(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.c
-
-.cpp$(OBJ_EXT):
-	$(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.cpp
-
-.cxx$(OBJ_EXT):
-	$(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.cxx
-
-.cc$(OBJ_EXT):
-	$(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.cc
-
-.C$(OBJ_EXT):
-	$(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.C
-
-
-# --- MakeMaker xs_c section:
-
-.xs.c:
-	$(XSUBPPRUN) $(XSPROTOARG) $(XSUBPPARGS) $(XSUBPP_EXTRA_ARGS) $*.xs > $*.xsc && $(MV) $*.xsc $*.c
-
-
-# --- MakeMaker xs_o section:
-
-.xs$(OBJ_EXT):
-	$(XSUBPPRUN) $(XSPROTOARG) $(XSUBPPARGS) $*.xs > $*.xsc && $(MV) $*.xsc $*.c
-	$(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.c
-
-
-# --- MakeMaker top_targets section:
-all :: pure_all manifypods
-	$(NOECHO) $(NOOP)
-
-
-pure_all :: config pm_to_blib subdirs linkext
-	$(NOECHO) $(NOOP)
-
-subdirs :: $(MYEXTLIB)
-	$(NOECHO) $(NOOP)
-
-config :: $(FIRST_MAKEFILE) blibdirs
-	$(NOECHO) $(NOOP)
-
-help :
-	perldoc ExtUtils::MakeMaker
-
-
-# --- MakeMaker blibdirs section:
-blibdirs : $(INST_LIBDIR)$(DFSEP).exists $(INST_ARCHLIB)$(DFSEP).exists $(INST_AUTODIR)$(DFSEP).exists $(INST_ARCHAUTODIR)$(DFSEP).exists $(INST_BIN)$(DFSEP).exists $(INST_SCRIPT)$(DFSEP).exists $(INST_MAN1DIR)$(DFSEP).exists $(INST_MAN3DIR)$(DFSEP).exists
-	$(NOECHO) $(NOOP)
-
-# Backwards compat with 6.18 through 6.25
-blibdirs.ts : blibdirs
-	$(NOECHO) $(NOOP)
-
-$(INST_LIBDIR)$(DFSEP).exists :: Makefile.PL
-	$(NOECHO) $(MKPATH) $(INST_LIBDIR)
-	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_LIBDIR)
-	$(NOECHO) $(TOUCH) $(INST_LIBDIR)$(DFSEP).exists
-
-$(INST_ARCHLIB)$(DFSEP).exists :: Makefile.PL
-	$(NOECHO) $(MKPATH) $(INST_ARCHLIB)
-	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_ARCHLIB)
-	$(NOECHO) $(TOUCH) $(INST_ARCHLIB)$(DFSEP).exists
-
-$(INST_AUTODIR)$(DFSEP).exists :: Makefile.PL
-	$(NOECHO) $(MKPATH) $(INST_AUTODIR)
-	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_AUTODIR)
-	$(NOECHO) $(TOUCH) $(INST_AUTODIR)$(DFSEP).exists
-
-$(INST_ARCHAUTODIR)$(DFSEP).exists :: Makefile.PL
-	$(NOECHO) $(MKPATH) $(INST_ARCHAUTODIR)
-	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_ARCHAUTODIR)
-	$(NOECHO) $(TOUCH) $(INST_ARCHAUTODIR)$(DFSEP).exists
-
-$(INST_BIN)$(DFSEP).exists :: Makefile.PL
-	$(NOECHO) $(MKPATH) $(INST_BIN)
-	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_BIN)
-	$(NOECHO) $(TOUCH) $(INST_BIN)$(DFSEP).exists
-
-$(INST_SCRIPT)$(DFSEP).exists :: Makefile.PL
-	$(NOECHO) $(MKPATH) $(INST_SCRIPT)
-	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_SCRIPT)
-	$(NOECHO) $(TOUCH) $(INST_SCRIPT)$(DFSEP).exists
-
-$(INST_MAN1DIR)$(DFSEP).exists :: Makefile.PL
-	$(NOECHO) $(MKPATH) $(INST_MAN1DIR)
-	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_MAN1DIR)
-	$(NOECHO) $(TOUCH) $(INST_MAN1DIR)$(DFSEP).exists
-
-$(INST_MAN3DIR)$(DFSEP).exists :: Makefile.PL
-	$(NOECHO) $(MKPATH) $(INST_MAN3DIR)
-	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_MAN3DIR)
-	$(NOECHO) $(TOUCH) $(INST_MAN3DIR)$(DFSEP).exists
-
-
-
-# --- MakeMaker linkext section:
-
-linkext :: $(LINKTYPE)
-	$(NOECHO) $(NOOP)
-
-
-# --- MakeMaker dlsyms section:
-
-
-# --- MakeMaker dynamic section:
-
-dynamic :: $(FIRST_MAKEFILE) $(INST_DYNAMIC) $(INST_BOOT)
-	$(NOECHO) $(NOOP)
-
-
-# --- MakeMaker dynamic_bs section:
-BOOTSTRAP = $(BASEEXT).bs
-
-# As Mkbootstrap might not write a file (if none is required)
-# we use touch to prevent make continually trying to remake it.
-# The DynaLoader only reads a non-empty file.
-$(BOOTSTRAP) : $(FIRST_MAKEFILE) $(BOOTDEP) $(INST_ARCHAUTODIR)$(DFSEP).exists
-	$(NOECHO) $(ECHO) "Running Mkbootstrap for $(NAME) ($(BSLOADLIBS))"
-	$(NOECHO) $(PERLRUN) \
-		"-MExtUtils::Mkbootstrap" \
-		-e "Mkbootstrap('$(BASEEXT)','$(BSLOADLIBS)');"
-	$(NOECHO) $(TOUCH) $@
-	$(CHMOD) $(PERM_RW) $@
-
-$(INST_BOOT) : $(BOOTSTRAP) $(INST_ARCHAUTODIR)$(DFSEP).exists
-	$(NOECHO) $(RM_RF) $@
-	- $(CP) $(BOOTSTRAP) $@
-	$(CHMOD) $(PERM_RW) $@
-
-
-# --- MakeMaker dynamic_lib section:
-
-# This section creates the dynamically loadable $(INST_DYNAMIC)
-# from $(OBJECT) and possibly $(MYEXTLIB).
-ARMAYBE = :
-OTHERLDFLAGS = 
-INST_DYNAMIC_DEP = 
-INST_DYNAMIC_FIX = 
-
-$(INST_DYNAMIC): $(OBJECT) $(MYEXTLIB) $(BOOTSTRAP) $(INST_ARCHAUTODIR)$(DFSEP).exists $(EXPORT_LIST) $(PERL_ARCHIVE) $(PERL_ARCHIVE_AFTER) $(INST_DYNAMIC_DEP)
-	$(RM_F) $@
-	$(LD)  $(LDDLFLAGS) $(LDFROM) $(OTHERLDFLAGS) -o $@ $(MYEXTLIB)	\
-	  $(PERL_ARCHIVE) $(LDLOADLIBS) $(PERL_ARCHIVE_AFTER) $(EXPORT_LIST)	\
-	  $(INST_DYNAMIC_FIX)
-	$(CHMOD) $(PERM_RWX) $@
-
-
-# --- MakeMaker static section:
-
-## $(INST_PM) has been moved to the all: target.
-## It remains here for awhile to allow for old usage: "make static"
-static :: $(FIRST_MAKEFILE) $(INST_STATIC)
-	$(NOECHO) $(NOOP)
-
-
-# --- MakeMaker static_lib section:
-
-$(INST_STATIC) : $(OBJECT) $(MYEXTLIB) $(INST_ARCHAUTODIR)$(DFSEP).exists
-	$(RM_RF) $@
-	$(FULL_AR) $(AR_STATIC_ARGS) $@ $(OBJECT) && $(RANLIB) $@
-	$(CHMOD) $(PERM_RWX) $@
-	$(NOECHO) $(ECHO) "$(EXTRALIBS)" > $(INST_ARCHAUTODIR)/extralibs.ld
-
-
-# --- MakeMaker manifypods section:
-
-POD2MAN_EXE = $(PERLRUN) "-MExtUtils::Command::MM" -e pod2man "--"
-POD2MAN = $(POD2MAN_EXE)
-
-
-manifypods : pure_all 
-	$(NOECHO) $(NOOP)
-
-
-
-
-# --- MakeMaker processPL section:
-
-
-# --- MakeMaker installbin section:
-
-
-# --- MakeMaker subdirs section:
-
-# none
-
-# --- MakeMaker clean_subdirs section:
-clean_subdirs :
-	$(NOECHO) $(NOOP)
-
-
-# --- MakeMaker clean section:
-
-# Delete temporary files but do not touch installed files. We don't delete
-# the Makefile here so a later make realclean still has a makefile to use.
-
-clean :: clean_subdirs
-	- $(RM_F) \
-	  *$(LIB_EXT) core \
-	  core.[0-9] $(INST_ARCHAUTODIR)/extralibs.all \
-	  core.[0-9][0-9] $(BASEEXT).bso \
-	  pm_to_blib.ts core.[0-9][0-9][0-9][0-9] \
-	  $(BASEEXT).x $(BOOTSTRAP) \
-	  perl$(EXE_EXT) tmon.out \
-	  *$(OBJ_EXT) pm_to_blib \
-	  $(INST_ARCHAUTODIR)/extralibs.ld blibdirs.ts \
-	  core.[0-9][0-9][0-9][0-9][0-9] *perl.core \
-	  core.*perl.*.? $(MAKE_APERL_FILE) \
-	  perl $(BASEEXT).def \
-	  core.[0-9][0-9][0-9] mon.out \
-	  lib$(BASEEXT).def perlmain.c \
-	  perl.exe so_locations \
-	  $(BASEEXT).exp 
-	- $(RM_RF) \
-	  blib 
-	- $(MV) $(FIRST_MAKEFILE) $(MAKEFILE_OLD) $(DEV_NULL)
-
-
-# --- MakeMaker realclean_subdirs section:
-realclean_subdirs :
-	$(NOECHO) $(NOOP)
-
-
-# --- MakeMaker realclean section:
-# Delete temporary files (via clean) and also delete dist files
-realclean purge ::  clean realclean_subdirs
-	- $(RM_F) \
-	  $(OBJECT) $(MAKEFILE_OLD) \
-	  $(FIRST_MAKEFILE) 
-	- $(RM_RF) \
-	  $(DISTVNAME) 
-
-
-# --- MakeMaker metafile section:
-metafile : create_distdir
-	$(NOECHO) $(ECHO) Generating META.yml
-	$(NOECHO) $(ECHO) '--- #YAML:1.0' > META_new.yml
-	$(NOECHO) $(ECHO) 'name:               CRFPP' >> META_new.yml
-	$(NOECHO) $(ECHO) 'version:            ' >> META_new.yml
-	$(NOECHO) $(ECHO) 'abstract:           ~' >> META_new.yml
-	$(NOECHO) $(ECHO) 'author:  []' >> META_new.yml
-	$(NOECHO) $(ECHO) 'license:            unknown' >> META_new.yml
-	$(NOECHO) $(ECHO) 'distribution_type:  module' >> META_new.yml
-	$(NOECHO) $(ECHO) 'configure_requires:' >> META_new.yml
-	$(NOECHO) $(ECHO) '    ExtUtils::MakeMaker:  0' >> META_new.yml
-	$(NOECHO) $(ECHO) 'build_requires:' >> META_new.yml
-	$(NOECHO) $(ECHO) '    ExtUtils::MakeMaker:  0' >> META_new.yml
-	$(NOECHO) $(ECHO) 'requires:  {}' >> META_new.yml
-	$(NOECHO) $(ECHO) 'no_index:' >> META_new.yml
-	$(NOECHO) $(ECHO) '    directory:' >> META_new.yml
-	$(NOECHO) $(ECHO) '        - t' >> META_new.yml
-	$(NOECHO) $(ECHO) '        - inc' >> META_new.yml
-	$(NOECHO) $(ECHO) 'generated_by:       ExtUtils::MakeMaker version 6.56' >> META_new.yml
-	$(NOECHO) $(ECHO) 'meta-spec:' >> META_new.yml
-	$(NOECHO) $(ECHO) '    url:      http://module-build.sourceforge.net/META-spec-v1.4.html' >> META_new.yml
-	$(NOECHO) $(ECHO) '    version:  1.4' >> META_new.yml
-	-$(NOECHO) $(MV) META_new.yml $(DISTVNAME)/META.yml
-
-
-# --- MakeMaker signature section:
-signature :
-	cpansign -s
-
-
-# --- MakeMaker dist_basics section:
-distclean :: realclean distcheck
-	$(NOECHO) $(NOOP)
-
-distcheck :
-	$(PERLRUN) "-MExtUtils::Manifest=fullcheck" -e fullcheck
-
-skipcheck :
-	$(PERLRUN) "-MExtUtils::Manifest=skipcheck" -e skipcheck
-
-manifest :
-	$(PERLRUN) "-MExtUtils::Manifest=mkmanifest" -e mkmanifest
-
-veryclean : realclean
-	$(RM_F) *~ */*~ *.orig */*.orig *.bak */*.bak *.old */*.old 
-
-
-
-# --- MakeMaker dist_core section:
-
-dist : $(DIST_DEFAULT) $(FIRST_MAKEFILE)
-	$(NOECHO) $(ABSPERLRUN) -l -e 'print '\''Warning: Makefile possibly out of date with $(VERSION_FROM)'\''' \
-	  -e '    if -e '\''$(VERSION_FROM)'\'' and -M '\''$(VERSION_FROM)'\'' < -M '\''$(FIRST_MAKEFILE)'\'';' --
-
-tardist : $(DISTVNAME).tar$(SUFFIX)
-	$(NOECHO) $(NOOP)
-
-uutardist : $(DISTVNAME).tar$(SUFFIX)
-	uuencode $(DISTVNAME).tar$(SUFFIX) $(DISTVNAME).tar$(SUFFIX) > $(DISTVNAME).tar$(SUFFIX)_uu
-
-$(DISTVNAME).tar$(SUFFIX) : distdir
-	$(PREOP)
-	$(TO_UNIX)
-	$(TAR) $(TARFLAGS) $(DISTVNAME).tar $(DISTVNAME)
-	$(RM_RF) $(DISTVNAME)
-	$(COMPRESS) $(DISTVNAME).tar
-	$(POSTOP)
-
-zipdist : $(DISTVNAME).zip
-	$(NOECHO) $(NOOP)
-
-$(DISTVNAME).zip : distdir
-	$(PREOP)
-	$(ZIP) $(ZIPFLAGS) $(DISTVNAME).zip $(DISTVNAME)
-	$(RM_RF) $(DISTVNAME)
-	$(POSTOP)
-
-shdist : distdir
-	$(PREOP)
-	$(SHAR) $(DISTVNAME) > $(DISTVNAME).shar
-	$(RM_RF) $(DISTVNAME)
-	$(POSTOP)
-
-
-# --- MakeMaker distdir section:
-create_distdir :
-	$(RM_RF) $(DISTVNAME)
-	$(PERLRUN) "-MExtUtils::Manifest=manicopy,maniread" \
-		-e "manicopy(maniread(),'$(DISTVNAME)', '$(DIST_CP)');"
-
-distdir : create_distdir distmeta 
-	$(NOECHO) $(NOOP)
-
-
-
-# --- MakeMaker dist_test section:
-disttest : distdir
-	cd $(DISTVNAME) && $(ABSPERLRUN) Makefile.PL 
-	cd $(DISTVNAME) && $(MAKE) $(PASTHRU)
-	cd $(DISTVNAME) && $(MAKE) test $(PASTHRU)
-
-
-
-# --- MakeMaker dist_ci section:
-
-ci :
-	$(PERLRUN) "-MExtUtils::Manifest=maniread" \
-	  -e "@all = keys %{ maniread() };" \
-	  -e "print(qq{Executing $(CI) @all\n}); system(qq{$(CI) @all});" \
-	  -e "print(qq{Executing $(RCS_LABEL) ...\n}); system(qq{$(RCS_LABEL) @all});"
-
-
-# --- MakeMaker distmeta section:
-distmeta : create_distdir metafile
-	$(NOECHO) cd $(DISTVNAME) && $(ABSPERLRUN) -MExtUtils::Manifest=maniadd -e 'eval { maniadd({q{META.yml} => q{Module meta-data (added by MakeMaker)}}) } ' \
-	  -e '    or print "Could not add META.yml to MANIFEST: $${'\''@'\''}\n"' --
-
-
-
-# --- MakeMaker distsignature section:
-distsignature : create_distdir
-	$(NOECHO) cd $(DISTVNAME) && $(ABSPERLRUN) -MExtUtils::Manifest=maniadd -e 'eval { maniadd({q{SIGNATURE} => q{Public-key signature (added by MakeMaker)}}) } ' \
-	  -e '    or print "Could not add SIGNATURE to MANIFEST: $${'\''@'\''}\n"' --
-	$(NOECHO) cd $(DISTVNAME) && $(TOUCH) SIGNATURE
-	cd $(DISTVNAME) && cpansign -s
-
-
-
-# --- MakeMaker install section:
-
-install :: pure_install doc_install
-	$(NOECHO) $(NOOP)
-
-install_perl :: pure_perl_install doc_perl_install
-	$(NOECHO) $(NOOP)
-
-install_site :: pure_site_install doc_site_install
-	$(NOECHO) $(NOOP)
-
-install_vendor :: pure_vendor_install doc_vendor_install
-	$(NOECHO) $(NOOP)
-
-pure_install :: pure_$(INSTALLDIRS)_install
-	$(NOECHO) $(NOOP)
-
-doc_install :: doc_$(INSTALLDIRS)_install
-	$(NOECHO) $(NOOP)
-
-pure__install : pure_site_install
-	$(NOECHO) $(ECHO) INSTALLDIRS not defined, defaulting to INSTALLDIRS=site
-
-doc__install : doc_site_install
-	$(NOECHO) $(ECHO) INSTALLDIRS not defined, defaulting to INSTALLDIRS=site
-
-pure_perl_install :: all
-	$(NOECHO) umask 022; $(MOD_INSTALL) \
-		$(INST_LIB) $(DESTINSTALLPRIVLIB) \
-		$(INST_ARCHLIB) $(DESTINSTALLARCHLIB) \
-		$(INST_BIN) $(DESTINSTALLBIN) \
-		$(INST_SCRIPT) $(DESTINSTALLSCRIPT) \
-		$(INST_MAN1DIR) $(DESTINSTALLMAN1DIR) \
-		$(INST_MAN3DIR) $(DESTINSTALLMAN3DIR)
-	$(NOECHO) $(WARN_IF_OLD_PACKLIST) \
-		$(SITEARCHEXP)/auto/$(FULLEXT)
-
-
-pure_site_install :: all
-	$(NOECHO) umask 02; $(MOD_INSTALL) \
-		read $(SITEARCHEXP)/auto/$(FULLEXT)/.packlist \
-		write $(DESTINSTALLSITEARCH)/auto/$(FULLEXT)/.packlist \
-		$(INST_LIB) $(DESTINSTALLSITELIB) \
-		$(INST_ARCHLIB) $(DESTINSTALLSITEARCH) \
-		$(INST_BIN) $(DESTINSTALLSITEBIN) \
-		$(INST_SCRIPT) $(DESTINSTALLSITESCRIPT) \
-		$(INST_MAN1DIR) $(DESTINSTALLSITEMAN1DIR) \
-		$(INST_MAN3DIR) $(DESTINSTALLSITEMAN3DIR)
-	$(NOECHO) $(WARN_IF_OLD_PACKLIST) \
-		$(PERL_ARCHLIB)/auto/$(FULLEXT)
-
-pure_vendor_install :: all
-	$(NOECHO) umask 022; $(MOD_INSTALL) \
-		$(INST_LIB) $(DESTINSTALLVENDORLIB) \
-		$(INST_ARCHLIB) $(DESTINSTALLVENDORARCH) \
-		$(INST_BIN) $(DESTINSTALLVENDORBIN) \
-		$(INST_SCRIPT) $(DESTINSTALLVENDORSCRIPT) \
-		$(INST_MAN1DIR) $(DESTINSTALLVENDORMAN1DIR) \
-		$(INST_MAN3DIR) $(DESTINSTALLVENDORMAN3DIR)
-
-doc_perl_install :: all
-
-doc_site_install :: all
-	$(NOECHO) $(ECHO) Appending installation info to $(DESTINSTALLSITEARCH)/perllocal.pod
-	-$(NOECHO) umask 02; $(MKPATH) $(DESTINSTALLSITEARCH)
-	-$(NOECHO) umask 02; $(DOC_INSTALL) \
-		"Module" "$(NAME)" \
-		"installed into" "$(INSTALLSITELIB)" \
-		LINKTYPE "$(LINKTYPE)" \
-		VERSION "$(VERSION)" \
-		EXE_FILES "$(EXE_FILES)" \
-		>> $(DESTINSTALLSITEARCH)/perllocal.pod
-
-doc_vendor_install :: all
-
-
-uninstall :: uninstall_from_$(INSTALLDIRS)dirs
-	$(NOECHO) $(NOOP)
-
-uninstall_from_perldirs ::
-
-uninstall_from_sitedirs ::
-	$(NOECHO) $(UNINSTALL) $(SITEARCHEXP)/auto/$(FULLEXT)/.packlist
-
-uninstall_from_vendordirs ::
-
-
-
-# --- MakeMaker force section:
-# Phony target to force checking subdirectories.
-FORCE :
-	$(NOECHO) $(NOOP)
-
-
-# --- MakeMaker perldepend section:
-
-PERL_HDRS = \
-	$(PERL_INC)/EXTERN.h		\
-	$(PERL_INC)/INTERN.h		\
-	$(PERL_INC)/XSUB.h		\
-	$(PERL_INC)/av.h		\
-	$(PERL_INC)/cc_runtime.h	\
-	$(PERL_INC)/config.h		\
-	$(PERL_INC)/cop.h		\
-	$(PERL_INC)/cv.h		\
-	$(PERL_INC)/dosish.h		\
-	$(PERL_INC)/embed.h		\
-	$(PERL_INC)/embedvar.h		\
-	$(PERL_INC)/fakethr.h		\
-	$(PERL_INC)/form.h		\
-	$(PERL_INC)/gv.h		\
-	$(PERL_INC)/handy.h		\
-	$(PERL_INC)/hv.h		\
-	$(PERL_INC)/intrpvar.h		\
-	$(PERL_INC)/iperlsys.h		\
-	$(PERL_INC)/keywords.h		\
-	$(PERL_INC)/mg.h		\
-	$(PERL_INC)/nostdio.h		\
-	$(PERL_INC)/op.h		\
-	$(PERL_INC)/opcode.h		\
-	$(PERL_INC)/patchlevel.h	\
-	$(PERL_INC)/perl.h		\
-	$(PERL_INC)/perlio.h		\
-	$(PERL_INC)/perlsdio.h		\
-	$(PERL_INC)/perlsfio.h		\
-	$(PERL_INC)/perlvars.h		\
-	$(PERL_INC)/perly.h		\
-	$(PERL_INC)/pp.h		\
-	$(PERL_INC)/pp_proto.h		\
-	$(PERL_INC)/proto.h		\
-	$(PERL_INC)/regcomp.h		\
-	$(PERL_INC)/regexp.h		\
-	$(PERL_INC)/regnodes.h		\
-	$(PERL_INC)/scope.h		\
-	$(PERL_INC)/sv.h		\
-	$(PERL_INC)/thread.h		\
-	$(PERL_INC)/unixish.h		\
-	$(PERL_INC)/util.h
-
-$(OBJECT) : $(PERL_HDRS)
-
-
-# --- MakeMaker makefile section:
-
-$(OBJECT) : $(FIRST_MAKEFILE)
-
-# We take a very conservative approach here, but it's worth it.
-# We move Makefile to Makefile.old here to avoid gnu make looping.
-$(FIRST_MAKEFILE) : Makefile.PL $(CONFIGDEP)
-	$(NOECHO) $(ECHO) "Makefile out-of-date with respect to $?"
-	$(NOECHO) $(ECHO) "Cleaning current config before rebuilding Makefile..."
-	-$(NOECHO) $(RM_F) $(MAKEFILE_OLD)
-	-$(NOECHO) $(MV)   $(FIRST_MAKEFILE) $(MAKEFILE_OLD)
-	- $(MAKE) $(USEMAKEFILE) $(MAKEFILE_OLD) clean $(DEV_NULL)
-	$(PERLRUN) Makefile.PL 
-	$(NOECHO) $(ECHO) "==> Your Makefile has been rebuilt. <=="
-	$(NOECHO) $(ECHO) "==> Please rerun the $(MAKE) command.  <=="
-	$(FALSE)
-
-
-
-# --- MakeMaker staticmake section:
-
-# --- MakeMaker makeaperl section ---
-MAP_TARGET    = perl
-FULLPERL      = /usr/bin/perl
-
-$(MAP_TARGET) :: static $(MAKE_APERL_FILE)
-	$(MAKE) $(USEMAKEFILE) $(MAKE_APERL_FILE) $@
-
-$(MAKE_APERL_FILE) : $(FIRST_MAKEFILE) pm_to_blib
-	$(NOECHO) $(ECHO) Writing \"$(MAKE_APERL_FILE)\" for this $(MAP_TARGET)
-	$(NOECHO) $(PERLRUNINST) \
-		Makefile.PL DIR= \
-		MAKEFILE=$(MAKE_APERL_FILE) LINKTYPE=static \
-		MAKEAPERL=1 NORECURS=1 CCCDLFLAGS=
-
-
-# --- MakeMaker test section:
-
-TEST_VERBOSE=0
-TEST_TYPE=test_$(LINKTYPE)
-TEST_FILE = test.pl
-TEST_FILES = 
-TESTDB_SW = -d
-
-testdb :: testdb_$(LINKTYPE)
-
-test :: $(TEST_TYPE) subdirs-test
-
-subdirs-test ::
-	$(NOECHO) $(NOOP)
-
-
-test_dynamic :: pure_all
-	PERL_DL_NONLAZY=1 $(FULLPERLRUN) "-I$(INST_LIB)" "-I$(INST_ARCHLIB)" $(TEST_FILE)
-
-testdb_dynamic :: pure_all
-	PERL_DL_NONLAZY=1 $(FULLPERLRUN) $(TESTDB_SW) "-I$(INST_LIB)" "-I$(INST_ARCHLIB)" $(TEST_FILE)
-
-test_ : test_dynamic
-
-test_static :: pure_all $(MAP_TARGET)
-	PERL_DL_NONLAZY=1 ./$(MAP_TARGET) "-I$(INST_LIB)" "-I$(INST_ARCHLIB)" $(TEST_FILE)
-
-testdb_static :: pure_all $(MAP_TARGET)
-	PERL_DL_NONLAZY=1 ./$(MAP_TARGET) $(TESTDB_SW) "-I$(INST_LIB)" "-I$(INST_ARCHLIB)" $(TEST_FILE)
-
-
-
-# --- MakeMaker ppd section:
-# Creates a PPD (Perl Package Description) for a binary distribution.
-ppd :
-	$(NOECHO) $(ECHO) '<SOFTPKG NAME="$(DISTNAME)" VERSION="">' > $(DISTNAME).ppd
-	$(NOECHO) $(ECHO) '    <ABSTRACT></ABSTRACT>' >> $(DISTNAME).ppd
-	$(NOECHO) $(ECHO) '    <AUTHOR></AUTHOR>' >> $(DISTNAME).ppd
-	$(NOECHO) $(ECHO) '    <IMPLEMENTATION>' >> $(DISTNAME).ppd
-	$(NOECHO) $(ECHO) '        <ARCHITECTURE NAME="x86_64-linux-gnu-thread-multi-5.12" />' >> $(DISTNAME).ppd
-	$(NOECHO) $(ECHO) '        <CODEBASE HREF="" />' >> $(DISTNAME).ppd
-	$(NOECHO) $(ECHO) '    </IMPLEMENTATION>' >> $(DISTNAME).ppd
-	$(NOECHO) $(ECHO) '</SOFTPKG>' >> $(DISTNAME).ppd
-
-
-# --- MakeMaker pm_to_blib section:
-
-pm_to_blib : $(FIRST_MAKEFILE) $(TO_INST_PM)
-	$(NOECHO) $(ABSPERLRUN) -MExtUtils::Install -e 'pm_to_blib({@ARGV}, '\''$(INST_LIB)/auto'\'', q[$(PM_FILTER)], '\''$(PERM_DIR)'\'')' -- \
-	  CRFPP.pm $(INST_LIB)/CRFPP.pm 
-	$(NOECHO) $(TOUCH) pm_to_blib
-
-
-# --- MakeMaker selfdocument section:
-
-
-# --- MakeMaker postamble section:
-
-
-# End.
+# This Makefile is for the CRFPP extension to perl.
+#
+# It was generated automatically by MakeMaker version
+# 6.56 (Revision: 65600) from the contents of
+# Makefile.PL. Don't edit this file, edit Makefile.PL instead.
+#
+#       ANY CHANGES MADE HERE WILL BE LOST!
+#
+#   MakeMaker ARGV: ()
+#
+
+#   MakeMaker Parameters:
+
+#     BUILD_REQUIRES => {  }
+#     CC => q[c++]
+#     INC => q[]
+#     LD => q[c++]
+#     LIBS => q[-lpthread -lcrfpp]
+#     NAME => q[CRFPP]
+#     OBJECT => q[CRFPP_wrap.o]
+#     PREREQ_PM => {  }
+
+# --- MakeMaker post_initialize section:
+
+
+# --- MakeMaker const_config section:
+
+# These definitions are from config.sh (via /usr/lib/perl/5.12/Config.pm).
+# They may have been overridden via Makefile.PL or on the command line.
+AR = ar
+CC = c++
+CCCDLFLAGS = -fPIC
+CCDLFLAGS = -Wl,-E
+DLEXT = so
+DLSRC = dl_dlopen.xs
+EXE_EXT = 
+FULL_AR = /usr/bin/ar
+LD = c++
+LDDLFLAGS = -shared -O2 -g -L/usr/local/lib -fstack-protector
+LDFLAGS =  -fstack-protector -L/usr/local/lib
+LIBC = 
+LIB_EXT = .a
+OBJ_EXT = .o
+OSNAME = linux
+OSVERS = 2.6.24-28-server
+RANLIB = :
+SITELIBEXP = /usr/local/share/perl/5.12.4
+SITEARCHEXP = /usr/local/lib/perl/5.12.4
+SO = so
+VENDORARCHEXP = /usr/lib/perl5
+VENDORLIBEXP = /usr/share/perl5
+
+
+# --- MakeMaker constants section:
+AR_STATIC_ARGS = cr
+DIRFILESEP = /
+DFSEP = $(DIRFILESEP)
+NAME = CRFPP
+NAME_SYM = CRFPP
+VERSION = 
+VERSION_MACRO = VERSION
+VERSION_SYM = 
+DEFINE_VERSION = -D$(VERSION_MACRO)=\"$(VERSION)\"
+XS_VERSION = 
+XS_VERSION_MACRO = XS_VERSION
+XS_DEFINE_VERSION = -D$(XS_VERSION_MACRO)=\"$(XS_VERSION)\"
+INST_ARCHLIB = blib/arch
+INST_SCRIPT = blib/script
+INST_BIN = blib/bin
+INST_LIB = blib/lib
+INST_MAN1DIR = blib/man1
+INST_MAN3DIR = blib/man3
+MAN1EXT = 1p
+MAN3EXT = 3pm
+INSTALLDIRS = site
+DESTDIR = 
+PREFIX = /usr
+PERLPREFIX = $(PREFIX)
+SITEPREFIX = $(PREFIX)/local
+VENDORPREFIX = $(PREFIX)
+INSTALLPRIVLIB = $(PERLPREFIX)/share/perl/5.12
+DESTINSTALLPRIVLIB = $(DESTDIR)$(INSTALLPRIVLIB)
+INSTALLSITELIB = $(SITEPREFIX)/share/perl/5.12.4
+DESTINSTALLSITELIB = $(DESTDIR)$(INSTALLSITELIB)
+INSTALLVENDORLIB = $(VENDORPREFIX)/share/perl5
+DESTINSTALLVENDORLIB = $(DESTDIR)$(INSTALLVENDORLIB)
+INSTALLARCHLIB = $(PERLPREFIX)/lib/perl/5.12
+DESTINSTALLARCHLIB = $(DESTDIR)$(INSTALLARCHLIB)
+INSTALLSITEARCH = $(SITEPREFIX)/lib/perl/5.12.4
+DESTINSTALLSITEARCH = $(DESTDIR)$(INSTALLSITEARCH)
+INSTALLVENDORARCH = $(VENDORPREFIX)/lib/perl5
+DESTINSTALLVENDORARCH = $(DESTDIR)$(INSTALLVENDORARCH)
+INSTALLBIN = $(PERLPREFIX)/bin
+DESTINSTALLBIN = $(DESTDIR)$(INSTALLBIN)
+INSTALLSITEBIN = $(SITEPREFIX)/bin
+DESTINSTALLSITEBIN = $(DESTDIR)$(INSTALLSITEBIN)
+INSTALLVENDORBIN = $(VENDORPREFIX)/bin
+DESTINSTALLVENDORBIN = $(DESTDIR)$(INSTALLVENDORBIN)
+INSTALLSCRIPT = $(PERLPREFIX)/bin
+DESTINSTALLSCRIPT = $(DESTDIR)$(INSTALLSCRIPT)
+INSTALLSITESCRIPT = $(SITEPREFIX)/bin
+DESTINSTALLSITESCRIPT = $(DESTDIR)$(INSTALLSITESCRIPT)
+INSTALLVENDORSCRIPT = $(VENDORPREFIX)/bin
+DESTINSTALLVENDORSCRIPT = $(DESTDIR)$(INSTALLVENDORSCRIPT)
+INSTALLMAN1DIR = $(PERLPREFIX)/share/man/man1
+DESTINSTALLMAN1DIR = $(DESTDIR)$(INSTALLMAN1DIR)
+INSTALLSITEMAN1DIR = $(SITEPREFIX)/man/man1
+DESTINSTALLSITEMAN1DIR = $(DESTDIR)$(INSTALLSITEMAN1DIR)
+INSTALLVENDORMAN1DIR = $(VENDORPREFIX)/share/man/man1
+DESTINSTALLVENDORMAN1DIR = $(DESTDIR)$(INSTALLVENDORMAN1DIR)
+INSTALLMAN3DIR = $(PERLPREFIX)/share/man/man3
+DESTINSTALLMAN3DIR = $(DESTDIR)$(INSTALLMAN3DIR)
+INSTALLSITEMAN3DIR = $(SITEPREFIX)/man/man3
+DESTINSTALLSITEMAN3DIR = $(DESTDIR)$(INSTALLSITEMAN3DIR)
+INSTALLVENDORMAN3DIR = $(VENDORPREFIX)/share/man/man3
+DESTINSTALLVENDORMAN3DIR = $(DESTDIR)$(INSTALLVENDORMAN3DIR)
+PERL_LIB = /usr/share/perl/5.12
+PERL_ARCHLIB = /usr/lib/perl/5.12
+LIBPERL_A = libperl.a
+FIRST_MAKEFILE = Makefile
+MAKEFILE_OLD = Makefile.old
+MAKE_APERL_FILE = Makefile.aperl
+PERLMAINCC = $(CC)
+PERL_INC = /usr/lib/perl/5.12/CORE
+PERL = /usr/bin/perl
+FULLPERL = /usr/bin/perl
+ABSPERL = $(PERL)
+PERLRUN = $(PERL)
+FULLPERLRUN = $(FULLPERL)
+ABSPERLRUN = $(ABSPERL)
+PERLRUNINST = $(PERLRUN) "-I$(INST_ARCHLIB)" "-I$(INST_LIB)"
+FULLPERLRUNINST = $(FULLPERLRUN) "-I$(INST_ARCHLIB)" "-I$(INST_LIB)"
+ABSPERLRUNINST = $(ABSPERLRUN) "-I$(INST_ARCHLIB)" "-I$(INST_LIB)"
+PERL_CORE = 0
+PERM_DIR = 755
+PERM_RW = 644
+PERM_RWX = 755
+
+MAKEMAKER   = /usr/share/perl/5.12/ExtUtils/MakeMaker.pm
+MM_VERSION  = 6.56
+MM_REVISION = 65600
+
+# FULLEXT = Pathname for extension directory (eg Foo/Bar/Oracle).
+# BASEEXT = Basename part of FULLEXT. May be just equal FULLEXT. (eg Oracle)
+# PARENT_NAME = NAME without BASEEXT and no trailing :: (eg Foo::Bar)
+# DLBASE  = Basename part of dynamic library. May be just equal BASEEXT.
+MAKE = make
+FULLEXT = CRFPP
+BASEEXT = CRFPP
+PARENT_NAME = 
+DLBASE = $(BASEEXT)
+VERSION_FROM = 
+INC = 
+OBJECT = CRFPP_wrap$(OBJ_EXT)
+LDFROM = $(OBJECT)
+LINKTYPE = dynamic
+BOOTDEP = 
+
+# Handy lists of source code files:
+XS_FILES = 
+C_FILES  = CRFPP_wrap.cxx
+O_FILES  = CRFPP_wrap.o
+H_FILES  = 
+MAN1PODS = 
+MAN3PODS = 
+
+# Where is the Config information that we are using/depend on
+CONFIGDEP = $(PERL_ARCHLIB)$(DFSEP)Config.pm $(PERL_INC)$(DFSEP)config.h
+
+# Where to build things
+INST_LIBDIR      = $(INST_LIB)
+INST_ARCHLIBDIR  = $(INST_ARCHLIB)
+
+INST_AUTODIR     = $(INST_LIB)/auto/$(FULLEXT)
+INST_ARCHAUTODIR = $(INST_ARCHLIB)/auto/$(FULLEXT)
+
+INST_STATIC      = $(INST_ARCHAUTODIR)/$(BASEEXT)$(LIB_EXT)
+INST_DYNAMIC     = $(INST_ARCHAUTODIR)/$(DLBASE).$(DLEXT)
+INST_BOOT        = $(INST_ARCHAUTODIR)/$(BASEEXT).bs
+
+# Extra linker info
+EXPORT_LIST        = 
+PERL_ARCHIVE       = 
+PERL_ARCHIVE_AFTER = 
+
+
+TO_INST_PM = CRFPP.pm
+
+PM_TO_BLIB = CRFPP.pm \
+	$(INST_LIB)/CRFPP.pm
+
+
+# --- MakeMaker platform_constants section:
+MM_Unix_VERSION = 6.56
+PERL_MALLOC_DEF = -DPERL_EXTMALLOC_DEF -Dmalloc=Perl_malloc -Dfree=Perl_mfree -Drealloc=Perl_realloc -Dcalloc=Perl_calloc
+
+
+# --- MakeMaker tool_autosplit section:
+# Usage: $(AUTOSPLITFILE) FileToSplit AutoDirToSplitInto
+AUTOSPLITFILE = $(ABSPERLRUN)  -e 'use AutoSplit;  autosplit($$ARGV[0], $$ARGV[1], 0, 1, 1)' --
+
+
+
+# --- MakeMaker tool_xsubpp section:
+
+XSUBPPDIR = /usr/share/perl/5.12/ExtUtils
+XSUBPP = $(XSUBPPDIR)$(DFSEP)xsubpp
+XSUBPPRUN = $(PERLRUN) $(XSUBPP)
+XSPROTOARG = 
+XSUBPPDEPS = /usr/share/perl/5.12/ExtUtils/typemap $(XSUBPP)
+XSUBPPARGS = -typemap /usr/share/perl/5.12/ExtUtils/typemap
+XSUBPP_EXTRA_ARGS = 
+
+
+# --- MakeMaker tools_other section:
+SHELL = /bin/sh
+CHMOD = chmod
+CP = cp
+MV = mv
+NOOP = $(TRUE)
+NOECHO = @
+RM_F = rm -f
+RM_RF = rm -rf
+TEST_F = test -f
+TOUCH = touch
+UMASK_NULL = umask 0
+DEV_NULL = > /dev/null 2>&1
+MKPATH = $(ABSPERLRUN) -MExtUtils::Command -e 'mkpath' --
+EQUALIZE_TIMESTAMP = $(ABSPERLRUN) -MExtUtils::Command -e 'eqtime' --
+FALSE = false
+TRUE = true
+ECHO = echo
+ECHO_N = echo -n
+UNINST = 0
+VERBINST = 0
+MOD_INSTALL = $(ABSPERLRUN) -MExtUtils::Install -e 'install([ from_to => {@ARGV}, verbose => '\''$(VERBINST)'\'', uninstall_shadows => '\''$(UNINST)'\'', dir_mode => '\''$(PERM_DIR)'\'' ]);' --
+DOC_INSTALL = $(ABSPERLRUN) -MExtUtils::Command::MM -e 'perllocal_install' --
+UNINSTALL = $(ABSPERLRUN) -MExtUtils::Command::MM -e 'uninstall' --
+WARN_IF_OLD_PACKLIST = $(ABSPERLRUN) -MExtUtils::Command::MM -e 'warn_if_old_packlist' --
+MACROSTART = 
+MACROEND = 
+USEMAKEFILE = -f
+FIXIN = $(ABSPERLRUN) -MExtUtils::MY -e 'MY->fixin(shift)' --
+
+
+# --- MakeMaker makemakerdflt section:
+makemakerdflt : all
+	$(NOECHO) $(NOOP)
+
+
+# --- MakeMaker dist section:
+TAR = tar
+TARFLAGS = cvf
+ZIP = zip
+ZIPFLAGS = -r
+COMPRESS = gzip --best
+SUFFIX = .gz
+SHAR = shar
+PREOP = $(NOECHO) $(NOOP)
+POSTOP = $(NOECHO) $(NOOP)
+TO_UNIX = $(NOECHO) $(NOOP)
+CI = ci -u
+RCS_LABEL = rcs -Nv$(VERSION_SYM): -q
+DIST_CP = best
+DIST_DEFAULT = tardist
+DISTNAME = CRFPP
+DISTVNAME = CRFPP-
+
+
+# --- MakeMaker macro section:
+
+
+# --- MakeMaker depend section:
+
+
+# --- MakeMaker cflags section:
+
+CCFLAGS = -D_REENTRANT -D_GNU_SOURCE -DDEBIAN -fno-strict-aliasing -pipe -fstack-protector -I/usr/local/include -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
+OPTIMIZE = -O2 -g
+PERLTYPE = 
+MPOLLUTE = 
+
+
+# --- MakeMaker const_loadlibs section:
+
+# CRFPP might depend on some other libraries:
+# See ExtUtils::Liblist for details
+#
+EXTRALIBS = -lcrfpp
+LDLOADLIBS = -lpthread -lcrfpp
+BSLOADLIBS = 
+
+
+# --- MakeMaker const_cccmd section:
+CCCMD = $(CC) -c $(PASTHRU_INC) $(INC) \
+	$(CCFLAGS) $(OPTIMIZE) \
+	$(PERLTYPE) $(MPOLLUTE) $(DEFINE_VERSION) \
+	$(XS_DEFINE_VERSION)
+
+# --- MakeMaker post_constants section:
+
+
+# --- MakeMaker pasthru section:
+
+PASTHRU = LIBPERL_A="$(LIBPERL_A)"\
+	LINKTYPE="$(LINKTYPE)"\
+	OPTIMIZE="$(OPTIMIZE)"\
+	PREFIX="$(PREFIX)"\
+	PASTHRU_INC="$(PASTHRU_INC)"
+
+
+# --- MakeMaker special_targets section:
+.SUFFIXES : .xs .c .C .cpp .i .s .cxx .cc $(OBJ_EXT)
+
+.PHONY: all config static dynamic test linkext manifest blibdirs clean realclean disttest distdir
+
+
+
+# --- MakeMaker c_o section:
+
+.c.i:
+	cc -E -c $(PASTHRU_INC) $(INC) \
+	$(CCFLAGS) $(OPTIMIZE) \
+	$(PERLTYPE) $(MPOLLUTE) $(DEFINE_VERSION) \
+	$(XS_DEFINE_VERSION) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.c > $*.i
+
+.c.s:
+	$(CCCMD) -S $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.c
+
+.c$(OBJ_EXT):
+	$(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.c
+
+.cpp$(OBJ_EXT):
+	$(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.cpp
+
+.cxx$(OBJ_EXT):
+	$(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.cxx
+
+.cc$(OBJ_EXT):
+	$(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.cc
+
+.C$(OBJ_EXT):
+	$(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.C
+
+
+# --- MakeMaker xs_c section:
+
+.xs.c:
+	$(XSUBPPRUN) $(XSPROTOARG) $(XSUBPPARGS) $(XSUBPP_EXTRA_ARGS) $*.xs > $*.xsc && $(MV) $*.xsc $*.c
+
+
+# --- MakeMaker xs_o section:
+
+.xs$(OBJ_EXT):
+	$(XSUBPPRUN) $(XSPROTOARG) $(XSUBPPARGS) $*.xs > $*.xsc && $(MV) $*.xsc $*.c
+	$(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.c
+
+
+# --- MakeMaker top_targets section:
+all :: pure_all manifypods
+	$(NOECHO) $(NOOP)
+
+
+pure_all :: config pm_to_blib subdirs linkext
+	$(NOECHO) $(NOOP)
+
+subdirs :: $(MYEXTLIB)
+	$(NOECHO) $(NOOP)
+
+config :: $(FIRST_MAKEFILE) blibdirs
+	$(NOECHO) $(NOOP)
+
+help :
+	perldoc ExtUtils::MakeMaker
+
+
+# --- MakeMaker blibdirs section:
+blibdirs : $(INST_LIBDIR)$(DFSEP).exists $(INST_ARCHLIB)$(DFSEP).exists $(INST_AUTODIR)$(DFSEP).exists $(INST_ARCHAUTODIR)$(DFSEP).exists $(INST_BIN)$(DFSEP).exists $(INST_SCRIPT)$(DFSEP).exists $(INST_MAN1DIR)$(DFSEP).exists $(INST_MAN3DIR)$(DFSEP).exists
+	$(NOECHO) $(NOOP)
+
+# Backwards compat with 6.18 through 6.25
+blibdirs.ts : blibdirs
+	$(NOECHO) $(NOOP)
+
+$(INST_LIBDIR)$(DFSEP).exists :: Makefile.PL
+	$(NOECHO) $(MKPATH) $(INST_LIBDIR)
+	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_LIBDIR)
+	$(NOECHO) $(TOUCH) $(INST_LIBDIR)$(DFSEP).exists
+
+$(INST_ARCHLIB)$(DFSEP).exists :: Makefile.PL
+	$(NOECHO) $(MKPATH) $(INST_ARCHLIB)
+	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_ARCHLIB)
+	$(NOECHO) $(TOUCH) $(INST_ARCHLIB)$(DFSEP).exists
+
+$(INST_AUTODIR)$(DFSEP).exists :: Makefile.PL
+	$(NOECHO) $(MKPATH) $(INST_AUTODIR)
+	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_AUTODIR)
+	$(NOECHO) $(TOUCH) $(INST_AUTODIR)$(DFSEP).exists
+
+$(INST_ARCHAUTODIR)$(DFSEP).exists :: Makefile.PL
+	$(NOECHO) $(MKPATH) $(INST_ARCHAUTODIR)
+	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_ARCHAUTODIR)
+	$(NOECHO) $(TOUCH) $(INST_ARCHAUTODIR)$(DFSEP).exists
+
+$(INST_BIN)$(DFSEP).exists :: Makefile.PL
+	$(NOECHO) $(MKPATH) $(INST_BIN)
+	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_BIN)
+	$(NOECHO) $(TOUCH) $(INST_BIN)$(DFSEP).exists
+
+$(INST_SCRIPT)$(DFSEP).exists :: Makefile.PL
+	$(NOECHO) $(MKPATH) $(INST_SCRIPT)
+	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_SCRIPT)
+	$(NOECHO) $(TOUCH) $(INST_SCRIPT)$(DFSEP).exists
+
+$(INST_MAN1DIR)$(DFSEP).exists :: Makefile.PL
+	$(NOECHO) $(MKPATH) $(INST_MAN1DIR)
+	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_MAN1DIR)
+	$(NOECHO) $(TOUCH) $(INST_MAN1DIR)$(DFSEP).exists
+
+$(INST_MAN3DIR)$(DFSEP).exists :: Makefile.PL
+	$(NOECHO) $(MKPATH) $(INST_MAN3DIR)
+	$(NOECHO) $(CHMOD) $(PERM_DIR) $(INST_MAN3DIR)
+	$(NOECHO) $(TOUCH) $(INST_MAN3DIR)$(DFSEP).exists
+
+
+
+# --- MakeMaker linkext section:
+
+linkext :: $(LINKTYPE)
+	$(NOECHO) $(NOOP)
+
+
+# --- MakeMaker dlsyms section:
+
+
+# --- MakeMaker dynamic section:
+
+dynamic :: $(FIRST_MAKEFILE) $(INST_DYNAMIC) $(INST_BOOT)
+	$(NOECHO) $(NOOP)
+
+
+# --- MakeMaker dynamic_bs section:
+BOOTSTRAP = $(BASEEXT).bs
+
+# As Mkbootstrap might not write a file (if none is required)
+# we use touch to prevent make continually trying to remake it.
+# The DynaLoader only reads a non-empty file.
+$(BOOTSTRAP) : $(FIRST_MAKEFILE) $(BOOTDEP) $(INST_ARCHAUTODIR)$(DFSEP).exists
+	$(NOECHO) $(ECHO) "Running Mkbootstrap for $(NAME) ($(BSLOADLIBS))"
+	$(NOECHO) $(PERLRUN) \
+		"-MExtUtils::Mkbootstrap" \
+		-e "Mkbootstrap('$(BASEEXT)','$(BSLOADLIBS)');"
+	$(NOECHO) $(TOUCH) $@
+	$(CHMOD) $(PERM_RW) $@
+
+$(INST_BOOT) : $(BOOTSTRAP) $(INST_ARCHAUTODIR)$(DFSEP).exists
+	$(NOECHO) $(RM_RF) $@
+	- $(CP) $(BOOTSTRAP) $@
+	$(CHMOD) $(PERM_RW) $@
+
+
+# --- MakeMaker dynamic_lib section:
+
+# This section creates the dynamically loadable $(INST_DYNAMIC)
+# from $(OBJECT) and possibly $(MYEXTLIB).
+ARMAYBE = :
+OTHERLDFLAGS = 
+INST_DYNAMIC_DEP = 
+INST_DYNAMIC_FIX = 
+
+$(INST_DYNAMIC): $(OBJECT) $(MYEXTLIB) $(BOOTSTRAP) $(INST_ARCHAUTODIR)$(DFSEP).exists $(EXPORT_LIST) $(PERL_ARCHIVE) $(PERL_ARCHIVE_AFTER) $(INST_DYNAMIC_DEP)
+	$(RM_F) $@
+	$(LD)  $(LDDLFLAGS) $(LDFROM) $(OTHERLDFLAGS) -o $@ $(MYEXTLIB)	\
+	  $(PERL_ARCHIVE) $(LDLOADLIBS) $(PERL_ARCHIVE_AFTER) $(EXPORT_LIST)	\
+	  $(INST_DYNAMIC_FIX)
+	$(CHMOD) $(PERM_RWX) $@
+
+
+# --- MakeMaker static section:
+
+## $(INST_PM) has been moved to the all: target.
+## It remains here for awhile to allow for old usage: "make static"
+static :: $(FIRST_MAKEFILE) $(INST_STATIC)
+	$(NOECHO) $(NOOP)
+
+
+# --- MakeMaker static_lib section:
+
+$(INST_STATIC) : $(OBJECT) $(MYEXTLIB) $(INST_ARCHAUTODIR)$(DFSEP).exists
+	$(RM_RF) $@
+	$(FULL_AR) $(AR_STATIC_ARGS) $@ $(OBJECT) && $(RANLIB) $@
+	$(CHMOD) $(PERM_RWX) $@
+	$(NOECHO) $(ECHO) "$(EXTRALIBS)" > $(INST_ARCHAUTODIR)/extralibs.ld
+
+
+# --- MakeMaker manifypods section:
+
+POD2MAN_EXE = $(PERLRUN) "-MExtUtils::Command::MM" -e pod2man "--"
+POD2MAN = $(POD2MAN_EXE)
+
+
+manifypods : pure_all 
+	$(NOECHO) $(NOOP)
+
+
+
+
+# --- MakeMaker processPL section:
+
+
+# --- MakeMaker installbin section:
+
+
+# --- MakeMaker subdirs section:
+
+# none
+
+# --- MakeMaker clean_subdirs section:
+clean_subdirs :
+	$(NOECHO) $(NOOP)
+
+
+# --- MakeMaker clean section:
+
+# Delete temporary files but do not touch installed files. We don't delete
+# the Makefile here so a later make realclean still has a makefile to use.
+
+clean :: clean_subdirs
+	- $(RM_F) \
+	  *$(LIB_EXT) core \
+	  core.[0-9] $(INST_ARCHAUTODIR)/extralibs.all \
+	  core.[0-9][0-9] $(BASEEXT).bso \
+	  pm_to_blib.ts core.[0-9][0-9][0-9][0-9] \
+	  $(BASEEXT).x $(BOOTSTRAP) \
+	  perl$(EXE_EXT) tmon.out \
+	  *$(OBJ_EXT) pm_to_blib \
+	  $(INST_ARCHAUTODIR)/extralibs.ld blibdirs.ts \
+	  core.[0-9][0-9][0-9][0-9][0-9] *perl.core \
+	  core.*perl.*.? $(MAKE_APERL_FILE) \
+	  perl $(BASEEXT).def \
+	  core.[0-9][0-9][0-9] mon.out \
+	  lib$(BASEEXT).def perlmain.c \
+	  perl.exe so_locations \
+	  $(BASEEXT).exp 
+	- $(RM_RF) \
+	  blib 
+	- $(MV) $(FIRST_MAKEFILE) $(MAKEFILE_OLD) $(DEV_NULL)
+
+
+# --- MakeMaker realclean_subdirs section:
+realclean_subdirs :
+	$(NOECHO) $(NOOP)
+
+
+# --- MakeMaker realclean section:
+# Delete temporary files (via clean) and also delete dist files
+realclean purge ::  clean realclean_subdirs
+	- $(RM_F) \
+	  $(OBJECT) $(MAKEFILE_OLD) \
+	  $(FIRST_MAKEFILE) 
+	- $(RM_RF) \
+	  $(DISTVNAME) 
+
+
+# --- MakeMaker metafile section:
+metafile : create_distdir
+	$(NOECHO) $(ECHO) Generating META.yml
+	$(NOECHO) $(ECHO) '--- #YAML:1.0' > META_new.yml
+	$(NOECHO) $(ECHO) 'name:               CRFPP' >> META_new.yml
+	$(NOECHO) $(ECHO) 'version:            ' >> META_new.yml
+	$(NOECHO) $(ECHO) 'abstract:           ~' >> META_new.yml
+	$(NOECHO) $(ECHO) 'author:  []' >> META_new.yml
+	$(NOECHO) $(ECHO) 'license:            unknown' >> META_new.yml
+	$(NOECHO) $(ECHO) 'distribution_type:  module' >> META_new.yml
+	$(NOECHO) $(ECHO) 'configure_requires:' >> META_new.yml
+	$(NOECHO) $(ECHO) '    ExtUtils::MakeMaker:  0' >> META_new.yml
+	$(NOECHO) $(ECHO) 'build_requires:' >> META_new.yml
+	$(NOECHO) $(ECHO) '    ExtUtils::MakeMaker:  0' >> META_new.yml
+	$(NOECHO) $(ECHO) 'requires:  {}' >> META_new.yml
+	$(NOECHO) $(ECHO) 'no_index:' >> META_new.yml
+	$(NOECHO) $(ECHO) '    directory:' >> META_new.yml
+	$(NOECHO) $(ECHO) '        - t' >> META_new.yml
+	$(NOECHO) $(ECHO) '        - inc' >> META_new.yml
+	$(NOECHO) $(ECHO) 'generated_by:       ExtUtils::MakeMaker version 6.56' >> META_new.yml
+	$(NOECHO) $(ECHO) 'meta-spec:' >> META_new.yml
+	$(NOECHO) $(ECHO) '    url:      http://module-build.sourceforge.net/META-spec-v1.4.html' >> META_new.yml
+	$(NOECHO) $(ECHO) '    version:  1.4' >> META_new.yml
+	-$(NOECHO) $(MV) META_new.yml $(DISTVNAME)/META.yml
+
+
+# --- MakeMaker signature section:
+signature :
+	cpansign -s
+
+
+# --- MakeMaker dist_basics section:
+distclean :: realclean distcheck
+	$(NOECHO) $(NOOP)
+
+distcheck :
+	$(PERLRUN) "-MExtUtils::Manifest=fullcheck" -e fullcheck
+
+skipcheck :
+	$(PERLRUN) "-MExtUtils::Manifest=skipcheck" -e skipcheck
+
+manifest :
+	$(PERLRUN) "-MExtUtils::Manifest=mkmanifest" -e mkmanifest
+
+veryclean : realclean
+	$(RM_F) *~ */*~ *.orig */*.orig *.bak */*.bak *.old */*.old 
+
+
+
+# --- MakeMaker dist_core section:
+
+dist : $(DIST_DEFAULT) $(FIRST_MAKEFILE)
+	$(NOECHO) $(ABSPERLRUN) -l -e 'print '\''Warning: Makefile possibly out of date with $(VERSION_FROM)'\''' \
+	  -e '    if -e '\''$(VERSION_FROM)'\'' and -M '\''$(VERSION_FROM)'\'' < -M '\''$(FIRST_MAKEFILE)'\'';' --
+
+tardist : $(DISTVNAME).tar$(SUFFIX)
+	$(NOECHO) $(NOOP)
+
+uutardist : $(DISTVNAME).tar$(SUFFIX)
+	uuencode $(DISTVNAME).tar$(SUFFIX) $(DISTVNAME).tar$(SUFFIX) > $(DISTVNAME).tar$(SUFFIX)_uu
+
+$(DISTVNAME).tar$(SUFFIX) : distdir
+	$(PREOP)
+	$(TO_UNIX)
+	$(TAR) $(TARFLAGS) $(DISTVNAME).tar $(DISTVNAME)
+	$(RM_RF) $(DISTVNAME)
+	$(COMPRESS) $(DISTVNAME).tar
+	$(POSTOP)
+
+zipdist : $(DISTVNAME).zip
+	$(NOECHO) $(NOOP)
+
+$(DISTVNAME).zip : distdir
+	$(PREOP)
+	$(ZIP) $(ZIPFLAGS) $(DISTVNAME).zip $(DISTVNAME)
+	$(RM_RF) $(DISTVNAME)
+	$(POSTOP)
+
+shdist : distdir
+	$(PREOP)
+	$(SHAR) $(DISTVNAME) > $(DISTVNAME).shar
+	$(RM_RF) $(DISTVNAME)
+	$(POSTOP)
+
+
+# --- MakeMaker distdir section:
+create_distdir :
+	$(RM_RF) $(DISTVNAME)
+	$(PERLRUN) "-MExtUtils::Manifest=manicopy,maniread" \
+		-e "manicopy(maniread(),'$(DISTVNAME)', '$(DIST_CP)');"
+
+distdir : create_distdir distmeta 
+	$(NOECHO) $(NOOP)
+
+
+
+# --- MakeMaker dist_test section:
+disttest : distdir
+	cd $(DISTVNAME) && $(ABSPERLRUN) Makefile.PL 
+	cd $(DISTVNAME) && $(MAKE) $(PASTHRU)
+	cd $(DISTVNAME) && $(MAKE) test $(PASTHRU)
+
+
+
+# --- MakeMaker dist_ci section:
+
+ci :
+	$(PERLRUN) "-MExtUtils::Manifest=maniread" \
+	  -e "@all = keys %{ maniread() };" \
+	  -e "print(qq{Executing $(CI) @all\n}); system(qq{$(CI) @all});" \
+	  -e "print(qq{Executing $(RCS_LABEL) ...\n}); system(qq{$(RCS_LABEL) @all});"
+
+
+# --- MakeMaker distmeta section:
+distmeta : create_distdir metafile
+	$(NOECHO) cd $(DISTVNAME) && $(ABSPERLRUN) -MExtUtils::Manifest=maniadd -e 'eval { maniadd({q{META.yml} => q{Module meta-data (added by MakeMaker)}}) } ' \
+	  -e '    or print "Could not add META.yml to MANIFEST: $${'\''@'\''}\n"' --
+
+
+
+# --- MakeMaker distsignature section:
+distsignature : create_distdir
+	$(NOECHO) cd $(DISTVNAME) && $(ABSPERLRUN) -MExtUtils::Manifest=maniadd -e 'eval { maniadd({q{SIGNATURE} => q{Public-key signature (added by MakeMaker)}}) } ' \
+	  -e '    or print "Could not add SIGNATURE to MANIFEST: $${'\''@'\''}\n"' --
+	$(NOECHO) cd $(DISTVNAME) && $(TOUCH) SIGNATURE
+	cd $(DISTVNAME) && cpansign -s
+
+
+
+# --- MakeMaker install section:
+
+install :: pure_install doc_install
+	$(NOECHO) $(NOOP)
+
+install_perl :: pure_perl_install doc_perl_install
+	$(NOECHO) $(NOOP)
+
+install_site :: pure_site_install doc_site_install
+	$(NOECHO) $(NOOP)
+
+install_vendor :: pure_vendor_install doc_vendor_install
+	$(NOECHO) $(NOOP)
+
+pure_install :: pure_$(INSTALLDIRS)_install
+	$(NOECHO) $(NOOP)
+
+doc_install :: doc_$(INSTALLDIRS)_install
+	$(NOECHO) $(NOOP)
+
+pure__install : pure_site_install
+	$(NOECHO) $(ECHO) INSTALLDIRS not defined, defaulting to INSTALLDIRS=site
+
+doc__install : doc_site_install
+	$(NOECHO) $(ECHO) INSTALLDIRS not defined, defaulting to INSTALLDIRS=site
+
+pure_perl_install :: all
+	$(NOECHO) umask 022; $(MOD_INSTALL) \
+		$(INST_LIB) $(DESTINSTALLPRIVLIB) \
+		$(INST_ARCHLIB) $(DESTINSTALLARCHLIB) \
+		$(INST_BIN) $(DESTINSTALLBIN) \
+		$(INST_SCRIPT) $(DESTINSTALLSCRIPT) \
+		$(INST_MAN1DIR) $(DESTINSTALLMAN1DIR) \
+		$(INST_MAN3DIR) $(DESTINSTALLMAN3DIR)
+	$(NOECHO) $(WARN_IF_OLD_PACKLIST) \
+		$(SITEARCHEXP)/auto/$(FULLEXT)
+
+
+pure_site_install :: all
+	$(NOECHO) umask 02; $(MOD_INSTALL) \
+		read $(SITEARCHEXP)/auto/$(FULLEXT)/.packlist \
+		write $(DESTINSTALLSITEARCH)/auto/$(FULLEXT)/.packlist \
+		$(INST_LIB) $(DESTINSTALLSITELIB) \
+		$(INST_ARCHLIB) $(DESTINSTALLSITEARCH) \
+		$(INST_BIN) $(DESTINSTALLSITEBIN) \
+		$(INST_SCRIPT) $(DESTINSTALLSITESCRIPT) \
+		$(INST_MAN1DIR) $(DESTINSTALLSITEMAN1DIR) \
+		$(INST_MAN3DIR) $(DESTINSTALLSITEMAN3DIR)
+	$(NOECHO) $(WARN_IF_OLD_PACKLIST) \
+		$(PERL_ARCHLIB)/auto/$(FULLEXT)
+
+pure_vendor_install :: all
+	$(NOECHO) umask 022; $(MOD_INSTALL) \
+		$(INST_LIB) $(DESTINSTALLVENDORLIB) \
+		$(INST_ARCHLIB) $(DESTINSTALLVENDORARCH) \
+		$(INST_BIN) $(DESTINSTALLVENDORBIN) \
+		$(INST_SCRIPT) $(DESTINSTALLVENDORSCRIPT) \
+		$(INST_MAN1DIR) $(DESTINSTALLVENDORMAN1DIR) \
+		$(INST_MAN3DIR) $(DESTINSTALLVENDORMAN3DIR)
+
+doc_perl_install :: all
+
+doc_site_install :: all
+	$(NOECHO) $(ECHO) Appending installation info to $(DESTINSTALLSITEARCH)/perllocal.pod
+	-$(NOECHO) umask 02; $(MKPATH) $(DESTINSTALLSITEARCH)
+	-$(NOECHO) umask 02; $(DOC_INSTALL) \
+		"Module" "$(NAME)" \
+		"installed into" "$(INSTALLSITELIB)" \
+		LINKTYPE "$(LINKTYPE)" \
+		VERSION "$(VERSION)" \
+		EXE_FILES "$(EXE_FILES)" \
+		>> $(DESTINSTALLSITEARCH)/perllocal.pod
+
+doc_vendor_install :: all
+
+
+uninstall :: uninstall_from_$(INSTALLDIRS)dirs
+	$(NOECHO) $(NOOP)
+
+uninstall_from_perldirs ::
+
+uninstall_from_sitedirs ::
+	$(NOECHO) $(UNINSTALL) $(SITEARCHEXP)/auto/$(FULLEXT)/.packlist
+
+uninstall_from_vendordirs ::
+
+
+
+# --- MakeMaker force section:
+# Phony target to force checking subdirectories.
+FORCE :
+	$(NOECHO) $(NOOP)
+
+
+# --- MakeMaker perldepend section:
+
+PERL_HDRS = \
+	$(PERL_INC)/EXTERN.h		\
+	$(PERL_INC)/INTERN.h		\
+	$(PERL_INC)/XSUB.h		\
+	$(PERL_INC)/av.h		\
+	$(PERL_INC)/cc_runtime.h	\
+	$(PERL_INC)/config.h		\
+	$(PERL_INC)/cop.h		\
+	$(PERL_INC)/cv.h		\
+	$(PERL_INC)/dosish.h		\
+	$(PERL_INC)/embed.h		\
+	$(PERL_INC)/embedvar.h		\
+	$(PERL_INC)/fakethr.h		\
+	$(PERL_INC)/form.h		\
+	$(PERL_INC)/gv.h		\
+	$(PERL_INC)/handy.h		\
+	$(PERL_INC)/hv.h		\
+	$(PERL_INC)/intrpvar.h		\
+	$(PERL_INC)/iperlsys.h		\
+	$(PERL_INC)/keywords.h		\
+	$(PERL_INC)/mg.h		\
+	$(PERL_INC)/nostdio.h		\
+	$(PERL_INC)/op.h		\
+	$(PERL_INC)/opcode.h		\
+	$(PERL_INC)/patchlevel.h	\
+	$(PERL_INC)/perl.h		\
+	$(PERL_INC)/perlio.h		\
+	$(PERL_INC)/perlsdio.h		\
+	$(PERL_INC)/perlsfio.h		\
+	$(PERL_INC)/perlvars.h		\
+	$(PERL_INC)/perly.h		\
+	$(PERL_INC)/pp.h		\
+	$(PERL_INC)/pp_proto.h		\
+	$(PERL_INC)/proto.h		\
+	$(PERL_INC)/regcomp.h		\
+	$(PERL_INC)/regexp.h		\
+	$(PERL_INC)/regnodes.h		\
+	$(PERL_INC)/scope.h		\
+	$(PERL_INC)/sv.h		\
+	$(PERL_INC)/thread.h		\
+	$(PERL_INC)/unixish.h		\
+	$(PERL_INC)/util.h
+
+$(OBJECT) : $(PERL_HDRS)
+
+
+# --- MakeMaker makefile section:
+
+$(OBJECT) : $(FIRST_MAKEFILE)
+
+# We take a very conservative approach here, but it's worth it.
+# We move Makefile to Makefile.old here to avoid gnu make looping.
+$(FIRST_MAKEFILE) : Makefile.PL $(CONFIGDEP)
+	$(NOECHO) $(ECHO) "Makefile out-of-date with respect to $?"
+	$(NOECHO) $(ECHO) "Cleaning current config before rebuilding Makefile..."
+	-$(NOECHO) $(RM_F) $(MAKEFILE_OLD)
+	-$(NOECHO) $(MV)   $(FIRST_MAKEFILE) $(MAKEFILE_OLD)
+	- $(MAKE) $(USEMAKEFILE) $(MAKEFILE_OLD) clean $(DEV_NULL)
+	$(PERLRUN) Makefile.PL 
+	$(NOECHO) $(ECHO) "==> Your Makefile has been rebuilt. <=="
+	$(NOECHO) $(ECHO) "==> Please rerun the $(MAKE) command.  <=="
+	$(FALSE)
+
+
+
+# --- MakeMaker staticmake section:
+
+# --- MakeMaker makeaperl section ---
+MAP_TARGET    = perl
+FULLPERL      = /usr/bin/perl
+
+$(MAP_TARGET) :: static $(MAKE_APERL_FILE)
+	$(MAKE) $(USEMAKEFILE) $(MAKE_APERL_FILE) $@
+
+$(MAKE_APERL_FILE) : $(FIRST_MAKEFILE) pm_to_blib
+	$(NOECHO) $(ECHO) Writing \"$(MAKE_APERL_FILE)\" for this $(MAP_TARGET)
+	$(NOECHO) $(PERLRUNINST) \
+		Makefile.PL DIR= \
+		MAKEFILE=$(MAKE_APERL_FILE) LINKTYPE=static \
+		MAKEAPERL=1 NORECURS=1 CCCDLFLAGS=
+
+
+# --- MakeMaker test section:
+
+TEST_VERBOSE=0
+TEST_TYPE=test_$(LINKTYPE)
+TEST_FILE = test.pl
+TEST_FILES = 
+TESTDB_SW = -d
+
+testdb :: testdb_$(LINKTYPE)
+
+test :: $(TEST_TYPE) subdirs-test
+
+subdirs-test ::
+	$(NOECHO) $(NOOP)
+
+
+test_dynamic :: pure_all
+	PERL_DL_NONLAZY=1 $(FULLPERLRUN) "-I$(INST_LIB)" "-I$(INST_ARCHLIB)" $(TEST_FILE)
+
+testdb_dynamic :: pure_all
+	PERL_DL_NONLAZY=1 $(FULLPERLRUN) $(TESTDB_SW) "-I$(INST_LIB)" "-I$(INST_ARCHLIB)" $(TEST_FILE)
+
+test_ : test_dynamic
+
+test_static :: pure_all $(MAP_TARGET)
+	PERL_DL_NONLAZY=1 ./$(MAP_TARGET) "-I$(INST_LIB)" "-I$(INST_ARCHLIB)" $(TEST_FILE)
+
+testdb_static :: pure_all $(MAP_TARGET)
+	PERL_DL_NONLAZY=1 ./$(MAP_TARGET) $(TESTDB_SW) "-I$(INST_LIB)" "-I$(INST_ARCHLIB)" $(TEST_FILE)
+
+
+
+# --- MakeMaker ppd section:
+# Creates a PPD (Perl Package Description) for a binary distribution.
+ppd :
+	$(NOECHO) $(ECHO) '<SOFTPKG NAME="$(DISTNAME)" VERSION="">' > $(DISTNAME).ppd
+	$(NOECHO) $(ECHO) '    <ABSTRACT></ABSTRACT>' >> $(DISTNAME).ppd
+	$(NOECHO) $(ECHO) '    <AUTHOR></AUTHOR>' >> $(DISTNAME).ppd
+	$(NOECHO) $(ECHO) '    <IMPLEMENTATION>' >> $(DISTNAME).ppd
+	$(NOECHO) $(ECHO) '        <ARCHITECTURE NAME="x86_64-linux-gnu-thread-multi-5.12" />' >> $(DISTNAME).ppd
+	$(NOECHO) $(ECHO) '        <CODEBASE HREF="" />' >> $(DISTNAME).ppd
+	$(NOECHO) $(ECHO) '    </IMPLEMENTATION>' >> $(DISTNAME).ppd
+	$(NOECHO) $(ECHO) '</SOFTPKG>' >> $(DISTNAME).ppd
+
+
+# --- MakeMaker pm_to_blib section:
+
+pm_to_blib : $(FIRST_MAKEFILE) $(TO_INST_PM)
+	$(NOECHO) $(ABSPERLRUN) -MExtUtils::Install -e 'pm_to_blib({@ARGV}, '\''$(INST_LIB)/auto'\'', q[$(PM_FILTER)], '\''$(PERM_DIR)'\'')' -- \
+	  CRFPP.pm $(INST_LIB)/CRFPP.pm 
+	$(NOECHO) $(TOUCH) pm_to_blib
+
+
+# --- MakeMaker selfdocument section:
+
+
+# --- MakeMaker postamble section:
+
+
+# End.

CRF/ruby/Makefile

@@ -1,157 +1,157 @@
-
-SHELL = /bin/sh
-
-#### Start of system configuration section. ####
-
-srcdir = .
-topdir = /usr/lib/ruby/1.8/x86_64-linux
-hdrdir = $(topdir)
-VPATH = $(srcdir):$(topdir):$(hdrdir)
-exec_prefix = $(prefix)
-prefix = $(DESTDIR)/usr
-sharedstatedir = $(prefix)/com
-mandir = $(prefix)/share/man
-psdir = $(docdir)
-oldincludedir = $(DESTDIR)/usr/include
-localedir = $(datarootdir)/locale
-bindir = $(exec_prefix)/bin
-libexecdir = $(prefix)/lib/ruby1.8
-sitedir = $(DESTDIR)/usr/local/lib/site_ruby
-htmldir = $(docdir)
-vendorarchdir = $(vendorlibdir)/$(sitearch)
-includedir = $(prefix)/include
-infodir = $(prefix)/share/info
-vendorlibdir = $(vendordir)/$(ruby_version)
-sysconfdir = $(DESTDIR)/etc
-libdir = $(exec_prefix)/lib
-sbindir = $(exec_prefix)/sbin
-rubylibdir = $(libdir)/ruby/$(ruby_version)
-docdir = $(datarootdir)/doc/$(PACKAGE)
-dvidir = $(docdir)
-vendordir = $(libdir)/ruby/vendor_ruby
-datarootdir = $(prefix)/share
-pdfdir = $(docdir)
-archdir = $(rubylibdir)/$(arch)
-sitearchdir = $(sitelibdir)/$(sitearch)
-datadir = $(datarootdir)
-localstatedir = $(DESTDIR)/var
-sitelibdir = $(sitedir)/$(ruby_version)
-
-CC = gcc
-LIBRUBY = $(LIBRUBY_SO)
-LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
-LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME)
-LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static
-
-RUBY_EXTCONF_H = 
-CFLAGS   =  -fPIC -fno-strict-aliasing -g -g -O2  -fPIC $(cflags) 
-INCFLAGS = -I. -I. -I/usr/lib/ruby/1.8/x86_64-linux -I.
-DEFS     = 
-CPPFLAGS = -DHAVE_CRFPP_H   
-CXXFLAGS = $(CFLAGS) 
-ldflags  = -L. -Wl,-Bsymbolic-functions -rdynamic -Wl,-export-dynamic
-dldflags = 
-archflag = 
-DLDFLAGS = $(ldflags) $(dldflags) $(archflag)
-LDSHARED = $(CC) -shared
-AR = ar
-EXEEXT = 
-
-RUBY_INSTALL_NAME = ruby1.8
-RUBY_SO_NAME = ruby1.8
-arch = x86_64-linux
-sitearch = x86_64-linux
-ruby_version = 1.8
-ruby = /usr/bin/ruby1.8
-RUBY = $(ruby)
-RM = rm -f
-MAKEDIRS = mkdir -p
-INSTALL = /usr/bin/install -c
-INSTALL_PROG = $(INSTALL) -m 0755
-INSTALL_DATA = $(INSTALL) -m 644
-COPY = cp
-
-#### End of system configuration section. ####
-
-preload = 
-
-libpath = . $(libdir)
-LIBPATH =  -L. -L$(libdir)
-DEFFILE = 
-
-CLEANFILES = mkmf.log
-DISTCLEANFILES = 
-
-extout = 
-extout_prefix = 
-target_prefix = 
-LOCAL_LIBS = 
-LIBS = $(LIBRUBYARG_SHARED) -lpthread -lcrfpp  -lpthread -lrt -ldl -lcrypt -lm   -lc
-SRCS = CRFPP_wrap.cpp
-OBJS = CRFPP_wrap.o
-TARGET = CRFPP
-DLLIB = $(TARGET).so
-EXTSTATIC = 
-STATIC_LIB = 
-
-BINDIR        = $(bindir)
-RUBYCOMMONDIR = $(sitedir)$(target_prefix)
-RUBYLIBDIR    = $(sitelibdir)$(target_prefix)
-RUBYARCHDIR   = $(sitearchdir)$(target_prefix)
-
-TARGET_SO     = $(DLLIB)
-CLEANLIBS     = $(TARGET).so $(TARGET).il? $(TARGET).tds $(TARGET).map
-CLEANOBJS     = *.o *.a *.s[ol] *.pdb *.exp *.bak
-
-all:		$(DLLIB)
-static:		$(STATIC_LIB)
-
-clean:
-		@-$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES)
-
-distclean:	clean
-		@-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
-		@-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
-
-realclean:	distclean
-install: install-so install-rb
-
-install-so: $(RUBYARCHDIR)
-install-so: $(RUBYARCHDIR)/$(DLLIB)
-$(RUBYARCHDIR)/$(DLLIB): $(DLLIB)
-	$(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
-install-rb: pre-install-rb install-rb-default
-install-rb-default: pre-install-rb-default
-pre-install-rb: Makefile
-pre-install-rb-default: Makefile
-$(RUBYARCHDIR):
-	$(MAKEDIRS) $@
-
-site-install: site-install-so site-install-rb
-site-install-so: install-so
-site-install-rb: install-rb
-
-.SUFFIXES: .c .m .cc .cxx .cpp .C .o
-
-.cc.o:
-	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
-
-.cxx.o:
-	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
-
-.cpp.o:
-	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
-
-.C.o:
-	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
-
-.c.o:
-	$(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c $<
-
-$(DLLIB): $(OBJS) Makefile
-	@-$(RM) $@
-	$(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
-
-
-
-$(OBJS): ruby.h defines.h
+
+SHELL = /bin/sh
+
+#### Start of system configuration section. ####
+
+srcdir = .
+topdir = /usr/lib/ruby/1.8/x86_64-linux
+hdrdir = $(topdir)
+VPATH = $(srcdir):$(topdir):$(hdrdir)
+exec_prefix = $(prefix)
+prefix = $(DESTDIR)/usr
+sharedstatedir = $(prefix)/com
+mandir = $(prefix)/share/man
+psdir = $(docdir)
+oldincludedir = $(DESTDIR)/usr/include
+localedir = $(datarootdir)/locale
+bindir = $(exec_prefix)/bin
+libexecdir = $(prefix)/lib/ruby1.8
+sitedir = $(DESTDIR)/usr/local/lib/site_ruby
+htmldir = $(docdir)
+vendorarchdir = $(vendorlibdir)/$(sitearch)
+includedir = $(prefix)/include
+infodir = $(prefix)/share/info
+vendorlibdir = $(vendordir)/$(ruby_version)
+sysconfdir = $(DESTDIR)/etc
+libdir = $(exec_prefix)/lib
+sbindir = $(exec_prefix)/sbin
+rubylibdir = $(libdir)/ruby/$(ruby_version)
+docdir = $(datarootdir)/doc/$(PACKAGE)
+dvidir = $(docdir)
+vendordir = $(libdir)/ruby/vendor_ruby
+datarootdir = $(prefix)/share
+pdfdir = $(docdir)
+archdir = $(rubylibdir)/$(arch)
+sitearchdir = $(sitelibdir)/$(sitearch)
+datadir = $(datarootdir)
+localstatedir = $(DESTDIR)/var
+sitelibdir = $(sitedir)/$(ruby_version)
+
+CC = gcc
+LIBRUBY = $(LIBRUBY_SO)
+LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
+LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME)
+LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static
+
+RUBY_EXTCONF_H = 
+CFLAGS   =  -fPIC -fno-strict-aliasing -g -g -O2  -fPIC $(cflags) 
+INCFLAGS = -I. -I. -I/usr/lib/ruby/1.8/x86_64-linux -I.
+DEFS     = 
+CPPFLAGS = -DHAVE_CRFPP_H   
+CXXFLAGS = $(CFLAGS) 
+ldflags  = -L. -Wl,-Bsymbolic-functions -rdynamic -Wl,-export-dynamic
+dldflags = 
+archflag = 
+DLDFLAGS = $(ldflags) $(dldflags) $(archflag)
+LDSHARED = $(CC) -shared
+AR = ar
+EXEEXT = 
+
+RUBY_INSTALL_NAME = ruby1.8
+RUBY_SO_NAME = ruby1.8
+arch = x86_64-linux
+sitearch = x86_64-linux
+ruby_version = 1.8
+ruby = /usr/bin/ruby1.8
+RUBY = $(ruby)
+RM = rm -f
+MAKEDIRS = mkdir -p
+INSTALL = /usr/bin/install -c
+INSTALL_PROG = $(INSTALL) -m 0755
+INSTALL_DATA = $(INSTALL) -m 644
+COPY = cp
+
+#### End of system configuration section. ####
+
+preload = 
+
+libpath = . $(libdir)
+LIBPATH =  -L. -L$(libdir)
+DEFFILE = 
+
+CLEANFILES = mkmf.log
+DISTCLEANFILES = 
+
+extout = 
+extout_prefix = 
+target_prefix = 
+LOCAL_LIBS = 
+LIBS = $(LIBRUBYARG_SHARED) -lpthread -lcrfpp  -lpthread -lrt -ldl -lcrypt -lm   -lc
+SRCS = CRFPP_wrap.cpp
+OBJS = CRFPP_wrap.o
+TARGET = CRFPP
+DLLIB = $(TARGET).so
+EXTSTATIC = 
+STATIC_LIB = 
+
+BINDIR        = $(bindir)
+RUBYCOMMONDIR = $(sitedir)$(target_prefix)
+RUBYLIBDIR    = $(sitelibdir)$(target_prefix)
+RUBYARCHDIR   = $(sitearchdir)$(target_prefix)
+
+TARGET_SO     = $(DLLIB)
+CLEANLIBS     = $(TARGET).so $(TARGET).il? $(TARGET).tds $(TARGET).map
+CLEANOBJS     = *.o *.a *.s[ol] *.pdb *.exp *.bak
+
+all:		$(DLLIB)
+static:		$(STATIC_LIB)
+
+clean:
+		@-$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES)
+
+distclean:	clean
+		@-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
+		@-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
+
+realclean:	distclean
+install: install-so install-rb
+
+install-so: $(RUBYARCHDIR)
+install-so: $(RUBYARCHDIR)/$(DLLIB)
+$(RUBYARCHDIR)/$(DLLIB): $(DLLIB)
+	$(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
+install-rb: pre-install-rb install-rb-default
+install-rb-default: pre-install-rb-default
+pre-install-rb: Makefile
+pre-install-rb-default: Makefile
+$(RUBYARCHDIR):
+	$(MAKEDIRS) $@
+
+site-install: site-install-so site-install-rb
+site-install-so: install-so
+site-install-rb: install-rb
+
+.SUFFIXES: .c .m .cc .cxx .cpp .C .o
+
+.cc.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.cxx.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.cpp.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.C.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.c.o:
+	$(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c $<
+
+$(DLLIB): $(OBJS) Makefile
+	@-$(RM) $@
+	$(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
+
+
+
+$(OBJS): ruby.h defines.h

CRF/winmain.h

@@ -1,69 +1,69 @@
-//
-//  CRF++ -- Yet Another CRF toolkit
-//
-//  $Id: common.h 1588 2007-02-12 09:03:39Z taku $;
-//
-//  Copyright(C) 2005-2007 Taku Kudo <taku@chasen.org>
-//
-#if defined(_WIN32) || defined(__CYGWIN__)
-
-#include <windows.h>
-#include <string>
-
-namespace {
-class CommandLine {
- public:
-  CommandLine(int argc, wchar_t **argv) : argc_(argc), argv_(0) {
-    argv_ = new char * [argc_];
-    for (int i = 0; i < argc_; ++i) {
-      const std::string arg = WideToUtf8(argv[i]);
-      argv_[i] = new char[arg.size() + 1];
-      ::memcpy(argv_[i], arg.data(), arg.size());
-      argv_[i][arg.size()] = '\0';
-    }
-  }
-  ~CommandLine() {
-    for (int i = 0; i < argc_; ++i) {
-      delete [] argv_[i];
-    }
-    delete [] argv_;
-  }
-
-  int argc() const { return argc_; }
-  char **argv() const { return argv_; }
-
- private:
-  static std::string WideToUtf8(const std::wstring &input) {
-    const int output_length = ::WideCharToMultiByte(CP_UTF8, 0,
-                                                    input.c_str(), -1, NULL, 0,
-                                                    NULL, NULL);
-    if (output_length == 0) {
-      return "";
-    }
-
-    char *input_encoded = new char[output_length + 1];
-    const int result = ::WideCharToMultiByte(CP_UTF8, 0, input.c_str(), -1,
-                                             input_encoded,
-                                             output_length + 1, NULL, NULL);
-    std::string output;
-    if (result > 0) {
-      output.assign(input_encoded);
-    }
-    delete [] input_encoded;
-    return output;
-  }
-
-  int argc_;
-  char **argv_;
-};
-}  // namespace
-
-#define main(argc, argv) wmain_to_main_wrapper(argc, argv)
-
-int wmain_to_main_wrapper(int argc, char **argv);
-
-int wmain(int argc, wchar_t **argv) {
-  CommandLine cmd(argc, argv);
-  return wmain_to_main_wrapper(cmd.argc(), cmd.argv());
-}
-#endif
+//
+//  CRF++ -- Yet Another CRF toolkit
+//
+//  $Id: common.h 1588 2007-02-12 09:03:39Z taku $;
+//
+//  Copyright(C) 2005-2007 Taku Kudo <taku@chasen.org>
+//
+#if defined(_WIN32) || defined(__CYGWIN__)
+
+#include <windows.h>
+#include <string>
+
+namespace {
+class CommandLine {
+ public:
+  CommandLine(int argc, wchar_t **argv) : argc_(argc), argv_(0) {
+    argv_ = new char * [argc_];
+    for (int i = 0; i < argc_; ++i) {
+      const std::string arg = WideToUtf8(argv[i]);
+      argv_[i] = new char[arg.size() + 1];
+      ::memcpy(argv_[i], arg.data(), arg.size());
+      argv_[i][arg.size()] = '\0';
+    }
+  }
+  ~CommandLine() {
+    for (int i = 0; i < argc_; ++i) {
+      delete [] argv_[i];
+    }
+    delete [] argv_;
+  }
+
+  int argc() const { return argc_; }
+  char **argv() const { return argv_; }
+
+ private:
+  static std::string WideToUtf8(const std::wstring &input) {
+    const int output_length = ::WideCharToMultiByte(CP_UTF8, 0,
+                                                    input.c_str(), -1, NULL, 0,
+                                                    NULL, NULL);
+    if (output_length == 0) {
+      return "";
+    }
+
+    char *input_encoded = new char[output_length + 1];
+    const int result = ::WideCharToMultiByte(CP_UTF8, 0, input.c_str(), -1,
+                                             input_encoded,
+                                             output_length + 1, NULL, NULL);
+    std::string output;
+    if (result > 0) {
+      output.assign(input_encoded);
+    }
+    delete [] input_encoded;
+    return output;
+  }
+
+  int argc_;
+  char **argv_;
+};
+}  // namespace
+
+#define main(argc, argv) wmain_to_main_wrapper(argc, argv)
+
+int wmain_to_main_wrapper(int argc, char **argv);
+
+int wmain(int argc, wchar_t **argv) {
+  CommandLine cmd(argc, argv);
+  return wmain_to_main_wrapper(cmd.argc(), cmd.argv());
+}
+#endif

GeneNER_SpeAss_run.py

@@ -1,746 +1,746 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Wed Jun  8 09:26:57 2022
-
-@author: luol2
-
-Pipeline: first gene NER, then species assignment
-input: species NER bioc xml file
-output: gene ner and species assignment results bioc xml file
-"""
-import argparse
-import os
-import io
-import time
-import sys
-import re
-import shutil
-from src_python.GeneNER import model_ner,ner_tag
-from src_python.SpeAss import model_sa,sa_tag
-
-import tensorflow as tf
-
-import bioc
-import stanza
-nlp_token = stanza.Pipeline(model_dir='gnorm_trained_models/stanza', lang='en', processors={'tokenize': 'spacy'},package='None', download_method=None) #package='craft' ;./gnorm_trained_models/stanza
-
-def NER_BioC(infolder,infile,outpath,nn_model):
-         
-    with open(infolder+"/"+infile, 'r',encoding='utf-8') as fin:
-        with open(outpath+"/"+infile,'w', encoding='utf8') as fout:
-            collection = bioc.load(fin)
-            
-            Total_n=len(collection.documents)
-            print('Total number of sub-documents:', Total_n)
-            pmid_n=0
-            for document in collection.documents:
-                print("Processing:{0}%".format(round(pmid_n * 100 / Total_n)), end="\r")
-                pmid_n+=1
-                # print(document.id)
-                mention_num_new=0
-                for passage in document.passages:
-                    if passage.text!='' and (not passage.text.isspace()) and passage.infons['type']!='ref': # have text and is not ref
-                        passage_offset=passage.offset
-                        tag_result=ner_tag.ML_Tag(passage.text,nn_model,nlp_token)
-                        mention_num=0
-                        for ele in tag_result:
-                            bioc_note = bioc.BioCAnnotation()
-                            bioc_note.id = str(mention_num)
-                            mention_num+=1
-                            bioc_note.infons['type'] = ele[2]
-                            start = int(ele[0])
-                            last = int(ele[1])
-                            loc = bioc.BioCLocation(offset=str(passage_offset+start), length= str(last-start))
-                            bioc_note.locations.append(loc)
-                            bioc_note.text = passage.text[start:last]
-                            passage.annotations.append(bioc_note)
-                    #update id
-                    for temp_annotation in passage.annotations:
-                        temp_annotation.id=str(mention_num_new)
-                        mention_num_new+=1
-            bioc.dump(collection, fout, pretty_print=True)
-            
-def NER_PubTator(infolder,infile,outpath,nn_model):
-    with open(infolder+"/"+infile, 'r',encoding='utf-8') as fin:
-        with open(outpath+"/"+infile,'w', encoding='utf-8') as fout:
-            title=''
-            abstract=''
-            all_text=fin.read().strip().split('\n\n')
-            Total_n=len(all_text)
-            print('Total number of sub-documents:', Total_n)
-            pmid_n=0
-            for doc in all_text:
-                print("Processing:{0}%".format(round(pmid_n * 100 / Total_n)), end="\r")
-                pmid_n+=1
-                lines = doc.split('\n')
-                seg=lines[0].split('|t|')
-                pmid=seg[0]
-                title=""
-                if len(seg)>1:
-                    title=seg[1]
-                abstract=""
-                if len(lines)>1:
-                    seg=lines[1].split('|a|')
-                    abstract=seg[1]
-                    if len(seg)>1:
-                        abstract=seg[1]
-               
-                intext=title+' '+abstract
-                tag_result=ner_tag.ML_Tag(intext,nn_model,nlp_token)
-                fout.write(doc+'\n')
-                for ele in tag_result:
-                    ent_start = ele[0]
-                    ent_last = ele[1]
-                    ent_mention = intext[int(ele[0]):int(ele[1])]
-                    ent_type=ele[2]
-                    fout.write(pmid+"\t"+ent_start+"\t"+ent_last+"\t"+ent_mention+"\t"+ent_type+"\n")
-                fout.write('\n')
-                title=''
-                abstract=''
-                
-def geneNER(infolder, outpath, modelfile):
-    
-    print('loading NER models........')    
-        
-    if modelfile.lower().find('bioformer')>=0:
-        vocabfiles={'labelfile':'./vocab/GeneNER_label.vocab',
-                        'checkpoint_path':'./gnorm_trained_models/bioformer-cased-v1.0/', #bioformer-cased-v1.0
-                        'lowercase':False,
-                        } 
-    else:
-        vocabfiles={'labelfile':'./vocab/GeneNER_label.vocab',
-                    'checkpoint_path':'./gnorm_trained_models/BiomedNLP-PubMedBERT-base-uncased-abstract/',
-                    'lowercase':True,
-                    } 
-        
-    nn_model=model_ner.HUGFACE_NER(vocabfiles)
-    nn_model.build_encoder()
-    nn_model.build_softmax_decoder()
-    nn_model.load_model(modelfile)
-        
-    #tagging text
-    print("begin GeneNER tagging........")
-    start_time=time.time()
-    
-    for infile in os.listdir(infolder):
-        if os.path.isfile(outpath+"/"+infile):
-            print(infile+' has exsited.')
-        else:
-            print('processing:',infile)             
-            fin = open(infolder+"/"+infile, 'r',encoding='utf-8')
-            input_format=""
-            for line in fin:
-                pattern_bioc = re.compile('.*<collection>.*')
-                pattern_pubtator = re.compile('^([^\|]+)\|[^\|]+\|(.*)')
-                if pattern_bioc.search(line):
-                    input_format="BioC"
-                    break
-                elif pattern_pubtator.search(line):
-                    input_format="PubTator"
-                    break
-            fin.close()
-            if(input_format == "PubTator"):
-                NER_PubTator(infolder,infile,outpath,nn_model)
-            elif(input_format == "BioC"):
-                NER_BioC(infolder,infile,outpath,nn_model)    
-    
-    print('tag done:',time.time()-start_time)
-    
-
-#SA for bioc format
-def SA_BioC(infolder,infile,outpath,nn_model,virus_set,prefix_dict):
-    
-    #BioC xml to pubtator
-    # pmid|t|text1
-    #pmid|a|text2
-    #pmid sid eid entity_txt entity_type entity_id (gene is blank)
-    fin = open(infolder+"/"+infile, 'r',encoding='utf-8')
-    # fout_pubtator=open(outpath+'tmp/input_xml.pubtator','w', encoding='utf-8')
-    fin_pubtator0=io.StringIO() #none *species
-    fin_pubtator1=io.StringIO() #one *species
-    fin_pubtator2=io.StringIO() #two or more species
-    collection = bioc.load(fin)
-    fin.close()
-    ori_ann_index={}  #{'pmid':{'ent.id':'ent_s-ent_e'}}
-    species_count={} #{pmid:{speid:num}}
-    gene_set=['Gene','FamilyName']
-    final_sa_results={} #{'pmid':{'entity_id':species_id}}
-    for document in collection.documents:
-        doc_pmid=document.id
-        doc_title=''
-        doc_abstract=''
-        doc_annotation=[]
-        _ann_index={}
-        _species_num={} #{*speciesid:num}
-        _gene_num=0
-        _passage_num=0              
-        if len(document.passages)<=2: #abstract xml or PMC only have title
-            for passage in document.passages:
-                passage_offset=passage.offset
-                _passage_num+=1              
-                #print(passage_offset,type(passage_offset))
-                #if passage.infons['type']=='title' or passage.infons['type']=='front':
-                if _passage_num==1:
-                    doc_title=passage.text
-                    for temp_annotation in passage.annotations:
-                        if temp_annotation.infons['type'] in gene_set:
-                            _gene_num+=1
-                        ent_start=temp_annotation.locations[0].offset-passage_offset
-                        ent_end=ent_start+temp_annotation.locations[0].length
-                        #print(ent_start,ent_end)
-                        _ann_index[temp_annotation.id]=str(ent_start)+'-'+str(ent_end)
-                        # print(temp_annotation.infons)
-                        if 'Identifier' in temp_annotation.infons.keys():
-                            # print(temp_annotation.infons.keys['Identifier'])
-                            species_ID=temp_annotation.infons['Identifier']
-                            if species_ID.find('*')>=0:
-                                if species_ID not in _species_num.keys():
-                                    _species_num[species_ID]=1
-                                else:
-                                    _species_num[species_ID]+=1
-                            doc_annotation.append(doc_pmid+'\t'+temp_annotation.id+'\t'+str(ent_start)+'\t'+str(ent_end)+'\t'+temp_annotation.text+'\t'+temp_annotation.infons['type']+'\t'+species_ID)
-                        else:
-                            doc_annotation.append(doc_pmid+'\t'+temp_annotation.id+'\t'+str(ent_start)+'\t'+str(ent_end)+'\t'+temp_annotation.text+'\t'+temp_annotation.infons['type'])
-                        
-                #elif passage.infons['type']=='abstract' or passage.infons['type']=='paragraph':
-                else:
-                    doc_abstract=passage.text
-                    for temp_annotation in passage.annotations:
-                        if temp_annotation.infons['type'] in gene_set:
-                            _gene_num+=1
-                        ent_start=len(doc_title)+1+temp_annotation.locations[0].offset-passage_offset
-                        ent_end=ent_start+temp_annotation.locations[0].length
-                        #print(ent_start,ent_end)
-                        _ann_index[temp_annotation.id]=str(ent_start)+'-'+str(ent_end)
-                        if 'Identifier' in temp_annotation.infons.keys():
-                            # print(temp_annotation.infons.keys['Identifier'])
-                            species_ID=temp_annotation.infons['Identifier']
-                            if species_ID.find('*')>=0:
-                                if species_ID not in _species_num.keys():
-                                    _species_num[species_ID]=1
-                                else:
-                                    _species_num[species_ID]+=1
-                            doc_annotation.append(doc_pmid+'\t'+temp_annotation.id+'\t'+str(ent_start)+'\t'+str(ent_end)+'\t'+temp_annotation.text+'\t'+temp_annotation.infons['type']+'\t'+species_ID)
-                        else:
-                            doc_annotation.append(doc_pmid+'\t'+temp_annotation.id+'\t'+str(ent_start)+'\t'+str(ent_end)+'\t'+temp_annotation.text+'\t'+temp_annotation.infons['type'])
-            
-            if len(_species_num)>=2 and _gene_num>0:
-                fin_pubtator2.write(doc_pmid+'|t|'+doc_title+'\n')
-                fin_pubtator2.write(doc_pmid+'|a|'+doc_abstract+'\n')
-                for ele in doc_annotation:
-                    fin_pubtator2.write(ele+'\n')
-                fin_pubtator2.write('\n')
-            elif len(_species_num)==1 and _gene_num>0: #可以直接给结果
-                fin_pubtator1.write(doc_pmid+'|t|'+doc_title+'\n')
-                fin_pubtator1.write(doc_pmid+'|a|'+doc_abstract+'\n')
-                major_speicesid,=_species_num
-                fin_pubtator1.write(major_speicesid[1:]+'\n')
-                for ele in doc_annotation:
-                    fin_pubtator1.write(ele+'\n')
-                fin_pubtator1.write('\n')
-            elif len(_species_num)==0 and _gene_num>0:
-                fin_pubtator0.write(doc_pmid+'|t|'+doc_title+'\n')
-                fin_pubtator0.write(doc_pmid+'|a|'+doc_abstract+'\n')
-                for ele in doc_annotation:
-                    fin_pubtator0.write(ele+'\n')
-                fin_pubtator0.write('\n')        
-        
-        else: # full text xml
-            for passage in document.passages:
-                passage_annotation=[]
-                _species_num_passage={}
-                _gene_num_passage=0
-                passage_offset=passage.offset
-                #print(passage_offset,type(passage_offset))
-                if passage.text!='' and (not passage.text.isspace()) and passage.infons['type']!='ref':
-                    doc_title=passage.text
-                    for temp_annotation in passage.annotations:
-                        if temp_annotation.infons['type'] in gene_set:
-                            _gene_num_passage+=1
-                        ent_start=temp_annotation.locations[0].offset-passage_offset
-                        ent_end=ent_start+temp_annotation.locations[0].length
-                        #print(ent_start,ent_end)
-                        _ann_index[temp_annotation.id]=str(ent_start)+'-'+str(ent_end)
-                        # print(temp_annotation.infons)
-                        if 'Identifier' in temp_annotation.infons.keys():
-                            # print(temp_annotation.infons.keys['Identifier'])
-                            species_ID=temp_annotation.infons['Identifier']
-                            if species_ID.find('*')>=0:
-                                if species_ID not in _species_num.keys():
-                                    _species_num[species_ID]=1
-                                else:
-                                    _species_num[species_ID]+=1
-                                if species_ID not in _species_num_passage.keys():
-                                    _species_num_passage[species_ID]=1
-                                else:
-                                    _species_num_passage[species_ID]+=1
-                            passage_annotation.append(doc_pmid+'\t'+temp_annotation.id+'\t'+str(ent_start)+'\t'+str(ent_end)+'\t'+temp_annotation.text+'\t'+temp_annotation.infons['type']+'\t'+species_ID)
-                        else:
-                            passage_annotation.append(doc_pmid+'\t'+temp_annotation.id+'\t'+str(ent_start)+'\t'+str(ent_end)+'\t'+temp_annotation.text+'\t'+temp_annotation.infons['type'])
-                        
-            
-                if len(_species_num_passage)>=2 and _gene_num_passage>0:
-                    fin_pubtator2.write(doc_pmid+'|t|'+doc_title+'\n')
-                    fin_pubtator2.write(doc_pmid+'|a|'+doc_abstract+'\n')
-                    for ele in passage_annotation:
-                        fin_pubtator2.write(ele+'\n')
-                    fin_pubtator2.write('\n')
-                elif len(_species_num_passage)==1 and _gene_num_passage>0: #可以直接给结果
-                    fin_pubtator1.write(doc_pmid+'|t|'+doc_title+'\n')
-                    fin_pubtator1.write(doc_pmid+'|a|'+doc_abstract+'\n')
-                    major_speicesid,=_species_num_passage
-                    fin_pubtator1.write(major_speicesid[1:]+'\n')
-                    for ele in passage_annotation:
-                        fin_pubtator1.write(ele+'\n')
-                    fin_pubtator1.write('\n')
-                elif len(_species_num_passage)==0 and _gene_num_passage>0:
-                    fin_pubtator0.write(doc_pmid+'|t|'+doc_title+'\n')
-                    fin_pubtator0.write(doc_pmid+'|a|'+doc_abstract+'\n')
-                    for ele in passage_annotation:
-                        fin_pubtator0.write(ele+'\n')
-                    fin_pubtator0.write('\n')        
-        # print(ori_ann_index)
-        
-        ori_ann_index[doc_pmid]=_ann_index
-        species_count[doc_pmid]=_species_num
-    
-            
-    cache_geneid={} #{pmid:{gene1:{id1:num,id2:num}}}
-    
-    if fin_pubtator2.getvalue()!='':
-        #pubtator format  ML tagging
-        # print(fin_pubtator2.getvalue())
-        ml_out= sa_tag.ml_tag_main(fin_pubtator2,nlp_token, nn_model)   
-        #print(ml_out.getvalue())
-        fin_result=io.StringIO(ml_out.getvalue())
-        all_in=fin_result.read().strip().split('\n\n')
-        #print('+2 species:',len(all_in))
-        fin_result.close()
-        
-        prefix_speid_allset=set(prefix_dict.keys())
-    
-        for doc in all_in:
-            lines=doc.split('\n')
-            pmid=lines[0].split('|t|')[0]
-            _prefix_str2id_dict={}
-            doc_species=list(species_count[pmid].keys())
-            for _spe_ele in doc_species:
-                if _spe_ele[1:] in prefix_speid_allset:
-                    for ele in prefix_dict[_spe_ele[1:]]:
-                        _prefix_str2id_dict[ele]=_spe_ele[1:]
-            
-            for i in range(2,len(lines)):
-                segs=lines[i].split('\t')
-                if pmid not in final_sa_results.keys():
-                    final_sa_results[pmid]={segs[1]:'Focus:'+segs[-1]}
-                else:
-                    final_sa_results[pmid][segs[1]]='Focus:'+segs[-1]
-                    
-                if segs[5] in gene_set:
-                    if segs[4][0:2] in _prefix_str2id_dict: #prefix rule
-                        #print('prefix rule:', pmid)
-                        # print(_prefix_str2id_dict)
-                        if pmid not in final_sa_results.keys():
-                            final_sa_results[pmid]={segs[1]:'Focus:'+_prefix_str2id_dict[segs[4][0:2]]}
-                        else:
-                            final_sa_results[pmid][segs[1]]='Focus:'+_prefix_str2id_dict[segs[4][0:2]]
-                    if pmid not in cache_geneid.keys():
-                        cache_geneid[pmid]={segs[4]:{'Focus:'+segs[-1]:1}}
-                    else:
-                        if segs[4] not in cache_geneid[pmid].keys():
-                            cache_geneid[pmid][segs[4]]={'Focus:'+segs[-1]:1}
-                        else:
-                            if segs[-1] not in cache_geneid[pmid][segs[4]].keys():
-                                cache_geneid[pmid][segs[4]]['Focus:'+segs[-1]]=1
-                            else:
-                                cache_geneid[pmid][segs[4]]['Focus:'+segs[-1]]+=1
-        
-    #print(final_sa_results)
-    
-    #one species  
-    if fin_pubtator1.getvalue()!='':
-        fin_result=io.StringIO(fin_pubtator1.getvalue())
-        all_in=fin_result.read().strip().split('\n\n')
-        fin_result.close()
-        #print('1 species:',len(all_in))
-        for doc in all_in:
-            lines=doc.split('\n')
-            pmid=lines[0].split('|t|')[0]
-            major_speicesid=lines[2]
-            for i in range(3,len(lines)):
-                segs=lines[i].split('\t')
-                if len(segs)>=7:#species
-                    if pmid not in final_sa_results.keys():
-                        final_sa_results[pmid]={segs[1]:segs[-1]}
-                    else:
-                        final_sa_results[pmid][segs[1]]=segs[-1]
-                else:#gene
-                    marjor_species='Focus:'+major_speicesid
-                    if pmid not in final_sa_results.keys():
-                        final_sa_results[pmid]={segs[1]:marjor_species}
-                    else:
-                        final_sa_results[pmid][segs[1]]=marjor_species
-                    if pmid not in cache_geneid.keys():
-                        cache_geneid[pmid]={segs[4]:{marjor_species:1}}
-                    else:
-                        if segs[4] not in cache_geneid[pmid].keys():
-                            cache_geneid[pmid][segs[4]]={marjor_species:1}
-                        else:
-                            if segs[-1] not in cache_geneid[pmid][segs[4]].keys():
-                                cache_geneid[pmid][segs[4]][marjor_species]=1
-                            else:
-                                cache_geneid[pmid][segs[4]][marjor_species]+=1
-
-    
-    #no species
-    fin_result=io.StringIO(fin_pubtator0.getvalue())
-    all_in=fin_result.read().strip().split('\n\n')
-    fin_result.close()
-    #print('no species:',len(all_in))
-    for doc in all_in:
-        lines=doc.split('\n')
-        pmid=lines[0].split('|t|')[0]
-
-        for i in range(2,len(lines)):
-            segs=lines[i].split('\t')
-            if (pmid in cache_geneid.keys()) and (segs[4] in cache_geneid[pmid].keys()):#same gene in doc
-                marjor_species = max(zip(cache_geneid[pmid][segs[4]].values(), cache_geneid[pmid][segs[4]].keys()))
-                if pmid not in final_sa_results.keys():
-                    final_sa_results[pmid]={segs[1]:marjor_species[1]}
-                else:
-                    final_sa_results[pmid][segs[1]]=marjor_species[1]
-            else: #marjor species in doc
-                if (pmid in species_count.keys()) and len(species_count[pmid])>0:#marjor species in doc
-                    marjor_species = max(zip(species_count[pmid].values(), species_count[pmid].keys()))
-
-                    if pmid not in final_sa_results.keys():
-                        final_sa_results[pmid]={segs[1]:'Focus:'+marjor_species[1][1:]}
-                    else:
-                        final_sa_results[pmid][segs[1]]='Focus:'+marjor_species[1][1:]
-                else:#no any species in doc,assign human
-                    if pmid not in final_sa_results.keys():
-                        final_sa_results[pmid]={segs[1]:'Focus:9606'}
-                    else:
-                        final_sa_results[pmid][segs[1]]='Focus:9606'
-
-    
-    
-    # print(final_sa_results)
-    fin = open(infolder+"/"+infile, 'r',encoding='utf-8')
-    fout_xml=open(outpath+"/"+infile,'w', encoding='utf8')
-    collection = bioc.load(fin)
-    for document in collection.documents:
-        doc_pmid=document.id
-        # print(final_sa_results[doc_pmid])
-        # print(doc_pmid)
-        for passage in document.passages:
-            for temp_annotation in passage.annotations:
-                if 'Identifier' not in temp_annotation.infons.keys():
-                    if temp_annotation.id in final_sa_results[doc_pmid].keys():
-                        if final_sa_results[doc_pmid][temp_annotation.id][6:] in virus_set:
-                            temp_annotation.infons['Identifier']=final_sa_results[doc_pmid][temp_annotation.id]+',9606'
-                            # print('!!! virus:', doc_pmid)
-                        else:
-                            temp_annotation.infons['Identifier']=final_sa_results[doc_pmid][temp_annotation.id]
-                    else: #same text bug
-                        if (doc_pmid in cache_geneid.keys()) and (temp_annotation.text in cache_geneid[doc_pmid].keys()):#same gene in doc
-                            marjor_species = max(zip(cache_geneid[doc_pmid][temp_annotation.text].values(), cache_geneid[doc_pmid][temp_annotation.text].keys()))
-                            temp_annotation.infons['Identifier']=marjor_species[1]
-                        else: 
-                            
-                            temp_annotation.infons['Identifier']='Focus:9606'
-    bioc.dump(collection, fout_xml, pretty_print=True)
-    fin.close()
-    fout_xml.close()
-    
-
-#SA for PubTator format
-def SA_PubTator(infolder,infile,outpath,nn_model,virus_set,prefix_dict):
-    
-
-    # pmid|t|text1
-    #pmid|a|text2
-    #pmid entity_id sid eid entity_txt entity_type  (gene is blank)
-    fin = open(infolder+"/"+infile, 'r',encoding='utf-8')
-    # fout_pubtator=open(outpath+'tmp/input_xml.pubtator','w', encoding='utf-8')
-    fin_pubtator2=io.StringIO() #two or more species
-    all_in_ori=fin.read().strip().split('\n\n')
-    fin.close()
-    species_gene_count={} #{pmid:{'spec':_species_num;'gene':_gene_num}}
-    gene_set=['Gene','FamilyName']
-    ML_results={} #{'pmid':{'sid-eid':species_id}}
-    
-    prefix_speid_allset=set(prefix_dict.keys())
-    
-    for document in all_in_ori:
-        lines=document.split('\n')
-        doc_pmid=lines[0].split('|t|')[0]
-        doc_title=lines[0].split('|t|')[1]
-        doc_abstract=lines[1].split('|a|')[1]
-        doc_annotation=[]
-        _species_num=set() #(*speciesid)
-        _gene_num=0
-        _ML_gene_num=0
-        _entity_num=0
-        _prefix_str2id_dict={} #{prestr:id}
-        for i in range(2,len(lines)):
-            segs=lines[i].split('\t')
-            if segs[4] in gene_set:
-                _gene_num+=1
-            if len(segs)>=6: #species
-                doc_annotation.append(segs[0]+'\t'+str(_entity_num)+'\t'+'\t'.join(segs[1:]))
-                species_ID=segs[-1]
-                if species_ID.find('*')>=0:
-                    _species_num.add(species_ID)
-                    if species_ID[1:] in prefix_speid_allset:
-                        for ele in prefix_dict[species_ID[1:]]:
-                            _prefix_str2id_dict[ele]=species_ID[1:]
-            else: #gene
-                if segs[3][0:2] in _prefix_str2id_dict:#prefix rule
-                    if _prefix_str2id_dict[segs[3][0:2]] in virus_set:
-                        doc_annotation.append(segs[0]+'\t'+str(_entity_num)+'\t'+'\t'.join(segs[1:])+'\tFocus:'+_prefix_str2id_dict[segs[3][0:2]]+',9606')
-                        if doc_pmid not in ML_results.keys():
-                            ML_results[doc_pmid]={segs[1]+'-'+segs[2]:_prefix_str2id_dict[segs[3][0:2]]+',9606'}
-                        else:
-                            ML_results[doc_pmid][segs[1]+'-'+segs[2]]=_prefix_str2id_dict[segs[3][0:2]]+',9606'
-
-                        # print('!!! prefixr and virus:', doc_pmid)
-                    else:
-                        doc_annotation.append(segs[0]+'\t'+str(_entity_num)+'\t'+'\t'.join(segs[1:])+'\tFocus:'+_prefix_str2id_dict[segs[3][0:2]])
-                        if doc_pmid not in ML_results.keys():
-                            ML_results[doc_pmid]={segs[1]+'-'+segs[2]:_prefix_str2id_dict[segs[3][0:2]]}
-                        else:
-                            ML_results[doc_pmid][segs[1]+'-'+segs[2]]=_prefix_str2id_dict[segs[3][0:2]]
-                    # print('prefix rule!!',_prefix_str2id_dict)
-                    # print(doc_pmid)
-                else:
-                    doc_annotation.append(segs[0]+'\t'+str(_entity_num)+'\t'+'\t'.join(segs[1:]))
-                    if segs[4] in gene_set:
-                        _ML_gene_num+=1
-            _entity_num+=1   
-                
-        if len(_species_num)>=2 and _ML_gene_num>0:
-            fin_pubtator2.write(doc_pmid+'|t|'+doc_title+'\n')
-            fin_pubtator2.write(doc_pmid+'|a|'+doc_abstract+'\n')
-            for ele in doc_annotation:
-                fin_pubtator2.write(ele+'\n')
-            fin_pubtator2.write('\n')
-            
-        species_gene_count[doc_pmid]={'spec':_species_num,'gene':_gene_num}
-            
-    if fin_pubtator2.getvalue()!='':
-        #pubtator format  ML tagging
-        #print(fin_pubtator2.getvalue())
-        ml_out= sa_tag.ml_tag_main(fin_pubtator2,nlp_token, nn_model)   
-        #print(ml_out.getvalue())
-        fin_result=io.StringIO(ml_out.getvalue())
-        all_in=fin_result.read().strip().split('\n\n')
-        #print('+2 species:',len(all_in))
-        fin_result.close()
-        for doc in all_in:
-            lines=doc.split('\n')
-            pmid=lines[0].split('|t|')[0]
-
-            for i in range(2,len(lines)):
-                segs=lines[i].split('\t')
-                if pmid not in ML_results.keys():
-                    ML_results[pmid]={segs[2]+'-'+segs[3]:segs[-1]}
-                else:
-                    ML_results[pmid][segs[2]+'-'+segs[3]]=segs[-1]
-    
-    #output
-    fout_pubtator=open(outpath+"/"+infile,'w', encoding='utf8')
-    for doc in all_in_ori:
-        lines=doc.split('\n')
-        pmid=lines[0].split('|t|')[0]
-        fout_pubtator.write(lines[0]+'\n'+lines[1]+'\n')
-        if len(species_gene_count[pmid]['spec'])>1 and species_gene_count[pmid]['gene']>0: # ML
-            for i in range(2,len(lines)):
-                segs=lines[i].split('\t')
-                if len(segs)>=6: #species
-                    fout_pubtator.write(lines[i]+'\n')
-                else:#gene
-                    if ML_results[pmid][segs[1]+'-'+segs[2]] in virus_set:
-                        fout_pubtator.write(lines[i]+'\tFocus:'+ML_results[pmid][segs[1]+'-'+segs[2]]+',9606'+'\n')
-                        # print('!!! virus:', pmid)
-                    else:
-                        fout_pubtator.write(lines[i]+'\tFocus:'+ML_results[pmid][segs[1]+'-'+segs[2]]+'\n')
-            fout_pubtator.write('\n')
-                    
-        elif len(species_gene_count[pmid]['spec'])==1 and species_gene_count[pmid]['gene']>0: #only one species
-            for i in range(2,len(lines)):
-                segs=lines[i].split('\t')
-                if len(segs)>=6: #species
-                    fout_pubtator.write(lines[i]+'\n')
-                else:#gene
-                    major_species,=species_gene_count[pmid]['spec']
-                    if major_species[1:] in virus_set:
-                        fout_pubtator.write(lines[i]+'\tFocus:'+major_species[1:]+',9606'+'\n')
-                        # print('!!! virus:', pmid)
-                    fout_pubtator.write(lines[i]+'\tFocus:'+major_species[1:]+'\n')
-            fout_pubtator.write('\n')
-            
-        elif len(species_gene_count[pmid]['spec'])==0 and species_gene_count[pmid]['gene']>0:#no species
-            for i in range(2,len(lines)):
-                segs=lines[i].split('\t')
-                if len(segs)>=6: #species
-                    fout_pubtator.write(lines[i]+'\n')
-                else:#gene
-                    fout_pubtator.write(lines[i]+'\tFocus:9606'+'\n')
-            fout_pubtator.write('\n')
-            
-        else:
-            for i in range(2,len(lines)):
-                fout_pubtator.write(lines[i]+'\n')
-            fout_pubtator.write('\n')   
-    fout_pubtator.close()
-
-    
-#SA main    
-def speciesAss(infolder,outpath, modelfile):
-
-    if modelfile.lower().find('bioformer')>=0:
-        model_type='bioformer'
-    else:
-        model_type='pubmedbert'
-    
-    print('loading SA models........')     
-    if model_type=='bioformer':
-        
-        vocabfiles={'labelfile':'./vocab/SpeAss_IO_label.vocab',
-                    'checkpoint_path':'./gnorm_trained_models/bioformer-cased-v1.0/',
-                    'lowercase':False,
-                    }
-    else:
-        vocabfiles={'labelfile':'./vocab/SpeAss_IO_label.vocab',
-                    'checkpoint_path':'./gnorm_trained_models/BiomedNLP-PubMedBERT-base-uncased-abstract/',
-                    'lowercase':True,
-                    }
-        
-    nn_model=model_sa.HUGFACE_NER(vocabfiles)
-    nn_model.build_encoder()
-    nn_model.build_softmax_decoder()
-    nn_model.load_model(modelfile)
-    
-    dict_filename={'prefix':'./Dictionary/SPPrefix.txt',
-                   'virus':'./Dictionary/SP_Virus2HumanList.txt'}
-    fin=open(dict_filename['virus'],'r',encoding='utf-8')
-    virus_set=set(fin.read().strip().split('\n'))
-    fin.close()
-    
-    prefix_dict={}#{id:[prefix1,prefix2]}
-    fin=open(dict_filename['prefix'],'r',encoding='utf-8')
-    for line in fin:
-        seg= line.strip().split('\t')
-        if seg[0] not in prefix_dict.keys():
-            prefix_dict[seg[0]]=seg[1].split('|')
-        else:
-            prefix_dict[seg[0]].extend(seg[1].split('|'))
-    fin.close()
-    
-
-    
-    print("begin species assignment........")
-    start_time=time.time() 
-    
-    for infile in os.listdir(infolder): 
-        if os.path.isfile(outpath+"/"+infile):
-            print(infile+' has exsited.')
-        else:
-            print('Processing:',infile)
-            fin=open(infolder+"/"+infile, 'r',encoding='utf-8')
-            file_format=""
-            for line in fin:
-                pattern_bioc = re.compile('.*<collection>.*')
-                pattern_pubtator = re.compile('^([^\|]+)\|[^\|]+\|(.*)')
-                if pattern_bioc.search(line):
-                    file_format="BioC"
-                    break
-                elif pattern_pubtator.search(line):
-                    file_format="PubTator"
-                    break
-            fin.close()
-            if(file_format == "PubTator"):
-                SA_PubTator(infolder,infile,outpath,nn_model,virus_set,prefix_dict)
-            elif(file_format == "BioC"):
-                SA_BioC(infolder,infile,outpath,nn_model,virus_set,prefix_dict)
-        
-
-    print('species assignment done:',time.time()-start_time)
-
-if __name__=='__main__':
-    
-    parser = argparse.ArgumentParser(description='run GeneNER and species assignment, python GeneNER_SpeAss_run.py -i input -n NERmodel -s SAmodel -r neroutput -a saoutput')
-    parser.add_argument('--infolder', '-i', help="input folder",default='./example/input/')
-    parser.add_argument('--NERmodel', '-n', help="trained deep learning NER model file",default='')
-    parser.add_argument('--SAmodel', '-s', help="trained deep learning species assignment model file",default='')
-    parser.add_argument('--NERoutpath', '-r', help="output folder to save the NER tagged results",default='./example/ner_output/')
-    parser.add_argument('--SAoutpath', '-a', help="output folder to save the SA tagged results",default='./example/sa_output/')
-    parser.add_argument('--NUM_THREADS', '-t', help="Number of threads",default='3')
-    args = parser.parse_args()
-       
-    
-    if args.NUM_THREADS.isdigit() == False:
-        args.NUM_THREADS='3'
-        
-    tf.config.threading.set_inter_op_parallelism_threads(int(args.NUM_THREADS))
-    tf.config.threading.set_intra_op_parallelism_threads(int(args.NUM_THREADS))
-
-    if args.NERmodel!='' and args.SAmodel!='':
-        
-        #pipleline
-        print('==============\n| GeneNER and SpeAss |\n==============')
-        
-        #creat output folder
-        
-        if args.infolder[-1]!='/':
-            args.infolder+='/'
-        if not os.path.exists(args.infolder):
-            os.makedirs(args.infolder)
-            
-        if args.NERoutpath[-1]!='/':
-            args.NERoutpath+='/'
-        if not os.path.exists(args.NERoutpath):
-            os.makedirs(args.NERoutpath)
-            
-        if args.SAoutpath[-1]!='/':
-            args.SAoutpath+='/'
-        if not os.path.exists(args.SAoutpath):
-            os.makedirs(args.SAoutpath)
-        
-        #1. gene NER, the results are saved in outpath/ner_tmp/
-        geneNER(args.infolder,args.NERoutpath, args.NERmodel)
-        
-        
-        #2. species assignment, the results are saved in outpath/sa_tmp/
-        speciesAss(args.NERoutpath,args.SAoutpath, args.SAmodel)
-        
-    elif args.NERmodel!='' and args.SAmodel=='':
-        if args.infolder[-1]!='/':
-            args.infolder+='/'
-        if not os.path.exists(args.infolder):
-            os.makedirs(args.infolder)
-            
-        # only geneNER
-        if args.NERoutpath[-1]!='/':
-            args.NERoutpath+='/'
-        if not os.path.exists(args.NERoutpath):
-            os.makedirs(args.NERoutpath)
-            
-        print('==============\n| GeneNER |\n==============')
-        geneNER(args.infolder,args.NERoutpath,args.NERmodel)
-        
-    elif args.NERmodel=='' and args.SAmodel!='':
-        # only speass
-        if args.SAoutpath[-1]!='/':
-            args.SAoutpath+='/'
-        if not os.path.exists(args.SAoutpath):
-            os.makedirs(args.SAoutpath)
-            
-        print('==============\n| SpeAss |\n==============')
-        speciesAss(args.infolder,args.SAoutpath,args.SAmodel)
-    else:
-        print('Please provide models!')
-        
-     
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Jun  8 09:26:57 2022
+
+@author: luol2
+
+Pipeline: first gene NER, then species assignment
+input: species NER bioc xml file
+output: gene ner and species assignment results bioc xml file
+"""
+import argparse
+import os
+import io
+import time
+import sys
+import re
+import shutil
+from src_python.GeneNER import model_ner,ner_tag
+from src_python.SpeAss import model_sa,sa_tag
+
+import tensorflow as tf
+
+import bioc
+import stanza
+nlp_token = stanza.Pipeline(model_dir='gnorm_trained_models/stanza', lang='en', processors={'tokenize': 'spacy'},package='None', download_method=None) #package='craft' ;./gnorm_trained_models/stanza
+
+def NER_BioC(infolder,infile,outpath,nn_model):
+         
+    with open(infolder+"/"+infile, 'r',encoding='utf-8') as fin:
+        with open(outpath+"/"+infile,'w', encoding='utf8') as fout:
+            collection = bioc.load(fin)
+            
+            Total_n=len(collection.documents)
+            print('Total number of sub-documents:', Total_n)
+            pmid_n=0
+            for document in collection.documents:
+                print("Processing:{0}%".format(round(pmid_n * 100 / Total_n)), end="\r")
+                pmid_n+=1
+                # print(document.id)
+                mention_num_new=0
+                for passage in document.passages:
+                    if passage.text!='' and (not passage.text.isspace()) and passage.infons['type']!='ref': # have text and is not ref
+                        passage_offset=passage.offset
+                        tag_result=ner_tag.ML_Tag(passage.text,nn_model,nlp_token)
+                        mention_num=0
+                        for ele in tag_result:
+                            bioc_note = bioc.BioCAnnotation()
+                            bioc_note.id = str(mention_num)
+                            mention_num+=1
+                            bioc_note.infons['type'] = ele[2]
+                            start = int(ele[0])
+                            last = int(ele[1])
+                            loc = bioc.BioCLocation(offset=str(passage_offset+start), length= str(last-start))
+                            bioc_note.locations.append(loc)
+                            bioc_note.text = passage.text[start:last]
+                            passage.annotations.append(bioc_note)
+                    #update id
+                    for temp_annotation in passage.annotations:
+                        temp_annotation.id=str(mention_num_new)
+                        mention_num_new+=1
+            bioc.dump(collection, fout, pretty_print=True)
+            
+def NER_PubTator(infolder,infile,outpath,nn_model):
+    with open(infolder+"/"+infile, 'r',encoding='utf-8') as fin:
+        with open(outpath+"/"+infile,'w', encoding='utf-8') as fout:
+            title=''
+            abstract=''
+            all_text=fin.read().strip().split('\n\n')
+            Total_n=len(all_text)
+            print('Total number of sub-documents:', Total_n)
+            pmid_n=0
+            for doc in all_text:
+                print("Processing:{0}%".format(round(pmid_n * 100 / Total_n)), end="\r")
+                pmid_n+=1
+                lines = doc.split('\n')
+                seg=lines[0].split('|t|')
+                pmid=seg[0]
+                title=""
+                if len(seg)>1:
+                    title=seg[1]
+                abstract=""
+                if len(lines)>1:
+                    seg=lines[1].split('|a|')
+                    abstract=seg[1]
+                    if len(seg)>1:
+                        abstract=seg[1]
+               
+                intext=title+' '+abstract
+                tag_result=ner_tag.ML_Tag(intext,nn_model,nlp_token)
+                fout.write(doc+'\n')
+                for ele in tag_result:
+                    ent_start = ele[0]
+                    ent_last = ele[1]
+                    ent_mention = intext[int(ele[0]):int(ele[1])]
+                    ent_type=ele[2]
+                    fout.write(pmid+"\t"+ent_start+"\t"+ent_last+"\t"+ent_mention+"\t"+ent_type+"\n")
+                fout.write('\n')
+                title=''
+                abstract=''
+                
+def geneNER(infolder, outpath, modelfile):
+    
+    print('loading NER models........')    
+        
+    if modelfile.lower().find('bioformer')>=0:
+        vocabfiles={'labelfile':'./vocab/GeneNER_label.vocab',
+                        'checkpoint_path':'./gnorm_trained_models/bioformer-cased-v1.0/', #bioformer-cased-v1.0
+                        'lowercase':False,
+                        } 
+    else:
+        vocabfiles={'labelfile':'./vocab/GeneNER_label.vocab',
+                    'checkpoint_path':'./gnorm_trained_models/BiomedNLP-PubMedBERT-base-uncased-abstract/',
+                    'lowercase':True,
+                    } 
+        
+    nn_model=model_ner.HUGFACE_NER(vocabfiles)
+    nn_model.build_encoder()
+    nn_model.build_softmax_decoder()
+    nn_model.load_model(modelfile)
+        
+    #tagging text
+    print("begin GeneNER tagging........")
+    start_time=time.time()
+    
+    for infile in os.listdir(infolder):
+        if os.path.isfile(outpath+"/"+infile):
+            print(infile+' has exsited.')
+        else:
+            print('processing:',infile)             
+            fin = open(infolder+"/"+infile, 'r',encoding='utf-8')
+            input_format=""
+            for line in fin:
+                pattern_bioc = re.compile('.*<collection>.*')
+                pattern_pubtator = re.compile('^([^\|]+)\|[^\|]+\|(.*)')
+                if pattern_bioc.search(line):
+                    input_format="BioC"
+                    break
+                elif pattern_pubtator.search(line):
+                    input_format="PubTator"
+                    break
+            fin.close()
+            if(input_format == "PubTator"):
+                NER_PubTator(infolder,infile,outpath,nn_model)
+            elif(input_format == "BioC"):
+                NER_BioC(infolder,infile,outpath,nn_model)    
+    
+    print('tag done:',time.time()-start_time)
+    
+
+#SA for bioc format
+def SA_BioC(infolder,infile,outpath,nn_model,virus_set,prefix_dict):
+    
+    #BioC xml to pubtator
+    # pmid|t|text1
+    #pmid|a|text2
+    #pmid sid eid entity_txt entity_type entity_id (gene is blank)
+    fin = open(infolder+"/"+infile, 'r',encoding='utf-8')
+    # fout_pubtator=open(outpath+'tmp/input_xml.pubtator','w', encoding='utf-8')
+    fin_pubtator0=io.StringIO() #none *species
+    fin_pubtator1=io.StringIO() #one *species
+    fin_pubtator2=io.StringIO() #two or more species
+    collection = bioc.load(fin)
+    fin.close()
+    ori_ann_index={}  #{'pmid':{'ent.id':'ent_s-ent_e'}}
+    species_count={} #{pmid:{speid:num}}
+    gene_set=['Gene','FamilyName']
+    final_sa_results={} #{'pmid':{'entity_id':species_id}}
+    for document in collection.documents:
+        doc_pmid=document.id
+        doc_title=''
+        doc_abstract=''
+        doc_annotation=[]
+        _ann_index={}
+        _species_num={} #{*speciesid:num}
+        _gene_num=0
+        _passage_num=0              
+        if len(document.passages)<=2: #abstract xml or PMC only have title
+            for passage in document.passages:
+                passage_offset=passage.offset
+                _passage_num+=1              
+                #print(passage_offset,type(passage_offset))
+                #if passage.infons['type']=='title' or passage.infons['type']=='front':
+                if _passage_num==1:
+                    doc_title=passage.text
+                    for temp_annotation in passage.annotations:
+                        if temp_annotation.infons['type'] in gene_set:
+                            _gene_num+=1
+                        ent_start=temp_annotation.locations[0].offset-passage_offset
+                        ent_end=ent_start+temp_annotation.locations[0].length
+                        #print(ent_start,ent_end)
+                        _ann_index[temp_annotation.id]=str(ent_start)+'-'+str(ent_end)
+                        # print(temp_annotation.infons)
+                        if 'Identifier' in temp_annotation.infons.keys():
+                            # print(temp_annotation.infons.keys['Identifier'])
+                            species_ID=temp_annotation.infons['Identifier']
+                            if species_ID.find('*')>=0:
+                                if species_ID not in _species_num.keys():
+                                    _species_num[species_ID]=1
+                                else:
+                                    _species_num[species_ID]+=1
+                            doc_annotation.append(doc_pmid+'\t'+temp_annotation.id+'\t'+str(ent_start)+'\t'+str(ent_end)+'\t'+temp_annotation.text+'\t'+temp_annotation.infons['type']+'\t'+species_ID)
+                        else:
+                            doc_annotation.append(doc_pmid+'\t'+temp_annotation.id+'\t'+str(ent_start)+'\t'+str(ent_end)+'\t'+temp_annotation.text+'\t'+temp_annotation.infons['type'])
+                        
+                #elif passage.infons['type']=='abstract' or passage.infons['type']=='paragraph':
+                else:
+                    doc_abstract=passage.text
+                    for temp_annotation in passage.annotations:
+                        if temp_annotation.infons['type'] in gene_set:
+                            _gene_num+=1
+                        ent_start=len(doc_title)+1+temp_annotation.locations[0].offset-passage_offset
+                        ent_end=ent_start+temp_annotation.locations[0].length
+                        #print(ent_start,ent_end)
+                        _ann_index[temp_annotation.id]=str(ent_start)+'-'+str(ent_end)
+                        if 'Identifier' in temp_annotation.infons.keys():
+                            # print(temp_annotation.infons.keys['Identifier'])
+                            species_ID=temp_annotation.infons['Identifier']
+                            if species_ID.find('*')>=0:
+                                if species_ID not in _species_num.keys():
+                                    _species_num[species_ID]=1
+                                else:
+                                    _species_num[species_ID]+=1
+                            doc_annotation.append(doc_pmid+'\t'+temp_annotation.id+'\t'+str(ent_start)+'\t'+str(ent_end)+'\t'+temp_annotation.text+'\t'+temp_annotation.infons['type']+'\t'+species_ID)
+                        else:
+                            doc_annotation.append(doc_pmid+'\t'+temp_annotation.id+'\t'+str(ent_start)+'\t'+str(ent_end)+'\t'+temp_annotation.text+'\t'+temp_annotation.infons['type'])
+            
+            if len(_species_num)>=2 and _gene_num>0:
+                fin_pubtator2.write(doc_pmid+'|t|'+doc_title+'\n')
+                fin_pubtator2.write(doc_pmid+'|a|'+doc_abstract+'\n')
+                for ele in doc_annotation:
+                    fin_pubtator2.write(ele+'\n')
+                fin_pubtator2.write('\n')
+            elif len(_species_num)==1 and _gene_num>0: #可以直接给结果
+                fin_pubtator1.write(doc_pmid+'|t|'+doc_title+'\n')
+                fin_pubtator1.write(doc_pmid+'|a|'+doc_abstract+'\n')
+                major_speicesid,=_species_num
+                fin_pubtator1.write(major_speicesid[1:]+'\n')
+                for ele in doc_annotation:
+                    fin_pubtator1.write(ele+'\n')
+                fin_pubtator1.write('\n')
+            elif len(_species_num)==0 and _gene_num>0:
+                fin_pubtator0.write(doc_pmid+'|t|'+doc_title+'\n')
+                fin_pubtator0.write(doc_pmid+'|a|'+doc_abstract+'\n')
+                for ele in doc_annotation:
+                    fin_pubtator0.write(ele+'\n')
+                fin_pubtator0.write('\n')        
+        
+        else: # full text xml
+            for passage in document.passages:
+                passage_annotation=[]
+                _species_num_passage={}
+                _gene_num_passage=0
+                passage_offset=passage.offset
+                #print(passage_offset,type(passage_offset))
+                if passage.text!='' and (not passage.text.isspace()) and passage.infons['type']!='ref':
+                    doc_title=passage.text
+                    for temp_annotation in passage.annotations:
+                        if temp_annotation.infons['type'] in gene_set:
+                            _gene_num_passage+=1
+                        ent_start=temp_annotation.locations[0].offset-passage_offset
+                        ent_end=ent_start+temp_annotation.locations[0].length
+                        #print(ent_start,ent_end)
+                        _ann_index[temp_annotation.id]=str(ent_start)+'-'+str(ent_end)
+                        # print(temp_annotation.infons)
+                        if 'Identifier' in temp_annotation.infons.keys():
+                            # print(temp_annotation.infons.keys['Identifier'])
+                            species_ID=temp_annotation.infons['Identifier']
+                            if species_ID.find('*')>=0:
+                                if species_ID not in _species_num.keys():
+                                    _species_num[species_ID]=1
+                                else:
+                                    _species_num[species_ID]+=1
+                                if species_ID not in _species_num_passage.keys():
+                                    _species_num_passage[species_ID]=1
+                                else:
+                                    _species_num_passage[species_ID]+=1
+                            passage_annotation.append(doc_pmid+'\t'+temp_annotation.id+'\t'+str(ent_start)+'\t'+str(ent_end)+'\t'+temp_annotation.text+'\t'+temp_annotation.infons['type']+'\t'+species_ID)
+                        else:
+                            passage_annotation.append(doc_pmid+'\t'+temp_annotation.id+'\t'+str(ent_start)+'\t'+str(ent_end)+'\t'+temp_annotation.text+'\t'+temp_annotation.infons['type'])
+                        
+            
+                if len(_species_num_passage)>=2 and _gene_num_passage>0:
+                    fin_pubtator2.write(doc_pmid+'|t|'+doc_title+'\n')
+                    fin_pubtator2.write(doc_pmid+'|a|'+doc_abstract+'\n')
+                    for ele in passage_annotation:
+                        fin_pubtator2.write(ele+'\n')
+                    fin_pubtator2.write('\n')
+                elif len(_species_num_passage)==1 and _gene_num_passage>0: #可以直��给结果
+                    fin_pubtator1.write(doc_pmid+'|t|'+doc_title+'\n')
+                    fin_pubtator1.write(doc_pmid+'|a|'+doc_abstract+'\n')
+                    major_speicesid,=_species_num_passage
+                    fin_pubtator1.write(major_speicesid[1:]+'\n')
+                    for ele in passage_annotation:
+                        fin_pubtator1.write(ele+'\n')
+                    fin_pubtator1.write('\n')
+                elif len(_species_num_passage)==0 and _gene_num_passage>0:
+                    fin_pubtator0.write(doc_pmid+'|t|'+doc_title+'\n')
+                    fin_pubtator0.write(doc_pmid+'|a|'+doc_abstract+'\n')
+                    for ele in passage_annotation:
+                        fin_pubtator0.write(ele+'\n')
+                    fin_pubtator0.write('\n')        
+        # print(ori_ann_index)
+        
+        ori_ann_index[doc_pmid]=_ann_index
+        species_count[doc_pmid]=_species_num
+    
+            
+    cache_geneid={} #{pmid:{gene1:{id1:num,id2:num}}}
+    
+    if fin_pubtator2.getvalue()!='':
+        #pubtator format  ML tagging
+        # print(fin_pubtator2.getvalue())
+        ml_out= sa_tag.ml_tag_main(fin_pubtator2,nlp_token, nn_model)   
+        #print(ml_out.getvalue())
+        fin_result=io.StringIO(ml_out.getvalue())
+        all_in=fin_result.read().strip().split('\n\n')
+        #print('+2 species:',len(all_in))
+        fin_result.close()
+        
+        prefix_speid_allset=set(prefix_dict.keys())
+    
+        for doc in all_in:
+            lines=doc.split('\n')
+            pmid=lines[0].split('|t|')[0]
+            _prefix_str2id_dict={}
+            doc_species=list(species_count[pmid].keys())
+            for _spe_ele in doc_species:
+                if _spe_ele[1:] in prefix_speid_allset:
+                    for ele in prefix_dict[_spe_ele[1:]]:
+                        _prefix_str2id_dict[ele]=_spe_ele[1:]
+            
+            for i in range(2,len(lines)):
+                segs=lines[i].split('\t')
+                if pmid not in final_sa_results.keys():
+                    final_sa_results[pmid]={segs[1]:'Focus:'+segs[-1]}
+                else:
+                    final_sa_results[pmid][segs[1]]='Focus:'+segs[-1]
+                    
+                if segs[5] in gene_set:
+                    if segs[4][0:2] in _prefix_str2id_dict: #prefix rule
+                        #print('prefix rule:', pmid)
+                        # print(_prefix_str2id_dict)
+                        if pmid not in final_sa_results.keys():
+                            final_sa_results[pmid]={segs[1]:'Focus:'+_prefix_str2id_dict[segs[4][0:2]]}
+                        else:
+                            final_sa_results[pmid][segs[1]]='Focus:'+_prefix_str2id_dict[segs[4][0:2]]
+                    if pmid not in cache_geneid.keys():
+                        cache_geneid[pmid]={segs[4]:{'Focus:'+segs[-1]:1}}
+                    else:
+                        if segs[4] not in cache_geneid[pmid].keys():
+                            cache_geneid[pmid][segs[4]]={'Focus:'+segs[-1]:1}
+                        else:
+                            if segs[-1] not in cache_geneid[pmid][segs[4]].keys():
+                                cache_geneid[pmid][segs[4]]['Focus:'+segs[-1]]=1
+                            else:
+                                cache_geneid[pmid][segs[4]]['Focus:'+segs[-1]]+=1
+        
+    #print(final_sa_results)
+    
+    #one species  
+    if fin_pubtator1.getvalue()!='':
+        fin_result=io.StringIO(fin_pubtator1.getvalue())
+        all_in=fin_result.read().strip().split('\n\n')
+        fin_result.close()
+        #print('1 species:',len(all_in))
+        for doc in all_in:
+            lines=doc.split('\n')
+            pmid=lines[0].split('|t|')[0]
+            major_speicesid=lines[2]
+            for i in range(3,len(lines)):
+                segs=lines[i].split('\t')
+                if len(segs)>=7:#species
+                    if pmid not in final_sa_results.keys():
+                        final_sa_results[pmid]={segs[1]:segs[-1]}
+                    else:
+                        final_sa_results[pmid][segs[1]]=segs[-1]
+                else:#gene
+                    marjor_species='Focus:'+major_speicesid
+                    if pmid not in final_sa_results.keys():
+                        final_sa_results[pmid]={segs[1]:marjor_species}
+                    else:
+                        final_sa_results[pmid][segs[1]]=marjor_species
+                    if pmid not in cache_geneid.keys():
+                        cache_geneid[pmid]={segs[4]:{marjor_species:1}}
+                    else:
+                        if segs[4] not in cache_geneid[pmid].keys():
+                            cache_geneid[pmid][segs[4]]={marjor_species:1}
+                        else:
+                            if segs[-1] not in cache_geneid[pmid][segs[4]].keys():
+                                cache_geneid[pmid][segs[4]][marjor_species]=1
+                            else:
+                                cache_geneid[pmid][segs[4]][marjor_species]+=1
+
+    
+    #no species
+    fin_result=io.StringIO(fin_pubtator0.getvalue())
+    all_in=fin_result.read().strip().split('\n\n')
+    fin_result.close()
+    #print('no species:',len(all_in))
+    for doc in all_in:
+        lines=doc.split('\n')
+        pmid=lines[0].split('|t|')[0]
+
+        for i in range(2,len(lines)):
+            segs=lines[i].split('\t')
+            if (pmid in cache_geneid.keys()) and (segs[4] in cache_geneid[pmid].keys()):#same gene in doc
+                marjor_species = max(zip(cache_geneid[pmid][segs[4]].values(), cache_geneid[pmid][segs[4]].keys()))
+                if pmid not in final_sa_results.keys():
+                    final_sa_results[pmid]={segs[1]:marjor_species[1]}
+                else:
+                    final_sa_results[pmid][segs[1]]=marjor_species[1]
+            else: #marjor species in doc
+                if (pmid in species_count.keys()) and len(species_count[pmid])>0:#marjor species in doc
+                    marjor_species = max(zip(species_count[pmid].values(), species_count[pmid].keys()))
+
+                    if pmid not in final_sa_results.keys():
+                        final_sa_results[pmid]={segs[1]:'Focus:'+marjor_species[1][1:]}
+                    else:
+                        final_sa_results[pmid][segs[1]]='Focus:'+marjor_species[1][1:]
+                else:#no any species in doc,assign human
+                    if pmid not in final_sa_results.keys():
+                        final_sa_results[pmid]={segs[1]:'Focus:9606'}
+                    else:
+                        final_sa_results[pmid][segs[1]]='Focus:9606'
+
+    
+    
+    # print(final_sa_results)
+    fin = open(infolder+"/"+infile, 'r',encoding='utf-8')
+    fout_xml=open(outpath+"/"+infile,'w', encoding='utf8')
+    collection = bioc.load(fin)
+    for document in collection.documents:
+        doc_pmid=document.id
+        # print(final_sa_results[doc_pmid])
+        # print(doc_pmid)
+        for passage in document.passages:
+            for temp_annotation in passage.annotations:
+                if 'Identifier' not in temp_annotation.infons.keys():
+                    if temp_annotation.id in final_sa_results[doc_pmid].keys():
+                        if final_sa_results[doc_pmid][temp_annotation.id][6:] in virus_set:
+                            temp_annotation.infons['Identifier']=final_sa_results[doc_pmid][temp_annotation.id]+',9606'
+                            # print('!!! virus:', doc_pmid)
+                        else:
+                            temp_annotation.infons['Identifier']=final_sa_results[doc_pmid][temp_annotation.id]
+                    else: #same text bug
+                        if (doc_pmid in cache_geneid.keys()) and (temp_annotation.text in cache_geneid[doc_pmid].keys()):#same gene in doc
+                            marjor_species = max(zip(cache_geneid[doc_pmid][temp_annotation.text].values(), cache_geneid[doc_pmid][temp_annotation.text].keys()))
+                            temp_annotation.infons['Identifier']=marjor_species[1]
+                        else: 
+                            
+                            temp_annotation.infons['Identifier']='Focus:9606'
+    bioc.dump(collection, fout_xml, pretty_print=True)
+    fin.close()
+    fout_xml.close()
+    
+
+#SA for PubTator format
+def SA_PubTator(infolder,infile,outpath,nn_model,virus_set,prefix_dict):
+    
+
+    # pmid|t|text1
+    #pmid|a|text2
+    #pmid entity_id sid eid entity_txt entity_type  (gene is blank)
+    fin = open(infolder+"/"+infile, 'r',encoding='utf-8')
+    # fout_pubtator=open(outpath+'tmp/input_xml.pubtator','w', encoding='utf-8')
+    fin_pubtator2=io.StringIO() #two or more species
+    all_in_ori=fin.read().strip().split('\n\n')
+    fin.close()
+    species_gene_count={} #{pmid:{'spec':_species_num;'gene':_gene_num}}
+    gene_set=['Gene','FamilyName']
+    ML_results={} #{'pmid':{'sid-eid':species_id}}
+    
+    prefix_speid_allset=set(prefix_dict.keys())
+    
+    for document in all_in_ori:
+        lines=document.split('\n')
+        doc_pmid=lines[0].split('|t|')[0]
+        doc_title=lines[0].split('|t|')[1]
+        doc_abstract=lines[1].split('|a|')[1]
+        doc_annotation=[]
+        _species_num=set() #(*speciesid)
+        _gene_num=0
+        _ML_gene_num=0
+        _entity_num=0
+        _prefix_str2id_dict={} #{prestr:id}
+        for i in range(2,len(lines)):
+            segs=lines[i].split('\t')
+            if segs[4] in gene_set:
+                _gene_num+=1
+            if len(segs)>=6: #species
+                doc_annotation.append(segs[0]+'\t'+str(_entity_num)+'\t'+'\t'.join(segs[1:]))
+                species_ID=segs[-1]
+                if species_ID.find('*')>=0:
+                    _species_num.add(species_ID)
+                    if species_ID[1:] in prefix_speid_allset:
+                        for ele in prefix_dict[species_ID[1:]]:
+                            _prefix_str2id_dict[ele]=species_ID[1:]
+            else: #gene
+                if segs[3][0:2] in _prefix_str2id_dict:#prefix rule
+                    if _prefix_str2id_dict[segs[3][0:2]] in virus_set:
+                        doc_annotation.append(segs[0]+'\t'+str(_entity_num)+'\t'+'\t'.join(segs[1:])+'\tFocus:'+_prefix_str2id_dict[segs[3][0:2]]+',9606')
+                        if doc_pmid not in ML_results.keys():
+                            ML_results[doc_pmid]={segs[1]+'-'+segs[2]:_prefix_str2id_dict[segs[3][0:2]]+',9606'}
+                        else:
+                            ML_results[doc_pmid][segs[1]+'-'+segs[2]]=_prefix_str2id_dict[segs[3][0:2]]+',9606'
+
+                        # print('!!! prefixr and virus:', doc_pmid)
+                    else:
+                        doc_annotation.append(segs[0]+'\t'+str(_entity_num)+'\t'+'\t'.join(segs[1:])+'\tFocus:'+_prefix_str2id_dict[segs[3][0:2]])
+                        if doc_pmid not in ML_results.keys():
+                            ML_results[doc_pmid]={segs[1]+'-'+segs[2]:_prefix_str2id_dict[segs[3][0:2]]}
+                        else:
+                            ML_results[doc_pmid][segs[1]+'-'+segs[2]]=_prefix_str2id_dict[segs[3][0:2]]
+                    # print('prefix rule!!',_prefix_str2id_dict)
+                    # print(doc_pmid)
+                else:
+                    doc_annotation.append(segs[0]+'\t'+str(_entity_num)+'\t'+'\t'.join(segs[1:]))
+                    if segs[4] in gene_set:
+                        _ML_gene_num+=1
+            _entity_num+=1   
+                
+        if len(_species_num)>=2 and _ML_gene_num>0:
+            fin_pubtator2.write(doc_pmid+'|t|'+doc_title+'\n')
+            fin_pubtator2.write(doc_pmid+'|a|'+doc_abstract+'\n')
+            for ele in doc_annotation:
+                fin_pubtator2.write(ele+'\n')
+            fin_pubtator2.write('\n')
+            
+        species_gene_count[doc_pmid]={'spec':_species_num,'gene':_gene_num}
+            
+    if fin_pubtator2.getvalue()!='':
+        #pubtator format  ML tagging
+        #print(fin_pubtator2.getvalue())
+        ml_out= sa_tag.ml_tag_main(fin_pubtator2,nlp_token, nn_model)   
+        #print(ml_out.getvalue())
+        fin_result=io.StringIO(ml_out.getvalue())
+        all_in=fin_result.read().strip().split('\n\n')
+        #print('+2 species:',len(all_in))
+        fin_result.close()
+        for doc in all_in:
+            lines=doc.split('\n')
+            pmid=lines[0].split('|t|')[0]
+
+            for i in range(2,len(lines)):
+                segs=lines[i].split('\t')
+                if pmid not in ML_results.keys():
+                    ML_results[pmid]={segs[2]+'-'+segs[3]:segs[-1]}
+                else:
+                    ML_results[pmid][segs[2]+'-'+segs[3]]=segs[-1]
+    
+    #output
+    fout_pubtator=open(outpath+"/"+infile,'w', encoding='utf8')
+    for doc in all_in_ori:
+        lines=doc.split('\n')
+        pmid=lines[0].split('|t|')[0]
+        fout_pubtator.write(lines[0]+'\n'+lines[1]+'\n')
+        if len(species_gene_count[pmid]['spec'])>1 and species_gene_count[pmid]['gene']>0: # ML
+            for i in range(2,len(lines)):
+                segs=lines[i].split('\t')
+                if len(segs)>=6: #species
+                    fout_pubtator.write(lines[i]+'\n')
+                else:#gene
+                    if ML_results[pmid][segs[1]+'-'+segs[2]] in virus_set:
+                        fout_pubtator.write(lines[i]+'\tFocus:'+ML_results[pmid][segs[1]+'-'+segs[2]]+',9606'+'\n')
+                        # print('!!! virus:', pmid)
+                    else:
+                        fout_pubtator.write(lines[i]+'\tFocus:'+ML_results[pmid][segs[1]+'-'+segs[2]]+'\n')
+            fout_pubtator.write('\n')
+                    
+        elif len(species_gene_count[pmid]['spec'])==1 and species_gene_count[pmid]['gene']>0: #only one species
+            for i in range(2,len(lines)):
+                segs=lines[i].split('\t')
+                if len(segs)>=6: #species
+                    fout_pubtator.write(lines[i]+'\n')
+                else:#gene
+                    major_species,=species_gene_count[pmid]['spec']
+                    if major_species[1:] in virus_set:
+                        fout_pubtator.write(lines[i]+'\tFocus:'+major_species[1:]+',9606'+'\n')
+                        # print('!!! virus:', pmid)
+                    fout_pubtator.write(lines[i]+'\tFocus:'+major_species[1:]+'\n')
+            fout_pubtator.write('\n')
+            
+        elif len(species_gene_count[pmid]['spec'])==0 and species_gene_count[pmid]['gene']>0:#no species
+            for i in range(2,len(lines)):
+                segs=lines[i].split('\t')
+                if len(segs)>=6: #species
+                    fout_pubtator.write(lines[i]+'\n')
+                else:#gene
+                    fout_pubtator.write(lines[i]+'\tFocus:9606'+'\n')
+            fout_pubtator.write('\n')
+            
+        else:
+            for i in range(2,len(lines)):
+                fout_pubtator.write(lines[i]+'\n')
+            fout_pubtator.write('\n')   
+    fout_pubtator.close()
+
+    
+#SA main    
+def speciesAss(infolder,outpath, modelfile):
+
+    if modelfile.lower().find('bioformer')>=0:
+        model_type='bioformer'
+    else:
+        model_type='pubmedbert'
+    
+    print('loading SA models........')     
+    if model_type=='bioformer':
+        
+        vocabfiles={'labelfile':'./vocab/SpeAss_IO_label.vocab',
+                    'checkpoint_path':'./gnorm_trained_models/bioformer-cased-v1.0/',
+                    'lowercase':False,
+                    }
+    else:
+        vocabfiles={'labelfile':'./vocab/SpeAss_IO_label.vocab',
+                    'checkpoint_path':'./gnorm_trained_models/BiomedNLP-PubMedBERT-base-uncased-abstract/',
+                    'lowercase':True,
+                    }
+        
+    nn_model=model_sa.HUGFACE_NER(vocabfiles)
+    nn_model.build_encoder()
+    nn_model.build_softmax_decoder()
+    nn_model.load_model(modelfile)
+    
+    dict_filename={'prefix':'./Dictionary/SPPrefix.txt',
+                   'virus':'./Dictionary/SP_Virus2HumanList.txt'}
+    fin=open(dict_filename['virus'],'r',encoding='utf-8')
+    virus_set=set(fin.read().strip().split('\n'))
+    fin.close()
+    
+    prefix_dict={}#{id:[prefix1,prefix2]}
+    fin=open(dict_filename['prefix'],'r',encoding='utf-8')
+    for line in fin:
+        seg= line.strip().split('\t')
+        if seg[0] not in prefix_dict.keys():
+            prefix_dict[seg[0]]=seg[1].split('|')
+        else:
+            prefix_dict[seg[0]].extend(seg[1].split('|'))
+    fin.close()
+    
+
+    
+    print("begin species assignment........")
+    start_time=time.time() 
+    
+    for infile in os.listdir(infolder): 
+        if os.path.isfile(outpath+"/"+infile):
+            print(infile+' has exsited.')
+        else:
+            print('Processing:',infile)
+            fin=open(infolder+"/"+infile, 'r',encoding='utf-8')
+            file_format=""
+            for line in fin:
+                pattern_bioc = re.compile('.*<collection>.*')
+                pattern_pubtator = re.compile('^([^\|]+)\|[^\|]+\|(.*)')
+                if pattern_bioc.search(line):
+                    file_format="BioC"
+                    break
+                elif pattern_pubtator.search(line):
+                    file_format="PubTator"
+                    break
+            fin.close()
+            if(file_format == "PubTator"):
+                SA_PubTator(infolder,infile,outpath,nn_model,virus_set,prefix_dict)
+            elif(file_format == "BioC"):
+                SA_BioC(infolder,infile,outpath,nn_model,virus_set,prefix_dict)
+        
+
+    print('species assignment done:',time.time()-start_time)
+
+if __name__=='__main__':
+    
+    parser = argparse.ArgumentParser(description='run GeneNER and species assignment, python GeneNER_SpeAss_run.py -i input -n NERmodel -s SAmodel -r neroutput -a saoutput')
+    parser.add_argument('--infolder', '-i', help="input folder",default='./example/input/')
+    parser.add_argument('--NERmodel', '-n', help="trained deep learning NER model file",default='')
+    parser.add_argument('--SAmodel', '-s', help="trained deep learning species assignment model file",default='')
+    parser.add_argument('--NERoutpath', '-r', help="output folder to save the NER tagged results",default='./example/ner_output/')
+    parser.add_argument('--SAoutpath', '-a', help="output folder to save the SA tagged results",default='./example/sa_output/')
+    parser.add_argument('--NUM_THREADS', '-t', help="Number of threads",default='3')
+    args = parser.parse_args()
+       
+    
+    if args.NUM_THREADS.isdigit() == False:
+        args.NUM_THREADS='3'
+        
+    tf.config.threading.set_inter_op_parallelism_threads(int(args.NUM_THREADS))
+    tf.config.threading.set_intra_op_parallelism_threads(int(args.NUM_THREADS))
+
+    if args.NERmodel!='' and args.SAmodel!='':
+        
+        #pipleline
+        print('==============\n| GeneNER and SpeAss |\n==============')
+        
+        #creat output folder
+        
+        if args.infolder[-1]!='/':
+            args.infolder+='/'
+        if not os.path.exists(args.infolder):
+            os.makedirs(args.infolder)
+            
+        if args.NERoutpath[-1]!='/':
+            args.NERoutpath+='/'
+        if not os.path.exists(args.NERoutpath):
+            os.makedirs(args.NERoutpath)
+            
+        if args.SAoutpath[-1]!='/':
+            args.SAoutpath+='/'
+        if not os.path.exists(args.SAoutpath):
+            os.makedirs(args.SAoutpath)
+        
+        #1. gene NER, the results are saved in outpath/ner_tmp/
+        geneNER(args.infolder,args.NERoutpath, args.NERmodel)
+        
+        
+        #2. species assignment, the results are saved in outpath/sa_tmp/
+        speciesAss(args.NERoutpath,args.SAoutpath, args.SAmodel)
+        
+    elif args.NERmodel!='' and args.SAmodel=='':
+        if args.infolder[-1]!='/':
+            args.infolder+='/'
+        if not os.path.exists(args.infolder):
+            os.makedirs(args.infolder)
+            
+        # only geneNER
+        if args.NERoutpath[-1]!='/':
+            args.NERoutpath+='/'
+        if not os.path.exists(args.NERoutpath):
+            os.makedirs(args.NERoutpath)
+            
+        print('==============\n| GeneNER |\n==============')
+        geneNER(args.infolder,args.NERoutpath,args.NERmodel)
+        
+    elif args.NERmodel=='' and args.SAmodel!='':
+        # only speass
+        if args.SAoutpath[-1]!='/':
+            args.SAoutpath+='/'
+        if not os.path.exists(args.SAoutpath):
+            os.makedirs(args.SAoutpath)
+            
+        print('==============\n| SpeAss |\n==============')
+        speciesAss(args.infolder,args.SAoutpath,args.SAmodel)
+    else:
+        print('Please provide models!')
+        
+     
     

Library/Ab3P.C

@@ -1,110 +1,110 @@
-#include "Ab3P.h"
-
-Ab3P::Ab3P ( void ) :
-  buffer(""),
-  wrdData( new WordData ) 
-{
-    
-  string sf_grp, sf_nchr, strat;
-  double value; 
-
-  char file_name[1000];
-  get_pathw( file_name, "Ab3P", "prec", "dat" );
-  ifstream fin(file_name);
-  if(!fin) {
-    cout << "Cannot open Ab3P_prec.dat\n";
-    exit(1);
-  }
-  //get precision of a given #-ch SF's strategy
-  while(fin>>sf_grp>>sf_nchr>>strat) {
-    fin>>value; //precision 
-    stratPrec.insert(pair<string, double>(sf_grp+sf_nchr+strat, value));
-    util.push_back_strat(sf_grp+sf_nchr, strat); //set strategy sequence
-  }
-}
-
-void Ab3P::get_abbrs( char * text, vector<AbbrOut> & abbrs ) {
-    abbrs.clear();
-
-    if( ! text[0] ) return; // skip empty line
-
-    ab.Proc(text); //extract potential SF & LF pairs
-    
-    for(int i=0; i<ab.numa; i++) {
-      AbbrOut result;
-      
-      try_pair( ab.abbs[i], ab.abbl[i], result );
-      
-      // preserve results
-      if ( result.prec > 0 ) {
-        abbrs.push_back( result );
-      }
-    }
-    ab.cleara();
-
-  }
-
-
-void Ab3P::try_pair( char * sf, char * lf, AbbrOut & result ) {
-  
-  //process i) lf (sf)
-  try_strats( sf, lf, false, result );
-  
-  //process ii) sf (lf)
-  ab.token(lf);
-  try_strats( ab.lst[ab.num-1], sf, true, result );
-}
-
-
-  /**
-     psf -- pointer short form
-     plf -- pointer long form
-  **/
-void Ab3P::try_strats ( char * psf, char * plf, bool swap,
-                        AbbrOut & result ) {
-      
-  string sfg; //SF group eg) Al1, Num2, Spec3
-  //false if sf is not ok, sfg will be assigned
-
-  if(!util.group_sf(psf,plf,sfg)) return;
-  if (swap) if(!util.exist_upperal(psf)) return;
-
-  char sf[1000], sfl[1000];
-
-  //strategy sequence for a given #-ch SF group
-  vector<string> strats = util.get_strats(sfg);
-  util.remove_nonAlnum(psf,sf); //sf will be w/o non-alnum
-
-  //go through strategies
-  for( int j=0; j<strats.size(); j++) { 
-    AbbrStra * strat =
-      util.strat_factory(strats[j]); //set a paticular strategy
-    strat->wData = wrdData; //set wordset, stopword
-    if(strat->strategy(sf,plf)) { //case sensitive
-      strat->str_tolower(sf,sfl);
-
-      if( strat->lf_ok(psf,strat->lf) ) {
-
-        map<string, double>::iterator p =
-          stratPrec.find(sfg+strats[j]);
-        if(p==stratPrec.end()) {
-          cout << "No precision assigned" << endl;
-          exit(1);
-        }
-
-        //add outputs 
-        if( p->second>result.prec ) {
-          result.sf = psf;
-          result.lf = strat->lf;
-          result.prec = p->second;
-          result.strat = strats[j];
-        }
-
-        delete strat;
-        return;
-      }
-    }
-    delete strat;
-  }
-
-}
+#include "Ab3P.h"
+
+Ab3P::Ab3P ( void ) :
+  buffer(""),
+  wrdData( new WordData ) 
+{
+    
+  string sf_grp, sf_nchr, strat;
+  double value; 
+
+  char file_name[1000];
+  get_pathw( file_name, "Ab3P", "prec", "dat" );
+  ifstream fin(file_name);
+  if(!fin) {
+    cout << "Cannot open Ab3P_prec.dat\n";
+    exit(1);
+  }
+  //get precision of a given #-ch SF's strategy
+  while(fin>>sf_grp>>sf_nchr>>strat) {
+    fin>>value; //precision 
+    stratPrec.insert(pair<string, double>(sf_grp+sf_nchr+strat, value));
+    util.push_back_strat(sf_grp+sf_nchr, strat); //set strategy sequence
+  }
+}
+
+void Ab3P::get_abbrs( char * text, vector<AbbrOut> & abbrs ) {
+    abbrs.clear();
+
+    if( ! text[0] ) return; // skip empty line
+
+    ab.Proc(text); //extract potential SF & LF pairs
+    
+    for(int i=0; i<ab.numa; i++) {
+      AbbrOut result;
+      
+      try_pair( ab.abbs[i], ab.abbl[i], result );
+      
+      // preserve results
+      if ( result.prec > 0 ) {
+        abbrs.push_back( result );
+      }
+    }
+    ab.cleara();
+
+  }
+
+
+void Ab3P::try_pair( char * sf, char * lf, AbbrOut & result ) {
+  
+  //process i) lf (sf)
+  try_strats( sf, lf, false, result );
+  
+  //process ii) sf (lf)
+  ab.token(lf);
+  try_strats( ab.lst[ab.num-1], sf, true, result );
+}
+
+
+  /**
+     psf -- pointer short form
+     plf -- pointer long form
+  **/
+void Ab3P::try_strats ( char * psf, char * plf, bool swap,
+                        AbbrOut & result ) {
+      
+  string sfg; //SF group eg) Al1, Num2, Spec3
+  //false if sf is not ok, sfg will be assigned
+
+  if(!util.group_sf(psf,plf,sfg)) return;
+  if (swap) if(!util.exist_upperal(psf)) return;
+
+  char sf[1000], sfl[1000];
+
+  //strategy sequence for a given #-ch SF group
+  vector<string> strats = util.get_strats(sfg);
+  util.remove_nonAlnum(psf,sf); //sf will be w/o non-alnum
+
+  //go through strategies
+  for( int j=0; j<strats.size(); j++) { 
+    AbbrStra * strat =
+      util.strat_factory(strats[j]); //set a paticular strategy
+    strat->wData = wrdData; //set wordset, stopword
+    if(strat->strategy(sf,plf)) { //case sensitive
+      strat->str_tolower(sf,sfl);
+
+      if( strat->lf_ok(psf,strat->lf) ) {
+
+        map<string, double>::iterator p =
+          stratPrec.find(sfg+strats[j]);
+        if(p==stratPrec.end()) {
+          cout << "No precision assigned" << endl;
+          exit(1);
+        }
+
+        //add outputs 
+        if( p->second>result.prec ) {
+          result.sf = psf;
+          result.lf = strat->lf;
+          result.prec = p->second;
+          result.strat = strats[j];
+        }
+
+        delete strat;
+        return;
+      }
+    }
+    delete strat;
+  }
+
+}

Library/Ab3P.h

@@ -1,83 +1,83 @@
-/*
-Identify sf & lf pairs from free text using multi-stage algorithm
-process one line at a time and print out:
-line
-  sf|lf|P-precision|strategy
-*/
-
-#include "AbbrvE.h"
-#include "AbbrStra.h"
-#include <vector>
-#include <map>
-#include <string>
-
-using namespace std;
-using namespace iret;
-
-namespace iret {
-
-class AbbrOut {
-public:
-  string sf, lf, strat;
-  double prec;
-
-  AbbrOut( void ) : sf(""), lf(""), strat(""), prec(0)
-  {}
-
-  void print ( ostream & out ) {
-    out << "  "  << sf << "|" << lf << "|" << prec;
-  }
-
-};
-
-
-class Ab3P {
-public:
-  Ab3P( void );
-  ~Ab3P(void) { delete wrdData; }
-
-  /**  Collect text for later abbreviation finding. **/
-  void add_text( const string & text ) {
-    buffer += text;
-  }
-  void add_text( char * text ) {
-    buffer += text;
-  }
-
-  /**  Sets abbrs to the abbreviations found in previous calls to add_text.
-       Afterwords, resets the text buffer.  **/
-  void get_abbrs( vector<AbbrOut> & abbrs ) {
-    get_abbrs( buffer, abbrs );
-    buffer = "";
-  }
-
-  /**  Sets abbrs to the abbreviations found in text
-       Does not interfere with the add_text buffer.  **/
-  void get_abbrs( const string & text, vector<AbbrOut> & abbrs ) {
-    abbrs.clear();
-
-    if(text.empty()) return; // skip empty line
-    // const_cast need so correct get_abbrs get called,
-    // otherwise, infinite loop
-    get_abbrs( const_cast<char*>(text.c_str()), abbrs );
-  }
-  void get_abbrs( char * text, vector<AbbrOut> & abbrs );
-
-  /**  Try a potential sf-lf form to find proper lf, strategy used,
-       and pseudo-precision of result **/
-  void try_pair( char * sf, char * lf, AbbrOut & abbr );
-
-  /**
-     psf -- pointer short form
-     plf -- pointer long form
-  **/
-  void try_strats ( char * psf, char * plf, bool swap, AbbrOut & result );
-
-  AbbrvE ab; //default # pairs = 10,000 
-  map<string, double> stratPrec;
-  StratUtil util;
-  WordData *wrdData;            //set data needed for AbbrStra
-  string buffer;                // collect text for later use
-};
-
-}
+/*
+Identify sf & lf pairs from free text using multi-stage algorithm
+process one line at a time and print out:
+line
+  sf|lf|P-precision|strategy
+*/
+
+#include "AbbrvE.h"
+#include "AbbrStra.h"
+#include <vector>
+#include <map>
+#include <string>
+
+using namespace std;
+using namespace iret;
+
+namespace iret {
+
+class AbbrOut {
+public:
+  string sf, lf, strat;
+  double prec;
+
+  AbbrOut( void ) : sf(""), lf(""), strat(""), prec(0)
+  {}
+
+  void print ( ostream & out ) {
+    out << "  "  << sf << "|" << lf << "|" << prec;
+  }
+
+};
+
+
+class Ab3P {
+public:
+  Ab3P( void );
+  ~Ab3P(void) { delete wrdData; }
+
+  /**  Collect text for later abbreviation finding. **/
+  void add_text( const string & text ) {
+    buffer += text;
+  }
+  void add_text( char * text ) {
+    buffer += text;
+  }
+
+  /**  Sets abbrs to the abbreviations found in previous calls to add_text.
+       Afterwords, resets the text buffer.  **/
+  void get_abbrs( vector<AbbrOut> & abbrs ) {
+    get_abbrs( buffer, abbrs );
+    buffer = "";
+  }
+
+  /**  Sets abbrs to the abbreviations found in text
+       Does not interfere with the add_text buffer.  **/
+  void get_abbrs( const string & text, vector<AbbrOut> & abbrs ) {
+    abbrs.clear();
+
+    if(text.empty()) return; // skip empty line
+    // const_cast need so correct get_abbrs get called,
+    // otherwise, infinite loop
+    get_abbrs( const_cast<char*>(text.c_str()), abbrs );
+  }
+  void get_abbrs( char * text, vector<AbbrOut> & abbrs );
+
+  /**  Try a potential sf-lf form to find proper lf, strategy used,
+       and pseudo-precision of result **/
+  void try_pair( char * sf, char * lf, AbbrOut & abbr );
+
+  /**
+     psf -- pointer short form
+     plf -- pointer long form
+  **/
+  void try_strats ( char * psf, char * plf, bool swap, AbbrOut & result );
+
+  AbbrvE ab; //default # pairs = 10,000 
+  map<string, double> stratPrec;
+  StratUtil util;
+  WordData *wrdData;            //set data needed for AbbrStra
+  string buffer;                // collect text for later use
+};
+
+}

Library/AbbrStra.C

@@ -1,1426 +1,1426 @@
-#include "AbbrStra.h"
-#include <runn.h>
-#include <vector>
-#include <fstream>
-#include <iostream>
-
-
-WordData::WordData(const char *wrdnam, const char *stpnam, 
-                   const char *lfsnam) :
-  wrdset(wrdnam), stp(stpnam), lfs(lfsnam)
-{
-  wrdset.set_path_name("Ab3P");
-  wrdset.gopen_ctable_map();
-  stp.set_path_name("Ab3P");
-  stp.gopen_htable_map();
-  lfs.set_path_name("Ab3P");
-  lfs.gopen_htable_map();
-}
-
-WordData::~WordData()
-{
-  wrdset.gclose_ctable_map();
-  stp.gclose_htable_map();
-  lfs.gclose_htable_map();
-}
-
-
-AbbrStra::AbbrStra()
-{
-  npairs = tpairs = nsfs = nmatchs = amatchs = 0;
-}
-
-
-AbbrStra::~AbbrStra()
-{
-}
-
-
-void AbbrStra::token(const char *str, char lst[1000][1000])
-{
-   long i,j=0,k=0;
-   long n=strlen(str)-1;
-
-   while(isblank(str[n])) n--; 
-
-   while(str[j]){
-      while(isblank(str[j]))j++;
-      i=j;
-      while((str[j])&&(!isblank(str[j])))j++;
-      strncpy(lst[k],str+i,j-i);
-      lst[k][j-i]='\0';
-      if(str[j]){
-	 k++; 
-         j++;
-      }
-   }
-   if((j-1)>n) k--; //added by Sohn (Jan-17-08): "ab cd " -> 2 tokens
-   ntk=k+1; //# tokens, ntk is data member
-}
-
-
-long AbbrStra::tokenize(const char *str, char lst[1000][1000])
-{
-   long i,j=0,k=0;
-   long n=strlen(str)-1;
-
-   while(isblank(str[n])) n--; 
-
-   while(str[j]){
-      while(isblank(str[j]))j++;
-      i=j;
-      while((str[j])&&(!isblank(str[j])))j++;
-      strncpy(lst[k],str+i,j-i);
-      lst[k][j-i]='\0';
-      if(str[j]){
-         k++; 
-         j++;
-      }
-   }
-   if((j-1)>n) k--; //added by Sohn (Jan-17-08): "ab cd " -> 2 tokens
-   return k+1; //# tokens
-}
-
-
-long AbbrStra::num_token(const char *str)
-{
-   long i,j=0,k=0;
-   long n=strlen(str)-1;
-
-   while(isblank(str[n])) n--; 
-
-   while(str[j]){
-      while(isblank(str[j]))j++;
-      i=j;
-      while((str[j])&&(!isblank(str[j])))j++;
-      if(str[j]){
-	 k++;
-         j++;
-      }
-   }
-   if((j-1)>n) k--; //added by Sohn (Jan-17-08): "ab cd " -> 2 tokens
-   return k+1; //# tokens
-}
-
-
-// fch is 1st char of str token from backward
-long AbbrStra::first_ch(const char *str, char *fch, long num)
-{
-  long i, j, numtk;
-  char tk[1000][1000];
-
-  numtk = tokenize(str,tk);
-  if(num>numtk) return 0;
-
-  for(i=0; i<num; i++) 
-    fch[i] = tk[numtk-num+i][0];
-  
-  return 1;
-}
-
-long AbbrStra::is_upperal(const char *str)
-{
-  for(long i=strlen(str)-1; i>=0; i--)
-    if(!isupper(str[i]) || !isalpha(str[i]))
-      return 0;
-  return 1;
-}
-
-long AbbrStra::is_alpha(const char *str)
-{
-  for(long i=strlen(str)-1; i>=0; i--)
-    if(!isalpha(str[i]))
-      return 0;
-  return 1;
-}
-
-
-// str2 will lower-case of str1
-void AbbrStra::str_tolower(const char *str1, char *str2)
-{
-  long i=0;
-
-  while(str1[i]) {
-    str2[i] = tolower(str1[i]);
-    i++;
-  }
-  str2[i] = '\0';
-}
-
-//copy num tokens from back of str1 to str2 
-long AbbrStra::get_str(const char *str1, char *str2, long num)
-{
-  char ch, tk[1000][1000];
-  long i, j, numtk;
-
-  if(num<0) { cout<<"num<0\n"; exit(1); }
-  numtk = tokenize(str1,tk);
-  if(numtk<num) return 0;
-
-  strcpy(str2,tk[numtk-num]);
-  for(i=1; i<num; i++) {
-    strcat(str2," ");
-    strcat(str2,tk[numtk-num+i]);
-  }
-
-  return 1;
-}
-
-bool AbbrStra::isupper_str(const char *str)
-{
-  long i, len=strlen(str);
-
-  for(i=0; i<len; i++) 
-    if(isalpha(str[i]) && !isupper(str[i])) 
-      return false;
-
-  return true;
-}
-
-bool AbbrStra::is_onealpha(const char *str)
-{
-  long i, j=0, len=strlen(str);
-
-  for(i=0; i<len; i++)
-    if(isalpha(str[i])) j++;
-
-  if(j==1) return true;
-  else return false;
-}
-
-long AbbrStra::count_upperstr(const char *str)
-{
-  long i, j, k, numtk;
-  char tk[1000][1000];
-
-  numtk = tokenize(str,tk);
-
-  j = 0;
-  for(i=numtk-1; i>=0; i--) {
-    if(isupper(tk[i][0])) j++;
-    else return j;
-  }
-
-  return j;
-}
-
-void AbbrStra::get_alpha(const char *str1, char *str2)
-{
-  long i = 0, j = 0;
-  long len = strlen(str1);
-
-  while(i<len) {
-    if(isalpha(str1[i])) {
-      str2[j] = str1[i];
-      j++;
-    }
-    i++;
-  } 
-  str2[j] = '\0';
-}
-
-
-bool AbbrStra::lf_ok(const char *shrtf, const char *longf)
-{
-  long i;
-  long paren=0, sbrac=0;
-  string s, l;
-
-  //false for one parenthesis or square bracket
-  for(i=strlen(longf)-1; i>=0; i--) { 
-    if(longf[i]=='(') paren++;
-    if(longf[i]==')') paren--;
-    if(longf[i]=='[') sbrac++;
-    if(longf[i]==']') sbrac--;
-  }
-  if(paren!=0 || sbrac!=0) return false;
-
-  s.assign(shrtf);
-  l.assign(longf);
-
-  for(i=0; i<s.length(); i++) s[i]=tolower(s[i]);
-  for(i=0; i<l.length(); i++) l[i]=tolower(l[i]);
-
-  //false if LF words contain SF  
-  if( (" "+l+" ").find(" "+s+" ")!=string::npos ) return false;    
-
-  return true;
-}
-
-
-//first=true: allow 1-ahpha, 0 don't allow
-long AbbrStra::search_backward(long sloc, long tinx, long tloc, const char *abbr, bool first)
-{
-  long sfloc=sloc, tkinx=tinx, tkloc=tloc; 
-
-   while(sfloc>=0) {
-      loop1: while((tkloc>=0)&&(tok[tkinx][tkloc]!=abbr[sfloc])) tkloc--;
-      if(tkloc<0) {  
-	tkinx--; 
-        if(tkinx<0) return 0; //moved to here (Sep-14-07)
-	tkloc=strlen(tok[tkinx])-1; 
-      }
-      else { 
-	if(sfloc==0) {
-	  if(tkloc!=0) {
-	    if(!first) { tkloc--; goto loop1; }
-	    else if(isalnum(tok[tkinx][tkloc-1])) { tkloc--; goto loop1; }
-	  }
-	}
-	mod[sfloc][0]=tkinx; 
-	mod[sfloc][1]=tkloc;
-	sfloc--; tkloc--;
-      }
-   }
-
-   return 1;
-}
-
-long AbbrStra::search_backward_adv(const char *abbr, bool flag)
-{
-   long i;
-   long lna=strlen(abbr);
-
-   i=0;
-   while(i<lna){
-      if(search_backward(i,mod[i][0],mod[i][1]-1,abbr,flag)) return 1;
-      i++;
-   }
-   return 0;
-}
-
-void AbbrStra::extract_lf(long begin, long end)
-{
-  strcpy(lf,tok[begin]);
-  for(long i=begin+1; i<=end; i++) { 
-    strcat(lf," "); 
-    strcat(lf,tok[i]); 
-  }
-}
-
-
-void AbbrStra::extract_lf(long begin, long end, const char *str)
-{
-  token(str,tok);
-  strcpy(lf,tok[begin]);
-  for(long i=begin+1; i<=end; i++) { 
-    strcat(lf," "); 
-    strcat(lf,tok[i]); 
-  }
-}
-
-//---
-bool AbbrStra::exist_skipword(long nsf)
-{
-  long i=0, j=0, k;
-
-  while(i<nsf) {
-    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
-    else k=mod[i+1][0]-mod[i][0]-1;
-    if(k>0) j+=k;
-    i++;
-  }
-
-  if(j>0) return true;
-  else return false;
-}
-
-
-bool AbbrStra::exist_n_skipwords(long nsf, long n)
-{
-  long i=0, j, k;
-  bool flag=false;
-
-  //k: # skip words
-  while(i<nsf) {
-    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
-    else k=mod[i+1][0]-mod[i][0]-1;
-    if(k>n) return false;
-    if(k==n) flag=true;
-    i++;
-  }
-
-  if(flag) return true;
-  else return false;
-}
-
-//exists n consecutive skip stopwords between tokens
-bool AbbrStra::exist_n_stopwords(long nsf, long n)
-{
-  long i=0, j, k;
-  bool flag=false;
-
-  while(i<nsf) {
-    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
-    else k=mod[i+1][0]-mod[i][0]-1;
-    if(k>n) return false;
-    if(k==n) flag=true;
-    if(k>0) { //skip word exists
-      while(k) {
-	if(!wData->stp.find(tok[mod[i][0]+k])) return false;
-	k--;
-      }
-    }
-    i++;
-  }
-
-  if(flag) return true;
-  else return false;
-}
-
-
-bool AbbrStra::stopword_ok(long nsf, long nsw)
-{
-  long i=0, j, k;
-
-  while(i<nsf) {
-    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
-    else k=mod[i+1][0]-mod[i][0]-1;
-    if(k>nsw) return false;
-    if(k>0) { //skip word exists
-      while(k) {
-	if(!wData->stp.find(tok[mod[i][0]+k])) return false;
-	k--;
-      }
-    }
-    i++;
-  }
-
-  return true;
-}
-
-bool AbbrStra::skip_stop_ok(long nsf, long nsw, long n)
-{
-  long i=0, j, k, nstp;
-
-  while(i<nsf) {
-    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
-    else k=mod[i+1][0]-mod[i][0]-1;
-    if(k>nsw) return false;
-    //if(k>0) { //skip word exists
-    if(k>(nsw-n)) { 
-      nstp=0; //# skiped stopword between tokens 
-      while(k) {
-	if(wData->stp.find(tok[mod[i][0]+k])) nstp++;
-	k--;
-      }
-      if(nstp<n) return false;
-    }
-    i++;
-  }
-
-  return true;
-}
-
-
-bool AbbrStra::skip_stop_ok2(long nsf, long nsw, long n)
-{
-  long i=0, j, k, nstp;
-
-  while(i<nsf) {
-    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
-    else k=mod[i+1][0]-mod[i][0]-1;
-    if((k>0)&&(k!=nsw)) return false;
-    if(k>0) { //skip word exists
-      nstp=0; //# skiped stopword between tokens 
-      while(k) {
-	if(wData->stp.find(tok[mod[i][0]+k])) nstp++;
-	k--;
-      }
-      if(nstp<n) return false;
-    }
-
-    i++;
-  }
-
-  return true;
-}
-
-
-bool AbbrStra::skipword_ok(long nsf, long nsw)
-{
-  long i=0, j, k;
-
-  while(i<nsf) {
-    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
-    else k=mod[i+1][0]-mod[i][0]-1;
-    if(k>nsw) return false;
-    i++;
-  }
-
-  return true;
-}
-
-
-bool AbbrStra::is_subword(long nsf)
-{
-  long i=0;
-  char word[1000];
-
-  while(i<nsf) {
-    if(mod[i][1]!=0) {
-      strcpy(word,tok[mod[i][0]]+mod[i][1]);
-      if(wData->wrdset.count(word)==0) return false;
-    }
-    i++;
-  }
-
-  return true;
-}
-
-
-bool AbbrStra::is_BeginWrdMatch(long nsf, bool general)
-{
-  long i=0, j;
-  bool *bwm = new bool [ntk]; //BeginWrdMatch of a given tok
-
-  for(j=0; j<ntk; j++) bwm[j] = false;
-
-  while(i<nsf) {
-    if(mod[i][1]==0)
-      bwm[mod[i][0]] = true;
-    else if( general && (!isalnum(tok[mod[i][0]][mod[i][1]-1])) ) 
-      bwm[mod[i][0]] = true; 
-    i++;
-  }
-
-  for(j=0; j<nsf; j++) 
-    if(!bwm[mod[j][0]]) { 
-      delete [] bwm; 
-      return false;
-    }
-
-  delete [] bwm;
-
-  return true;
-}
-
-
-bool AbbrStra::is_WithinWrdMatch(long nsf, bool general)
-{
-  long i=0, wwm=0;
-
-  while(i<nsf) {
-    if(!general) { 
-      if(mod[i][1]>0) wwm++; 
-    }
-    else {
-      if(mod[i][1]>0 && isalnum(tok[mod[i][0]][mod[i][1]-1])) wwm++;
-    }
-    i++;
-  }
-
-  if(wwm>0) return true;
-  else return false;
-}
-
-
-bool AbbrStra::is_FirstLetMatch(long nsf, bool general)
-{
-  long i=0, flm=0, flm2=0;
-
-  while(i<nsf) {
-    if(mod[i][1]==0) flm++;
-    else if( general && (!isalnum(tok[mod[i][0]][mod[i][1]-1])) ) { 
-      flm++; flm2++;
-    }
-    i++;
-  }
-
-  if(flm==nsf) return true;
-  else return false;
-}
-
-
-bool AbbrStra::is_FirstLetMatch2(long nsf, bool general)
-{
-  long i=0, flm=0, flm2=0;
-
-  while(i<nsf) {
-    if(mod[i][1]==0) flm++;
-    else if( general && (!isalnum(tok[mod[i][0]][mod[i][1]-1])) ) { 
-      flm++; flm2++;
-    }
-    i++;
-  }
-  
-  if( (flm==nsf) && (flm2>=1) ) return true;
-  else return false;
-}
-
-
-bool AbbrStra::is_FirstLetSMatch(const char *abbr, bool general)
-{
-  long i=0, j=strlen(abbr)-1, flm=0, lsm=0;
- 
-  while(i<j) {
-    if(mod[i][1]==0) flm++;
-    else if( general && (!isalnum(tok[mod[i][0]][mod[i][1]-1])) ) flm++;
-    i++;
-  }
-
-  if( (tok[mod[j][0]][mod[j][1]]=='s') &&
-      (mod[j][1]==(strlen(tok[mod[j][0]])-1)) && 
-      mod[j][0]==mod[j-1][0] ) lsm++;
-
-  if((flm==j) && (lsm==1)) return true;
-  else return false;
-}
-
-
-bool AbbrStra::is_ContLetMatch(long nsf)
-{
-  long i=0, cl=1;
-
-  while(i<(nsf-1)) {
-    if( mod[i][0]==mod[i+1][0] &&
-	(mod[i][1]+1)==mod[i+1][1] ) cl++;
-    i++;
-  }
-
-  if(cl>=2) return true;
-  else return false;
-}
-//----
-
-
-//---1st ch must be alnum & at least one alphabet for all
-//str1: sf
-bool AbbrStra::set_condition(const char *str1)
-{
-  int n=0, m=0, o=0;
-
-  switch(setCondition) {
-    case 1: //all alphabet SFs
-      for(long i=strlen(str1)-1; i>=0; i--)
-	if(!isalpha(str1[i]))
-	  return false;
-      return true;
-      break;
-    case 2: //at least one non-alphabet
-      if(!isalnum(str1[0])) return false;
-      for(long i=strlen(str1)-1; i>=0; i--) {
-	if(isalpha(str1[i])) n++;
-	else m++;
-      }
-      if( (n>0) && (m>0) ) return true;
-      else return false;
-      break;
-    case 3: //only alnum & at least one num 
-      for(long i=strlen(str1)-1; i>=0; i--) {
-	if(!isalnum(str1[i])) return false;
-	if(isalpha(str1[i])) n++;
-	if(isdigit(str1[i])) m++;
-      }
-      if( (n>0) && (m>0) ) return true;
-      else return false;
-      break;
-    case 4: //only alpha and non-alnum & at least one non-alnum 
-      if(!isalpha(str1[0])) return false;
-      for(long i=strlen(str1)-1; i>=0; i--) {
-	if(isdigit(str1[i])) return false;
-	if(!isalnum(str1[i])) n++;
-      }
-      if(n>0) return true;
-      else return false;
-      break;
-    case 5: //at least one non-alnum 
-      if(!isalnum(str1[0])) return false;
-      for(long i=strlen(str1)-1; i>0; i--) {
-	if(!isalnum(str1[i])) return true;
-      }
-      return false;
-      break;
-    case 6: //at least one num and non-alnum
-      if(!isalnum(str1[0])) return false;
-      for(long i=strlen(str1)-1; i>=0; i--) {
-	if(isalpha(str1[i])) n++;
-	if(isdigit(str1[i])) m++;
-	if(!isalnum(str1[i])) o++;
-      }
-      if( (n>0) && (m>0) && (o>0) ) return true;
-      else return false;
-      break;
-    case 7: //1+2 (SH algorithm)
-      if(!isalnum(str1[0])) return false;
-      for(long i=strlen(str1)-1; i>=0; i--)
-	if(isalpha(str1[i])) return true;
-      return false;
-      break;
-    default:
-      cout << "Not defined set condition\n"; 
-      exit(1);
-  }
-}
-
-//---
-//same as FirstLet::set_condition
-//but requires extra set conditions 
-bool FirstLetOneChSF::set_condition(const char *shrtf, const char *longf, char *str)
-{
-  long i=0, len=strlen(shrtf), numtk;
-  char tk[1000][1000];
-
-  //sf conditions: all alphabet
-  while(i<len && isalpha(shrtf[i])) i++;
-  if(i!=len) return false; 
-
-  //lf conditions: #tok>=|SF|, 1st ch of words must be alphabet
-  numtk = tokenize(longf,tk);
-  if(len>numtk) return false;
-
-  for(i=0; i<len; i++) 
-    str[i] = tk[numtk-len+i][0];
-  str[i] = '\0';
-
-  if(!is_alpha(str)) return false;
-
-  return true;
-}
-
-
-long FirstLetOneChSF::strategy(const char *sf_, const char *str_) {
-  long lna,lnt,flag;
-  bool genFL=false; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-  char phr[10000], phrl[10000];    
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  get_str(str_,phr,1); //phr: 1st token of str from back
-  str_tolower(phr,phrl);  
-  //conditions
-  if(is_onealpha(phr)) return 0; //last token includes 1 alphabet
-  if(isupper_str(phr)) return 0; //last token is all upper-case alphabet
-  if(wData->stp.find(phrl)) return 0; //last token is stopword
-  if(!wData->lfs.find(phrl)) return 0; //lfs (1-ch sf) for FirstLet match cases < 2
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!skipword_ok(lna,0)) continue; 
-    if(!is_FirstLetMatch(lna,genFL)) continue; //not allow 1-alpha
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-//---
-
-bool FirstLet::set_condition(const char *shrtf, const char *longf, char *str)
-{
-  long i=0, len=strlen(shrtf), numtk;
-  char tk[1000][1000];
-
-  //sf conditions
-  while(i<len && isalpha(shrtf[i])) i++;
-  if(i!=len) return false; 
-
-  //lf conditions
-  numtk = tokenize(longf,tk);
-  if(len>numtk) return false;
-
-  for(i=0; i<len; i++) 
-    str[i] = tk[numtk-len+i][0];
-  str[i] = '\0';
-
-  if(!is_alpha(str)) return false;
-
-  return true;
-}
-
-
-long FirstLet::strategy(const char *sf_, const char *str_) {
-  long lna,lnt,flag;
-  bool genFL=false; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!skipword_ok(lna,0)) continue; 
-    if(!is_FirstLetMatch(lna,genFL)) continue; //not allow 1-alpha
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long FirstLetGen::strategy(const char *sf_, const char *str_)
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!skipword_ok(lna,0)) continue; 
-    if(!is_FirstLetMatch2(lna,genFL)) continue; //at least 1-alpha
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long FirstLetGen2::strategy(const char *sf_, const char *str_)
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!skipword_ok(lna,0)) continue; 
-    if(!is_FirstLetMatch(lna,genFL)) continue; 
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-bool FirstLetGenS::set_condition(const char *str)
-{
-  if(str[strlen(str)-1]!='s') return false;
-
-  for(long i=strlen(str)-2; i>=0; i--) {
-    if(!isupper(str[i])) return false;
-    if(!isalpha(str[i])) return false; //necessary?
-  }
-
-  return true;
-}
-
-
-long FirstLetGenS::strategy(const char *sf_, const char *str_)
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  if(!set_condition(sf_)) return 0;
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!skipword_ok(lna,0)) continue; 
-    if(!is_FirstLetSMatch(sf,genFL)) continue;
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long FirstLetGenStp::strategy(const char *sf_, const char *str_)
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!exist_skipword(lna)) continue; 
-    if(!stopword_ok(lna,1)) continue; 
-    if(!is_FirstLetMatch(lna,genFL)) continue; 
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long FirstLetGenStp2::strategy(const char *sf_, const char *str_)
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!exist_n_stopwords(lna,2)) continue; 
-    if(!is_FirstLetMatch(lna,genFL)) continue; 
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long FirstLetGenSkp::strategy(const char *sf_, const char *str_)
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!exist_skipword(lna)) continue; 
-    if(!skipword_ok(lna,1)) continue; 
-    if(!is_FirstLetMatch(lna,genFL)) continue; 
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long WithinWrdWrd::strategy(const char *sf_, const char *str_) 
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!skipword_ok(lna,0)) continue; 
-    if(!is_subword(lna)) continue; 
-    if(!is_WithinWrdMatch(lna,genFL)) continue;
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long WithinWrdFWrd::strategy(const char *sf_, const char *str_) 
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!skipword_ok(lna,0)) continue; 
-    if(!is_subword(lna)) continue; 
-    if(!is_BeginWrdMatch(lna,genFL)) continue; 
-    if(!is_WithinWrdMatch(lna,genFL)) continue;
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long WithinWrdFWrdSkp::strategy(const char *sf_, const char *str_) 
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!exist_skipword(lna)) continue; 
-    if(!skipword_ok(lna,1)) continue; 
-    if(!is_subword(lna)) continue; 
-    if(!is_BeginWrdMatch(lna,genFL)) continue; 
-    if(!is_WithinWrdMatch(lna,genFL)) continue;
-    
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long WithinWrdLet::strategy(const char *sf_, const char *str_)
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!skipword_ok(lna,0)) continue; 
-    if(!is_WithinWrdMatch(lna,genFL)) continue;
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long WithinWrdFLet::strategy(const char *sf_, const char *str_) 
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!skipword_ok(lna,0)) continue; 
-    if(!is_BeginWrdMatch(lna,genFL)) continue; 
-    if(!is_WithinWrdMatch(lna,genFL)) continue;
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long WithinWrdFLetSkp::strategy(const char *sf_, const char *str_) 
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!exist_skipword(lna)) continue; 
-    if(!skipword_ok(lna,1)) continue; 
-    if(!is_BeginWrdMatch(lna,genFL)) continue; 
-    if(!is_WithinWrdMatch(lna,genFL)) continue;
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long ContLet::strategy(const char *sf_, const char *str_) 
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!skipword_ok(lna,0)) continue; 
-    if(!is_BeginWrdMatch(lna,genFL)) continue; 
-    if(!is_ContLetMatch(lna)) continue; 
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long ContLetSkp::strategy(const char *sf_, const char *str_) 
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!exist_skipword(lna)) continue; 
-    if(!skipword_ok(lna,1)) continue; 
-    if(!is_BeginWrdMatch(lna,genFL)) continue; 
-    if(!is_ContLetMatch(lna)) continue; 
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-long AnyLet::strategy(const char *sf_, const char *str_) 
-{
-  long lna,lnt,flag;
-  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
-
-  str_tolower(sf_,sf);
-  str_tolower(str_,text);
-
-  token(text,tok);
-  lna = strlen(sf);
-  lnt = strlen(tok[ntk-1]);
-
-  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
-  if(!flag) return 0;
-
-  do {
-    if(!skipword_ok(lna,1)) continue; 
-
-    extract_lf(mod[0][0],ntk-1,str_);
-    return 1;
-  } while(search_backward_adv(sf,genFL));
-
-  return 0;
-}
-
-
-
-//-----
-AbbrStra * StratUtil::strat_factory(string name)
-{
-  if(name=="FirstLetOneChSF") return new FirstLetOneChSF;
-  else if(name=="FirstLet") return new FirstLet;
-  else if(name=="FirstLetGen") return new FirstLetGen;
-  else if(name=="FirstLetGen2") return new FirstLetGen2;
-  else if(name=="FirstLetGenS") return new FirstLetGenS;
-  else if(name=="FirstLetGenStp") return new FirstLetGenStp;
-  else if(name=="FirstLetGenStp2") return new FirstLetGenStp2;
-  else if(name=="FirstLetGenSkp") return new FirstLetGenSkp;
-  else if(name=="WithinWrdWrd") return new WithinWrdWrd;
-  else if(name=="WithinWrdFWrd") return new WithinWrdFWrd;
-  else if(name=="WithinWrdFWrdSkp") return new WithinWrdFWrdSkp;
-  else if(name=="WithinWrdLet") return new WithinWrdLet;
-  else if(name=="WithinWrdFLet") return new WithinWrdFLet;
-  else if(name=="WithinWrdFLetSkp") return new WithinWrdFLetSkp;
-  else if(name=="ContLet") return new ContLet;
-  else if(name=="ContLetSkp") return new ContLetSkp;
-  else if(name=="AnyLet") return new AnyLet;
-  else { cout << "Fail strat_factory\n"; exit(1); }
-}
-
-
-//check if sf is ok and assign a group
-//if sf length > 5, use 5!!
-//grp will be Al+#ChInSF, Num+#ChInSF, or Spec+#ChInSF
-bool StratUtil::group_sf(const char *sf,  string &grp)
-{
-  long i, j, len=strlen(sf);
-  long al=0, num=0, nonalnum=0;
-  long paren=0, sbrac=0;
-
-  grp = "";                      // if failure, no group
-
-  if(!isalnum(sf[0])) return false; //1sf ch must alnum
-  for(i=0; i<len; i++) {
-    if(isalpha(sf[i])) al++;
-    else if(isdigit(sf[i])) num++;
-    else nonalnum++;
-  }
-  if(al<1) return false; //at least one alphabet
-
-  //false for one parenthesis or square bracket
-  for(i=len-1; i>=0; i--) { 
-    if(sf[i]=='(') paren++;
-    if(sf[i]==')') paren--;
-    if(sf[i]=='[') sbrac++;
-    if(sf[i]==']') sbrac--;
-  }
-  if(paren!=0 || sbrac!=0) return false;
-
-  if(al==len) grp.assign("Al");
-  else if(num>0) grp.assign("Num");
-  else if(nonalnum>0) grp.assign("Spec");
-  else { cout << "No sf group\n"; exit(1); }
-
-  //append sf length
-  len = len>5 ? 5 : len;
-
-    switch(len) {
-      case 1:
-        grp.append("1");
-	break;
-      case 2:
-        grp.append("2");
-	break;
-      case 3:
-        grp.append("3");
-	break;
-      case 4:
-        grp.append("4");
-	break;
-      case 5:
-        grp.append("5");
-	break;
-      default:
-	cout << "Not defined #-ch SF" << endl;
-	exit(1);
-    }
-
-  return true;
-}
-
-//add the condition |lf|>|sf|
-bool StratUtil::group_sf(const char *sf,  const char *lf, string &grp)
-{
-  long i, j, len=strlen(sf);
-  long al=0, num=0, nonalnum=0;
-  long paren=0, sbrac=0;
-
-  if(strlen(lf)<len) return false; //|lf|>|sf|
-  if(!isalnum(sf[0])) return false; //1sf ch must alnum
-  for(i=0; i<len; i++) {
-    if(isalpha(sf[i])) al++;
-    else if(isdigit(sf[i])) num++;
-    else nonalnum++;
-  }
-  if(al<1) return false; //at least one alphabet
-  if(al>10) return false; //|alpha sf| is at most 10
-  if(num_token(sf)>2) return false; //added Feb-21-08
-
-  //false for one parenthesis or square bracket
-  for(i=len-1; i>=0; i--) { 
-    if(sf[i]=='(') paren++;
-    if(sf[i]==')') paren--;
-    if(sf[i]=='[') sbrac++;
-    if(sf[i]==']') sbrac--;
-  }
-  if(paren!=0 || sbrac!=0) return false;
-
-  if(al==len) grp.assign("Al");
-  else if(num>0) grp.assign("Num");
-  else if(nonalnum>0) grp.assign("Spec");
-  else { cout << "No sf group\n"; exit(1); }
-
-  //append sf length
-  len = len>5 ? 5 : len;
-
-    switch(len) {
-      case 1:
-        grp.append("1");
-	break;
-      case 2:
-        grp.append("2");
-	break;
-      case 3:
-        grp.append("3");
-	break;
-      case 4:
-        grp.append("4");
-	break;
-      case 5:
-        grp.append("5");
-	break;
-      default:
-	cout << "Not defined #-ch SF" << endl;
-	exit(1);
-    }
-
-  return true;
-}
-
-
-//remove non-alnum in str1 and save it to str2
-void StratUtil::remove_nonAlnum(const char *str1, char *str2)
-{
-  long i=0, j=0;
-
-  while(str1[i]) {
-    if(isalnum(str1[i])) {
-      str2[j] = str1[i];
-      j++;
-    }
-    i++;
-  } 
-  str2[j] = '\0';
-}
-
-
-vector<string> StratUtil::get_strats(string s)
-{
-  if(s=="Al1") return Al1;
-  else if(s=="Al2") return Al2;
-  else if(s=="Al3") return Al3;
-  else if(s=="Al4") return Al4;
-  else if(s=="Al5") return Al5;
-  else if(s=="Num2") return Num2;
-  else if(s=="Num3") return Num3;
-  else if(s=="Num4") return Num4;
-  else if(s=="Num5") return Num5;
-  else if(s=="Spec2") return Spec2;
-  else if(s=="Spec3") return Spec3;
-  else if(s=="Spec4") return Spec4;
-  else if(s=="Spec5") return Spec5;
-  else { cout << "Incorrect name\n"; exit(1); }
-}
-
-
-void StratUtil::push_back_strat(string sgp, string strat)
-{
-  if(sgp=="Al1") Al1.push_back(strat);
-  else if(sgp=="Al2") Al2.push_back(strat);
-  else if(sgp=="Al3") Al3.push_back(strat);
-  else if(sgp=="Al4") Al4.push_back(strat);
-  else if(sgp=="Al5") Al5.push_back(strat);
-  else if(sgp=="Num2") Num2.push_back(strat);
-  else if(sgp=="Num3") Num3.push_back(strat);
-  else if(sgp=="Num4") Num4.push_back(strat);
-  else if(sgp=="Num5") Num5.push_back(strat);
-  else if(sgp=="Spec2") Spec2.push_back(strat);
-  else if(sgp=="Spec3") Spec3.push_back(strat);
-  else if(sgp=="Spec4") Spec4.push_back(strat);
-  else if(sgp=="Spec5") Spec5.push_back(strat);
-}
-
-
-long StratUtil::exist_upperal(const char *str)
-{
-  long i, len=strlen(str);
-
-  for(i=0; i<len; i++)
-    if(isupper(str[i]))
-      return 1;
-  return 0;
-}
-
-long StratUtil::num_token(const char *str)
-{
-   long i,j=0,k=0;
-   long n=strlen(str)-1;
-
-   while(isblank(str[n])) n--; 
-
-   while(str[j]){
-      while(isblank(str[j]))j++;
-      i=j;
-      while((str[j])&&(!isblank(str[j])))j++;
-      if(str[j]){
-	 k++;
-         j++;
-      }
-   }
-   if((j-1)>n) k--; //added by Sohn (Jan-17-08): "ab cd " -> 2 tokens
-   return k+1; //# tokens
-}
-//-----
+#include "AbbrStra.h"
+#include <runn.h>
+#include <vector>
+#include <fstream>
+#include <iostream>
+
+
+WordData::WordData(const char *wrdnam, const char *stpnam, 
+                   const char *lfsnam) :
+  wrdset(wrdnam), stp(stpnam), lfs(lfsnam)
+{
+  wrdset.set_path_name("Ab3P");
+  wrdset.gopen_ctable_map();
+  stp.set_path_name("Ab3P");
+  stp.gopen_htable_map();
+  lfs.set_path_name("Ab3P");
+  lfs.gopen_htable_map();
+}
+
+WordData::~WordData()
+{
+  wrdset.gclose_ctable_map();
+  stp.gclose_htable_map();
+  lfs.gclose_htable_map();
+}
+
+
+AbbrStra::AbbrStra()
+{
+  npairs = tpairs = nsfs = nmatchs = amatchs = 0;
+}
+
+
+AbbrStra::~AbbrStra()
+{
+}
+
+
+void AbbrStra::token(const char *str, char lst[1000][1000])
+{
+   long i,j=0,k=0;
+   long n=strlen(str)-1;
+
+   while(isblank(str[n])) n--; 
+
+   while(str[j]){
+      while(isblank(str[j]))j++;
+      i=j;
+      while((str[j])&&(!isblank(str[j])))j++;
+      strncpy(lst[k],str+i,j-i);
+      lst[k][j-i]='\0';
+      if(str[j]){
+	 k++; 
+         j++;
+      }
+   }
+   if((j-1)>n) k--; //added by Sohn (Jan-17-08): "ab cd " -> 2 tokens
+   ntk=k+1; //# tokens, ntk is data member
+}
+
+
+long AbbrStra::tokenize(const char *str, char lst[1000][1000])
+{
+   long i,j=0,k=0;
+   long n=strlen(str)-1;
+
+   while(isblank(str[n])) n--; 
+
+   while(str[j]){
+      while(isblank(str[j]))j++;
+      i=j;
+      while((str[j])&&(!isblank(str[j])))j++;
+      strncpy(lst[k],str+i,j-i);
+      lst[k][j-i]='\0';
+      if(str[j]){
+         k++; 
+         j++;
+      }
+   }
+   if((j-1)>n) k--; //added by Sohn (Jan-17-08): "ab cd " -> 2 tokens
+   return k+1; //# tokens
+}
+
+
+long AbbrStra::num_token(const char *str)
+{
+   long i,j=0,k=0;
+   long n=strlen(str)-1;
+
+   while(isblank(str[n])) n--; 
+
+   while(str[j]){
+      while(isblank(str[j]))j++;
+      i=j;
+      while((str[j])&&(!isblank(str[j])))j++;
+      if(str[j]){
+	 k++;
+         j++;
+      }
+   }
+   if((j-1)>n) k--; //added by Sohn (Jan-17-08): "ab cd " -> 2 tokens
+   return k+1; //# tokens
+}
+
+
+// fch is 1st char of str token from backward
+long AbbrStra::first_ch(const char *str, char *fch, long num)
+{
+  long i, j, numtk;
+  char tk[1000][1000];
+
+  numtk = tokenize(str,tk);
+  if(num>numtk) return 0;
+
+  for(i=0; i<num; i++) 
+    fch[i] = tk[numtk-num+i][0];
+  
+  return 1;
+}
+
+long AbbrStra::is_upperal(const char *str)
+{
+  for(long i=strlen(str)-1; i>=0; i--)
+    if(!isupper(str[i]) || !isalpha(str[i]))
+      return 0;
+  return 1;
+}
+
+long AbbrStra::is_alpha(const char *str)
+{
+  for(long i=strlen(str)-1; i>=0; i--)
+    if(!isalpha(str[i]))
+      return 0;
+  return 1;
+}
+
+
+// str2 will lower-case of str1
+void AbbrStra::str_tolower(const char *str1, char *str2)
+{
+  long i=0;
+
+  while(str1[i]) {
+    str2[i] = tolower(str1[i]);
+    i++;
+  }
+  str2[i] = '\0';
+}
+
+//copy num tokens from back of str1 to str2 
+long AbbrStra::get_str(const char *str1, char *str2, long num)
+{
+  char ch, tk[1000][1000];
+  long i, j, numtk;
+
+  if(num<0) { cout<<"num<0\n"; exit(1); }
+  numtk = tokenize(str1,tk);
+  if(numtk<num) return 0;
+
+  strcpy(str2,tk[numtk-num]);
+  for(i=1; i<num; i++) {
+    strcat(str2," ");
+    strcat(str2,tk[numtk-num+i]);
+  }
+
+  return 1;
+}
+
+bool AbbrStra::isupper_str(const char *str)
+{
+  long i, len=strlen(str);
+
+  for(i=0; i<len; i++) 
+    if(isalpha(str[i]) && !isupper(str[i])) 
+      return false;
+
+  return true;
+}
+
+bool AbbrStra::is_onealpha(const char *str)
+{
+  long i, j=0, len=strlen(str);
+
+  for(i=0; i<len; i++)
+    if(isalpha(str[i])) j++;
+
+  if(j==1) return true;
+  else return false;
+}
+
+long AbbrStra::count_upperstr(const char *str)
+{
+  long i, j, k, numtk;
+  char tk[1000][1000];
+
+  numtk = tokenize(str,tk);
+
+  j = 0;
+  for(i=numtk-1; i>=0; i--) {
+    if(isupper(tk[i][0])) j++;
+    else return j;
+  }
+
+  return j;
+}
+
+void AbbrStra::get_alpha(const char *str1, char *str2)
+{
+  long i = 0, j = 0;
+  long len = strlen(str1);
+
+  while(i<len) {
+    if(isalpha(str1[i])) {
+      str2[j] = str1[i];
+      j++;
+    }
+    i++;
+  } 
+  str2[j] = '\0';
+}
+
+
+bool AbbrStra::lf_ok(const char *shrtf, const char *longf)
+{
+  long i;
+  long paren=0, sbrac=0;
+  string s, l;
+
+  //false for one parenthesis or square bracket
+  for(i=strlen(longf)-1; i>=0; i--) { 
+    if(longf[i]=='(') paren++;
+    if(longf[i]==')') paren--;
+    if(longf[i]=='[') sbrac++;
+    if(longf[i]==']') sbrac--;
+  }
+  if(paren!=0 || sbrac!=0) return false;
+
+  s.assign(shrtf);
+  l.assign(longf);
+
+  for(i=0; i<s.length(); i++) s[i]=tolower(s[i]);
+  for(i=0; i<l.length(); i++) l[i]=tolower(l[i]);
+
+  //false if LF words contain SF  
+  if( (" "+l+" ").find(" "+s+" ")!=string::npos ) return false;    
+
+  return true;
+}
+
+
+//first=true: allow 1-ahpha, 0 don't allow
+long AbbrStra::search_backward(long sloc, long tinx, long tloc, const char *abbr, bool first)
+{
+  long sfloc=sloc, tkinx=tinx, tkloc=tloc; 
+
+   while(sfloc>=0) {
+      loop1: while((tkloc>=0)&&(tok[tkinx][tkloc]!=abbr[sfloc])) tkloc--;
+      if(tkloc<0) {  
+	tkinx--; 
+        if(tkinx<0) return 0; //moved to here (Sep-14-07)
+	tkloc=strlen(tok[tkinx])-1; 
+      }
+      else { 
+	if(sfloc==0) {
+	  if(tkloc!=0) {
+	    if(!first) { tkloc--; goto loop1; }
+	    else if(isalnum(tok[tkinx][tkloc-1])) { tkloc--; goto loop1; }
+	  }
+	}
+	mod[sfloc][0]=tkinx; 
+	mod[sfloc][1]=tkloc;
+	sfloc--; tkloc--;
+      }
+   }
+
+   return 1;
+}
+
+long AbbrStra::search_backward_adv(const char *abbr, bool flag)
+{
+   long i;
+   long lna=strlen(abbr);
+
+   i=0;
+   while(i<lna){
+      if(search_backward(i,mod[i][0],mod[i][1]-1,abbr,flag)) return 1;
+      i++;
+   }
+   return 0;
+}
+
+void AbbrStra::extract_lf(long begin, long end)
+{
+  strcpy(lf,tok[begin]);
+  for(long i=begin+1; i<=end; i++) { 
+    strcat(lf," "); 
+    strcat(lf,tok[i]); 
+  }
+}
+
+
+void AbbrStra::extract_lf(long begin, long end, const char *str)
+{
+  token(str,tok);
+  strcpy(lf,tok[begin]);
+  for(long i=begin+1; i<=end; i++) { 
+    strcat(lf," "); 
+    strcat(lf,tok[i]); 
+  }
+}
+
+//---
+bool AbbrStra::exist_skipword(long nsf)
+{
+  long i=0, j=0, k;
+
+  while(i<nsf) {
+    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
+    else k=mod[i+1][0]-mod[i][0]-1;
+    if(k>0) j+=k;
+    i++;
+  }
+
+  if(j>0) return true;
+  else return false;
+}
+
+
+bool AbbrStra::exist_n_skipwords(long nsf, long n)
+{
+  long i=0, j, k;
+  bool flag=false;
+
+  //k: # skip words
+  while(i<nsf) {
+    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
+    else k=mod[i+1][0]-mod[i][0]-1;
+    if(k>n) return false;
+    if(k==n) flag=true;
+    i++;
+  }
+
+  if(flag) return true;
+  else return false;
+}
+
+//exists n consecutive skip stopwords between tokens
+bool AbbrStra::exist_n_stopwords(long nsf, long n)
+{
+  long i=0, j, k;
+  bool flag=false;
+
+  while(i<nsf) {
+    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
+    else k=mod[i+1][0]-mod[i][0]-1;
+    if(k>n) return false;
+    if(k==n) flag=true;
+    if(k>0) { //skip word exists
+      while(k) {
+	if(!wData->stp.find(tok[mod[i][0]+k])) return false;
+	k--;
+      }
+    }
+    i++;
+  }
+
+  if(flag) return true;
+  else return false;
+}
+
+
+bool AbbrStra::stopword_ok(long nsf, long nsw)
+{
+  long i=0, j, k;
+
+  while(i<nsf) {
+    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
+    else k=mod[i+1][0]-mod[i][0]-1;
+    if(k>nsw) return false;
+    if(k>0) { //skip word exists
+      while(k) {
+	if(!wData->stp.find(tok[mod[i][0]+k])) return false;
+	k--;
+      }
+    }
+    i++;
+  }
+
+  return true;
+}
+
+bool AbbrStra::skip_stop_ok(long nsf, long nsw, long n)
+{
+  long i=0, j, k, nstp;
+
+  while(i<nsf) {
+    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
+    else k=mod[i+1][0]-mod[i][0]-1;
+    if(k>nsw) return false;
+    //if(k>0) { //skip word exists
+    if(k>(nsw-n)) { 
+      nstp=0; //# skiped stopword between tokens 
+      while(k) {
+	if(wData->stp.find(tok[mod[i][0]+k])) nstp++;
+	k--;
+      }
+      if(nstp<n) return false;
+    }
+    i++;
+  }
+
+  return true;
+}
+
+
+bool AbbrStra::skip_stop_ok2(long nsf, long nsw, long n)
+{
+  long i=0, j, k, nstp;
+
+  while(i<nsf) {
+    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
+    else k=mod[i+1][0]-mod[i][0]-1;
+    if((k>0)&&(k!=nsw)) return false;
+    if(k>0) { //skip word exists
+      nstp=0; //# skiped stopword between tokens 
+      while(k) {
+	if(wData->stp.find(tok[mod[i][0]+k])) nstp++;
+	k--;
+      }
+      if(nstp<n) return false;
+    }
+
+    i++;
+  }
+
+  return true;
+}
+
+
+bool AbbrStra::skipword_ok(long nsf, long nsw)
+{
+  long i=0, j, k;
+
+  while(i<nsf) {
+    if(i==(nsf-1)) k=ntk-mod[i][0]-1;
+    else k=mod[i+1][0]-mod[i][0]-1;
+    if(k>nsw) return false;
+    i++;
+  }
+
+  return true;
+}
+
+
+bool AbbrStra::is_subword(long nsf)
+{
+  long i=0;
+  char word[1000];
+
+  while(i<nsf) {
+    if(mod[i][1]!=0) {
+      strcpy(word,tok[mod[i][0]]+mod[i][1]);
+      if(wData->wrdset.count(word)==0) return false;
+    }
+    i++;
+  }
+
+  return true;
+}
+
+
+bool AbbrStra::is_BeginWrdMatch(long nsf, bool general)
+{
+  long i=0, j;
+  bool *bwm = new bool [ntk]; //BeginWrdMatch of a given tok
+
+  for(j=0; j<ntk; j++) bwm[j] = false;
+
+  while(i<nsf) {
+    if(mod[i][1]==0)
+      bwm[mod[i][0]] = true;
+    else if( general && (!isalnum(tok[mod[i][0]][mod[i][1]-1])) ) 
+      bwm[mod[i][0]] = true; 
+    i++;
+  }
+
+  for(j=0; j<nsf; j++) 
+    if(!bwm[mod[j][0]]) { 
+      delete [] bwm; 
+      return false;
+    }
+
+  delete [] bwm;
+
+  return true;
+}
+
+
+bool AbbrStra::is_WithinWrdMatch(long nsf, bool general)
+{
+  long i=0, wwm=0;
+
+  while(i<nsf) {
+    if(!general) { 
+      if(mod[i][1]>0) wwm++; 
+    }
+    else {
+      if(mod[i][1]>0 && isalnum(tok[mod[i][0]][mod[i][1]-1])) wwm++;
+    }
+    i++;
+  }
+
+  if(wwm>0) return true;
+  else return false;
+}
+
+
+bool AbbrStra::is_FirstLetMatch(long nsf, bool general)
+{
+  long i=0, flm=0, flm2=0;
+
+  while(i<nsf) {
+    if(mod[i][1]==0) flm++;
+    else if( general && (!isalnum(tok[mod[i][0]][mod[i][1]-1])) ) { 
+      flm++; flm2++;
+    }
+    i++;
+  }
+
+  if(flm==nsf) return true;
+  else return false;
+}
+
+
+bool AbbrStra::is_FirstLetMatch2(long nsf, bool general)
+{
+  long i=0, flm=0, flm2=0;
+
+  while(i<nsf) {
+    if(mod[i][1]==0) flm++;
+    else if( general && (!isalnum(tok[mod[i][0]][mod[i][1]-1])) ) { 
+      flm++; flm2++;
+    }
+    i++;
+  }
+  
+  if( (flm==nsf) && (flm2>=1) ) return true;
+  else return false;
+}
+
+
+bool AbbrStra::is_FirstLetSMatch(const char *abbr, bool general)
+{
+  long i=0, j=strlen(abbr)-1, flm=0, lsm=0;
+ 
+  while(i<j) {
+    if(mod[i][1]==0) flm++;
+    else if( general && (!isalnum(tok[mod[i][0]][mod[i][1]-1])) ) flm++;
+    i++;
+  }
+
+  if( (tok[mod[j][0]][mod[j][1]]=='s') &&
+      (mod[j][1]==(strlen(tok[mod[j][0]])-1)) && 
+      mod[j][0]==mod[j-1][0] ) lsm++;
+
+  if((flm==j) && (lsm==1)) return true;
+  else return false;
+}
+
+
+bool AbbrStra::is_ContLetMatch(long nsf)
+{
+  long i=0, cl=1;
+
+  while(i<(nsf-1)) {
+    if( mod[i][0]==mod[i+1][0] &&
+	(mod[i][1]+1)==mod[i+1][1] ) cl++;
+    i++;
+  }
+
+  if(cl>=2) return true;
+  else return false;
+}
+//----
+
+
+//---1st ch must be alnum & at least one alphabet for all
+//str1: sf
+bool AbbrStra::set_condition(const char *str1)
+{
+  int n=0, m=0, o=0;
+
+  switch(setCondition) {
+    case 1: //all alphabet SFs
+      for(long i=strlen(str1)-1; i>=0; i--)
+	if(!isalpha(str1[i]))
+	  return false;
+      return true;
+      break;
+    case 2: //at least one non-alphabet
+      if(!isalnum(str1[0])) return false;
+      for(long i=strlen(str1)-1; i>=0; i--) {
+	if(isalpha(str1[i])) n++;
+	else m++;
+      }
+      if( (n>0) && (m>0) ) return true;
+      else return false;
+      break;
+    case 3: //only alnum & at least one num 
+      for(long i=strlen(str1)-1; i>=0; i--) {
+	if(!isalnum(str1[i])) return false;
+	if(isalpha(str1[i])) n++;
+	if(isdigit(str1[i])) m++;
+      }
+      if( (n>0) && (m>0) ) return true;
+      else return false;
+      break;
+    case 4: //only alpha and non-alnum & at least one non-alnum 
+      if(!isalpha(str1[0])) return false;
+      for(long i=strlen(str1)-1; i>=0; i--) {
+	if(isdigit(str1[i])) return false;
+	if(!isalnum(str1[i])) n++;
+      }
+      if(n>0) return true;
+      else return false;
+      break;
+    case 5: //at least one non-alnum 
+      if(!isalnum(str1[0])) return false;
+      for(long i=strlen(str1)-1; i>0; i--) {
+	if(!isalnum(str1[i])) return true;
+      }
+      return false;
+      break;
+    case 6: //at least one num and non-alnum
+      if(!isalnum(str1[0])) return false;
+      for(long i=strlen(str1)-1; i>=0; i--) {
+	if(isalpha(str1[i])) n++;
+	if(isdigit(str1[i])) m++;
+	if(!isalnum(str1[i])) o++;
+      }
+      if( (n>0) && (m>0) && (o>0) ) return true;
+      else return false;
+      break;
+    case 7: //1+2 (SH algorithm)
+      if(!isalnum(str1[0])) return false;
+      for(long i=strlen(str1)-1; i>=0; i--)
+	if(isalpha(str1[i])) return true;
+      return false;
+      break;
+    default:
+      cout << "Not defined set condition\n"; 
+      exit(1);
+  }
+}
+
+//---
+//same as FirstLet::set_condition
+//but requires extra set conditions 
+bool FirstLetOneChSF::set_condition(const char *shrtf, const char *longf, char *str)
+{
+  long i=0, len=strlen(shrtf), numtk;
+  char tk[1000][1000];
+
+  //sf conditions: all alphabet
+  while(i<len && isalpha(shrtf[i])) i++;
+  if(i!=len) return false; 
+
+  //lf conditions: #tok>=|SF|, 1st ch of words must be alphabet
+  numtk = tokenize(longf,tk);
+  if(len>numtk) return false;
+
+  for(i=0; i<len; i++) 
+    str[i] = tk[numtk-len+i][0];
+  str[i] = '\0';
+
+  if(!is_alpha(str)) return false;
+
+  return true;
+}
+
+
+long FirstLetOneChSF::strategy(const char *sf_, const char *str_) {
+  long lna,lnt,flag;
+  bool genFL=false; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+  char phr[10000], phrl[10000];    
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  get_str(str_,phr,1); //phr: 1st token of str from back
+  str_tolower(phr,phrl);  
+  //conditions
+  if(is_onealpha(phr)) return 0; //last token includes 1 alphabet
+  if(isupper_str(phr)) return 0; //last token is all upper-case alphabet
+  if(wData->stp.find(phrl)) return 0; //last token is stopword
+  if(!wData->lfs.find(phrl)) return 0; //lfs (1-ch sf) for FirstLet match cases < 2
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!skipword_ok(lna,0)) continue; 
+    if(!is_FirstLetMatch(lna,genFL)) continue; //not allow 1-alpha
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+//---
+
+bool FirstLet::set_condition(const char *shrtf, const char *longf, char *str)
+{
+  long i=0, len=strlen(shrtf), numtk;
+  char tk[1000][1000];
+
+  //sf conditions
+  while(i<len && isalpha(shrtf[i])) i++;
+  if(i!=len) return false; 
+
+  //lf conditions
+  numtk = tokenize(longf,tk);
+  if(len>numtk) return false;
+
+  for(i=0; i<len; i++) 
+    str[i] = tk[numtk-len+i][0];
+  str[i] = '\0';
+
+  if(!is_alpha(str)) return false;
+
+  return true;
+}
+
+
+long FirstLet::strategy(const char *sf_, const char *str_) {
+  long lna,lnt,flag;
+  bool genFL=false; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!skipword_ok(lna,0)) continue; 
+    if(!is_FirstLetMatch(lna,genFL)) continue; //not allow 1-alpha
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long FirstLetGen::strategy(const char *sf_, const char *str_)
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!skipword_ok(lna,0)) continue; 
+    if(!is_FirstLetMatch2(lna,genFL)) continue; //at least 1-alpha
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long FirstLetGen2::strategy(const char *sf_, const char *str_)
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!skipword_ok(lna,0)) continue; 
+    if(!is_FirstLetMatch(lna,genFL)) continue; 
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+bool FirstLetGenS::set_condition(const char *str)
+{
+  if(str[strlen(str)-1]!='s') return false;
+
+  for(long i=strlen(str)-2; i>=0; i--) {
+    if(!isupper(str[i])) return false;
+    if(!isalpha(str[i])) return false; //necessary?
+  }
+
+  return true;
+}
+
+
+long FirstLetGenS::strategy(const char *sf_, const char *str_)
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  if(!set_condition(sf_)) return 0;
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!skipword_ok(lna,0)) continue; 
+    if(!is_FirstLetSMatch(sf,genFL)) continue;
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long FirstLetGenStp::strategy(const char *sf_, const char *str_)
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!exist_skipword(lna)) continue; 
+    if(!stopword_ok(lna,1)) continue; 
+    if(!is_FirstLetMatch(lna,genFL)) continue; 
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long FirstLetGenStp2::strategy(const char *sf_, const char *str_)
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!exist_n_stopwords(lna,2)) continue; 
+    if(!is_FirstLetMatch(lna,genFL)) continue; 
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long FirstLetGenSkp::strategy(const char *sf_, const char *str_)
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!exist_skipword(lna)) continue; 
+    if(!skipword_ok(lna,1)) continue; 
+    if(!is_FirstLetMatch(lna,genFL)) continue; 
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long WithinWrdWrd::strategy(const char *sf_, const char *str_) 
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!skipword_ok(lna,0)) continue; 
+    if(!is_subword(lna)) continue; 
+    if(!is_WithinWrdMatch(lna,genFL)) continue;
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long WithinWrdFWrd::strategy(const char *sf_, const char *str_) 
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!skipword_ok(lna,0)) continue; 
+    if(!is_subword(lna)) continue; 
+    if(!is_BeginWrdMatch(lna,genFL)) continue; 
+    if(!is_WithinWrdMatch(lna,genFL)) continue;
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long WithinWrdFWrdSkp::strategy(const char *sf_, const char *str_) 
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!exist_skipword(lna)) continue; 
+    if(!skipword_ok(lna,1)) continue; 
+    if(!is_subword(lna)) continue; 
+    if(!is_BeginWrdMatch(lna,genFL)) continue; 
+    if(!is_WithinWrdMatch(lna,genFL)) continue;
+    
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long WithinWrdLet::strategy(const char *sf_, const char *str_)
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!skipword_ok(lna,0)) continue; 
+    if(!is_WithinWrdMatch(lna,genFL)) continue;
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long WithinWrdFLet::strategy(const char *sf_, const char *str_) 
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!skipword_ok(lna,0)) continue; 
+    if(!is_BeginWrdMatch(lna,genFL)) continue; 
+    if(!is_WithinWrdMatch(lna,genFL)) continue;
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long WithinWrdFLetSkp::strategy(const char *sf_, const char *str_) 
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!exist_skipword(lna)) continue; 
+    if(!skipword_ok(lna,1)) continue; 
+    if(!is_BeginWrdMatch(lna,genFL)) continue; 
+    if(!is_WithinWrdMatch(lna,genFL)) continue;
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long ContLet::strategy(const char *sf_, const char *str_) 
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!skipword_ok(lna,0)) continue; 
+    if(!is_BeginWrdMatch(lna,genFL)) continue; 
+    if(!is_ContLetMatch(lna)) continue; 
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long ContLetSkp::strategy(const char *sf_, const char *str_) 
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!exist_skipword(lna)) continue; 
+    if(!skipword_ok(lna,1)) continue; 
+    if(!is_BeginWrdMatch(lna,genFL)) continue; 
+    if(!is_ContLetMatch(lna)) continue; 
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+long AnyLet::strategy(const char *sf_, const char *str_) 
+{
+  long lna,lnt,flag;
+  bool genFL=true; //1:allow 1-ahpha for 1ch of SF match, 0:don't
+
+  str_tolower(sf_,sf);
+  str_tolower(str_,text);
+
+  token(text,tok);
+  lna = strlen(sf);
+  lnt = strlen(tok[ntk-1]);
+
+  flag = search_backward(lna-1,ntk-1,lnt-1,sf,genFL);
+  if(!flag) return 0;
+
+  do {
+    if(!skipword_ok(lna,1)) continue; 
+
+    extract_lf(mod[0][0],ntk-1,str_);
+    return 1;
+  } while(search_backward_adv(sf,genFL));
+
+  return 0;
+}
+
+
+
+//-----
+AbbrStra * StratUtil::strat_factory(string name)
+{
+  if(name=="FirstLetOneChSF") return new FirstLetOneChSF;
+  else if(name=="FirstLet") return new FirstLet;
+  else if(name=="FirstLetGen") return new FirstLetGen;
+  else if(name=="FirstLetGen2") return new FirstLetGen2;
+  else if(name=="FirstLetGenS") return new FirstLetGenS;
+  else if(name=="FirstLetGenStp") return new FirstLetGenStp;
+  else if(name=="FirstLetGenStp2") return new FirstLetGenStp2;
+  else if(name=="FirstLetGenSkp") return new FirstLetGenSkp;
+  else if(name=="WithinWrdWrd") return new WithinWrdWrd;
+  else if(name=="WithinWrdFWrd") return new WithinWrdFWrd;
+  else if(name=="WithinWrdFWrdSkp") return new WithinWrdFWrdSkp;
+  else if(name=="WithinWrdLet") return new WithinWrdLet;
+  else if(name=="WithinWrdFLet") return new WithinWrdFLet;
+  else if(name=="WithinWrdFLetSkp") return new WithinWrdFLetSkp;
+  else if(name=="ContLet") return new ContLet;
+  else if(name=="ContLetSkp") return new ContLetSkp;
+  else if(name=="AnyLet") return new AnyLet;
+  else { cout << "Fail strat_factory\n"; exit(1); }
+}
+
+
+//check if sf is ok and assign a group
+//if sf length > 5, use 5!!
+//grp will be Al+#ChInSF, Num+#ChInSF, or Spec+#ChInSF
+bool StratUtil::group_sf(const char *sf,  string &grp)
+{
+  long i, j, len=strlen(sf);
+  long al=0, num=0, nonalnum=0;
+  long paren=0, sbrac=0;
+
+  grp = "";                      // if failure, no group
+
+  if(!isalnum(sf[0])) return false; //1sf ch must alnum
+  for(i=0; i<len; i++) {
+    if(isalpha(sf[i])) al++;
+    else if(isdigit(sf[i])) num++;
+    else nonalnum++;
+  }
+  if(al<1) return false; //at least one alphabet
+
+  //false for one parenthesis or square bracket
+  for(i=len-1; i>=0; i--) { 
+    if(sf[i]=='(') paren++;
+    if(sf[i]==')') paren--;
+    if(sf[i]=='[') sbrac++;
+    if(sf[i]==']') sbrac--;
+  }
+  if(paren!=0 || sbrac!=0) return false;
+
+  if(al==len) grp.assign("Al");
+  else if(num>0) grp.assign("Num");
+  else if(nonalnum>0) grp.assign("Spec");
+  else { cout << "No sf group\n"; exit(1); }
+
+  //append sf length
+  len = len>5 ? 5 : len;
+
+    switch(len) {
+      case 1:
+        grp.append("1");
+	break;
+      case 2:
+        grp.append("2");
+	break;
+      case 3:
+        grp.append("3");
+	break;
+      case 4:
+        grp.append("4");
+	break;
+      case 5:
+        grp.append("5");
+	break;
+      default:
+	cout << "Not defined #-ch SF" << endl;
+	exit(1);
+    }
+
+  return true;
+}
+
+//add the condition |lf|>|sf|
+bool StratUtil::group_sf(const char *sf,  const char *lf, string &grp)
+{
+  long i, j, len=strlen(sf);
+  long al=0, num=0, nonalnum=0;
+  long paren=0, sbrac=0;
+
+  if(strlen(lf)<len) return false; //|lf|>|sf|
+  if(!isalnum(sf[0])) return false; //1sf ch must alnum
+  for(i=0; i<len; i++) {
+    if(isalpha(sf[i])) al++;
+    else if(isdigit(sf[i])) num++;
+    else nonalnum++;
+  }
+  if(al<1) return false; //at least one alphabet
+  if(al>10) return false; //|alpha sf| is at most 10
+  if(num_token(sf)>2) return false; //added Feb-21-08
+
+  //false for one parenthesis or square bracket
+  for(i=len-1; i>=0; i--) { 
+    if(sf[i]=='(') paren++;
+    if(sf[i]==')') paren--;
+    if(sf[i]=='[') sbrac++;
+    if(sf[i]==']') sbrac--;
+  }
+  if(paren!=0 || sbrac!=0) return false;
+
+  if(al==len) grp.assign("Al");
+  else if(num>0) grp.assign("Num");
+  else if(nonalnum>0) grp.assign("Spec");
+  else { cout << "No sf group\n"; exit(1); }
+
+  //append sf length
+  len = len>5 ? 5 : len;
+
+    switch(len) {
+      case 1:
+        grp.append("1");
+	break;
+      case 2:
+        grp.append("2");
+	break;
+      case 3:
+        grp.append("3");
+	break;
+      case 4:
+        grp.append("4");
+	break;
+      case 5:
+        grp.append("5");
+	break;
+      default:
+	cout << "Not defined #-ch SF" << endl;
+	exit(1);
+    }
+
+  return true;
+}
+
+
+//remove non-alnum in str1 and save it to str2
+void StratUtil::remove_nonAlnum(const char *str1, char *str2)
+{
+  long i=0, j=0;
+
+  while(str1[i]) {
+    if(isalnum(str1[i])) {
+      str2[j] = str1[i];
+      j++;
+    }
+    i++;
+  } 
+  str2[j] = '\0';
+}
+
+
+vector<string> StratUtil::get_strats(string s)
+{
+  if(s=="Al1") return Al1;
+  else if(s=="Al2") return Al2;
+  else if(s=="Al3") return Al3;
+  else if(s=="Al4") return Al4;
+  else if(s=="Al5") return Al5;
+  else if(s=="Num2") return Num2;
+  else if(s=="Num3") return Num3;
+  else if(s=="Num4") return Num4;
+  else if(s=="Num5") return Num5;
+  else if(s=="Spec2") return Spec2;
+  else if(s=="Spec3") return Spec3;
+  else if(s=="Spec4") return Spec4;
+  else if(s=="Spec5") return Spec5;
+  else { cout << "Incorrect name\n"; exit(1); }
+}
+
+
+void StratUtil::push_back_strat(string sgp, string strat)
+{
+  if(sgp=="Al1") Al1.push_back(strat);
+  else if(sgp=="Al2") Al2.push_back(strat);
+  else if(sgp=="Al3") Al3.push_back(strat);
+  else if(sgp=="Al4") Al4.push_back(strat);
+  else if(sgp=="Al5") Al5.push_back(strat);
+  else if(sgp=="Num2") Num2.push_back(strat);
+  else if(sgp=="Num3") Num3.push_back(strat);
+  else if(sgp=="Num4") Num4.push_back(strat);
+  else if(sgp=="Num5") Num5.push_back(strat);
+  else if(sgp=="Spec2") Spec2.push_back(strat);
+  else if(sgp=="Spec3") Spec3.push_back(strat);
+  else if(sgp=="Spec4") Spec4.push_back(strat);
+  else if(sgp=="Spec5") Spec5.push_back(strat);
+}
+
+
+long StratUtil::exist_upperal(const char *str)
+{
+  long i, len=strlen(str);
+
+  for(i=0; i<len; i++)
+    if(isupper(str[i]))
+      return 1;
+  return 0;
+}
+
+long StratUtil::num_token(const char *str)
+{
+   long i,j=0,k=0;
+   long n=strlen(str)-1;
+
+   while(isblank(str[n])) n--; 
+
+   while(str[j]){
+      while(isblank(str[j]))j++;
+      i=j;
+      while((str[j])&&(!isblank(str[j])))j++;
+      if(str[j]){
+	 k++;
+         j++;
+      }
+   }
+   if((j-1)>n) k--; //added by Sohn (Jan-17-08): "ab cd " -> 2 tokens
+   return k+1; //# tokens
+}
+//-----

Library/AbbrStra.h

@@ -1,332 +1,332 @@
-#ifndef ABBRSTRA_H
-#define ABBRSTRA_H
-
-#include <vector>
-#include <string>
-#include <Hash.h>
-
-using namespace std;
-using namespace iret;
-
-
-class WordData {
-public:
-  WordData(const char *wrdname="wrdset3", const char *stpname="stop", 
-           const char *lfsname="Lf1chSf");
-  
-  ~WordData();
-
-  Chash wrdset; //sigle word in MEDLINE 
-  Hash stp; //stopword 
-  Hash lfs; //lfs (1-ch sf) for FirstLet match cases >=2
-};
-
-
-class AbbrStra {
-public:
-  AbbrStra();
-  ~AbbrStra();
-  void token(const char *str, char lst[1000][1000]); // tokennize & set ntk 
-  long tokenize(const char *str, char lst[1000][1000]); //tokennize & return # tokens
-  long num_token(const char *str); //return # tokens
-  long first_ch(const char *str, char *fch, long num);
-  long is_upperal(const char *str);
-  long is_alpha(const char *str);
-  void str_tolower(const char *str1, char *str2);
-  long get_str(const char *str1, char *str2, long num);
-  bool isupper_str(const char *str);
-  bool is_onealpha(const char *str);
-  long count_upperstr(const char *str);
-    //return # upper-case 1st letter of consecutive tokens (backward)
-  void get_alpha(const char *str1, char *str2);
-    //set str2 with only alphabet of str1
-  bool lf_ok(const char *shrtf, const char *longf);
-   
-  virtual bool set_condition(const char *sf);
-    //must set nonAlphaSF=true if want to use SF containing non-alphabet
-  virtual long strategy(const char *sf, const char *str) = 0;
-    //sf & str will be lower-cased (OCt-25-2007)
-  long search_backward(long sloc, long tnum, long tloc, const char *sf, bool first); 
-     //search backward to find match starting from sf[sloc] 
-     //Returns 1 if matches. sf[0] must match with begin word
-  long search_backward_adv(const char *sf, bool first); 
-     //Searches for next model setting. Returns 1 if finds one.
-  void extract_lf(long begin, long end); 
-     //save strings from begin to end of tok to lf
-  void extract_lf(long begin, long end, const char *str); 
-     //save strings from begin to end of str's tok to lf
-
-  //---after set mod check conditions
-  //nsf:# ch in sf, nsw:# allowed skipword, general:true allow 1st ch match after non-alnum 
-  bool exist_skipword(long nsf);
-     //true if at least one skip word exists
-  bool exist_n_skipwords(long nsf, long n);
-     //true if exist n consecutive skip words between tokens but cannot be more than n
-  bool exist_n_stopwords(long nsf, long n);
-     //true if exist n consecutive skip stopwords between tokens but cannot be more than n
-  bool stopword_ok(long nsf, long nsw);
-     //true if at most (can be 0) nsw skip stopword in row exists
-  bool skip_stop_ok(long nsf, long nsw, long n);
-     //true if at most (can be 0) nsw skip word, which include at least n stopwords, in row exists
-  bool skip_stop_ok2(long nsf, long nsw, long n);
-     //true if nsw skip word, which include at least n stopwords, in row exists
-  bool skipword_ok(long nsf, long nsw);
-     //true if at most (can be 0) nsw skip word in row exists
-  bool is_subword(long nsf);
-     //true if matching string is begin of a tok or a word in wrdlist
-  bool is_BeginWrdMatch(long nsf, bool general);
-     //true if begining ch of a word match 
-     //if general is true, allow match after non-alnum (eg, 1-alpha)
-  bool is_WithinWrdMatch(long nsf, bool general);
-     //true if within word match
-     //if general is true, 1-Alpha: 'A' is not within word match
-  bool is_FirstLetMatch(long nsf, bool general);
-     //true if each ch of sf match with 1st ch of word
-     //(true: Alpha anyword Beta (AB))
-     //if general=true, true: 1-Alpha Beta, Alpha-Beta
-  bool is_FirstLetMatch2(long nsf, bool general);
-     //at least one 1-Alpha
-  bool is_FirstLetSMatch(const char *sf, bool general);
-     //true if first letter match & 's' match with last ch of lf
-  bool is_ContLetMatch(long nsf);
-     //true if two or more consecutive letter match
-  //---
-
-  char *pch; //sf applied to a strategy
-  char *ps, *pl; //sf, potential lf
-  char sf[100], text[10000]; //sf & potential lf used in a strategy
-  char lf[10000]; //lf found by a strategy 
-  char tok[1000][1000]; //token of potential lf 
-                        //lower after strategy, original after extract_lf(b,e,str)
-  long ntk; //# tokens
-  long mod[100][2]; //match locations of tok with a given sf
-     //mod[sf_inx][0]=tok inx, mod[sf_inx][1]=match loc in tok[mod[sf_inx][0]] 
-
-  //for each n_ch-SF
-  long npairs; //selected pairs for this strategy 
-  long tpairs; //total pairs
-  long nsfs; //# selected unique sfs for this strategy 
-  long nmatchs; //# matchs (success strategy & given sf == real sf)
-  long amatchs; //# accumulated matchs up to this strategy
-  long setCondition;  //SF condition
-  long greaterEqNsf; //if 1 select SF  |SF|>=nsf
-
-  WordData *wData;
-};
-
-
-/*
-alpha beta gamma (ABG)
- */
-class FirstLet : public AbbrStra {
-public:
-  virtual bool set_condition(const char *str1, const char *str2, char *str);
-  virtual long strategy(const char *sf, const char *str); 
-};
-
-
-class FirstLetOneChSF : public AbbrStra {
-public:
-  virtual bool set_condition(const char *str1, const char *str2, char *str);
-  virtual long strategy(const char *sf, const char *str); 
-};
-
-
-/*
-- sf ch matchs with 1st ch or ch right after non-alphanum of lf 
-  but at least one match right after non-alphanum 
-  (eg, success: 1-alpha 2-beta (AB), alpha-beta(AB), 
-       fail: alpha beta(AB))
-*/
-class FirstLetGen : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str); 
-};
-
-
-/*
-- sf ch matchs with 1st ch or ch right after non-alphanum of lf 
-  (eg, success: 1-alpha 2-beta (AB), alpha-beta(AB), 
-                alpha beta(AB))
-*/
-class FirstLetGen2 : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str); 
-};
-
-
-/*
-For sf consisting of capital letters & lower-case 's'
-- First letter & 's' in the last token of lf
-(success: Alpha Betas (ABs), 1-Alpha Betas (ABs), 
-          1-Alpha-Betas (ABs), Alpha BetaS (ABs)
- fail: Alpha Beta xxs (ABs) )
-*/
-class FirstLetGenS : public AbbrStra {
-public:
-  virtual bool set_condition(const char *sf); //sf must be an original sf
-                                              //true if sf is like ABCs
-  virtual long strategy(const char *sf, const char *str);
-};
-
-
-/*
-- sf ch matches with 1st ch or ch right after non-alphanum of lf
-- allowing one skip stopword between tokens (no more than one in row)
-  at least one skip stopword in total 
-  (eg, success: alpha and beta (AB), 1-alpha and beta (AB)
-       fail: alpha beta (AB), alpha word beta (AB))
-*/
-class FirstLetGenStp : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str); 
-};
-
-
-/*
-- same as FirstLetGenStp except for 2 skip stopwords
-  & at least one two consecutive skip stopwords
- */
-class FirstLetGenStp2 : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str); 
-};
-
-
-/*
-- same as FirstLetGenStp except using skip any word instead of stopword
- */
-class FirstLetGenSkp : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str); 
-};
-
-
-/*
-- a matching sub-string must be word 
-   (eg, success: AlphaBeta (AB), Beta is word 
-                 x-AlphaBeta (AB) )
-- at least one within word match
-  (eg,fail: Alpha Beta Word (ABW), Alpha x-Beta x-Word (ABW) 
-      success: AlphaBeta Word (ABW), x-AlphaBeta inWord (ABW))
- */
-class WithinWrdWrd : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str);
-};
-
-
-/*
-- WithinWrdWrd w/ Begin Word Match
-  (success: AlphaBeta x-Word (ABW)
-   fail: AlphaBeta inWord (ABW) )
- */
-class WithinWrdFWrd : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str);
-};
-
-
-/*
-- WithinWrdFWrd w/ allowing one skip word between tokens (no more than one in row)
-  at least one skip word in total
-  (success: AlphaBeta zzz x-Word zzz (ABW)
-   fail: AlphaBeta x-Word (ABW), AlphaBeta zzz yyy x-Word (ABW))
-*/
-class WithinWrdFWrdSkp : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str);
-};
-
-
-/*
-- at least one within word match
-  ( success: Alpha InXyy (AX), x-Alpha InXyy (AX))
-    fail: Alpha Xyy (AX), Alpha 1-Xyy (AX))
-*/
-class WithinWrdLet : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str);
-};
-
-
-/*
-- WithinWrdLet w/ Begin Word Match
-  (fail: Alpha InXyy (AX), x-Alpha InXyy (AX)
-   success: AlphaXyy Word (AXW), x-AlphaXyy 1-Word (AXW))
-*/
-class WithinWrdFLet : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str);
-};
-
-
-/*
-- WithinWrdFLet w/ allowing one skip word between tokens (no more than one in row)
-  at least one skip word in total
-  (success: AlphaXyy zzz Word zzz (AXW)
-   fail: AlphaXyy Word (AXW), AlphaXyy zzz yyy Word (AXW))
-*/
-class WithinWrdFLetSkp : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str);
-};
-
-
-/* 
-- any two consecutive letter matching  w/ begin word match  
-eg) ABxxx (AB), 1-ABxxx (AB), ABxxx Cxxx (ABC), Axxx BCxxx (ABC)
-    prolactin (PRL), succinylcholine (SCh)
-*/
-class ContLet : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str);
-};
-
-
-/*
-- ContLet w/ allowing one skip word between tokens (no more than one in row) 
-  at least one skip word in total 
-*/
-class ContLetSkp : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str);
-};
-
-
-/*
--  match can occur anywhere 
--  allow one skip word between tokens (no more than one in row)
-   (success: Alpha yXyy (AX), Alpha yXyy word (AX)
-             1-Alpha yXyy word (AX)) 
-*/
-class AnyLet : public AbbrStra {
-public:
-  virtual long strategy(const char *sf, const char *str);
-};
-
-
-class StratUtil {
-public:
-  AbbrStra *strat_factory(string name);
-  vector<string> get_strats(string s);
-    //get the strategy sequence for a given #-ch SF group
-  void push_back_strat(string sgp, string strat);
-  bool group_sf(const char *sf,  string &grp);
-    //check if sf is ok and assign a group
-  bool group_sf(const char *sf,  const char *lf, string &grp);
-    //add the contion |lf|>|sf|
-  void remove_nonAlnum(const char *str1, char *str2);
-    //remove non-alnum in str1 and save it to str2
-  long exist_upperal(const char *str); //return 1 if exists upper char, 0 ow
-  long num_token(const char *str); //return # tokens
-
-  vector<string> Al1, Al2, Al3, Al4, Al5;
-  vector<string> Num2, Num3, Num4, Num5;
-  vector<string> Spec2, Spec3, Spec4, Spec5;
-};
-
-
-#endif
-
-
+#ifndef ABBRSTRA_H
+#define ABBRSTRA_H
+
+#include <vector>
+#include <string>
+#include <Hash.h>
+
+using namespace std;
+using namespace iret;
+
+
+class WordData {
+public:
+  WordData(const char *wrdname="wrdset3", const char *stpname="stop", 
+           const char *lfsname="Lf1chSf");
+  
+  ~WordData();
+
+  Chash wrdset; //sigle word in MEDLINE 
+  Hash stp; //stopword 
+  Hash lfs; //lfs (1-ch sf) for FirstLet match cases >=2
+};
+
+
+class AbbrStra {
+public:
+  AbbrStra();
+  ~AbbrStra();
+  void token(const char *str, char lst[1000][1000]); // tokennize & set ntk 
+  long tokenize(const char *str, char lst[1000][1000]); //tokennize & return # tokens
+  long num_token(const char *str); //return # tokens
+  long first_ch(const char *str, char *fch, long num);
+  long is_upperal(const char *str);
+  long is_alpha(const char *str);
+  void str_tolower(const char *str1, char *str2);
+  long get_str(const char *str1, char *str2, long num);
+  bool isupper_str(const char *str);
+  bool is_onealpha(const char *str);
+  long count_upperstr(const char *str);
+    //return # upper-case 1st letter of consecutive tokens (backward)
+  void get_alpha(const char *str1, char *str2);
+    //set str2 with only alphabet of str1
+  bool lf_ok(const char *shrtf, const char *longf);
+   
+  virtual bool set_condition(const char *sf);
+    //must set nonAlphaSF=true if want to use SF containing non-alphabet
+  virtual long strategy(const char *sf, const char *str) = 0;
+    //sf & str will be lower-cased (OCt-25-2007)
+  long search_backward(long sloc, long tnum, long tloc, const char *sf, bool first); 
+     //search backward to find match starting from sf[sloc] 
+     //Returns 1 if matches. sf[0] must match with begin word
+  long search_backward_adv(const char *sf, bool first); 
+     //Searches for next model setting. Returns 1 if finds one.
+  void extract_lf(long begin, long end); 
+     //save strings from begin to end of tok to lf
+  void extract_lf(long begin, long end, const char *str); 
+     //save strings from begin to end of str's tok to lf
+
+  //---after set mod check conditions
+  //nsf:# ch in sf, nsw:# allowed skipword, general:true allow 1st ch match after non-alnum 
+  bool exist_skipword(long nsf);
+     //true if at least one skip word exists
+  bool exist_n_skipwords(long nsf, long n);
+     //true if exist n consecutive skip words between tokens but cannot be more than n
+  bool exist_n_stopwords(long nsf, long n);
+     //true if exist n consecutive skip stopwords between tokens but cannot be more than n
+  bool stopword_ok(long nsf, long nsw);
+     //true if at most (can be 0) nsw skip stopword in row exists
+  bool skip_stop_ok(long nsf, long nsw, long n);
+     //true if at most (can be 0) nsw skip word, which include at least n stopwords, in row exists
+  bool skip_stop_ok2(long nsf, long nsw, long n);
+     //true if nsw skip word, which include at least n stopwords, in row exists
+  bool skipword_ok(long nsf, long nsw);
+     //true if at most (can be 0) nsw skip word in row exists
+  bool is_subword(long nsf);
+     //true if matching string is begin of a tok or a word in wrdlist
+  bool is_BeginWrdMatch(long nsf, bool general);
+     //true if begining ch of a word match 
+     //if general is true, allow match after non-alnum (eg, 1-alpha)
+  bool is_WithinWrdMatch(long nsf, bool general);
+     //true if within word match
+     //if general is true, 1-Alpha: 'A' is not within word match
+  bool is_FirstLetMatch(long nsf, bool general);
+     //true if each ch of sf match with 1st ch of word
+     //(true: Alpha anyword Beta (AB))
+     //if general=true, true: 1-Alpha Beta, Alpha-Beta
+  bool is_FirstLetMatch2(long nsf, bool general);
+     //at least one 1-Alpha
+  bool is_FirstLetSMatch(const char *sf, bool general);
+     //true if first letter match & 's' match with last ch of lf
+  bool is_ContLetMatch(long nsf);
+     //true if two or more consecutive letter match
+  //---
+
+  char *pch; //sf applied to a strategy
+  char *ps, *pl; //sf, potential lf
+  char sf[100], text[10000]; //sf & potential lf used in a strategy
+  char lf[10000]; //lf found by a strategy 
+  char tok[1000][1000]; //token of potential lf 
+                        //lower after strategy, original after extract_lf(b,e,str)
+  long ntk; //# tokens
+  long mod[100][2]; //match locations of tok with a given sf
+     //mod[sf_inx][0]=tok inx, mod[sf_inx][1]=match loc in tok[mod[sf_inx][0]] 
+
+  //for each n_ch-SF
+  long npairs; //selected pairs for this strategy 
+  long tpairs; //total pairs
+  long nsfs; //# selected unique sfs for this strategy 
+  long nmatchs; //# matchs (success strategy & given sf == real sf)
+  long amatchs; //# accumulated matchs up to this strategy
+  long setCondition;  //SF condition
+  long greaterEqNsf; //if 1 select SF  |SF|>=nsf
+
+  WordData *wData;
+};
+
+
+/*
+alpha beta gamma (ABG)
+ */
+class FirstLet : public AbbrStra {
+public:
+  virtual bool set_condition(const char *str1, const char *str2, char *str);
+  virtual long strategy(const char *sf, const char *str); 
+};
+
+
+class FirstLetOneChSF : public AbbrStra {
+public:
+  virtual bool set_condition(const char *str1, const char *str2, char *str);
+  virtual long strategy(const char *sf, const char *str); 
+};
+
+
+/*
+- sf ch matchs with 1st ch or ch right after non-alphanum of lf 
+  but at least one match right after non-alphanum 
+  (eg, success: 1-alpha 2-beta (AB), alpha-beta(AB), 
+       fail: alpha beta(AB))
+*/
+class FirstLetGen : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str); 
+};
+
+
+/*
+- sf ch matchs with 1st ch or ch right after non-alphanum of lf 
+  (eg, success: 1-alpha 2-beta (AB), alpha-beta(AB), 
+                alpha beta(AB))
+*/
+class FirstLetGen2 : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str); 
+};
+
+
+/*
+For sf consisting of capital letters & lower-case 's'
+- First letter & 's' in the last token of lf
+(success: Alpha Betas (ABs), 1-Alpha Betas (ABs), 
+          1-Alpha-Betas (ABs), Alpha BetaS (ABs)
+ fail: Alpha Beta xxs (ABs) )
+*/
+class FirstLetGenS : public AbbrStra {
+public:
+  virtual bool set_condition(const char *sf); //sf must be an original sf
+                                              //true if sf is like ABCs
+  virtual long strategy(const char *sf, const char *str);
+};
+
+
+/*
+- sf ch matches with 1st ch or ch right after non-alphanum of lf
+- allowing one skip stopword between tokens (no more than one in row)
+  at least one skip stopword in total 
+  (eg, success: alpha and beta (AB), 1-alpha and beta (AB)
+       fail: alpha beta (AB), alpha word beta (AB))
+*/
+class FirstLetGenStp : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str); 
+};
+
+
+/*
+- same as FirstLetGenStp except for 2 skip stopwords
+  & at least one two consecutive skip stopwords
+ */
+class FirstLetGenStp2 : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str); 
+};
+
+
+/*
+- same as FirstLetGenStp except using skip any word instead of stopword
+ */
+class FirstLetGenSkp : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str); 
+};
+
+
+/*
+- a matching sub-string must be word 
+   (eg, success: AlphaBeta (AB), Beta is word 
+                 x-AlphaBeta (AB) )
+- at least one within word match
+  (eg,fail: Alpha Beta Word (ABW), Alpha x-Beta x-Word (ABW) 
+      success: AlphaBeta Word (ABW), x-AlphaBeta inWord (ABW))
+ */
+class WithinWrdWrd : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str);
+};
+
+
+/*
+- WithinWrdWrd w/ Begin Word Match
+  (success: AlphaBeta x-Word (ABW)
+   fail: AlphaBeta inWord (ABW) )
+ */
+class WithinWrdFWrd : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str);
+};
+
+
+/*
+- WithinWrdFWrd w/ allowing one skip word between tokens (no more than one in row)
+  at least one skip word in total
+  (success: AlphaBeta zzz x-Word zzz (ABW)
+   fail: AlphaBeta x-Word (ABW), AlphaBeta zzz yyy x-Word (ABW))
+*/
+class WithinWrdFWrdSkp : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str);
+};
+
+
+/*
+- at least one within word match
+  ( success: Alpha InXyy (AX), x-Alpha InXyy (AX))
+    fail: Alpha Xyy (AX), Alpha 1-Xyy (AX))
+*/
+class WithinWrdLet : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str);
+};
+
+
+/*
+- WithinWrdLet w/ Begin Word Match
+  (fail: Alpha InXyy (AX), x-Alpha InXyy (AX)
+   success: AlphaXyy Word (AXW), x-AlphaXyy 1-Word (AXW))
+*/
+class WithinWrdFLet : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str);
+};
+
+
+/*
+- WithinWrdFLet w/ allowing one skip word between tokens (no more than one in row)
+  at least one skip word in total
+  (success: AlphaXyy zzz Word zzz (AXW)
+   fail: AlphaXyy Word (AXW), AlphaXyy zzz yyy Word (AXW))
+*/
+class WithinWrdFLetSkp : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str);
+};
+
+
+/* 
+- any two consecutive letter matching  w/ begin word match  
+eg) ABxxx (AB), 1-ABxxx (AB), ABxxx Cxxx (ABC), Axxx BCxxx (ABC)
+    prolactin (PRL), succinylcholine (SCh)
+*/
+class ContLet : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str);
+};
+
+
+/*
+- ContLet w/ allowing one skip word between tokens (no more than one in row) 
+  at least one skip word in total 
+*/
+class ContLetSkp : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str);
+};
+
+
+/*
+-  match can occur anywhere 
+-  allow one skip word between tokens (no more than one in row)
+   (success: Alpha yXyy (AX), Alpha yXyy word (AX)
+             1-Alpha yXyy word (AX)) 
+*/
+class AnyLet : public AbbrStra {
+public:
+  virtual long strategy(const char *sf, const char *str);
+};
+
+
+class StratUtil {
+public:
+  AbbrStra *strat_factory(string name);
+  vector<string> get_strats(string s);
+    //get the strategy sequence for a given #-ch SF group
+  void push_back_strat(string sgp, string strat);
+  bool group_sf(const char *sf,  string &grp);
+    //check if sf is ok and assign a group
+  bool group_sf(const char *sf,  const char *lf, string &grp);
+    //add the contion |lf|>|sf|
+  void remove_nonAlnum(const char *str1, char *str2);
+    //remove non-alnum in str1 and save it to str2
+  long exist_upperal(const char *str); //return 1 if exists upper char, 0 ow
+  long num_token(const char *str); //return # tokens
+
+  vector<string> Al1, Al2, Al3, Al4, Al5;
+  vector<string> Num2, Num3, Num4, Num5;
+  vector<string> Spec2, Spec3, Spec4, Spec5;
+};
+
+
+#endif
+
+

Library/AbbrvE.C

@@ -1,629 +1,629 @@
-#include "AbbrvE.h"
-#include <sstream>
-
-namespace iret {
-
-  Find_Seq::Find_Seq( void  )
-  /* initializers work in C++0x
-     :
-     seq_i ( { "i", "ii", "iii", "iv", "v", "vi" } ),
-     seq_I ( { "I", "II", "III", "IV", "V", "VI" } ),
-     seq_a ( { "a", "b", "c", "d", "e", "f" } ),
-     seq_A ( { "A", "B", "C", "D", "E", "F" } )
-  */
-  {
-    seq_i.push_back("i");
-    seq_i.push_back("ii");
-    seq_i.push_back("iii");
-    seq_i.push_back("iv");
-    seq_i.push_back("v");
-    seq_i.push_back("vi");
-    
-    seq_I.push_back("I");
-    seq_I.push_back("II");
-    seq_I.push_back("III");
-    seq_I.push_back("IV");
-    seq_I.push_back("V");
-    seq_I.push_back("VI");
-    
-    seq_a.push_back("a");
-    seq_a.push_back("b");
-    seq_a.push_back("c");
-    seq_a.push_back("d");
-    seq_a.push_back("e");
-    seq_a.push_back("f");
-    
-    seq_A.push_back("A");
-    seq_A.push_back("B");
-    seq_A.push_back("C");
-    seq_A.push_back("D");
-    seq_A.push_back("E");
-    seq_A.push_back("F");
-    
-  }
-
-  void 
-  Find_Seq::flag_seq( int numa, char* abbs[] ) {
-
-    my_numa = numa;
-    my_abbs = abbs;
-    
-    my_rate.resize(numa);
-    for ( int i = 0; i < numa; ++i )
-      my_rate[i] = true;
-
-    find_seq(seq_i);
-    find_seq(seq_I);
-    find_seq(seq_a);
-    find_seq(seq_A);
-
-    create_seq();
-  }
-
-
-  void
-  Find_Seq::find_seq( const vector<string> & seq ) {
-    
-    for ( int i_abbr = 0; i_abbr < my_numa-1; ++i_abbr ) {
-      // need to see at least 2 in sequence
-      
-      if ( seq[0] == my_abbs[i_abbr] ) {
-        int i_seq = 1;
-        while ( i_seq < seq.size()  and
-                i_seq + i_abbr < my_numa  and
-                seq[i_seq] == my_abbs[i_abbr + i_seq ] )
-          ++i_seq;
-      
-        if ( i_seq > 1 )
-          for ( int i = 0; i < i_seq; ++i )
-            my_rate[i_abbr+i] = false;
-      }
-      
-    }
-  }
-  
-  void
-  Find_Seq::create_seq( void ) {
-    
-    for ( int i_abbr = 0; i_abbr < my_numa; ++i_abbr ) {
-      
-      size_t len = std::strlen( my_abbs[i_abbr] );
-      
-      if ( my_abbs[i_abbr][len-1] == '1' ) {
-        // create sequence and test
-        
-        string prefix( my_abbs[i_abbr], len-1 );
-        size_t seq_len = my_numa - i_abbr; // max possible length
-        vector<string> seq;
-        // sequence starts with 1
-        for ( int i= 1; i <= seq_len; ++i ) {
-          std::ostringstream stream(prefix,std::ios::app);
-          stream << i;
-          seq.push_back( stream.str() );
-        }
-
-        //        cout << seq << '\n';
-        find_seq(seq);
-      }
-    }
-  }
-  
-  
-  AbbrvE::AbbrvE(long ta,long wrd_spc){
-    tta=ta;
-    word_space=wrd_spc;
-    abbl=new char*[tta];
-    abbs=new char*[tta];
-    nt=new int[tta];
-    lst=new char*[word_space];
-    numa=num=0;
-    pMt=new MPtok;
-    setup_Test();
-  }
-
-  AbbrvE::~AbbrvE(){
-    if(numa)cleara();
-    clear();
-    delete [] abbl;
-    delete [] abbs;
-    delete [] nt;
-    delete [] lst;
-    delete pMt;
-  }
-  
-void AbbrvE::Extract(char*pch){
-   long i,j,k,u,flag;
-   int ix;
-
-   if ( strlen(pch) <= 0 )	// no text to look at
-     return;
-
-   token(pch);
-
-   i=j=k=0;
-   flag=0;
-   while(i<num){
-      if(!strcmp("(",lst[i])){
-         if(flag)k=j+1;
-         if((i>k)&&(strcmp(")",lst[i-1]))){
-            j=i;
-            flag=1;
-         }
-      }
-      if(!strcmp(")",lst[i])){
-         if(!flag){j=k=i+1;}
-         else {
-            if(((j>k)&&(i<j+12))&&(i>j+1)){
-               if(k<j-10)k=j-10;
-               strcpy(cnam,lst[k]);
-               for(u=k+1;u<j;u++){
-                  strcat(cnam," ");
-                  strcat(cnam,lst[u]);
-               }
-               ix=strlen(cnam);
-               abbl[numa]=new char[ix+1];
-               strcpy(abbl[numa],cnam);
-
-               strcpy(cnam,lst[j+1]);
-               for(u=j+2;u<i;u++){
-                  strcat(cnam," ");
-                  strcat(cnam,lst[u]);
-               }
-               nt[numa]=i-j-1;
-               ix=strlen(cnam);
-               abbs[numa]=new char[ix+1];
-               strcpy(abbs[numa],cnam);
-               if(Test(abbs[numa]))numa++;
-	       else{ //if test done earlier would not need to allocate memory
-		     //until known to be needed
-		 delete [] abbs[numa];
-		 delete [] abbl[numa];
-	       }
-               flag=0;
-            }
-            else {
-               flag=0;
-               k=i+1;
-            }
-         }
-      }
-      i++;
-   }
-}
-
-
-//modified  Jan-9-2008
-//extract SF in [], parse until ';' or ',' in () or []
-void AbbrvE::Extract2(const char*pch){
-   long i,j,k,u,ii,jj,kk,flag;
-   int ix;
-   char openCh[2], closeCh[2];
-
-   token2(pch); // alpha beta (AB) -> alpha beta ( AB )
- 
-   for(jj=0; jj<2; jj++) {//deal with both () & []
-   i=j=k=0;
-   flag=0;
-   
-   if(jj==0) { strcpy(openCh,"("); strcpy(closeCh,")"); }
-   else if(jj==1) { strcpy(openCh,"["); strcpy(closeCh,"]"); }
-
-   while(i<num){
-      if(!strcmp(openCh,lst[i])){
-         if(flag)k=j+1; //increment after seeing both '(' and ')'
-         if((i>k)&&(strcmp(closeCh,lst[i-1]))){
-            j=i; //index of '(' 
-            flag=1;
-         }
-      }
-      if(!strcmp(closeCh,lst[i])){
-         if(!flag){j=k=i+1;} //next token
-         else {
-            if(((j>k)&&(i<j+12))&&(i>j+1)){ 
-               if(k<j-10)k=j-10; 
-               strcpy(cnam,lst[k]);
-               for(u=k+1;u<j;u++){
-                  strcat(cnam," ");
-                  strcat(cnam,lst[u]);
-               }
-               ix=strlen(cnam);
-               abbl[numa]=new char[ix+1];
-               strcpy(abbl[numa],cnam);
-
-               strcpy(cnam,lst[j+1]);
-               for(u=j+2;u<i;u++){
-                  strcat(cnam," ");
-                  strcat(cnam,lst[u]);
-               }
-               nt[numa]=i-j-1; //# abbr tokens
-               ix=strlen(cnam);
-	       
-	       //---- parse until ';' or ',' 
-	       ii=0;
-	       while(ii<ix) {
-		 if( ((cnam[ii]==';')&&(cnam[ii+1]==' ')) || 
-                     ((cnam[ii]==',')&&(cnam[ii+1]==' ')) ) {
-                    ix=ii+1;
-		    cnam[ii]='\0';
-		    break;
-		  }
-		  ii++;
-	       }
-	       //----
-	       
-               abbs[numa]=new char[ix+1];
-               strcpy(abbs[numa],cnam);
-                 if(Test(abbs[numa]))numa++;
-		 else{ //if test done earlier would not need to allocate memory
-		   //until known to be needed
-		   delete [] abbs[numa];
-		   delete [] abbl[numa];
-		 }
-               flag=0;
-            }
-            else {
-               flag=0;
-               k=i+1;
-            }
-         }
-      }
-      i++;
-   }
-   }
-}
-
-
-void AbbrvE::token(const char *pch){
-   long i=1,j=0,k=0;
-   long u=1,flag=0;
-   char c,*str=cnam;
-   int size=strlen(pch);
-   if(size>cnam_size) {
-     cerr<<"Scratch space "<<cnam_size<<", needed "<<size<<endl;
-     exit(1);
-   }
-   clear();                     // ready space for tokens
-   cnam[0]=pch[0];
-   while(c=pch[i]){
-      switch(c){
-         case '(': if(isblank(str[u-1])){
-                      str[u++]=pch[i++];
-                      if(!isblank(pch[i])){
-                         str[u++]=' ';
-                      }
-                      flag=1;
-                   }
-                   else str[u++]=pch[i++];
-                   break;
-         case ')': if(flag){
-                      if(!isblank(str[u-1])){
-                         str[u++]=' ';
-                         str[u++]=pch[i++];
-                      }
-                      if(!isblank(pch[i]))str[u++]=' ';
-                      flag=0;
-                   }
-                   else str[u++]=pch[i++];
-                   break;
-         default: str[u++]=pch[i++];
-      }
-   }
-   while((u>0)&&(isblank(str[u-1])))u--;
-   str[u]='\0';    
-  
-   while(str[j]){
-      while(isblank(str[j]))j++;
-      i=j;
-      while((str[j])&&(!isblank(str[j])))j++;
-      lst[k]=new char[j-i+1];
-      strncpy(lst[k],str+i,j-i);
-      lst[k][j-i]='\0';
-      if(str[j]){
-         k++;
-         j++;
-      }
-   }
-   num=k+1;
-}
-
-
-//both () & [] Jan-9-2008
-//(G(1)) -> ( G(1) ) Jan-28-2008
-void AbbrvE::token2(const char *pch){
-   long i=1,j=0,k=0;
-   long u=1;
-   vector<bool> openChFlag1,openChFlag2;
-   long cflag;
-   long ii, jj, kk, sz;
-   char c,*str=cnam;
-   clear();                     // ready space for tokens
-   cnam[0]=pch[0];
-   while(c=pch[i]){
-      switch(c){
-        case '(':   	   
-                   //--- (h)alpha -> (h)alpha, (h)-alpha -> ( h ) -alpha
-                   ii=kk=i;
-		   cflag=0; 
-		   while(pch[ii] && !isblank(pch[ii])) { //pch[ii] can be '\0'
-		     if(pch[ii]=='(') cflag -= 1;
-		     else if(pch[ii]==')') { cflag += 1; kk=ii; }
-		     ii++; 
-		   }
-
-		   if(!cflag && isalnum(pch[kk+1])) { //if alnum right after ')'
-		     while(i<ii) str[u++]=pch[i++];
-		     break;
-		   }
-		   //---
-		   
-	           if(isblank(str[u-1])){
-                      str[u++]=pch[i++];
-                      if(!isblank(pch[i])){ 
-                         str[u++]=' ';
-                      }
-                      openChFlag1.push_back(true);
-                   }
-                   else {
-		      str[u++]=pch[i++];
-                      openChFlag1.push_back(false);
-		   }
-
-                   break;
-
-        case ')':  sz = openChFlag1.size();
-	           if(sz>0 && openChFlag1[sz-1]){ //modified Jan-28-08
-		      if(!isblank(str[u-1])){ 
-			 str[u++]=' ';
-                         str[u++]=pch[i++]; //pch[i++] is ')'
-                      }
-		      //---added (Jan-11-08): (BIV; ), -> ( BIV; ) ,
-		      else if(!isblank(pch[i+1])){ 
-			 str[u++]=pch[i++]; //pch[i++] is ')'
-		      }
-	              //---
-
-                      if(!isblank(pch[i]))str[u++]=' '; //pch[i] must be after ')'
-                   }
-                   else str[u++]=pch[i++];
-
-		   if(sz>0) openChFlag1.pop_back();
-
-                   break;
-
-         case '[': 
-                   //--- [h]alpha -> [h]alpha
-                   ii=kk=i;
-		   cflag=0; 
-		   while(pch[ii] && !isblank(pch[ii])) { //pch[ii] can be '\0'
-		     if(pch[ii]=='[') cflag -= 1;
-		     else if(pch[ii]==']') { cflag += 1; kk=ii; }
-		     ii++; 
-		   }
-
-		   if(!cflag && isalnum(pch[kk+1])) { //if alnum right after ')'
-		     while(i<ii) str[u++]=pch[i++];
-		     break;
-		   }
-		   //---
-
-                   if(isblank(str[u-1])){
-                      str[u++]=pch[i++];
-                      if(!isblank(pch[i])){
-                         str[u++]=' ';
-                      }
-                      openChFlag2.push_back(true);
-                   }
-                   else {
-		      str[u++]=pch[i++];
-                      openChFlag2.push_back(false);
-		   }
-
-                   break;
-
-        case ']':  sz=openChFlag2.size(); 
-	           if(sz>0 && openChFlag2[sz-1]){ //modified Jan-28-08
-                      if(!isblank(str[u-1])){ 
-                         str[u++]=' ';
-                         str[u++]=pch[i++];
-                      }
-		      //---added (Jan-11-08): [BIV; ], -> [ BIV; ] ,
-		      else if(!isblank(pch[i+1])){ 
-			 str[u++]=pch[i++];
-		      }
-	              //---
-                      if(!isblank(pch[i]))str[u++]=' ';
-                   }
-                   else str[u++]=pch[i++];
-
-		   if(sz>0) openChFlag2.pop_back();
-		   
-                   break;
-         default: str[u++]=pch[i++];
-      }
-   }
-   while((u>0)&&(isblank(str[u-1])))u--;
-   str[u]='\0';    
-      
-   while(str[j]){
-      while(isblank(str[j]))j++;
-      i=j;
-      while((str[j])&&(!isblank(str[j])))j++;
-      lst[k]=new char[j-i+1];
-      strncpy(lst[k],str+i,j-i);
-      lst[k][j-i]='\0';
-      if(str[j]){
-         k++;
-         j++;
-      }
-   }
-   num=k+1;
-}
-
-
-  void AbbrvE::clear(void){
-    for ( int i=0; i<num; i++ ) {
-      delete [] lst[i];
-    }
-    num=0;
-  }
-
-void AbbrvE::cleara(void){
-   long i;
-   for(i=0;i<numa;i++){
-      delete [] abbl[i];
-      delete [] abbs[i];
-   }
-   numa=0;
-}
-
-#if 0
-
-//no space before and after abbs[] (because of using token)
-int AbbrvE::Test(const char *str){
-   long i,j,k;
-   char b,c;
-
-   if(strchr(str,'='))return(0);
-   if(!strcmp(str,"author's transl"))return(0);
-   if(!strcmp(str,"proceedings"))return(0);
-   //---added (Jan-11-08) & (Apr 08)
-   if(!strcmp(str,"see"))return(0);
-   if(!strcmp(str,"and"))return(0);
-   if(!strcmp(str,"comment"))return(0);
-   if(!strcmp(str,"letter"))return(0);
-   //---
-   if((str[0]=='e')&&(str[1]=='g')){
-      if(!(c=str[2])||(c=='.')||(c==','))return(0);
-   }
-   if((str[0]=='s')&&(str[1]=='e')&&(str[2]=='e')&&(((b=str[3])==' ')||(b==',')))return(0);
-   if('p'==tolower(str[0])){
-      if(strchr(str+1,'<'))return(0);
-   }
-   i=j=k=0;
-   while((c=str[i])&&(c!=' ')){
-      i++;
-      if(isdigit(c))j++;
-      if(isalpha(c))k++;
-      if((i==j)&&(i==3))return(0);
-   }
-   if((i==j)||(k==0))return(0);
-   else return(1);
-} 
-
-#endif
-
-  bool AbbrvE::prefix_match( const char *str ) {
-    size_t size = strlen(str);
-    for ( int i = 0; i < prefix.size(); ++i ) {
-      string& pre = prefix[i];
-      if ( size > pre.size()  and
-           0 == pre.compare( 0, pre.size(), str, pre.size() ) )
-        return true;
-    }
-    return false;
-  }
-
-    
-//no space before and after abbs[] (because of using token)
-bool AbbrvE::Test(const char *str){
-
-   if ( match.find(str) != match.end() ) return false;
-   if ( prefix_match(str) ) return false;
-
-   size_t length, letters, digits;
-   length = letters = digits = 0;
-
-   char c;
-   while((c=str[length])&&(c!=' ')){
-     length++;
-     if ( isdigit(c) ) digits++;
-     if ( isalpha(c) ) letters++;
-     
-     if( length==digits  and  length>=3 ) return false;
-   }
-   if ( digits == length ) return false;
-   if ( letters <= 0 ) return false;
-
-   return true;
-} 
-
-  void AbbrvE::setup_Test( void ) {
-
-    match.insert("author's transl");
-    match.insert("proceedings");
-    match.insert("see");
-    match.insert("and");
-    match.insert("comment");
-    match.insert("letter");
-    match.insert("eg");
-
-    prefix.push_back("=");
-    prefix.push_back("eg.");
-    prefix.push_back("eg,");
-    prefix.push_back("see ");
-    prefix.push_back("see,");
-    prefix.push_back("p<");
-    prefix.push_back("P<");
-
-    // rules added in 2010
-    match.insert("e.g.");
-    match.insert("ie");
-    match.insert("i.e.");
-    match.insert("mean");
-    match.insert("age");
-    match.insert("std");
-    match.insert("range");
-    match.insert("young");
-    match.insert("old");
-    match.insert("male");
-    match.insert("female");
-
-  }
-
-void AbbrvE::Proc(char *pxh){
-   long i,j;
-   char *pch,*ptr;
-   pMt->segment(pxh);
-   for(i=0;i<pMt->sent.size();i++){
-     Extract2( (pMt->sent[i]).c_str() );
-   }
-   
-   seq.flag_seq( numa, abbs );
-   j=0;
-   for(i=0;i<numa;i++){
-     if( seq.rate(i) ){
-         if(j<i){
-            pch=abbl[i];
-            if(ptr=strchr(pch,'|')){
-               *ptr='/';
-               ptr++;
-               while(ptr=strchr(pch,'|')){
-                  *ptr='/';
-                  ptr++;
-               }
-            }
-            abbl[j]=pch;
-            pch=abbs[i];
-            if(ptr=strchr(pch,'|')){
-               *ptr='/';
-               ptr++;
-               while(ptr=strchr(pch,'|')){
-                  *ptr='/';
-                  ptr++;
-               }
-            }
-            abbs[j]=pch;
-            nt[j]=nt[i];
-         }
-         j++;
-      }
-      else {
-         delete [] abbl[i];
-         delete [] abbs[i];
-      }
-   }
-
-   numa=j;
-}
-
-}
+#include "AbbrvE.h"
+#include <sstream>
+
+namespace iret {
+
+  Find_Seq::Find_Seq( void  )
+  /* initializers work in C++0x
+     :
+     seq_i ( { "i", "ii", "iii", "iv", "v", "vi" } ),
+     seq_I ( { "I", "II", "III", "IV", "V", "VI" } ),
+     seq_a ( { "a", "b", "c", "d", "e", "f" } ),
+     seq_A ( { "A", "B", "C", "D", "E", "F" } )
+  */
+  {
+    seq_i.push_back("i");
+    seq_i.push_back("ii");
+    seq_i.push_back("iii");
+    seq_i.push_back("iv");
+    seq_i.push_back("v");
+    seq_i.push_back("vi");
+    
+    seq_I.push_back("I");
+    seq_I.push_back("II");
+    seq_I.push_back("III");
+    seq_I.push_back("IV");
+    seq_I.push_back("V");
+    seq_I.push_back("VI");
+    
+    seq_a.push_back("a");
+    seq_a.push_back("b");
+    seq_a.push_back("c");
+    seq_a.push_back("d");
+    seq_a.push_back("e");
+    seq_a.push_back("f");
+    
+    seq_A.push_back("A");
+    seq_A.push_back("B");
+    seq_A.push_back("C");
+    seq_A.push_back("D");
+    seq_A.push_back("E");
+    seq_A.push_back("F");
+    
+  }
+
+  void 
+  Find_Seq::flag_seq( int numa, char* abbs[] ) {
+
+    my_numa = numa;
+    my_abbs = abbs;
+    
+    my_rate.resize(numa);
+    for ( int i = 0; i < numa; ++i )
+      my_rate[i] = true;
+
+    find_seq(seq_i);
+    find_seq(seq_I);
+    find_seq(seq_a);
+    find_seq(seq_A);
+
+    create_seq();
+  }
+
+
+  void
+  Find_Seq::find_seq( const vector<string> & seq ) {
+    
+    for ( int i_abbr = 0; i_abbr < my_numa-1; ++i_abbr ) {
+      // need to see at least 2 in sequence
+      
+      if ( seq[0] == my_abbs[i_abbr] ) {
+        int i_seq = 1;
+        while ( i_seq < seq.size()  and
+                i_seq + i_abbr < my_numa  and
+                seq[i_seq] == my_abbs[i_abbr + i_seq ] )
+          ++i_seq;
+      
+        if ( i_seq > 1 )
+          for ( int i = 0; i < i_seq; ++i )
+            my_rate[i_abbr+i] = false;
+      }
+      
+    }
+  }
+  
+  void
+  Find_Seq::create_seq( void ) {
+    
+    for ( int i_abbr = 0; i_abbr < my_numa; ++i_abbr ) {
+      
+      size_t len = std::strlen( my_abbs[i_abbr] );
+      
+      if ( my_abbs[i_abbr][len-1] == '1' ) {
+        // create sequence and test
+        
+        string prefix( my_abbs[i_abbr], len-1 );
+        size_t seq_len = my_numa - i_abbr; // max possible length
+        vector<string> seq;
+        // sequence starts with 1
+        for ( int i= 1; i <= seq_len; ++i ) {
+          std::ostringstream stream(prefix,std::ios::app);
+          stream << i;
+          seq.push_back( stream.str() );
+        }
+
+        //        cout << seq << '\n';
+        find_seq(seq);
+      }
+    }
+  }
+  
+  
+  AbbrvE::AbbrvE(long ta,long wrd_spc){
+    tta=ta;
+    word_space=wrd_spc;
+    abbl=new char*[tta];
+    abbs=new char*[tta];
+    nt=new int[tta];
+    lst=new char*[word_space];
+    numa=num=0;
+    pMt=new MPtok;
+    setup_Test();
+  }
+
+  AbbrvE::~AbbrvE(){
+    if(numa)cleara();
+    clear();
+    delete [] abbl;
+    delete [] abbs;
+    delete [] nt;
+    delete [] lst;
+    delete pMt;
+  }
+  
+void AbbrvE::Extract(char*pch){
+   long i,j,k,u,flag;
+   int ix;
+
+   if ( strlen(pch) <= 0 )	// no text to look at
+     return;
+
+   token(pch);
+
+   i=j=k=0;
+   flag=0;
+   while(i<num){
+      if(!strcmp("(",lst[i])){
+         if(flag)k=j+1;
+         if((i>k)&&(strcmp(")",lst[i-1]))){
+            j=i;
+            flag=1;
+         }
+      }
+      if(!strcmp(")",lst[i])){
+         if(!flag){j=k=i+1;}
+         else {
+            if(((j>k)&&(i<j+12))&&(i>j+1)){
+               if(k<j-10)k=j-10;
+               strcpy(cnam,lst[k]);
+               for(u=k+1;u<j;u++){
+                  strcat(cnam," ");
+                  strcat(cnam,lst[u]);
+               }
+               ix=strlen(cnam);
+               abbl[numa]=new char[ix+1];
+               strcpy(abbl[numa],cnam);
+
+               strcpy(cnam,lst[j+1]);
+               for(u=j+2;u<i;u++){
+                  strcat(cnam," ");
+                  strcat(cnam,lst[u]);
+               }
+               nt[numa]=i-j-1;
+               ix=strlen(cnam);
+               abbs[numa]=new char[ix+1];
+               strcpy(abbs[numa],cnam);
+               if(Test(abbs[numa]))numa++;
+	       else{ //if test done earlier would not need to allocate memory
+		     //until known to be needed
+		 delete [] abbs[numa];
+		 delete [] abbl[numa];
+	       }
+               flag=0;
+            }
+            else {
+               flag=0;
+               k=i+1;
+            }
+         }
+      }
+      i++;
+   }
+}
+
+
+//modified  Jan-9-2008
+//extract SF in [], parse until ';' or ',' in () or []
+void AbbrvE::Extract2(const char*pch){
+   long i,j,k,u,ii,jj,kk,flag;
+   int ix;
+   char openCh[2], closeCh[2];
+
+   token2(pch); // alpha beta (AB) -> alpha beta ( AB )
+ 
+   for(jj=0; jj<2; jj++) {//deal with both () & []
+   i=j=k=0;
+   flag=0;
+   
+   if(jj==0) { strcpy(openCh,"("); strcpy(closeCh,")"); }
+   else if(jj==1) { strcpy(openCh,"["); strcpy(closeCh,"]"); }
+
+   while(i<num){
+      if(!strcmp(openCh,lst[i])){
+         if(flag)k=j+1; //increment after seeing both '(' and ')'
+         if((i>k)&&(strcmp(closeCh,lst[i-1]))){
+            j=i; //index of '(' 
+            flag=1;
+         }
+      }
+      if(!strcmp(closeCh,lst[i])){
+         if(!flag){j=k=i+1;} //next token
+         else {
+            if(((j>k)&&(i<j+12))&&(i>j+1)){ 
+               if(k<j-10)k=j-10; 
+               strcpy(cnam,lst[k]);
+               for(u=k+1;u<j;u++){
+                  strcat(cnam," ");
+                  strcat(cnam,lst[u]);
+               }
+               ix=strlen(cnam);
+               abbl[numa]=new char[ix+1];
+               strcpy(abbl[numa],cnam);
+
+               strcpy(cnam,lst[j+1]);
+               for(u=j+2;u<i;u++){
+                  strcat(cnam," ");
+                  strcat(cnam,lst[u]);
+               }
+               nt[numa]=i-j-1; //# abbr tokens
+               ix=strlen(cnam);
+	       
+	       //---- parse until ';' or ',' 
+	       ii=0;
+	       while(ii<ix) {
+		 if( ((cnam[ii]==';')&&(cnam[ii+1]==' ')) || 
+                     ((cnam[ii]==',')&&(cnam[ii+1]==' ')) ) {
+                    ix=ii+1;
+		    cnam[ii]='\0';
+		    break;
+		  }
+		  ii++;
+	       }
+	       //----
+	       
+               abbs[numa]=new char[ix+1];
+               strcpy(abbs[numa],cnam);
+                 if(Test(abbs[numa]))numa++;
+		 else{ //if test done earlier would not need to allocate memory
+		   //until known to be needed
+		   delete [] abbs[numa];
+		   delete [] abbl[numa];
+		 }
+               flag=0;
+            }
+            else {
+               flag=0;
+               k=i+1;
+            }
+         }
+      }
+      i++;
+   }
+   }
+}
+
+
+void AbbrvE::token(const char *pch){
+   long i=1,j=0,k=0;
+   long u=1,flag=0;
+   char c,*str=cnam;
+   int size=strlen(pch);
+   if(size>cnam_size) {
+     cerr<<"Scratch space "<<cnam_size<<", needed "<<size<<endl;
+     exit(1);
+   }
+   clear();                     // ready space for tokens
+   cnam[0]=pch[0];
+   while(c=pch[i]){
+      switch(c){
+         case '(': if(isblank(str[u-1])){
+                      str[u++]=pch[i++];
+                      if(!isblank(pch[i])){
+                         str[u++]=' ';
+                      }
+                      flag=1;
+                   }
+                   else str[u++]=pch[i++];
+                   break;
+         case ')': if(flag){
+                      if(!isblank(str[u-1])){
+                         str[u++]=' ';
+                         str[u++]=pch[i++];
+                      }
+                      if(!isblank(pch[i]))str[u++]=' ';
+                      flag=0;
+                   }
+                   else str[u++]=pch[i++];
+                   break;
+         default: str[u++]=pch[i++];
+      }
+   }
+   while((u>0)&&(isblank(str[u-1])))u--;
+   str[u]='\0';    
+  
+   while(str[j]){
+      while(isblank(str[j]))j++;
+      i=j;
+      while((str[j])&&(!isblank(str[j])))j++;
+      lst[k]=new char[j-i+1];
+      strncpy(lst[k],str+i,j-i);
+      lst[k][j-i]='\0';
+      if(str[j]){
+         k++;
+         j++;
+      }
+   }
+   num=k+1;
+}
+
+
+//both () & [] Jan-9-2008
+//(G(1)) -> ( G(1) ) Jan-28-2008
+void AbbrvE::token2(const char *pch){
+   long i=1,j=0,k=0;
+   long u=1;
+   vector<bool> openChFlag1,openChFlag2;
+   long cflag;
+   long ii, jj, kk, sz;
+   char c,*str=cnam;
+   clear();                     // ready space for tokens
+   cnam[0]=pch[0];
+   while(c=pch[i]){
+      switch(c){
+        case '(':   	   
+                   //--- (h)alpha -> (h)alpha, (h)-alpha -> ( h ) -alpha
+                   ii=kk=i;
+		   cflag=0; 
+		   while(pch[ii] && !isblank(pch[ii])) { //pch[ii] can be '\0'
+		     if(pch[ii]=='(') cflag -= 1;
+		     else if(pch[ii]==')') { cflag += 1; kk=ii; }
+		     ii++; 
+		   }
+
+		   if(!cflag && isalnum(pch[kk+1])) { //if alnum right after ')'
+		     while(i<ii) str[u++]=pch[i++];
+		     break;
+		   }
+		   //---
+		   
+	           if(isblank(str[u-1])){
+                      str[u++]=pch[i++];
+                      if(!isblank(pch[i])){ 
+                         str[u++]=' ';
+                      }
+                      openChFlag1.push_back(true);
+                   }
+                   else {
+		      str[u++]=pch[i++];
+                      openChFlag1.push_back(false);
+		   }
+
+                   break;
+
+        case ')':  sz = openChFlag1.size();
+	           if(sz>0 && openChFlag1[sz-1]){ //modified Jan-28-08
+		      if(!isblank(str[u-1])){ 
+			 str[u++]=' ';
+                         str[u++]=pch[i++]; //pch[i++] is ')'
+                      }
+		      //---added (Jan-11-08): (BIV; ), -> ( BIV; ) ,
+		      else if(!isblank(pch[i+1])){ 
+			 str[u++]=pch[i++]; //pch[i++] is ')'
+		      }
+	              //---
+
+                      if(!isblank(pch[i]))str[u++]=' '; //pch[i] must be after ')'
+                   }
+                   else str[u++]=pch[i++];
+
+		   if(sz>0) openChFlag1.pop_back();
+
+                   break;
+
+         case '[': 
+                   //--- [h]alpha -> [h]alpha
+                   ii=kk=i;
+		   cflag=0; 
+		   while(pch[ii] && !isblank(pch[ii])) { //pch[ii] can be '\0'
+		     if(pch[ii]=='[') cflag -= 1;
+		     else if(pch[ii]==']') { cflag += 1; kk=ii; }
+		     ii++; 
+		   }
+
+		   if(!cflag && isalnum(pch[kk+1])) { //if alnum right after ')'
+		     while(i<ii) str[u++]=pch[i++];
+		     break;
+		   }
+		   //---
+
+                   if(isblank(str[u-1])){
+                      str[u++]=pch[i++];
+                      if(!isblank(pch[i])){
+                         str[u++]=' ';
+                      }
+                      openChFlag2.push_back(true);
+                   }
+                   else {
+		      str[u++]=pch[i++];
+                      openChFlag2.push_back(false);
+		   }
+
+                   break;
+
+        case ']':  sz=openChFlag2.size(); 
+	           if(sz>0 && openChFlag2[sz-1]){ //modified Jan-28-08
+                      if(!isblank(str[u-1])){ 
+                         str[u++]=' ';
+                         str[u++]=pch[i++];
+                      }
+		      //---added (Jan-11-08): [BIV; ], -> [ BIV; ] ,
+		      else if(!isblank(pch[i+1])){ 
+			 str[u++]=pch[i++];
+		      }
+	              //---
+                      if(!isblank(pch[i]))str[u++]=' ';
+                   }
+                   else str[u++]=pch[i++];
+
+		   if(sz>0) openChFlag2.pop_back();
+		   
+                   break;
+         default: str[u++]=pch[i++];
+      }
+   }
+   while((u>0)&&(isblank(str[u-1])))u--;
+   str[u]='\0';    
+      
+   while(str[j]){
+      while(isblank(str[j]))j++;
+      i=j;
+      while((str[j])&&(!isblank(str[j])))j++;
+      lst[k]=new char[j-i+1];
+      strncpy(lst[k],str+i,j-i);
+      lst[k][j-i]='\0';
+      if(str[j]){
+         k++;
+         j++;
+      }
+   }
+   num=k+1;
+}
+
+
+  void AbbrvE::clear(void){
+    for ( int i=0; i<num; i++ ) {
+      delete [] lst[i];
+    }
+    num=0;
+  }
+
+void AbbrvE::cleara(void){
+   long i;
+   for(i=0;i<numa;i++){
+      delete [] abbl[i];
+      delete [] abbs[i];
+   }
+   numa=0;
+}
+
+#if 0
+
+//no space before and after abbs[] (because of using token)
+int AbbrvE::Test(const char *str){
+   long i,j,k;
+   char b,c;
+
+   if(strchr(str,'='))return(0);
+   if(!strcmp(str,"author's transl"))return(0);
+   if(!strcmp(str,"proceedings"))return(0);
+   //---added (Jan-11-08) & (Apr 08)
+   if(!strcmp(str,"see"))return(0);
+   if(!strcmp(str,"and"))return(0);
+   if(!strcmp(str,"comment"))return(0);
+   if(!strcmp(str,"letter"))return(0);
+   //---
+   if((str[0]=='e')&&(str[1]=='g')){
+      if(!(c=str[2])||(c=='.')||(c==','))return(0);
+   }
+   if((str[0]=='s')&&(str[1]=='e')&&(str[2]=='e')&&(((b=str[3])==' ')||(b==',')))return(0);
+   if('p'==tolower(str[0])){
+      if(strchr(str+1,'<'))return(0);
+   }
+   i=j=k=0;
+   while((c=str[i])&&(c!=' ')){
+      i++;
+      if(isdigit(c))j++;
+      if(isalpha(c))k++;
+      if((i==j)&&(i==3))return(0);
+   }
+   if((i==j)||(k==0))return(0);
+   else return(1);
+} 
+
+#endif
+
+  bool AbbrvE::prefix_match( const char *str ) {
+    size_t size = strlen(str);
+    for ( int i = 0; i < prefix.size(); ++i ) {
+      string& pre = prefix[i];
+      if ( size > pre.size()  and
+           0 == pre.compare( 0, pre.size(), str, pre.size() ) )
+        return true;
+    }
+    return false;
+  }
+
+    
+//no space before and after abbs[] (because of using token)
+bool AbbrvE::Test(const char *str){
+
+   if ( match.find(str) != match.end() ) return false;
+   if ( prefix_match(str) ) return false;
+
+   size_t length, letters, digits;
+   length = letters = digits = 0;
+
+   char c;
+   while((c=str[length])&&(c!=' ')){
+     length++;
+     if ( isdigit(c) ) digits++;
+     if ( isalpha(c) ) letters++;
+     
+     if( length==digits  and  length>=3 ) return false;
+   }
+   if ( digits == length ) return false;
+   if ( letters <= 0 ) return false;
+
+   return true;
+} 
+
+  void AbbrvE::setup_Test( void ) {
+
+    match.insert("author's transl");
+    match.insert("proceedings");
+    match.insert("see");
+    match.insert("and");
+    match.insert("comment");
+    match.insert("letter");
+    match.insert("eg");
+
+    prefix.push_back("=");
+    prefix.push_back("eg.");
+    prefix.push_back("eg,");
+    prefix.push_back("see ");
+    prefix.push_back("see,");
+    prefix.push_back("p<");
+    prefix.push_back("P<");
+
+    // rules added in 2010
+    match.insert("e.g.");
+    match.insert("ie");
+    match.insert("i.e.");
+    match.insert("mean");
+    match.insert("age");
+    match.insert("std");
+    match.insert("range");
+    match.insert("young");
+    match.insert("old");
+    match.insert("male");
+    match.insert("female");
+
+  }
+
+void AbbrvE::Proc(char *pxh){
+   long i,j;
+   char *pch,*ptr;
+   pMt->segment(pxh);
+   for(i=0;i<pMt->sent.size();i++){
+     Extract2( (pMt->sent[i]).c_str() );
+   }
+   
+   seq.flag_seq( numa, abbs );
+   j=0;
+   for(i=0;i<numa;i++){
+     if( seq.rate(i) ){
+         if(j<i){
+            pch=abbl[i];
+            if(ptr=strchr(pch,'|')){
+               *ptr='/';
+               ptr++;
+               while(ptr=strchr(pch,'|')){
+                  *ptr='/';
+                  ptr++;
+               }
+            }
+            abbl[j]=pch;
+            pch=abbs[i];
+            if(ptr=strchr(pch,'|')){
+               *ptr='/';
+               ptr++;
+               while(ptr=strchr(pch,'|')){
+                  *ptr='/';
+                  ptr++;
+               }
+            }
+            abbs[j]=pch;
+            nt[j]=nt[i];
+         }
+         j++;
+      }
+      else {
+         delete [] abbl[i];
+         delete [] abbs[i];
+      }
+   }
+
+   numa=j;
+}
+
+}

Library/AbbrvE.h

@@ -1,93 +1,93 @@
-#ifndef ABBRVE_H
-#define ABBRVE_H
-#include <fstream>
-#include <iostream>
-#include <runn.h>
-#include <MPtok.h>
-#include <vector>
-using namespace std;
-namespace iret {
-
-typedef vector<string> strings;
-
-
-class Find_Seq {
-public:
-
-  Find_Seq( void  );
-
-  // flag the SFs whether part of sequence or not
-  void flag_seq( int numa, char* abbs[] );
-
-  // true if good SF, false if part of sequence
-  bool rate( int i ) const { my_rate[i]; }
-
-private:
-  void find_seq( const vector<string> & seq );
-  void create_seq( void );
-
-  // const works with c++0x
-  /* const */ strings seq_i;
-  /* const */ strings seq_I;
-  /* const */ strings seq_a;
-  /* const */ strings seq_A;
-  
-  vector<bool> my_rate;
-  int my_numa;
-  char ** my_abbs;              // really char *[], but that doesn't work
-  
-};
-
-
-class AbbrvE {
-   public:
-      AbbrvE(long ta=10000,long wrd_spc=10000); //Sets space for extracted
-         //potential abbreviations to ta & word_space to wrd_spc
-     ~AbbrvE(void);
-      void Extract(char *pch); //Extracts possible long-short form
-         //pairs, but does not attempt to find the relationship
-      void Extract2(const char *pch); //extened version (Jan-9-2008)
-      bool Test(const char *str); //Tests a single token and returns true
-         //if the token should be a possible first token of a short form
-      void Rate(void); //Sets ratings for the proposed pairs. Effort to 
-         //remove (a), (b), etc., sequence markers
-      void token(const char *str); //Produces a list of tokens in order of
-         //of occurrence in the string.
-      void token2(const char *str); //extended version (Jan-9-2008)
-      void cleara(void); //Clear the abbl & abbs memory of strings
-      void clear(void); //Clear the lst memory of words
-
-      //Application functions
-      void Proc(char *pch); //Accepts a natural language statement and
-         //processes to final results stored in tta, abbs, and abbl
-         //Need to call cleara function after each use of this function
-
-      // Internal routines:
-      // setup data for Test method
-      void setup_Test( void );
-      bool prefix_match( const char *str ); // does str begins with a prefix?
-
-      //Data
-      long tta; //Total possible abbreviations extracted
-         //default 10k
-      long numa; //number of abbreviations in current extract
-      char **abbl; //Long form space, hold up to 10 tokens
-      char **abbs; //Short form space, hold up to 10 tokens
-      Find_Seq seq;             // identify sequences to ignore
-      int  *nt; //Number of tokens within parentheses
-      long word_space; //Space in lst for tokens
-         //default 10k
-      long num; //Number of tokens
-      char **lst; //Holds the tokens
-
-      static const int cnam_size=100000;
-      char cnam[cnam_size]; //Work space
-      MPtok *pMt; //Pointer at tokenizer class. Used to segment text
-         //in Proc function.
-
-      // Test data
-      set<string> match;        // bad SF to match exactly
-      vector<string> prefix;    // bad SF to match prefix
-};
-}
-#endif
+#ifndef ABBRVE_H
+#define ABBRVE_H
+#include <fstream>
+#include <iostream>
+#include <runn.h>
+#include <MPtok.h>
+#include <vector>
+using namespace std;
+namespace iret {
+
+typedef vector<string> strings;
+
+
+class Find_Seq {
+public:
+
+  Find_Seq( void  );
+
+  // flag the SFs whether part of sequence or not
+  void flag_seq( int numa, char* abbs[] );
+
+  // true if good SF, false if part of sequence
+  bool rate( int i ) const { my_rate[i]; }
+
+private:
+  void find_seq( const vector<string> & seq );
+  void create_seq( void );
+
+  // const works with c++0x
+  /* const */ strings seq_i;
+  /* const */ strings seq_I;
+  /* const */ strings seq_a;
+  /* const */ strings seq_A;
+  
+  vector<bool> my_rate;
+  int my_numa;
+  char ** my_abbs;              // really char *[], but that doesn't work
+  
+};
+
+
+class AbbrvE {
+   public:
+      AbbrvE(long ta=10000,long wrd_spc=10000); //Sets space for extracted
+         //potential abbreviations to ta & word_space to wrd_spc
+     ~AbbrvE(void);
+      void Extract(char *pch); //Extracts possible long-short form
+         //pairs, but does not attempt to find the relationship
+      void Extract2(const char *pch); //extened version (Jan-9-2008)
+      bool Test(const char *str); //Tests a single token and returns true
+         //if the token should be a possible first token of a short form
+      void Rate(void); //Sets ratings for the proposed pairs. Effort to 
+         //remove (a), (b), etc., sequence markers
+      void token(const char *str); //Produces a list of tokens in order of
+         //of occurrence in the string.
+      void token2(const char *str); //extended version (Jan-9-2008)
+      void cleara(void); //Clear the abbl & abbs memory of strings
+      void clear(void); //Clear the lst memory of words
+
+      //Application functions
+      void Proc(char *pch); //Accepts a natural language statement and
+         //processes to final results stored in tta, abbs, and abbl
+         //Need to call cleara function after each use of this function
+
+      // Internal routines:
+      // setup data for Test method
+      void setup_Test( void );
+      bool prefix_match( const char *str ); // does str begins with a prefix?
+
+      //Data
+      long tta; //Total possible abbreviations extracted
+         //default 10k
+      long numa; //number of abbreviations in current extract
+      char **abbl; //Long form space, hold up to 10 tokens
+      char **abbs; //Short form space, hold up to 10 tokens
+      Find_Seq seq;             // identify sequences to ignore
+      int  *nt; //Number of tokens within parentheses
+      long word_space; //Space in lst for tokens
+         //default 10k
+      long num; //Number of tokens
+      char **lst; //Holds the tokens
+
+      static const int cnam_size=100000;
+      char cnam[cnam_size]; //Work space
+      MPtok *pMt; //Pointer at tokenizer class. Used to segment text
+         //in Proc function.
+
+      // Test data
+      set<string> match;        // bad SF to match exactly
+      vector<string> prefix;    // bad SF to match prefix
+};
+}
+#endif

Library/Btree.C

@@ -1,1304 +1,1304 @@
-#include <iostream>
-#include <fstream>
-#include <cstdio>
-#include <cstdlib>
-#include <cmath>
-#include <cstring>
-#include <cassert>
-#include "Btree.h"
-#include "runn.h"
-
-using namespace std;
-namespace iret {
-
-Node::Node(void){
-   str=NULL;
-   rel=NULL;
-   pdn=NULL;
-}
-
-Node::Node(const char *ptr){
-   int i=strlen(ptr);
-   str = new char[i+1];
-   strcpy(str,ptr);
-   rel=NULL;
-   pdn=NULL;
-}
-
-Node::Node(char const *ptr,void *dtr){
-   int i=strlen(ptr);
-   str = new char[i+1];
-   strcpy(str,ptr);
-   rel = dtr;
-   pdn=NULL;
-}
-
-Node::~Node(){
-   if(str)delete [] str;
-}
-
-void Node::debug(void){
-   cout << "Node {" << endl;
-   cout << "   str: " << this->str << endl;
-   if(rel==NULL)cout << "   rel: NULL" << endl;
-   else cout << "   rel: " << (long)rel << endl;
-   if(pdn==NULL)cout << "   pdn: NULL" << endl;
-   else cout << "   pdn: " << (long)pdn << endl;
-   cout << " }" << endl;
-}
-
-Page::Page(){
-   pdn=NULL;
-   ndnm='\0';
-}
-
-Page::Page(Page *const pz,Page *const pn,const int n){
-   pdn=pn;
-   int j=(int)(pz->ndnm)-n;
-   ndnm=(char)(j>0 ? j : 0);
-   for(int i=0;i<(int)ndnm;i++){pnd[i]=(pz->pnd)[n+i];}
-}
-
-Page::~Page(){ 
-   for(int i=0;i<(int)ndnm;i++){
-      delete pnd[i];
-   }
-}
-
-void Page::clean(void){
-   for(int i=0;i<(int)ndnm;i++){
-      pnd[i]->str=NULL;
-   }
-}
-
-void Page::insert(const int n,Node * const nd,const int j){
-   assert(j<ord2);
-   assert(n<=j);
-   if(n==j){
-      pnd[j]=nd;
-   } 
-   else {
-      for(int i=j;i>n;i--)pnd[i]=pnd[i-1];
-      pnd[n]=nd;
-   }   
-   ndnm++;
-}
-
-int Page::search(int &a,int &b,const char *str,int &p){
-   int j;
-   if((j=stc_my(a,b,str,pnd[0]->str))<0){
-      p=0;
-      return(0);
-   }
-   else if(j==0){
-      p=0;
-      return(1);
-   }
-   if((j=stc_my(a,b,str,pnd[(int)(ndnm-1)]->str))>0){
-      p=(int)ndnm;
-      return(0);
-   }
-   else if(j==0){
-      p=(int)(ndnm-1);
-      return(1);
-   }
-   int x=0,i;
-   int y=(int)(ndnm-1);
-   while(y-x>1){
-      i=(y+x)/2;
-      if((j=stc_my(a,b,str,pnd[i]->str))==0){p=i;return(1);}
-      else if(j<0)y=i;
-      else x=i;
-   }
-   p=y;
-   return(0);
-}
-
-int Page::search(int &a,int &b,char *str,int &p,Partial_match *btr){
-   int j;
-   if((j=btr->stc_my_long(a,b,str,pnd[0]->str,0))<0){
-      p=0;
-      return(0);
-   }
-   else if(j==0){
-      p=0;
-      return(1);
-   }
-   if((j=btr->stc_my_long(a,b,str,pnd[(int)(ndnm-1)]->str,(int)(ndnm-1)))>0){
-      p=(int)ndnm;
-      return(0);
-   }
-   else if(j==0){
-      p=(int)(ndnm-1);
-      return(1);
-   }
-   int x=0,i;
-   int y=(int)(ndnm-1);
-   while(y-x>1){
-      i=(y+x)/2;
-      if((j=btr->stc_my_long(a,b,str,pnd[i]->str,i))==0){p=i;return(1);}
-      else if(j<0)y=i;
-      else x=i;
-   }
-   p=y;
-   return(0);
-}
-
-void Page::debug(void){
-   cout << "Page {" << endl;
-   cout << "   ndnm: " << (int)ndnm << endl;
-   if(pdn==NULL)cout << "   pdn: NULL" << endl;
-   else cout << "   pdn: " << (long)pdn << endl;
-   for(int i=0;i<(int)ndnm;i++){
-      cout << i << " ";
-      (this->pnd[i])->debug();
-   }
-   cout << " }" << endl;
-}
-
-int stc_my(int &a,int &b,const char *str,const char *ptr)
-   {register int i=(a<b) ? a : b;
-   register const char *p1=str+i;
-   register const char *p2=ptr+i;
-   register int j=0;
-   while((*p1==*p2)&&(*p1!='\0')){
-      j++;
-      p1++;
-      p2++;
-   }
-   if(*p1==*p2)return(0);
-   else if(*p1<*p2){
-      b=i+j;
-      return(-1);
-   }
-   else {
-      a=i+j;
-      return(1);
-   }
-}
-
-Btree::Btree(){
-   iclean=0;
-   copy=false;
-   depth=0;
-   root=new Page;
-   root->ndnm = 1;
-   (root->pnd)[0]=new Node("");
-}
-
-int Btree::search(const char *str){
-   depth=-1;
-   Page *pu=root;
-   register int a=0,b=0,i,j;
-   while(pu!=NULL){
-      depth++;
-      pg[depth]=pu;
-      j=(pu->search)(a,b,str,i);
-      cnd[depth]=i;
-      if(j==1)return(1);
-      if(i==0)pu=pu->pdn;        
-      else pu=(pu->pnd)[i-1]->pdn;
-   }
-return(0);
-}
-
-int Btree::insert(Node *nd){
-   int w,k;
-   Page *pm,*pz;
-   while((nd!=NULL)&&(depth)){
-      pm=pg[depth];
-      w=pm->ndnm;
-      if(w<ord2){
-         pm->insert(cnd[depth],nd,w);
-         nd=NULL;
-      }
-      else {
-         k=cnd[depth];
-         if(k<order){
-             pz=new Page(pm,((pm->pnd)[order-1])->pdn,order);
-             pm->insert(k,nd,order);
-             nd=pm->pnd[order];
-             nd->pdn=pz;
-             pm->ndnm=order;
-         }
-         else if(k>order){
-             pz=new Page(pm,((pm->pnd)[order])->pdn,order+1);
-             pz->insert(k-order-1,nd,order-1);
-             nd=pm->pnd[order];
-             nd->pdn=pz;
-             pm->ndnm=order;
-         }
-         else {
-             pz=new Page(pm,nd->pdn,order);    
-             nd->pdn=pz;
-             pm->ndnm=order;
-         }
-      }
-   depth--;
-   }
-   if(nd!=NULL){
-      pm=pg[depth];
-      w=pm->ndnm;
-      if(w<ord2)pm->insert(cnd[depth],nd,w);
-      else {
-         root=new Page();
-         root->pdn=pm;
-         k=cnd[depth];
-         if(k<order){
-            pz=new Page(pm,((pm->pnd)[order-1])->pdn,order);
-            pm->insert(k,nd,order);
-            (root->pnd)[0]=pm->pnd[order];
-            ((root->pnd)[0])->pdn=pz;
-            root->ndnm=1;
-            pm->ndnm=order;
-         }
-         else if(k>order){
-            pz=new Page(pm,((pm->pnd)[order])->pdn,order+1);
-            pz->insert(k-order-1,nd,order-1);
-            (root->pnd)[0]=pm->pnd[order];
-            ((root->pnd)[0])->pdn=pz;
-            root->ndnm=1;
-            pm->ndnm=order;
-         }
-         else {
-            pz=new Page(pm,nd->pdn,order);
-            (root->pnd)[0]=nd;
-            nd->pdn=pz;
-            root->ndnm=1;
-            pm->ndnm=order;
-         }
-      }
-   }
-return(1);
-}
-            
-void Btree::node_first(void){
-   depth=0;
-   pg[depth]=root;
-   cnd[depth]=0;
-   Page *pm;
-   while((pm=(pg[depth]->pdn))!=NULL){
-      depth++;
-      pg[depth]=pm;
-      cnd[depth]=0;
-   }
-}
-
-int Btree::node_next(){
-   int i=cnd[depth];
-   Page *pd=((pg[depth]->pnd)[i])->pdn;
-   if(pd!=NULL){
-      (cnd[depth])++;
-      depth++;
-      pg[depth]=pd;
-      cnd[depth]=0;
-      while((pd=(pg[depth]->pdn))!=NULL){
-         depth++;
-         pg[depth]=pd;
-         cnd[depth]=0;
-      }
-   }
-   else {
-      cnd[depth]=++i;
-      while((depth>=1)&&(i==(pg[depth]->ndnm))){depth--;i=cnd[depth];}
-      if((depth==0)&&(i==(pg[depth]->ndnm)))depth--;
-      if(depth<0)return(0);
-   }
-return(1);
-}
-      
-char *Btree::show_str(){
-   return(((pg[depth]->pnd)[cnd[depth]])->str);
-}
-
-void *Btree::give_ptr(){
-   return(((pg[depth]->pnd)[cnd[depth]])->rel);
-}
-
-void Btree::set_ptr(void *dtr){
-   ((pg[depth]->pnd)[cnd[depth]])->rel=dtr;
-}
-
-Btree::~Btree(){
-   int pflag=get_qflag();
-   long k=0;
-   if (copy) return;           // only delete original
-   if(!iclean){
-      node_first();
-      int i=depth,j;
-      do{
-         j=node_next();
-         if(depth<i){
-            while(i>depth){
-               delete pg[i];
-               i--;
-               mark(pflag,++k,1000,"pages deleted");
-            }
-         }
-         else i=depth;
-      } while(j); 
-   }
-   else {
-      node_first();
-      int i=depth,j;
-      do{
-         j=node_next();
-         if(depth<i){
-            while(i>depth){
-               pg[i]->clean();
-               delete pg[i];
-               i--;
-               mark(pflag,++k,1000,"pages deleted");
-            }
-         }
-         else i=depth;
-      } while(j);
-   }
-}
-
-long Btree::list_write(ofstream &fout){
-   int pflag=get_qflag();
-   long ct=0;
-   node_first();
-   while(node_next()){
-      fout << show_str() << endl;
-      mark(pflag,++ct,1000,"strings written");
-   }
-   fout.close();
-   return((int)fout.good());
-}
-
-Btree::Btree(ifstream &fin){
-   copy=false;
-   char cnam[256];
-   int pflag=get_qflag();
-   depth=0;
-   pg[0]=root=new Page();
-   cnd[0]=root->ndnm = 1;
-   (root->pnd)[0]=new Node("");
-   Node *pno;
-   long ct=0;
-   while(get_string(cnam,fin,'\n')){
-      pno = new Node(cnam);  
-      add(pno);
-      mark(pflag,++ct,10000,"strings read");
-   }
-   fin.close();
-}
-
-int Btree::add(Node *nd){
-   int w,k,dp;
-   Page *pm,*pz;
-   dp=depth; //uses dp in place of depth in insert.
-   while((nd!=NULL)&&(dp)){
-      pm=pg[dp];
-      w=pm->ndnm;
-      if(w<ord2){
-         pm->insert(cnd[dp],nd,w);
-         nd=NULL;
-      (cnd[dp])++; //variation from insert.
-      }
-      else {
-         k=cnd[dp];
-         if(k<order){
-             pz=new Page(pm,((pm->pnd)[order-1])->pdn,order);
-             pm->insert(k,nd,order);
-             nd=pm->pnd[order];
-             nd->pdn=pz;
-             pm->ndnm=order;
-         }
-         else if(k>order){
-             pz=new Page(pm,((pm->pnd)[order])->pdn,order+1);
-             pz->insert(k-order-1,nd,order-1);
-             nd=pm->pnd[order];
-             nd->pdn=pz;
-             pm->ndnm=order;
-         }
-         else {
-             pz=new Page(pm,nd->pdn,order);    
-             nd->pdn=pz;
-             pm->ndnm=order;
-         }
-         pg[dp]=pz; //2 lines of variation from insert.
-         cnd[dp]=order;
-      }
-   dp--;
-   }
-   if(nd!=NULL){
-      pm=pg[dp];
-      w=pm->ndnm;
-      if(w<ord2){
-         pm->insert(cnd[dp],nd,w);
-         (cnd[dp])++; //variation from insert.
-      }
-      else {
-         root=new Page();
-         root->pdn=pm;
-         k=cnd[dp];
-         if(k<order){
-            pz=new Page(pm,((pm->pnd)[order-1])->pdn,order);
-            pm->insert(k,nd,order);
-            (root->pnd)[0]=pm->pnd[order];
-            ((root->pnd)[0])->pdn=pz;
-            root->ndnm=1;
-            pm->ndnm=order;
-         }
-         else if(k>order){
-            pz=new Page(pm,((pm->pnd)[order])->pdn,order+1);
-            pz->insert(k-order-1,nd,order-1);
-            (root->pnd)[0]=pm->pnd[order];
-            ((root->pnd)[0])->pdn=pz;
-            root->ndnm=1;
-            pm->ndnm=order;
-         }
-         else {
-            pz=new Page(pm,nd->pdn,order);
-            (root->pnd)[0]=nd;
-            nd->pdn=pz;
-            root->ndnm=1;
-            pm->ndnm=order;
-         }
-      next_empty(); //variation from insert.
-      }
-   }
-return(1);
-}
-            
-void Btree::next_empty(){
-   depth=0;
-   pg[depth]=root;
-   int i=cnd[depth]=root->ndnm;
-   Page *pm;
-   while((pm=((pg[depth]->pnd)[i-1])->pdn)!=NULL){
-      depth++;
-      pg[depth]=pm;
-      i=cnd[depth]=pm->ndnm;
-   }
-}
-
-Str_str::Str_str() : Btree() {
-}
-
-Str_str::~Str_str(){
-   if(copy)return;
-   this->node_first();
-   while(this->node_next())delete [] (char*)this->give_ptr();
-}
-
-void Str_str::add_pair(const char *one,const char *two){
-   Node *pnd;
-   if(search(one)){
-      cout << "Duplicate string in keys list = " << one << endl;
-      exit(0);
-   }   
-   else {
-      int i=strlen(two);
-      char *st=new char[i+1];
-      strcpy(st,two);
-      pnd=new Node(one,(void *)st);
-      add(pnd);
-   }
-}
-
-char *Str_str::match(const char *one){
-   if(search(one)){
-      return((char*)give_ptr());
-   }
-   else {
-      cout << "String not a key = " << one << endl;
-      exit(0);
-   }
-}
-
-List::List() : Btree() {
-   cnt_key=0;
-}
-
-List::~List(){
-}
-
-void List::add_key(const char *str){
-   Node *pnd;
-   if(!search(str)){
-      pnd=new Node(str);
-      add(pnd);
-   }
-}
-
-void List::add_key_count(const char *str){
-   Node *pnd;
-   if(!search(str)){
-      pnd=new Node(str);
-      add(pnd);
-      cnt_key++;
-   }
-}
-
-void List::addp_key_count(char *str){
-   Node *pnd;
-   if(!search(str)){
-      pnd=new Node;
-      pnd->str=str;
-      add(pnd);
-      cnt_key++;
-   }
-}
-
-Num_num::Num_num() : Btree() {
-}
-
-Num_num::~Num_num(){
-   if(copy)return;
-   this->node_first();
-   while(this->node_next())delete (long*)this->give_ptr();
-}
-
-void Num_num::add_pair(long i,long j){
-   Node *pnd;
-   char cnam[256];
-   long_str(cnam,i);
-   if(!search(cnam)){
-      long *st=new long;
-      *st=j;
-      pnd=new Node(cnam,(void *)st);
-      add(pnd);
-   }
-}
-
-long Num_num::match(long i){
-   char cnam[256];
-   long_str(cnam,i);
-   if(search(cnam)){
-      return(*((long*)give_ptr()));
-   }
-   else return(LNEG);
-}
-
-Count::Count() : List() {
-   total=0;
-}
-
-Count::~Count(){
-   if(copy)return;
-   long *pk;
-   this->node_first();
-   while(this->node_next()){
-      pk=(long*)(this->give_ptr());
-      if(pk)delete pk;
-   }
-}
-
-void Count::add_count(const char *pch,long n){
-   long *ppt;
-   Node *np;
-   total+=n;
-   if(this->search(pch)==0){
-      ppt = new long;
-      (*ppt) =n;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-   }
-   else {
-      (*(long*) this->give_ptr())+=n;
-   }
-}
-
-void Count::add_countz(const char *pch,long n){
-   long *ppt;
-   Node *np;
-   if(this->search(pch)==0){
-      ppt = new long;
-      (*ppt) =n;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      (*(long*) this->give_ptr())+=n;
-   }
-}
-
-void Count::add_count2(const char *pch,long n){
-   long *ppt;
-   Node *np;
-   total+=n;
-   if(this->search(pch)==0){
-      ppt = new long;
-      (*ppt) =n;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      (*(long*) this->give_ptr())+=n;
-   }
-}
-
-void Count::addp_count2(char *pch,long n){
-   long *ppt;
-   Node *np;
-   total+=n;
-   if(this->search(pch)==0){
-      ppt = new long;
-      (*ppt) =n;
-      np=new Node;
-      np->str=pch;
-      np->rel=ppt;
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      (*(long*) this->give_ptr())+=n;
-   }
-}
-
-void Count::correct(const char *pch,long n){
-   if(this->search(pch)){
-      (*(long*) this->give_ptr())=n;
-   }
-}
-
-long Count::count(const char *pch){
-   if(this->search(pch)==0){
-      return(0);
-   }
-   else {
-      return(*((long*) this->give_ptr()));
-   }
-}
-
-long Count::count(void){
-   return(*((long*) this->give_ptr()));
-}
-
-void Count::max_count(const char *pch,long n){
-   long *ppt,i;
-   Node *np;
-   total+=n;
-   if(!search(pch)){
-      ppt = new long;
-      (*ppt) =n;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-   }
-   else {
-      ppt=(long*)give_ptr();
-      if(*ppt<n)*ppt=n;
-   }
-}
-
-void Count::max_count2(const char *pch,long n){
-   long *ppt,i;
-   Node *np;
-   total+=n;
-   if(!search(pch)){
-      ppt = new long;
-      (*ppt) =n;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      ppt=(long*)give_ptr();
-      if(*ppt<n)*ppt=n;
-   }
-}
-
-void Count::maxp_count2(char *pch,long n){
-   long *ppt,i;
-   Node *np;
-   total+=n;
-   if(!search(pch)){
-      ppt = new long;
-      (*ppt) =n;
-      np=new Node;
-      np->str=pch;
-      np->rel=ppt;
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      ppt=(long*)give_ptr();
-      if(*ppt<n)*ppt=n;
-   }
-}
-
-void Count::min_count(const char *pch,long n){
-   long *ppt,i;
-   Node *np;
-   total+=n;
-   if(!search(pch)){
-      ppt = new long;
-      (*ppt) =n;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-   }
-   else {
-      ppt=(long*)give_ptr();
-      if(*ppt>n)*ppt=n;
-   }
-}
-
-void Count::min_count2(const char *pch,long n){
-   long *ppt,i;
-   Node *np;
-   total+=n;
-   if(!search(pch)){
-      ppt = new long;
-      (*ppt) =n;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      ppt=(long*)give_ptr();
-      if(*ppt>n)*ppt=n;
-   }
-}
-
-void Count::minp_count2(char *pch,long n){
-   long *ppt,i;
-   Node *np;
-   total+=n;
-   if(!search(pch)){
-      ppt = new long;
-      (*ppt) =n;
-      np=new Node;
-      np->str=pch;
-      np->rel=ppt;
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      ppt=(long*)give_ptr();
-      if(*ppt>n)*ppt=n;
-   }
-}
-
-//FCount (float count tree)
-
-FCount::FCount() : List() {
-   total=0;
-}
-
-FCount::~FCount(){
-   if(copy)return;
-   float *pk;
-   this->node_first();
-   while(this->node_next()){
-      pk=(float*)(this->give_ptr());
-      if(pk)delete pk;
-   }
-}
-
-void FCount::Copy(FCount &Fc){
-   char *pch;
-   float *xx,*zz;
-   Node *pN;
-
-   pg[0]=root;
-   cnd[0]=root->ndnm;
-
-   Fc.node_first();
-   while(Fc.node_next()){
-      pch=Fc.show_str();
-      xx=(float*)Fc.give_ptr();
-      zz=new float;
-      *zz=*xx;
-      pN=new Node(pch,(void*)zz);
-      add(pN);
-   }
-}
-      
-void FCount::add_count(const char *pch,float z){
-   float *ppt;
-   Node *np;
-   total+=z;
-   if(this->search(pch)==0){
-      ppt = new float;
-      (*ppt) =z;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-   }
-   else {
-      (*(float*) this->give_ptr())+=z;
-   }
-}
-
-void FCount::add_count2(const char *pch,float z){
-   float *ppt;
-   Node *np;
-   total+=z;
-   if(this->search(pch)==0){
-      ppt = new float;
-      (*ppt) =z;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      (*(float*) this->give_ptr())+=z;
-   }
-}
-
-void FCount::addp_count2(char *pch,float z){
-   float *ppt;
-   Node *np;
-   total+=z;
-   if(this->search(pch)==0){
-      ppt = new float;
-      (*ppt) =z;
-      np=new Node;
-      np->str=pch;
-      np->rel=ppt;
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      (*(float*) this->give_ptr())+=z;
-   }
-}
-
-float FCount::count(const char *pch){
-   if(this->search(pch)==0){
-      return(0);
-   }
-   else {
-      return(*((float*) this->give_ptr()));
-   }
-}
-
-float FCount::count(void){
-   return(*((float*) this->give_ptr()));
-}
-
-//DCount (double precision count tree)
-
-DCount::DCount() : List() {
-   total=0;
-}
-
-DCount::~DCount(){
-   if(copy)return;
-   double *pk;
-   this->node_first();
-   while(this->node_next()){
-      pk=(double*)(this->give_ptr());
-      if(pk)delete pk;
-   }
-}
-
-void DCount::Copy(DCount &Dc){
-   char *pch;
-   double *xx,*zz;
-   Node *pN;
-
-   pg[0]=root;
-   cnd[0]=root->ndnm;
-
-   Dc.node_first();
-   while(Dc.node_next()){
-      pch=Dc.show_str();
-      xx=(double*)Dc.give_ptr();
-      zz=new double;
-      *zz=*xx;
-      pN=new Node(pch,(void*)zz);
-      add(pN);
-   }
-}
-      
-void DCount::add_count(const char *pch,double z){
-   double *ppt;
-   Node *np;
-   total+=z;
-   if(this->search(pch)==0){
-      ppt = new double;
-      (*ppt) =z;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-   }
-   else {
-      (*(double*) this->give_ptr())+=z;
-   }
-}
-
-void DCount::add_count2(const char *pch,double z){
-   double *ppt;
-   Node *np;
-   total+=z;
-   if(this->search(pch)==0){
-      ppt = new double;
-      (*ppt) =z;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      (*(double*) this->give_ptr())+=z;
-   }
-}
-
-void DCount::addp_count2(char *pch,double z){
-   double *ppt;
-   Node *np;
-   total+=z;
-   if(this->search(pch)==0){
-      ppt = new double;
-      (*ppt) =z;
-      np=new Node;
-      np->str=pch;
-      np->rel=ppt;
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      (*(double*) this->give_ptr())+=z;
-   }
-}
-
-double DCount::count(const char *pch){
-   if(this->search(pch)==0){
-      return(0);
-   }
-   else {
-      return(*((double*) this->give_ptr()));
-   }
-}
-
-double DCount::count(void){
-   return(*((double*) this->give_ptr()));
-}
-
-void DCount::max_count(const char *pch,double z){
-   double *ppt;
-   Node *np;
-   total+=z;
-   if(!search(pch)){
-      ppt = new double;
-      (*ppt) =z;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-   }
-   else {
-      ppt=(double*)give_ptr();
-      if(*ppt<z)*ppt=z;
-   }
-}
-
-void DCount::max_count2(const char *pch,double z){
-   double *ppt;
-   Node *np;
-   total+=z;
-   if(!search(pch)){
-      ppt = new double;
-      (*ppt) =z;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      ppt=(double*)give_ptr();
-      if(*ppt<z)*ppt=z;
-   }
-}
-
-void DCount::maxp_count2(char *pch,double z){
-   double *ppt;
-   Node *np;
-   total+=z;
-   if(!search(pch)){
-      ppt = new double;
-      (*ppt) =z;
-      np=new Node;
-      np->str=pch;
-      np->rel=ppt;
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      ppt=(double*)give_ptr();
-      if(*ppt<z)*ppt=z;
-   }
-}
-
-void DCount::min_count(const char *pch,double z){
-   double *ppt;
-   Node *np;
-   total+=z;
-   if(!search(pch)){
-      ppt = new double;
-      (*ppt) =z;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-   }
-   else {
-      ppt=(double*)give_ptr();
-      if(*ppt>z)*ppt=z;
-   }
-}
-
-void DCount::min_count2(const char *pch,double z){
-   double *ppt;
-   Node *np;
-   total+=z;
-   if(!search(pch)){
-      ppt = new double;
-      (*ppt) =z;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      ppt=(double*)give_ptr();
-      if(*ppt>z)*ppt=z;
-   }
-}
-
-void DCount::minp_count2(char *pch,double z){
-   double *ppt;
-   Node *np;
-   total+=z;
-   if(!search(pch)){
-      ppt = new double;
-      (*ppt) =z;
-      np=new Node;
-      np->str=pch;
-      np->rel=ppt;
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      ppt=(double*)give_ptr();
-      if(*ppt>z)*ppt=z;
-   }
-}
-
-void DCount::debug(void){
-   node_first();
-   while(node_next()){
-      cout << count() << " " << show_str() << endl;
-   }
-}
-
-//Partial Match
-
-Partial_match::Partial_match() : Count() {
-}
-
-Partial_match::~Partial_match(){
-}
-
-void Partial_match::long_match(char *str,List &Lst){
-   char *pch;
-   while(*str!='\0'){
-      if(this->search_long(str)){
-         pch=this->show_str();
-         Lst.add_key_count(pch);
-      }
-      if((pch=strchr(str,' '))!=NULL)str=pch+1;
-      else str=str+strlen(str);
-   }   
-}
-
-void Partial_match::local_match(char *str,List &Lst){
-   char *pch;
-   int i,j;
-   if(*str!='\0'){
-      if(this->search_long(str)){
-         pch=this->show_str();
-         Lst.add_key_count(pch);
-         i=strlen(pch)-1;
-         while(0<i){
-            while((0<i)&&(*(str+i)!=' '))i--;
-            if(0<i){
-               *(str+i)='\0';
-               j=this->search(str);
-               *(str+i)=' ';
-               if(j){
-                  pch=this->show_str();
-                  Lst.add_key_count(pch);
-               }
-               i--;
-            }
-         }
-      }
-   }
-}
-
-void Partial_match::all_match(char *str,List &Lst){
-   char *pch;
-   int i,j;
-   while(*str!='\0'){
-      if(this->search_long(str)){
-         pch=this->show_str();
-         Lst.add_key_count(pch);
-         i=strlen(pch)-1;
-         while(0<i){
-            while((0<i)&&(*(str+i)!=' '))i--;
-            if(0<i){
-               *(str+i)='\0';
-               j=this->search(str);
-               *(str+i)=' ';
-               if(j){
-                  pch=this->show_str();
-                  Lst.add_key_count(pch);
-               }
-               i--;
-            }
-         }
-      }
-      if((pch=strchr(str,' '))!=NULL)str=pch+1;
-      else str=str+strlen(str);
-   }   
-}
-
-void Partial_match::long_match(char *str,Count &Cnt,long n){
-   char *pch;
-   while(*str!='\0'){
-      if(this->search_long(str)){
-         pch=this->show_str();
-         Cnt.add_count2(pch,n);
-      }
-      if((pch=strchr(str,' '))!=NULL)str=pch+1;
-      else str=str+strlen(str);
-   }   
-}
-
-void Partial_match::local_match(char *str,Count &Cnt,long n){
-   char *pch;
-   int i,j;
-   if(*str!='\0'){
-      if(this->search_long(str)){
-         pch=this->show_str();
-         Cnt.add_count2(pch,n);
-         i=strlen(pch)-1;
-         while(0<i){
-            while((0<i)&&(*(str+i)!=' '))i--;
-            if(0<i){
-               *(str+i)='\0';
-               j=this->search(str);
-               *(str+i)=' ';
-               if(j){
-                  pch=this->show_str();
-                  Cnt.add_count2(pch,n);
-               }
-               i--;
-            }
-         }
-      }
-   } 
-}
-
-void Partial_match::all_match(char *str,Count &Cnt,long n){
-   char *pch;
-   int i,j;
-   while(*str!='\0'){
-      if(this->search_long(str)){
-         pch=this->show_str();
-         Cnt.add_count2(pch,n);
-         i=strlen(pch)-1;
-         while(0<i){
-            while((0<i)&&(*(str+i)!=' '))i--;
-            if(0<i){
-               *(str+i)='\0';
-               j=this->search(str);
-               *(str+i)=' ';
-               if(j){
-                  pch=this->show_str();
-                  Cnt.add_count2(pch,n);
-               }
-               i--;
-            }
-         }
-      }
-      if((pch=strchr(str,' '))!=NULL)str=pch+1;
-      else str=str+strlen(str);
-   }   
-}
-
-int Partial_match::search_long(char *str){
-   int a=0,b=0,i,j;
-   len=strlen(str);
-   if(this->step_one(a,b,str))return(1);
-   i=(a<b)?b:a;
-   while(cln_o<i){
-      while((cln_o<i)&&(*(str+i)!=' '))i--;
-      if(cln_o<i){
-         *(str+i)='\0';
-         j=this->search(str);
-         *(str+i)=' ';
-         if(j)return(1);
-         i--;
-      }
-   }
-   if(cln_o){
-      depth=depth_o;
-      cnd[depth]=index_o;
-      return(1);
-   }
-   else return(0);
-}
-
-int Partial_match::step_one(int &a,int &b,char *str){
-   char c;
-   cln_o=0;
-   cln=0;
-   while((c=*(str+cln))&&c!=32)cln++;
-   *(str+cln)='\0';
-   depth=-1;
-   Page *pu=root;
-   int i,j;
-   while(pu!=NULL){
-      depth++;
-      pg[depth]=pu;
-      j=(pu->search)(a,b,str,i,this);
-      cnd[depth]=i;
-      if(j==1)return(1);
-      if(i==0)pu=pu->pdn;        
-      else pu=(pu->pnd)[i-1]->pdn;
-   }
-
-if(cln<len)*(str+cln)=' ';
-return(0);
-}
-
-int Partial_match::stc_my_long(int &a,int &b,char *str,const char *ptr,int index)
-   {char c;
-   int i=(a<b) ? a : b;
-   const char *p1=str+i;
-   const char *p2=ptr+i;
-   int j=0;
-   while((*p1==*p2)&&(*p1!='\0')){
-      j++;
-      p1++;
-      p2++;
-      if((*p1=='\0'&&*p2!='\0')&&(cln<len)){
-         *(str+cln++)=' ';
-         while((c=*(str+cln))&&c!=32)cln++;
-         *(str+cln)='\0';
-      }
-   }
-   if(*p1==*p2){
-      if(cln<len){
-         depth_o=depth;
-         index_o=index;
-         cln_o=cln;
-         *(str+cln++)=' ';
-         while((c=*(str+cln))&&c!=32)cln++;
-         *(str+cln)='\0';
-         a=i+j;
-         return(1);
-      }
-      else return(0);
-   }
-   else if(*p1<*p2){
-      b=i+j;
-      return(-1);
-   }
-   else {
-      a=i+j;
-      return(1);
-   }
-}
-
-}
+#include <iostream>
+#include <fstream>
+#include <cstdio>
+#include <cstdlib>
+#include <cmath>
+#include <cstring>
+#include <cassert>
+#include "Btree.h"
+#include "runn.h"
+
+using namespace std;
+namespace iret {
+
+Node::Node(void){
+   str=NULL;
+   rel=NULL;
+   pdn=NULL;
+}
+
+Node::Node(const char *ptr){
+   int i=strlen(ptr);
+   str = new char[i+1];
+   strcpy(str,ptr);
+   rel=NULL;
+   pdn=NULL;
+}
+
+Node::Node(char const *ptr,void *dtr){
+   int i=strlen(ptr);
+   str = new char[i+1];
+   strcpy(str,ptr);
+   rel = dtr;
+   pdn=NULL;
+}
+
+Node::~Node(){
+   if(str)delete [] str;
+}
+
+void Node::debug(void){
+   cout << "Node {" << endl;
+   cout << "   str: " << this->str << endl;
+   if(rel==NULL)cout << "   rel: NULL" << endl;
+   else cout << "   rel: " << (long)rel << endl;
+   if(pdn==NULL)cout << "   pdn: NULL" << endl;
+   else cout << "   pdn: " << (long)pdn << endl;
+   cout << " }" << endl;
+}
+
+Page::Page(){
+   pdn=NULL;
+   ndnm='\0';
+}
+
+Page::Page(Page *const pz,Page *const pn,const int n){
+   pdn=pn;
+   int j=(int)(pz->ndnm)-n;
+   ndnm=(char)(j>0 ? j : 0);
+   for(int i=0;i<(int)ndnm;i++){pnd[i]=(pz->pnd)[n+i];}
+}
+
+Page::~Page(){ 
+   for(int i=0;i<(int)ndnm;i++){
+      delete pnd[i];
+   }
+}
+
+void Page::clean(void){
+   for(int i=0;i<(int)ndnm;i++){
+      pnd[i]->str=NULL;
+   }
+}
+
+void Page::insert(const int n,Node * const nd,const int j){
+   assert(j<ord2);
+   assert(n<=j);
+   if(n==j){
+      pnd[j]=nd;
+   } 
+   else {
+      for(int i=j;i>n;i--)pnd[i]=pnd[i-1];
+      pnd[n]=nd;
+   }   
+   ndnm++;
+}
+
+int Page::search(int &a,int &b,const char *str,int &p){
+   int j;
+   if((j=stc_my(a,b,str,pnd[0]->str))<0){
+      p=0;
+      return(0);
+   }
+   else if(j==0){
+      p=0;
+      return(1);
+   }
+   if((j=stc_my(a,b,str,pnd[(int)(ndnm-1)]->str))>0){
+      p=(int)ndnm;
+      return(0);
+   }
+   else if(j==0){
+      p=(int)(ndnm-1);
+      return(1);
+   }
+   int x=0,i;
+   int y=(int)(ndnm-1);
+   while(y-x>1){
+      i=(y+x)/2;
+      if((j=stc_my(a,b,str,pnd[i]->str))==0){p=i;return(1);}
+      else if(j<0)y=i;
+      else x=i;
+   }
+   p=y;
+   return(0);
+}
+
+int Page::search(int &a,int &b,char *str,int &p,Partial_match *btr){
+   int j;
+   if((j=btr->stc_my_long(a,b,str,pnd[0]->str,0))<0){
+      p=0;
+      return(0);
+   }
+   else if(j==0){
+      p=0;
+      return(1);
+   }
+   if((j=btr->stc_my_long(a,b,str,pnd[(int)(ndnm-1)]->str,(int)(ndnm-1)))>0){
+      p=(int)ndnm;
+      return(0);
+   }
+   else if(j==0){
+      p=(int)(ndnm-1);
+      return(1);
+   }
+   int x=0,i;
+   int y=(int)(ndnm-1);
+   while(y-x>1){
+      i=(y+x)/2;
+      if((j=btr->stc_my_long(a,b,str,pnd[i]->str,i))==0){p=i;return(1);}
+      else if(j<0)y=i;
+      else x=i;
+   }
+   p=y;
+   return(0);
+}
+
+void Page::debug(void){
+   cout << "Page {" << endl;
+   cout << "   ndnm: " << (int)ndnm << endl;
+   if(pdn==NULL)cout << "   pdn: NULL" << endl;
+   else cout << "   pdn: " << (long)pdn << endl;
+   for(int i=0;i<(int)ndnm;i++){
+      cout << i << " ";
+      (this->pnd[i])->debug();
+   }
+   cout << " }" << endl;
+}
+
+int stc_my(int &a,int &b,const char *str,const char *ptr)
+   {register int i=(a<b) ? a : b;
+   register const char *p1=str+i;
+   register const char *p2=ptr+i;
+   register int j=0;
+   while((*p1==*p2)&&(*p1!='\0')){
+      j++;
+      p1++;
+      p2++;
+   }
+   if(*p1==*p2)return(0);
+   else if(*p1<*p2){
+      b=i+j;
+      return(-1);
+   }
+   else {
+      a=i+j;
+      return(1);
+   }
+}
+
+Btree::Btree(){
+   iclean=0;
+   copy=false;
+   depth=0;
+   root=new Page;
+   root->ndnm = 1;
+   (root->pnd)[0]=new Node("");
+}
+
+int Btree::search(const char *str){
+   depth=-1;
+   Page *pu=root;
+   register int a=0,b=0,i,j;
+   while(pu!=NULL){
+      depth++;
+      pg[depth]=pu;
+      j=(pu->search)(a,b,str,i);
+      cnd[depth]=i;
+      if(j==1)return(1);
+      if(i==0)pu=pu->pdn;        
+      else pu=(pu->pnd)[i-1]->pdn;
+   }
+return(0);
+}
+
+int Btree::insert(Node *nd){
+   int w,k;
+   Page *pm,*pz;
+   while((nd!=NULL)&&(depth)){
+      pm=pg[depth];
+      w=pm->ndnm;
+      if(w<ord2){
+         pm->insert(cnd[depth],nd,w);
+         nd=NULL;
+      }
+      else {
+         k=cnd[depth];
+         if(k<order){
+             pz=new Page(pm,((pm->pnd)[order-1])->pdn,order);
+             pm->insert(k,nd,order);
+             nd=pm->pnd[order];
+             nd->pdn=pz;
+             pm->ndnm=order;
+         }
+         else if(k>order){
+             pz=new Page(pm,((pm->pnd)[order])->pdn,order+1);
+             pz->insert(k-order-1,nd,order-1);
+             nd=pm->pnd[order];
+             nd->pdn=pz;
+             pm->ndnm=order;
+         }
+         else {
+             pz=new Page(pm,nd->pdn,order);    
+             nd->pdn=pz;
+             pm->ndnm=order;
+         }
+      }
+   depth--;
+   }
+   if(nd!=NULL){
+      pm=pg[depth];
+      w=pm->ndnm;
+      if(w<ord2)pm->insert(cnd[depth],nd,w);
+      else {
+         root=new Page();
+         root->pdn=pm;
+         k=cnd[depth];
+         if(k<order){
+            pz=new Page(pm,((pm->pnd)[order-1])->pdn,order);
+            pm->insert(k,nd,order);
+            (root->pnd)[0]=pm->pnd[order];
+            ((root->pnd)[0])->pdn=pz;
+            root->ndnm=1;
+            pm->ndnm=order;
+         }
+         else if(k>order){
+            pz=new Page(pm,((pm->pnd)[order])->pdn,order+1);
+            pz->insert(k-order-1,nd,order-1);
+            (root->pnd)[0]=pm->pnd[order];
+            ((root->pnd)[0])->pdn=pz;
+            root->ndnm=1;
+            pm->ndnm=order;
+         }
+         else {
+            pz=new Page(pm,nd->pdn,order);
+            (root->pnd)[0]=nd;
+            nd->pdn=pz;
+            root->ndnm=1;
+            pm->ndnm=order;
+         }
+      }
+   }
+return(1);
+}
+            
+void Btree::node_first(void){
+   depth=0;
+   pg[depth]=root;
+   cnd[depth]=0;
+   Page *pm;
+   while((pm=(pg[depth]->pdn))!=NULL){
+      depth++;
+      pg[depth]=pm;
+      cnd[depth]=0;
+   }
+}
+
+int Btree::node_next(){
+   int i=cnd[depth];
+   Page *pd=((pg[depth]->pnd)[i])->pdn;
+   if(pd!=NULL){
+      (cnd[depth])++;
+      depth++;
+      pg[depth]=pd;
+      cnd[depth]=0;
+      while((pd=(pg[depth]->pdn))!=NULL){
+         depth++;
+         pg[depth]=pd;
+         cnd[depth]=0;
+      }
+   }
+   else {
+      cnd[depth]=++i;
+      while((depth>=1)&&(i==(pg[depth]->ndnm))){depth--;i=cnd[depth];}
+      if((depth==0)&&(i==(pg[depth]->ndnm)))depth--;
+      if(depth<0)return(0);
+   }
+return(1);
+}
+      
+char *Btree::show_str(){
+   return(((pg[depth]->pnd)[cnd[depth]])->str);
+}
+
+void *Btree::give_ptr(){
+   return(((pg[depth]->pnd)[cnd[depth]])->rel);
+}
+
+void Btree::set_ptr(void *dtr){
+   ((pg[depth]->pnd)[cnd[depth]])->rel=dtr;
+}
+
+Btree::~Btree(){
+   int pflag=get_qflag();
+   long k=0;
+   if (copy) return;           // only delete original
+   if(!iclean){
+      node_first();
+      int i=depth,j;
+      do{
+         j=node_next();
+         if(depth<i){
+            while(i>depth){
+               delete pg[i];
+               i--;
+               mark(pflag,++k,1000,"pages deleted");
+            }
+         }
+         else i=depth;
+      } while(j); 
+   }
+   else {
+      node_first();
+      int i=depth,j;
+      do{
+         j=node_next();
+         if(depth<i){
+            while(i>depth){
+               pg[i]->clean();
+               delete pg[i];
+               i--;
+               mark(pflag,++k,1000,"pages deleted");
+            }
+         }
+         else i=depth;
+      } while(j);
+   }
+}
+
+long Btree::list_write(ofstream &fout){
+   int pflag=get_qflag();
+   long ct=0;
+   node_first();
+   while(node_next()){
+      fout << show_str() << endl;
+      mark(pflag,++ct,1000,"strings written");
+   }
+   fout.close();
+   return((int)fout.good());
+}
+
+Btree::Btree(ifstream &fin){
+   copy=false;
+   char cnam[256];
+   int pflag=get_qflag();
+   depth=0;
+   pg[0]=root=new Page();
+   cnd[0]=root->ndnm = 1;
+   (root->pnd)[0]=new Node("");
+   Node *pno;
+   long ct=0;
+   while(get_string(cnam,fin,'\n')){
+      pno = new Node(cnam);  
+      add(pno);
+      mark(pflag,++ct,10000,"strings read");
+   }
+   fin.close();
+}
+
+int Btree::add(Node *nd){
+   int w,k,dp;
+   Page *pm,*pz;
+   dp=depth; //uses dp in place of depth in insert.
+   while((nd!=NULL)&&(dp)){
+      pm=pg[dp];
+      w=pm->ndnm;
+      if(w<ord2){
+         pm->insert(cnd[dp],nd,w);
+         nd=NULL;
+      (cnd[dp])++; //variation from insert.
+      }
+      else {
+         k=cnd[dp];
+         if(k<order){
+             pz=new Page(pm,((pm->pnd)[order-1])->pdn,order);
+             pm->insert(k,nd,order);
+             nd=pm->pnd[order];
+             nd->pdn=pz;
+             pm->ndnm=order;
+         }
+         else if(k>order){
+             pz=new Page(pm,((pm->pnd)[order])->pdn,order+1);
+             pz->insert(k-order-1,nd,order-1);
+             nd=pm->pnd[order];
+             nd->pdn=pz;
+             pm->ndnm=order;
+         }
+         else {
+             pz=new Page(pm,nd->pdn,order);    
+             nd->pdn=pz;
+             pm->ndnm=order;
+         }
+         pg[dp]=pz; //2 lines of variation from insert.
+         cnd[dp]=order;
+      }
+   dp--;
+   }
+   if(nd!=NULL){
+      pm=pg[dp];
+      w=pm->ndnm;
+      if(w<ord2){
+         pm->insert(cnd[dp],nd,w);
+         (cnd[dp])++; //variation from insert.
+      }
+      else {
+         root=new Page();
+         root->pdn=pm;
+         k=cnd[dp];
+         if(k<order){
+            pz=new Page(pm,((pm->pnd)[order-1])->pdn,order);
+            pm->insert(k,nd,order);
+            (root->pnd)[0]=pm->pnd[order];
+            ((root->pnd)[0])->pdn=pz;
+            root->ndnm=1;
+            pm->ndnm=order;
+         }
+         else if(k>order){
+            pz=new Page(pm,((pm->pnd)[order])->pdn,order+1);
+            pz->insert(k-order-1,nd,order-1);
+            (root->pnd)[0]=pm->pnd[order];
+            ((root->pnd)[0])->pdn=pz;
+            root->ndnm=1;
+            pm->ndnm=order;
+         }
+         else {
+            pz=new Page(pm,nd->pdn,order);
+            (root->pnd)[0]=nd;
+            nd->pdn=pz;
+            root->ndnm=1;
+            pm->ndnm=order;
+         }
+      next_empty(); //variation from insert.
+      }
+   }
+return(1);
+}
+            
+void Btree::next_empty(){
+   depth=0;
+   pg[depth]=root;
+   int i=cnd[depth]=root->ndnm;
+   Page *pm;
+   while((pm=((pg[depth]->pnd)[i-1])->pdn)!=NULL){
+      depth++;
+      pg[depth]=pm;
+      i=cnd[depth]=pm->ndnm;
+   }
+}
+
+Str_str::Str_str() : Btree() {
+}
+
+Str_str::~Str_str(){
+   if(copy)return;
+   this->node_first();
+   while(this->node_next())delete [] (char*)this->give_ptr();
+}
+
+void Str_str::add_pair(const char *one,const char *two){
+   Node *pnd;
+   if(search(one)){
+      cout << "Duplicate string in keys list = " << one << endl;
+      exit(0);
+   }   
+   else {
+      int i=strlen(two);
+      char *st=new char[i+1];
+      strcpy(st,two);
+      pnd=new Node(one,(void *)st);
+      add(pnd);
+   }
+}
+
+char *Str_str::match(const char *one){
+   if(search(one)){
+      return((char*)give_ptr());
+   }
+   else {
+      cout << "String not a key = " << one << endl;
+      exit(0);
+   }
+}
+
+List::List() : Btree() {
+   cnt_key=0;
+}
+
+List::~List(){
+}
+
+void List::add_key(const char *str){
+   Node *pnd;
+   if(!search(str)){
+      pnd=new Node(str);
+      add(pnd);
+   }
+}
+
+void List::add_key_count(const char *str){
+   Node *pnd;
+   if(!search(str)){
+      pnd=new Node(str);
+      add(pnd);
+      cnt_key++;
+   }
+}
+
+void List::addp_key_count(char *str){
+   Node *pnd;
+   if(!search(str)){
+      pnd=new Node;
+      pnd->str=str;
+      add(pnd);
+      cnt_key++;
+   }
+}
+
+Num_num::Num_num() : Btree() {
+}
+
+Num_num::~Num_num(){
+   if(copy)return;
+   this->node_first();
+   while(this->node_next())delete (long*)this->give_ptr();
+}
+
+void Num_num::add_pair(long i,long j){
+   Node *pnd;
+   char cnam[256];
+   long_str(cnam,i);
+   if(!search(cnam)){
+      long *st=new long;
+      *st=j;
+      pnd=new Node(cnam,(void *)st);
+      add(pnd);
+   }
+}
+
+long Num_num::match(long i){
+   char cnam[256];
+   long_str(cnam,i);
+   if(search(cnam)){
+      return(*((long*)give_ptr()));
+   }
+   else return(LNEG);
+}
+
+Count::Count() : List() {
+   total=0;
+}
+
+Count::~Count(){
+   if(copy)return;
+   long *pk;
+   this->node_first();
+   while(this->node_next()){
+      pk=(long*)(this->give_ptr());
+      if(pk)delete pk;
+   }
+}
+
+void Count::add_count(const char *pch,long n){
+   long *ppt;
+   Node *np;
+   total+=n;
+   if(this->search(pch)==0){
+      ppt = new long;
+      (*ppt) =n;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+   }
+   else {
+      (*(long*) this->give_ptr())+=n;
+   }
+}
+
+void Count::add_countz(const char *pch,long n){
+   long *ppt;
+   Node *np;
+   if(this->search(pch)==0){
+      ppt = new long;
+      (*ppt) =n;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      (*(long*) this->give_ptr())+=n;
+   }
+}
+
+void Count::add_count2(const char *pch,long n){
+   long *ppt;
+   Node *np;
+   total+=n;
+   if(this->search(pch)==0){
+      ppt = new long;
+      (*ppt) =n;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      (*(long*) this->give_ptr())+=n;
+   }
+}
+
+void Count::addp_count2(char *pch,long n){
+   long *ppt;
+   Node *np;
+   total+=n;
+   if(this->search(pch)==0){
+      ppt = new long;
+      (*ppt) =n;
+      np=new Node;
+      np->str=pch;
+      np->rel=ppt;
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      (*(long*) this->give_ptr())+=n;
+   }
+}
+
+void Count::correct(const char *pch,long n){
+   if(this->search(pch)){
+      (*(long*) this->give_ptr())=n;
+   }
+}
+
+long Count::count(const char *pch){
+   if(this->search(pch)==0){
+      return(0);
+   }
+   else {
+      return(*((long*) this->give_ptr()));
+   }
+}
+
+long Count::count(void){
+   return(*((long*) this->give_ptr()));
+}
+
+void Count::max_count(const char *pch,long n){
+   long *ppt,i;
+   Node *np;
+   total+=n;
+   if(!search(pch)){
+      ppt = new long;
+      (*ppt) =n;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+   }
+   else {
+      ppt=(long*)give_ptr();
+      if(*ppt<n)*ppt=n;
+   }
+}
+
+void Count::max_count2(const char *pch,long n){
+   long *ppt,i;
+   Node *np;
+   total+=n;
+   if(!search(pch)){
+      ppt = new long;
+      (*ppt) =n;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      ppt=(long*)give_ptr();
+      if(*ppt<n)*ppt=n;
+   }
+}
+
+void Count::maxp_count2(char *pch,long n){
+   long *ppt,i;
+   Node *np;
+   total+=n;
+   if(!search(pch)){
+      ppt = new long;
+      (*ppt) =n;
+      np=new Node;
+      np->str=pch;
+      np->rel=ppt;
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      ppt=(long*)give_ptr();
+      if(*ppt<n)*ppt=n;
+   }
+}
+
+void Count::min_count(const char *pch,long n){
+   long *ppt,i;
+   Node *np;
+   total+=n;
+   if(!search(pch)){
+      ppt = new long;
+      (*ppt) =n;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+   }
+   else {
+      ppt=(long*)give_ptr();
+      if(*ppt>n)*ppt=n;
+   }
+}
+
+void Count::min_count2(const char *pch,long n){
+   long *ppt,i;
+   Node *np;
+   total+=n;
+   if(!search(pch)){
+      ppt = new long;
+      (*ppt) =n;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      ppt=(long*)give_ptr();
+      if(*ppt>n)*ppt=n;
+   }
+}
+
+void Count::minp_count2(char *pch,long n){
+   long *ppt,i;
+   Node *np;
+   total+=n;
+   if(!search(pch)){
+      ppt = new long;
+      (*ppt) =n;
+      np=new Node;
+      np->str=pch;
+      np->rel=ppt;
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      ppt=(long*)give_ptr();
+      if(*ppt>n)*ppt=n;
+   }
+}
+
+//FCount (float count tree)
+
+FCount::FCount() : List() {
+   total=0;
+}
+
+FCount::~FCount(){
+   if(copy)return;
+   float *pk;
+   this->node_first();
+   while(this->node_next()){
+      pk=(float*)(this->give_ptr());
+      if(pk)delete pk;
+   }
+}
+
+void FCount::Copy(FCount &Fc){
+   char *pch;
+   float *xx,*zz;
+   Node *pN;
+
+   pg[0]=root;
+   cnd[0]=root->ndnm;
+
+   Fc.node_first();
+   while(Fc.node_next()){
+      pch=Fc.show_str();
+      xx=(float*)Fc.give_ptr();
+      zz=new float;
+      *zz=*xx;
+      pN=new Node(pch,(void*)zz);
+      add(pN);
+   }
+}
+      
+void FCount::add_count(const char *pch,float z){
+   float *ppt;
+   Node *np;
+   total+=z;
+   if(this->search(pch)==0){
+      ppt = new float;
+      (*ppt) =z;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+   }
+   else {
+      (*(float*) this->give_ptr())+=z;
+   }
+}
+
+void FCount::add_count2(const char *pch,float z){
+   float *ppt;
+   Node *np;
+   total+=z;
+   if(this->search(pch)==0){
+      ppt = new float;
+      (*ppt) =z;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      (*(float*) this->give_ptr())+=z;
+   }
+}
+
+void FCount::addp_count2(char *pch,float z){
+   float *ppt;
+   Node *np;
+   total+=z;
+   if(this->search(pch)==0){
+      ppt = new float;
+      (*ppt) =z;
+      np=new Node;
+      np->str=pch;
+      np->rel=ppt;
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      (*(float*) this->give_ptr())+=z;
+   }
+}
+
+float FCount::count(const char *pch){
+   if(this->search(pch)==0){
+      return(0);
+   }
+   else {
+      return(*((float*) this->give_ptr()));
+   }
+}
+
+float FCount::count(void){
+   return(*((float*) this->give_ptr()));
+}
+
+//DCount (double precision count tree)
+
+DCount::DCount() : List() {
+   total=0;
+}
+
+DCount::~DCount(){
+   if(copy)return;
+   double *pk;
+   this->node_first();
+   while(this->node_next()){
+      pk=(double*)(this->give_ptr());
+      if(pk)delete pk;
+   }
+}
+
+void DCount::Copy(DCount &Dc){
+   char *pch;
+   double *xx,*zz;
+   Node *pN;
+
+   pg[0]=root;
+   cnd[0]=root->ndnm;
+
+   Dc.node_first();
+   while(Dc.node_next()){
+      pch=Dc.show_str();
+      xx=(double*)Dc.give_ptr();
+      zz=new double;
+      *zz=*xx;
+      pN=new Node(pch,(void*)zz);
+      add(pN);
+   }
+}
+      
+void DCount::add_count(const char *pch,double z){
+   double *ppt;
+   Node *np;
+   total+=z;
+   if(this->search(pch)==0){
+      ppt = new double;
+      (*ppt) =z;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+   }
+   else {
+      (*(double*) this->give_ptr())+=z;
+   }
+}
+
+void DCount::add_count2(const char *pch,double z){
+   double *ppt;
+   Node *np;
+   total+=z;
+   if(this->search(pch)==0){
+      ppt = new double;
+      (*ppt) =z;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      (*(double*) this->give_ptr())+=z;
+   }
+}
+
+void DCount::addp_count2(char *pch,double z){
+   double *ppt;
+   Node *np;
+   total+=z;
+   if(this->search(pch)==0){
+      ppt = new double;
+      (*ppt) =z;
+      np=new Node;
+      np->str=pch;
+      np->rel=ppt;
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      (*(double*) this->give_ptr())+=z;
+   }
+}
+
+double DCount::count(const char *pch){
+   if(this->search(pch)==0){
+      return(0);
+   }
+   else {
+      return(*((double*) this->give_ptr()));
+   }
+}
+
+double DCount::count(void){
+   return(*((double*) this->give_ptr()));
+}
+
+void DCount::max_count(const char *pch,double z){
+   double *ppt;
+   Node *np;
+   total+=z;
+   if(!search(pch)){
+      ppt = new double;
+      (*ppt) =z;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+   }
+   else {
+      ppt=(double*)give_ptr();
+      if(*ppt<z)*ppt=z;
+   }
+}
+
+void DCount::max_count2(const char *pch,double z){
+   double *ppt;
+   Node *np;
+   total+=z;
+   if(!search(pch)){
+      ppt = new double;
+      (*ppt) =z;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      ppt=(double*)give_ptr();
+      if(*ppt<z)*ppt=z;
+   }
+}
+
+void DCount::maxp_count2(char *pch,double z){
+   double *ppt;
+   Node *np;
+   total+=z;
+   if(!search(pch)){
+      ppt = new double;
+      (*ppt) =z;
+      np=new Node;
+      np->str=pch;
+      np->rel=ppt;
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      ppt=(double*)give_ptr();
+      if(*ppt<z)*ppt=z;
+   }
+}
+
+void DCount::min_count(const char *pch,double z){
+   double *ppt;
+   Node *np;
+   total+=z;
+   if(!search(pch)){
+      ppt = new double;
+      (*ppt) =z;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+   }
+   else {
+      ppt=(double*)give_ptr();
+      if(*ppt>z)*ppt=z;
+   }
+}
+
+void DCount::min_count2(const char *pch,double z){
+   double *ppt;
+   Node *np;
+   total+=z;
+   if(!search(pch)){
+      ppt = new double;
+      (*ppt) =z;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      ppt=(double*)give_ptr();
+      if(*ppt>z)*ppt=z;
+   }
+}
+
+void DCount::minp_count2(char *pch,double z){
+   double *ppt;
+   Node *np;
+   total+=z;
+   if(!search(pch)){
+      ppt = new double;
+      (*ppt) =z;
+      np=new Node;
+      np->str=pch;
+      np->rel=ppt;
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      ppt=(double*)give_ptr();
+      if(*ppt>z)*ppt=z;
+   }
+}
+
+void DCount::debug(void){
+   node_first();
+   while(node_next()){
+      cout << count() << " " << show_str() << endl;
+   }
+}
+
+//Partial Match
+
+Partial_match::Partial_match() : Count() {
+}
+
+Partial_match::~Partial_match(){
+}
+
+void Partial_match::long_match(char *str,List &Lst){
+   char *pch;
+   while(*str!='\0'){
+      if(this->search_long(str)){
+         pch=this->show_str();
+         Lst.add_key_count(pch);
+      }
+      if((pch=strchr(str,' '))!=NULL)str=pch+1;
+      else str=str+strlen(str);
+   }   
+}
+
+void Partial_match::local_match(char *str,List &Lst){
+   char *pch;
+   int i,j;
+   if(*str!='\0'){
+      if(this->search_long(str)){
+         pch=this->show_str();
+         Lst.add_key_count(pch);
+         i=strlen(pch)-1;
+         while(0<i){
+            while((0<i)&&(*(str+i)!=' '))i--;
+            if(0<i){
+               *(str+i)='\0';
+               j=this->search(str);
+               *(str+i)=' ';
+               if(j){
+                  pch=this->show_str();
+                  Lst.add_key_count(pch);
+               }
+               i--;
+            }
+         }
+      }
+   }
+}
+
+void Partial_match::all_match(char *str,List &Lst){
+   char *pch;
+   int i,j;
+   while(*str!='\0'){
+      if(this->search_long(str)){
+         pch=this->show_str();
+         Lst.add_key_count(pch);
+         i=strlen(pch)-1;
+         while(0<i){
+            while((0<i)&&(*(str+i)!=' '))i--;
+            if(0<i){
+               *(str+i)='\0';
+               j=this->search(str);
+               *(str+i)=' ';
+               if(j){
+                  pch=this->show_str();
+                  Lst.add_key_count(pch);
+               }
+               i--;
+            }
+         }
+      }
+      if((pch=strchr(str,' '))!=NULL)str=pch+1;
+      else str=str+strlen(str);
+   }   
+}
+
+void Partial_match::long_match(char *str,Count &Cnt,long n){
+   char *pch;
+   while(*str!='\0'){
+      if(this->search_long(str)){
+         pch=this->show_str();
+         Cnt.add_count2(pch,n);
+      }
+      if((pch=strchr(str,' '))!=NULL)str=pch+1;
+      else str=str+strlen(str);
+   }   
+}
+
+void Partial_match::local_match(char *str,Count &Cnt,long n){
+   char *pch;
+   int i,j;
+   if(*str!='\0'){
+      if(this->search_long(str)){
+         pch=this->show_str();
+         Cnt.add_count2(pch,n);
+         i=strlen(pch)-1;
+         while(0<i){
+            while((0<i)&&(*(str+i)!=' '))i--;
+            if(0<i){
+               *(str+i)='\0';
+               j=this->search(str);
+               *(str+i)=' ';
+               if(j){
+                  pch=this->show_str();
+                  Cnt.add_count2(pch,n);
+               }
+               i--;
+            }
+         }
+      }
+   } 
+}
+
+void Partial_match::all_match(char *str,Count &Cnt,long n){
+   char *pch;
+   int i,j;
+   while(*str!='\0'){
+      if(this->search_long(str)){
+         pch=this->show_str();
+         Cnt.add_count2(pch,n);
+         i=strlen(pch)-1;
+         while(0<i){
+            while((0<i)&&(*(str+i)!=' '))i--;
+            if(0<i){
+               *(str+i)='\0';
+               j=this->search(str);
+               *(str+i)=' ';
+               if(j){
+                  pch=this->show_str();
+                  Cnt.add_count2(pch,n);
+               }
+               i--;
+            }
+         }
+      }
+      if((pch=strchr(str,' '))!=NULL)str=pch+1;
+      else str=str+strlen(str);
+   }   
+}
+
+int Partial_match::search_long(char *str){
+   int a=0,b=0,i,j;
+   len=strlen(str);
+   if(this->step_one(a,b,str))return(1);
+   i=(a<b)?b:a;
+   while(cln_o<i){
+      while((cln_o<i)&&(*(str+i)!=' '))i--;
+      if(cln_o<i){
+         *(str+i)='\0';
+         j=this->search(str);
+         *(str+i)=' ';
+         if(j)return(1);
+         i--;
+      }
+   }
+   if(cln_o){
+      depth=depth_o;
+      cnd[depth]=index_o;
+      return(1);
+   }
+   else return(0);
+}
+
+int Partial_match::step_one(int &a,int &b,char *str){
+   char c;
+   cln_o=0;
+   cln=0;
+   while((c=*(str+cln))&&c!=32)cln++;
+   *(str+cln)='\0';
+   depth=-1;
+   Page *pu=root;
+   int i,j;
+   while(pu!=NULL){
+      depth++;
+      pg[depth]=pu;
+      j=(pu->search)(a,b,str,i,this);
+      cnd[depth]=i;
+      if(j==1)return(1);
+      if(i==0)pu=pu->pdn;        
+      else pu=(pu->pnd)[i-1]->pdn;
+   }
+
+if(cln<len)*(str+cln)=' ';
+return(0);
+}
+
+int Partial_match::stc_my_long(int &a,int &b,char *str,const char *ptr,int index)
+   {char c;
+   int i=(a<b) ? a : b;
+   const char *p1=str+i;
+   const char *p2=ptr+i;
+   int j=0;
+   while((*p1==*p2)&&(*p1!='\0')){
+      j++;
+      p1++;
+      p2++;
+      if((*p1=='\0'&&*p2!='\0')&&(cln<len)){
+         *(str+cln++)=' ';
+         while((c=*(str+cln))&&c!=32)cln++;
+         *(str+cln)='\0';
+      }
+   }
+   if(*p1==*p2){
+      if(cln<len){
+         depth_o=depth;
+         index_o=index;
+         cln_o=cln;
+         *(str+cln++)=' ';
+         while((c=*(str+cln))&&c!=32)cln++;
+         *(str+cln)='\0';
+         a=i+j;
+         return(1);
+      }
+      else return(0);
+   }
+   else if(*p1<*p2){
+      b=i+j;
+      return(-1);
+   }
+   else {
+      a=i+j;
+      return(1);
+   }
+}
+
+}

Library/Btree.h

@@ -1,547 +1,547 @@
-#ifndef BTREE_H
-#define BTREE_H
-
-#define LNEG -100000000
-
-#include <iostream>
-#include <fstream>
-using namespace std;
-namespace iret {
-
-const int order = 5; //Half the order of the Btree that we build.
-const int height_limit =12; //Limit on the height of the Btree.
-const int ord2  = order*2; //The order of the Btree.
-
-int stc_my(int &,int &,const char *,const char *); //Function used to compare 
-   //two strings. The first two arguments hold information about how much the 
-   //string can be ignored in the comparison.
-
-class Page; //forward declaration
-class Btree; //forward declaration
-class Partial_match; //forward declaration
-
-class Node {
-   friend int stc_my(int &,int &,const char *,const char *);
-   friend class Page;
-   friend class Btree;
-   friend class List;
-   friend class Count;
-   friend class FCount;
-   friend class DCount;
-   template<class Z> friend class BCount;
-   friend class Partial_match;
-   friend class Thes;
-public:
-   Node(void); //Sets all points to NULL.
-   Node(const char * ); //Argument is the string for this node.
-   Node(const char * ,void *); //Arguments are first the string and then the 
-   //data pointer.
-   ~Node();
-   void debug(); //Prints out the node in simple format.
-private:
-   char *str; //String pointer.
-   void *rel; //Data pointer.
-   Page *pdn; //Points down to the page below or to NULL.
-};
-
-class Page {
-   friend int stc_my(int &,int &,const char *,const char *);
-   friend class Btree;
-   friend class Partial_match;
-   friend class FCount;
-   friend class DCount;
-public:
-   Page(); //Constructs a new empty page. Only happens at the root.
-   Page(Page * const pz,Page * const pn,const int n); //Constructs a page that 
-      //holds the right half of a full page. The full page is pointed at by the 
-      //pz. The new pages downward pointer is set to pn. 
-      //n tells how much of the full page is to remain or where to begin removal.
-   ~Page();
-   void clean(void); //Used to delete without touching search keys in the nodes
-      //which were created with addp functions and do not belong to the tree.
-   void insert(const int n,Node * const nd,const int j); //inserts in partially empty 
-      //page. n is insertion point, j is number of nodes on page that are viable.
-   int search(int &a,int &b,const char *,int &p); //searches for string on 
-      //the page. Returns 1 if found, 0 otherwise. If found p is the index, otherwise
-      //if p is 0 then the page downward pointer is to next page to search, but if
-      //p is positive then p-1 is number of node that has the downward pointer to 
-      //next page to search. 
-   int search(int &a,int &b,char *,int &p,Partial_match *btr); //Looks for longest 
-      //partial match. 
-   void debug(); //Prints out the page for debugging purposes.
-
-private:
-   char ndnm; //Indicates the number of Nodes on the page.
-   Page *pdn; //Pointer that points to the page below and also lexically below. 
-   //May be NULL.
-   Node *pnd[ord2]; //Pointers to the nodes on the page. Some may be NULL.
-};
-
-class Btree {
-   friend class Page;
-public:
-   Btree(void);
-   Btree(ifstream &); //Reads in a Btree in form of list written out by 
-          //list_write() from disc. String arguments mark the path in proj file.
-   Btree( const Btree & btree ) {copy = true; root = btree.root;} // Actually
-   // creates another reference to the same tree.  Take great care to
-   // avoid simultaneously modifying both copies.
-   ~Btree(void);
-   int search(const char *); //Searches for a string and sets the path to that 
-                    //string or its insertion point.
-   int insert(Node *);//Only to be called after a search has failed to find the 
-                       //string.
-   void node_first();//Finds the first node in the tree and sets the path to it.
-   int node_next(); //Given the path is already set to a node, this function 
-                    //finds the next node in lexicographic order.
-   char *show_str();//Used to show the string after a call to next is successful.
-   void *give_ptr();//Used to give the data pointer in the current node.
-   void set_ptr(void *); //Used to set the data pointer after a call to search 
-                         //has found string.
-   int add(Node *); //Only to be used to construct a tree from a lexical list 
-                    //as written out by list_write();
-   void next_empty(); //Only used to reset the pointer arrays when the root is 
-                      //split. Used in add().
-   long list_write(ofstream &); //Writes out a lexical list of the strings in 
-                                //the tree. 
-   int iclean; //Default 0, but set to 1 if want to have destructor run without
-       //touching key strings (if addp used in making tree).
-protected:
-   int depth; //Tells the depth in the tree that marks the current location.
-   Page *root; //Points at the root page of the tree.
-   Page *pg[height_limit]; //Descending list of pointers that mark the pages.
-   int cnd[height_limit]; //Mark the positions of the nodes just above the 
-       //downard page pointer at each level. Thus 0 marks the page's downward 
-       //pointer, but a nonzero value must have 1 subtracted and then it gives 
-       //the node whose downward pointer is the correct downward pointer.
-   bool copy; //flags copies of a tree with true.
-};
-
-class List : public Btree {
-public:
-   List();
-   List(const List & list) : Btree(list) {}
-   ~List();
-   void add_key(const char *str); //Adds the string *str to the tree if not already in list
-   void add_key_count(const char *str); //Adds the string *str to the tree if 
-      //not already in list and counts it.
-   void addp_key_count(char *str); //Adds the string *str to the tree if 
-      //not already in list and counts it. Uses the actual string pointer instead 
-      //of making a copy
-   long cnt_key; //Used to count the number of keys.
-};
-
-class Count : public List {
-public:
-   Count();
-   Count(const Count & Ct) : List(Ct){} 
-   ~Count();
-   void add_count(const char *str,long n); //Adds the string *str with its count
-      //to the tree if not already in list. String is key and count is data.
-      //If string is already a key the count is incremented by n.
-   void add_countz(const char *str,long n); //Adds the string *str with its count
-      //just as add_count, but also counts number of unique keys in count.
-      //Does not add count to the total variable, unlike add_count2.
-   void add_count2(const char *str,long n); //Adds the string *str with its count
-      //just as add_count, but also counts number of unique keys in count.
-   void addp_count2(char *str,long n); //Adds the string *str with its count
-      //just as add_count, but also counts number of unique keys in count.
-      //Does not make copy of string, but uses the pointer str as key pointer.
-   void correct(const char *str,long n); //If str is in the tree the count is
-      //changed to n. Otherwise nothing is done.
-
-   //Functions for maximum calculation
-   void max_count(const char *str,long n); //Adds the string *str with its count
-      //to the tree if not already in list. String is key and count is data.
-      //If string is already a key the count is max of n and prior value.
-   void max_count2(const char *str,long n); //Adds the string *str with its count
-      //just as max_count, but also counts number of unique keys in count.
-   void maxp_count2(char *str,long n); //Adds the string *str with its count
-      //just as max_count, but also counts number of unique keys in count.
-      //Does not make copy of string, but uses the pointer str as key pointer.
-
-   //Functions for minium calculation
-   void min_count(const char *str,long n); //Adds the string *str with its count
-      //to the tree if not already in list. String is key and count is data.
-      //If string is already a key the count is min of n and prior value.
-   void min_count2(const char *str,long n); //Adds the string *str with its count
-      //just as min_count, but also counts number of unique keys in count.
-   void minp_count2(char *str,long n); //Adds the string *str with its count
-      //just as min_count, but also counts number of unique keys in count.
-      //Does not make copy of string, but uses the pointer str as key pointer.
-
-   long count(const char *str); //Returns the count if a key (in list) otherwise
-      //returns 0.
-   long count(void); //Returns the count of the current string. Assumes the
-      //pointers have already been set by a search or node_next call.
-   long total; //Holds the total of all counts added for all keys.
-};
-
-class FCount : public List {
-public:
-   FCount();
-   FCount(const FCount & Ct) : List(Ct){}
-   ~FCount();
-   void Copy(FCount &Dc); //Makes a copy of the tree Dc in the current tree.
-   void add_count(const char *str,float z); //Adds the string *str with its count
-      //to the tree if not already in list. String is key and count is data.
-      //If string is already a key the count is incremented by z.
-   void add_count2(const char *str,float z); //Adds the string *str with its count
-      //just as add_count, but also counts number of unique keys in count.
-   void addp_count2(char *str,float z); //Adds the string *str with its count
-      //just as add_count, but also counts number of unique keys in count.
-      //Does not make copy of string, but uses the pointer str as key pointer.
-   float count(const char *str); //Returns the count if a key (in list) otherwise
-      //returns 0.
-   float count(void); //Returns the count of the current string. Assumes the
-      //pointers have already been set by a search or node_next call.
-   float total; //Holds the total of all counts added for all keys.
-};
-
-class DCount : public List {
-public:
-   DCount();
-   DCount(const DCount & Ct) : List(Ct){}
-   ~DCount();
-   void Copy(DCount &Dc); //Makes a copy of the tree Dc in the current tree.
-   void add_count(const char *str,double z); //Adds the string *str with its count
-      //to the tree if not already in list. String is key and count is data.
-      //If string is already a key the count is incremented by z.
-   void add_count2(const char *str,double z); //Adds the string *str with its count
-      //just as add_count, but also counts number of unique keys in count.
-   void addp_count2(char *str,double z); //Adds the string *str with its count
-      //just as add_count, but also counts number of unique keys in count.
-      //Does not make copy of string, but uses the pointer str as key pointer.
-   double count(const char *str); //Returns the count if a key (in list) otherwise
-      //returns 0.
-   double count(void); //Returns the count of the current string. Assumes the
-      //pointers have already been set by a search or node_next call.
-
-   //Functions for maximum calculation
-   void max_count(const char *str,double z); //Adds the string *str with its count
-      //to the tree if not already in list. String is key and count is data.
-      //If string is already a key the count is max of z and prior value.
-   void max_count2(const char *str,double z); //Adds the string *str with its count
-      //just as max_count, but also counts number of unique keys in count.
-   void maxp_count2(char *str,double z); //Adds the string *str with its count
-      //just as max_count, but also counts number of unique keys in count.
-      //Does not make copy of string, but uses the pointer str as key pointer.
-
-   //Functions for minium calculation
-   void min_count(const char *str,double z); //Adds the string *str with its count
-      //to the tree if not already in list. String is key and count is data.
-      //If string is already a key the count is min of z and prior value.
-   void min_count2(const char *str,double z); //Adds the string *str with its count
-      //just as min_count, but also counts number of unique keys in count.
-   void minp_count2(char *str,double z); //Adds the string *str with its count
-      //just as min_count, but also counts number of unique keys in count.
-      //Does not make copy of string, but uses the pointer str as key pointer.
-
-   void debug(void); //Prints to stdout a list "i str[i]"
-   double total; //Holds the total of all counts added for all keys.
-};
-
-class Partial_match : public Count {
-   friend class Page;
-public:
-   Partial_match();
-   Partial_match(const Partial_match & Par_mat) : Count(Par_mat){}
-   ~Partial_match();
-   void long_match(char *,List &); //Finds the longest matches for all word
-       //starts in the string and adds them to the list.
-   void local_match(char *,List &); //Finds all matches that start at 
-       //beginning of the string and adds them to the list.
-   void all_match(char *,List &); //Finds all matches within the string and
-       //adds them to the list.
-   void long_match(char *,Count &,long n); //Finds the longest matches for all word
-       //starts in the string and adds them to the list in Count.
-   void local_match(char *,Count &,long n); //Finds all matches that start at
-       //beginning of string and adds them to the list in Count.
-   void all_match(char *,Count &,long n); //Finds all matches within the string and
-       //adds them to the list in Count.
-   int search_long(char *); //Searches for longest partial match to an initial
-       //segment of a string that ends at a word boundary and 
-       //sets the path to that string or its insertion point.
-
-private:
-   int stc_my_long(int &,int &,char *,const char *,int); //Function used to compare 
-      //two strings. The first two arguments hold information about how much the 
-      //string can be ignored in the comparison. The last argument holds the index
-      //or number of the string's node on the page.
-   int step_one(int &,int &,char *); //Looks for partial or complete match and
-      //returns 1 if complete found. Partial is reflected in parameters.
-
-    //Special parameters used in partial matching.
-   int depth_o; //Depth of longest partial match thus far.
-   int index_o; //index of longest partial match thus far.
-   int cln_o;  //String length of longest partial match thus far.
-   int len; //Length of query string. 
-   int cln; //Current null position in string.
-};
-
-class Str_str : public Btree {
-public:
-   Str_str();
-   Str_str(const Str_str & Stst) : Btree(Stst){}
-   ~Str_str();
-   void add_pair(const char *one,const char *two); //Adds the string *one to the tree and stores
-      //the string *two at that node.
-   char *match(const char *one); //Returns pointer to the string stored under string *one.
-};
-
-class Num_num : public Btree {
-public:
-   Num_num();
-   Num_num(const Num_num & Nmnm) : Btree(Nmnm){}
-   ~Num_num();
-   void add_pair(long i, long j); //Adds the string for i to the tree and
-      //stores the  number j at that node.
-   long match(long i); //Returns the number stored under the string for i.
-};
-
-template<class Z>
-class BCount : public List {
-public:
-   BCount();
-   BCount(const BCount<Z> & Ct) : List(Ct){} 
-   ~BCount();
-   void add_count(const char *str,Z n); //Adds the string *str with its count
-      //to the tree if not already in list. String is key and count is data.
-      //If string is already a key the count is incremented by n.
-   void add_count2(const char *str,Z n); //Adds the string *str with its count
-      //just as add_count, but also counts number of unique keys in count.
-   void addp_count2(char *str,Z n); //Adds the string *str with its count
-      //just as add_count, but also counts number of unique keys in count.
-      //Does not make copy of string, but uses the pointer str as key pointer.
-   void correct(const char *str,Z n); //If str is in the tree the count is
-      //changed to n. Otherwise nothing is done.
-
-   //Functions for maximum calculation
-   void max_count(const char *str,Z n); //Adds the string *str with its count
-      //to the tree if not already in list. String is key and count is data.
-      //If string is already a key the count is max of n and prior value.
-   void max_count2(const char *str,Z n); //Adds the string *str with its count
-      //just as max_count, but also counts number of unique keys in count.
-   void maxp_count2(char *str,Z n); //Adds the string *str with its count
-      //just as max_count, but also counts number of unique keys in count.
-      //Does not make copy of string, but uses the pointer str as key pointer.
-
-   //Functions for minium calculation
-   void min_count(const char *str,Z n); //Adds the string *str with its count
-      //to the tree if not already in list. String is key and count is data.
-      //If string is already a key the count is min of n and prior value.
-   void min_count2(const char *str,Z n); //Adds the string *str with its count
-      //just as min_count, but also counts number of unique keys in count.
-   void minp_count2(char *str,Z n); //Adds the string *str with its count
-      //just as min_count, but also counts number of unique keys in count.
-      //Does not make copy of string, but uses the pointer str as key pointer.
-
-   Z count(const char *str); //Returns the count if a key (in list) otherwise
-      //returns 0.
-   Z count(void); //Returns the count of the current string. Assumes the
-      //pointers have already been set by a search or node_next call.
-   Z total; //Holds the total of all counts added for all keys.
-};
-
-template<class Z>
-BCount<Z>::BCount() : List() {
-   total=0;
-}  
-   
-template<class Z>
-BCount<Z>::~BCount(){
-   if(copy)return;
-   Z *pk; 
-   this->node_first();
-   while(this->node_next()){
-      pk=(Z *)(this->give_ptr());
-      if(pk)delete pk;
-   }
-}  
-
-template<class Z>
-void BCount<Z>::add_count(const char *pch,Z n){
-   Z *ppt;
-   Node *np;
-   total+=n;
-   if(this->search(pch)==0){
-      ppt = new Z;
-      (*ppt) =n;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-   }
-   else {
-      (*(Z *) this->give_ptr())+=n;
-   }
-}
-
-template<class Z>
-void BCount<Z>::add_count2(const char *pch,Z n){
-   Z *ppt;
-   Node *np;
-   total+=n;
-   if(this->search(pch)==0){
-      ppt = new Z;
-      (*ppt) =n;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      (*(Z *) this->give_ptr())+=n;
-   }
-}
-
-template<class Z>
-void BCount<Z>::addp_count2(char *pch,Z n){
-   Z *ppt;
-   Node *np;
-   total+=n;
-   if(this->search(pch)==0){
-      ppt = new Z;
-      (*ppt) =n;
-      np=new Node;
-      np->str=pch;
-      np->rel=ppt;
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      (*(Z *) this->give_ptr())+=n;
-   }
-}
-
-template<class Z>
-void BCount<Z>::correct(const char *pch,Z n){
-   if(this->search(pch)){
-      (*(Z *) this->give_ptr())=n;
-   }
-}
-
-template<class Z>
-Z BCount<Z>::count(const char *pch){
-   if(this->search(pch)==0){
-      return(0);
-   }
-   else {
-      return(*((Z *) this->give_ptr()));
-   }
-}
-
-template<class Z>
-Z BCount<Z>::count(void){
-   return(*((Z *) this->give_ptr()));
-}
-
-template<class Z>
-void BCount<Z>::max_count(const char *pch,Z n){
-   Z *ppt,i;
-   Node *np;
-   total+=n;
-   if(!search(pch)){
-      ppt = new Z;
-      (*ppt) =n;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-   }
-   else {
-      ppt=(Z *)give_ptr();
-      if(*ppt<n)*ppt=n;
-   }
-}
-
-template<class Z>
-void BCount<Z>::max_count2(const char *pch,Z n){
-   Z *ppt,i;
-   Node *np;
-   total+=n;
-   if(!search(pch)){
-      ppt = new Z;
-      (*ppt) =n;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      ppt=(Z *)give_ptr();
-      if(*ppt<n)*ppt=n;
-   }
-}
-
-template<class Z>
-void BCount<Z>::maxp_count2(char *pch,Z n){
-   Z *ppt,i;
-   Node *np;
-   total+=n;
-   if(!search(pch)){
-      ppt = new Z;
-      (*ppt) =n;
-      np=new Node;
-      np->str=pch;
-      np->rel=ppt;
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      ppt=(Z *)give_ptr();
-      if(*ppt<n)*ppt=n;
-   }
-}
-
-template<class Z>
-void BCount<Z>::min_count(const char *pch,Z n){
-   Z *ppt,i;
-   Node *np;
-   total+=n;
-   if(!search(pch)){
-      ppt = new Z;
-      (*ppt) =n;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-   }
-   else {
-      ppt=(Z *)give_ptr();
-      if(*ppt>n)*ppt=n;
-   }
-}
-
-template<class Z>
-void BCount<Z>::min_count2(const char *pch,Z n){
-   Z *ppt,i;
-   Node *np;
-   total+=n;
-   if(!search(pch)){
-      ppt = new Z;
-      (*ppt) =n;
-      np=new Node(pch,(void*)ppt);
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      ppt=(Z *)give_ptr();
-      if(*ppt>n)*ppt=n;
-   }
-}
-
-template<class Z>
-void BCount<Z>::minp_count2(char *pch,Z n){
-   Z *ppt,i;
-   Node *np;
-   total+=n;
-   if(!search(pch)){
-      ppt = new Z;
-      (*ppt) =n;
-      np=new Node;
-      np->str=pch;
-      np->rel=ppt;
-      this->insert(np);
-      cnt_key++;
-   }
-   else {
-      ppt=(Z *)give_ptr();
-      if(*ppt>n)*ppt=n;
-   }
-}
-
-}
-#endif
+#ifndef BTREE_H
+#define BTREE_H
+
+#define LNEG -100000000
+
+#include <iostream>
+#include <fstream>
+using namespace std;
+namespace iret {
+
+const int order = 5; //Half the order of the Btree that we build.
+const int height_limit =12; //Limit on the height of the Btree.
+const int ord2  = order*2; //The order of the Btree.
+
+int stc_my(int &,int &,const char *,const char *); //Function used to compare 
+   //two strings. The first two arguments hold information about how much the 
+   //string can be ignored in the comparison.
+
+class Page; //forward declaration
+class Btree; //forward declaration
+class Partial_match; //forward declaration
+
+class Node {
+   friend int stc_my(int &,int &,const char *,const char *);
+   friend class Page;
+   friend class Btree;
+   friend class List;
+   friend class Count;
+   friend class FCount;
+   friend class DCount;
+   template<class Z> friend class BCount;
+   friend class Partial_match;
+   friend class Thes;
+public:
+   Node(void); //Sets all points to NULL.
+   Node(const char * ); //Argument is the string for this node.
+   Node(const char * ,void *); //Arguments are first the string and then the 
+   //data pointer.
+   ~Node();
+   void debug(); //Prints out the node in simple format.
+private:
+   char *str; //String pointer.
+   void *rel; //Data pointer.
+   Page *pdn; //Points down to the page below or to NULL.
+};
+
+class Page {
+   friend int stc_my(int &,int &,const char *,const char *);
+   friend class Btree;
+   friend class Partial_match;
+   friend class FCount;
+   friend class DCount;
+public:
+   Page(); //Constructs a new empty page. Only happens at the root.
+   Page(Page * const pz,Page * const pn,const int n); //Constructs a page that 
+      //holds the right half of a full page. The full page is pointed at by the 
+      //pz. The new pages downward pointer is set to pn. 
+      //n tells how much of the full page is to remain or where to begin removal.
+   ~Page();
+   void clean(void); //Used to delete without touching search keys in the nodes
+      //which were created with addp functions and do not belong to the tree.
+   void insert(const int n,Node * const nd,const int j); //inserts in partially empty 
+      //page. n is insertion point, j is number of nodes on page that are viable.
+   int search(int &a,int &b,const char *,int &p); //searches for string on 
+      //the page. Returns 1 if found, 0 otherwise. If found p is the index, otherwise
+      //if p is 0 then the page downward pointer is to next page to search, but if
+      //p is positive then p-1 is number of node that has the downward pointer to 
+      //next page to search. 
+   int search(int &a,int &b,char *,int &p,Partial_match *btr); //Looks for longest 
+      //partial match. 
+   void debug(); //Prints out the page for debugging purposes.
+
+private:
+   char ndnm; //Indicates the number of Nodes on the page.
+   Page *pdn; //Pointer that points to the page below and also lexically below. 
+   //May be NULL.
+   Node *pnd[ord2]; //Pointers to the nodes on the page. Some may be NULL.
+};
+
+class Btree {
+   friend class Page;
+public:
+   Btree(void);
+   Btree(ifstream &); //Reads in a Btree in form of list written out by 
+          //list_write() from disc. String arguments mark the path in proj file.
+   Btree( const Btree & btree ) {copy = true; root = btree.root;} // Actually
+   // creates another reference to the same tree.  Take great care to
+   // avoid simultaneously modifying both copies.
+   ~Btree(void);
+   int search(const char *); //Searches for a string and sets the path to that 
+                    //string or its insertion point.
+   int insert(Node *);//Only to be called after a search has failed to find the 
+                       //string.
+   void node_first();//Finds the first node in the tree and sets the path to it.
+   int node_next(); //Given the path is already set to a node, this function 
+                    //finds the next node in lexicographic order.
+   char *show_str();//Used to show the string after a call to next is successful.
+   void *give_ptr();//Used to give the data pointer in the current node.
+   void set_ptr(void *); //Used to set the data pointer after a call to search 
+                         //has found string.
+   int add(Node *); //Only to be used to construct a tree from a lexical list 
+                    //as written out by list_write();
+   void next_empty(); //Only used to reset the pointer arrays when the root is 
+                      //split. Used in add().
+   long list_write(ofstream &); //Writes out a lexical list of the strings in 
+                                //the tree. 
+   int iclean; //Default 0, but set to 1 if want to have destructor run without
+       //touching key strings (if addp used in making tree).
+protected:
+   int depth; //Tells the depth in the tree that marks the current location.
+   Page *root; //Points at the root page of the tree.
+   Page *pg[height_limit]; //Descending list of pointers that mark the pages.
+   int cnd[height_limit]; //Mark the positions of the nodes just above the 
+       //downard page pointer at each level. Thus 0 marks the page's downward 
+       //pointer, but a nonzero value must have 1 subtracted and then it gives 
+       //the node whose downward pointer is the correct downward pointer.
+   bool copy; //flags copies of a tree with true.
+};
+
+class List : public Btree {
+public:
+   List();
+   List(const List & list) : Btree(list) {}
+   ~List();
+   void add_key(const char *str); //Adds the string *str to the tree if not already in list
+   void add_key_count(const char *str); //Adds the string *str to the tree if 
+      //not already in list and counts it.
+   void addp_key_count(char *str); //Adds the string *str to the tree if 
+      //not already in list and counts it. Uses the actual string pointer instead 
+      //of making a copy
+   long cnt_key; //Used to count the number of keys.
+};
+
+class Count : public List {
+public:
+   Count();
+   Count(const Count & Ct) : List(Ct){} 
+   ~Count();
+   void add_count(const char *str,long n); //Adds the string *str with its count
+      //to the tree if not already in list. String is key and count is data.
+      //If string is already a key the count is incremented by n.
+   void add_countz(const char *str,long n); //Adds the string *str with its count
+      //just as add_count, but also counts number of unique keys in count.
+      //Does not add count to the total variable, unlike add_count2.
+   void add_count2(const char *str,long n); //Adds the string *str with its count
+      //just as add_count, but also counts number of unique keys in count.
+   void addp_count2(char *str,long n); //Adds the string *str with its count
+      //just as add_count, but also counts number of unique keys in count.
+      //Does not make copy of string, but uses the pointer str as key pointer.
+   void correct(const char *str,long n); //If str is in the tree the count is
+      //changed to n. Otherwise nothing is done.
+
+   //Functions for maximum calculation
+   void max_count(const char *str,long n); //Adds the string *str with its count
+      //to the tree if not already in list. String is key and count is data.
+      //If string is already a key the count is max of n and prior value.
+   void max_count2(const char *str,long n); //Adds the string *str with its count
+      //just as max_count, but also counts number of unique keys in count.
+   void maxp_count2(char *str,long n); //Adds the string *str with its count
+      //just as max_count, but also counts number of unique keys in count.
+      //Does not make copy of string, but uses the pointer str as key pointer.
+
+   //Functions for minium calculation
+   void min_count(const char *str,long n); //Adds the string *str with its count
+      //to the tree if not already in list. String is key and count is data.
+      //If string is already a key the count is min of n and prior value.
+   void min_count2(const char *str,long n); //Adds the string *str with its count
+      //just as min_count, but also counts number of unique keys in count.
+   void minp_count2(char *str,long n); //Adds the string *str with its count
+      //just as min_count, but also counts number of unique keys in count.
+      //Does not make copy of string, but uses the pointer str as key pointer.
+
+   long count(const char *str); //Returns the count if a key (in list) otherwise
+      //returns 0.
+   long count(void); //Returns the count of the current string. Assumes the
+      //pointers have already been set by a search or node_next call.
+   long total; //Holds the total of all counts added for all keys.
+};
+
+class FCount : public List {
+public:
+   FCount();
+   FCount(const FCount & Ct) : List(Ct){}
+   ~FCount();
+   void Copy(FCount &Dc); //Makes a copy of the tree Dc in the current tree.
+   void add_count(const char *str,float z); //Adds the string *str with its count
+      //to the tree if not already in list. String is key and count is data.
+      //If string is already a key the count is incremented by z.
+   void add_count2(const char *str,float z); //Adds the string *str with its count
+      //just as add_count, but also counts number of unique keys in count.
+   void addp_count2(char *str,float z); //Adds the string *str with its count
+      //just as add_count, but also counts number of unique keys in count.
+      //Does not make copy of string, but uses the pointer str as key pointer.
+   float count(const char *str); //Returns the count if a key (in list) otherwise
+      //returns 0.
+   float count(void); //Returns the count of the current string. Assumes the
+      //pointers have already been set by a search or node_next call.
+   float total; //Holds the total of all counts added for all keys.
+};
+
+class DCount : public List {
+public:
+   DCount();
+   DCount(const DCount & Ct) : List(Ct){}
+   ~DCount();
+   void Copy(DCount &Dc); //Makes a copy of the tree Dc in the current tree.
+   void add_count(const char *str,double z); //Adds the string *str with its count
+      //to the tree if not already in list. String is key and count is data.
+      //If string is already a key the count is incremented by z.
+   void add_count2(const char *str,double z); //Adds the string *str with its count
+      //just as add_count, but also counts number of unique keys in count.
+   void addp_count2(char *str,double z); //Adds the string *str with its count
+      //just as add_count, but also counts number of unique keys in count.
+      //Does not make copy of string, but uses the pointer str as key pointer.
+   double count(const char *str); //Returns the count if a key (in list) otherwise
+      //returns 0.
+   double count(void); //Returns the count of the current string. Assumes the
+      //pointers have already been set by a search or node_next call.
+
+   //Functions for maximum calculation
+   void max_count(const char *str,double z); //Adds the string *str with its count
+      //to the tree if not already in list. String is key and count is data.
+      //If string is already a key the count is max of z and prior value.
+   void max_count2(const char *str,double z); //Adds the string *str with its count
+      //just as max_count, but also counts number of unique keys in count.
+   void maxp_count2(char *str,double z); //Adds the string *str with its count
+      //just as max_count, but also counts number of unique keys in count.
+      //Does not make copy of string, but uses the pointer str as key pointer.
+
+   //Functions for minium calculation
+   void min_count(const char *str,double z); //Adds the string *str with its count
+      //to the tree if not already in list. String is key and count is data.
+      //If string is already a key the count is min of z and prior value.
+   void min_count2(const char *str,double z); //Adds the string *str with its count
+      //just as min_count, but also counts number of unique keys in count.
+   void minp_count2(char *str,double z); //Adds the string *str with its count
+      //just as min_count, but also counts number of unique keys in count.
+      //Does not make copy of string, but uses the pointer str as key pointer.
+
+   void debug(void); //Prints to stdout a list "i str[i]"
+   double total; //Holds the total of all counts added for all keys.
+};
+
+class Partial_match : public Count {
+   friend class Page;
+public:
+   Partial_match();
+   Partial_match(const Partial_match & Par_mat) : Count(Par_mat){}
+   ~Partial_match();
+   void long_match(char *,List &); //Finds the longest matches for all word
+       //starts in the string and adds them to the list.
+   void local_match(char *,List &); //Finds all matches that start at 
+       //beginning of the string and adds them to the list.
+   void all_match(char *,List &); //Finds all matches within the string and
+       //adds them to the list.
+   void long_match(char *,Count &,long n); //Finds the longest matches for all word
+       //starts in the string and adds them to the list in Count.
+   void local_match(char *,Count &,long n); //Finds all matches that start at
+       //beginning of string and adds them to the list in Count.
+   void all_match(char *,Count &,long n); //Finds all matches within the string and
+       //adds them to the list in Count.
+   int search_long(char *); //Searches for longest partial match to an initial
+       //segment of a string that ends at a word boundary and 
+       //sets the path to that string or its insertion point.
+
+private:
+   int stc_my_long(int &,int &,char *,const char *,int); //Function used to compare 
+      //two strings. The first two arguments hold information about how much the 
+      //string can be ignored in the comparison. The last argument holds the index
+      //or number of the string's node on the page.
+   int step_one(int &,int &,char *); //Looks for partial or complete match and
+      //returns 1 if complete found. Partial is reflected in parameters.
+
+    //Special parameters used in partial matching.
+   int depth_o; //Depth of longest partial match thus far.
+   int index_o; //index of longest partial match thus far.
+   int cln_o;  //String length of longest partial match thus far.
+   int len; //Length of query string. 
+   int cln; //Current null position in string.
+};
+
+class Str_str : public Btree {
+public:
+   Str_str();
+   Str_str(const Str_str & Stst) : Btree(Stst){}
+   ~Str_str();
+   void add_pair(const char *one,const char *two); //Adds the string *one to the tree and stores
+      //the string *two at that node.
+   char *match(const char *one); //Returns pointer to the string stored under string *one.
+};
+
+class Num_num : public Btree {
+public:
+   Num_num();
+   Num_num(const Num_num & Nmnm) : Btree(Nmnm){}
+   ~Num_num();
+   void add_pair(long i, long j); //Adds the string for i to the tree and
+      //stores the  number j at that node.
+   long match(long i); //Returns the number stored under the string for i.
+};
+
+template<class Z>
+class BCount : public List {
+public:
+   BCount();
+   BCount(const BCount<Z> & Ct) : List(Ct){} 
+   ~BCount();
+   void add_count(const char *str,Z n); //Adds the string *str with its count
+      //to the tree if not already in list. String is key and count is data.
+      //If string is already a key the count is incremented by n.
+   void add_count2(const char *str,Z n); //Adds the string *str with its count
+      //just as add_count, but also counts number of unique keys in count.
+   void addp_count2(char *str,Z n); //Adds the string *str with its count
+      //just as add_count, but also counts number of unique keys in count.
+      //Does not make copy of string, but uses the pointer str as key pointer.
+   void correct(const char *str,Z n); //If str is in the tree the count is
+      //changed to n. Otherwise nothing is done.
+
+   //Functions for maximum calculation
+   void max_count(const char *str,Z n); //Adds the string *str with its count
+      //to the tree if not already in list. String is key and count is data.
+      //If string is already a key the count is max of n and prior value.
+   void max_count2(const char *str,Z n); //Adds the string *str with its count
+      //just as max_count, but also counts number of unique keys in count.
+   void maxp_count2(char *str,Z n); //Adds the string *str with its count
+      //just as max_count, but also counts number of unique keys in count.
+      //Does not make copy of string, but uses the pointer str as key pointer.
+
+   //Functions for minium calculation
+   void min_count(const char *str,Z n); //Adds the string *str with its count
+      //to the tree if not already in list. String is key and count is data.
+      //If string is already a key the count is min of n and prior value.
+   void min_count2(const char *str,Z n); //Adds the string *str with its count
+      //just as min_count, but also counts number of unique keys in count.
+   void minp_count2(char *str,Z n); //Adds the string *str with its count
+      //just as min_count, but also counts number of unique keys in count.
+      //Does not make copy of string, but uses the pointer str as key pointer.
+
+   Z count(const char *str); //Returns the count if a key (in list) otherwise
+      //returns 0.
+   Z count(void); //Returns the count of the current string. Assumes the
+      //pointers have already been set by a search or node_next call.
+   Z total; //Holds the total of all counts added for all keys.
+};
+
+template<class Z>
+BCount<Z>::BCount() : List() {
+   total=0;
+}  
+   
+template<class Z>
+BCount<Z>::~BCount(){
+   if(copy)return;
+   Z *pk; 
+   this->node_first();
+   while(this->node_next()){
+      pk=(Z *)(this->give_ptr());
+      if(pk)delete pk;
+   }
+}  
+
+template<class Z>
+void BCount<Z>::add_count(const char *pch,Z n){
+   Z *ppt;
+   Node *np;
+   total+=n;
+   if(this->search(pch)==0){
+      ppt = new Z;
+      (*ppt) =n;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+   }
+   else {
+      (*(Z *) this->give_ptr())+=n;
+   }
+}
+
+template<class Z>
+void BCount<Z>::add_count2(const char *pch,Z n){
+   Z *ppt;
+   Node *np;
+   total+=n;
+   if(this->search(pch)==0){
+      ppt = new Z;
+      (*ppt) =n;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      (*(Z *) this->give_ptr())+=n;
+   }
+}
+
+template<class Z>
+void BCount<Z>::addp_count2(char *pch,Z n){
+   Z *ppt;
+   Node *np;
+   total+=n;
+   if(this->search(pch)==0){
+      ppt = new Z;
+      (*ppt) =n;
+      np=new Node;
+      np->str=pch;
+      np->rel=ppt;
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      (*(Z *) this->give_ptr())+=n;
+   }
+}
+
+template<class Z>
+void BCount<Z>::correct(const char *pch,Z n){
+   if(this->search(pch)){
+      (*(Z *) this->give_ptr())=n;
+   }
+}
+
+template<class Z>
+Z BCount<Z>::count(const char *pch){
+   if(this->search(pch)==0){
+      return(0);
+   }
+   else {
+      return(*((Z *) this->give_ptr()));
+   }
+}
+
+template<class Z>
+Z BCount<Z>::count(void){
+   return(*((Z *) this->give_ptr()));
+}
+
+template<class Z>
+void BCount<Z>::max_count(const char *pch,Z n){
+   Z *ppt,i;
+   Node *np;
+   total+=n;
+   if(!search(pch)){
+      ppt = new Z;
+      (*ppt) =n;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+   }
+   else {
+      ppt=(Z *)give_ptr();
+      if(*ppt<n)*ppt=n;
+   }
+}
+
+template<class Z>
+void BCount<Z>::max_count2(const char *pch,Z n){
+   Z *ppt,i;
+   Node *np;
+   total+=n;
+   if(!search(pch)){
+      ppt = new Z;
+      (*ppt) =n;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      ppt=(Z *)give_ptr();
+      if(*ppt<n)*ppt=n;
+   }
+}
+
+template<class Z>
+void BCount<Z>::maxp_count2(char *pch,Z n){
+   Z *ppt,i;
+   Node *np;
+   total+=n;
+   if(!search(pch)){
+      ppt = new Z;
+      (*ppt) =n;
+      np=new Node;
+      np->str=pch;
+      np->rel=ppt;
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      ppt=(Z *)give_ptr();
+      if(*ppt<n)*ppt=n;
+   }
+}
+
+template<class Z>
+void BCount<Z>::min_count(const char *pch,Z n){
+   Z *ppt,i;
+   Node *np;
+   total+=n;
+   if(!search(pch)){
+      ppt = new Z;
+      (*ppt) =n;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+   }
+   else {
+      ppt=(Z *)give_ptr();
+      if(*ppt>n)*ppt=n;
+   }
+}
+
+template<class Z>
+void BCount<Z>::min_count2(const char *pch,Z n){
+   Z *ppt,i;
+   Node *np;
+   total+=n;
+   if(!search(pch)){
+      ppt = new Z;
+      (*ppt) =n;
+      np=new Node(pch,(void*)ppt);
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      ppt=(Z *)give_ptr();
+      if(*ppt>n)*ppt=n;
+   }
+}
+
+template<class Z>
+void BCount<Z>::minp_count2(char *pch,Z n){
+   Z *ppt,i;
+   Node *np;
+   total+=n;
+   if(!search(pch)){
+      ppt = new Z;
+      (*ppt) =n;
+      np=new Node;
+      np->str=pch;
+      np->rel=ppt;
+      this->insert(np);
+      cnt_key++;
+   }
+   else {
+      ppt=(Z *)give_ptr();
+      if(*ppt>n)*ppt=n;
+   }
+}
+
+}
+#endif

Library/FBase.C

@@ -1,600 +1,600 @@
-#include <iostream>
-#include <fstream>
-#include <cstdlib>
-#include <iomanip>
-#include <cstring>
-#include <cmath>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/mman.h>
-#include "runn.h"
-#include "FBase.h"
-
-using namespace std;
-namespace iret {
-
-FBase::FBase(const char *typ,const char *nam){
-   int lxn=strlen(typ);
-   type=new char[lxn+1];
-   tpnm=-1;
-   nmnm=-1;
-   strcpy(type,typ);
-   lxn=strlen(nam);
-   name=new char[lxn+1];
-   strcpy(name,nam);
-   cflag=0;
-   oflag=0;
-   pflag=get_qflag();
-   eflag=1;
-}
-
-FBase::FBase(const char *typ,int tpn,const char *nam){
-   int lxn=strlen(typ);
-   type=new char[lxn+1];
-   tpnm=tpn;
-   nmnm=-1;
-   strcpy(type,typ);
-   lxn=strlen(nam);
-   name=new char[lxn+1];
-   strcpy(name,nam);
-   cflag=0;
-   oflag=0;
-   pflag=get_qflag();
-   eflag=1;
-}
-
-FBase::FBase(const char *typ,const char *nam,const char *pt){
-   int lxn=strlen(typ);
-   type=new char[lxn+1];
-   tpnm=-1;
-   nmnm=-1;
-   strcpy(type,typ);
-   lxn=strlen(nam);
-   name=new char[lxn+1];
-   strcpy(name,nam);
-   cflag=0;
-   oflag=0;
-   pflag=get_qflag();
-   if(*pt!=':')set_path_name(pt);
-   else set_path_internal(pt+1);
-}
-
-FBase::~FBase(void){
-   delete [] type;
-   delete [] name;
-}
-
-void FBase::set_type_num(int tn){tpnm=tn;}
-
-void FBase::set_name_num(int nn){nmnm=nn;}
-
-void FBase::change_type(const char *typ){
-   if(type!=NULL)delete [] type;
-   int lxn=strlen(typ);
-   type=new char[lxn+1];
-   strcpy(type,typ);
-}
-
-void FBase::change_name(const char *nam){
-   if(name!=NULL)delete [] name;
-   int lxn=strlen(nam);
-   name=new char[lxn+1];
-   strcpy(name,nam);
-}
-
-void FBase::set_name(const char *nam){
-   if(name!=NULL)delete [] name;
-   int lxn=strlen(nam);
-   name=new char[lxn+1];
-   strcpy(name,nam);
-}
-
-void FBase::subname(const char *tph,const char *tpl,const char *nm){
-   char cnam[max_str];
-   long i=strlen(tpl);
-   strcpy(cnam,tpl);
-   cnam[i]='_';
-   cnam[i+1]='\0';
-   strcat(cnam,nm);
-   change_type(tph);
-   change_name(cnam);
-}
-
-void FBase::set_path_internal(const char *pt){
-   long len;
-   if(pt&&(len=strlen(pt))){
-      eflag=0;
-      path=new char[len+1];
-      strcpy(path,pt);
-   }
-   else eflag=1;
-}
-
-void FBase::set_path_name(const char *pa){
-   long len;
-   if(pa&&(len=strlen(pa))){
-      eflag=2;
-      pnam=new char[len+1];
-      strcpy(pnam,pa);
-   }
-   else eflag=1;
-}
-
-void FBase::map_down(FBase *pFb){
-   pFb->change_type(type);
-   pFb->change_name(name);
-   pFb->set_type_num(tpnm);
-   pFb->set_name_num(nmnm);
-   pFb->pflag=pflag;
-   if(eflag==2)pFb->set_path_name(pnam);
-   else if(!eflag)pFb->set_path_internal(path);
-}
-
-void FBase::map_down_sub(FBase *pFb,const char *subtype){
-   pFb->subname(type,name,subtype);
-   pFb->set_type_num(tpnm);
-   pFb->set_name_num(nmnm);
-   pFb->pflag=pflag;
-   if(eflag==2)pFb->set_path_name(pnam);
-   else if(!eflag)pFb->set_path_internal(path);
-}
-
-void FBase::get_pathx(char *nam,const char *ch){
-   char cnam[256];
-   ifstream fin;
-
-   if(eflag==2){
-      strcpy(cnam,"path_");
-      strcat(cnam,pnam);
-      fin.open(cnam,ios::in);
-      if(!fin.is_open()){
-         fin.clear();
-         strcpy(cnam,"path");
-         fin.open(cnam,ios::in);
-         if(!fin.is_open()){
-            cout << "Path file for type " << type << " does not exist!" << endl;
-            exit(0);
-         }
-      }
-      fin.getline(nam,256);
-      fin.close();
-   }
-   else if(eflag){
-      strcpy(cnam,"path_");
-      strcat(cnam,type);
-      strcat(cnam,"_");
-      strcat(cnam,name);
-      strcat(cnam,".");
-      strcat(cnam,ch);
-      fin.open(cnam,ios::in);
-      if(!fin.is_open()){
-         fin.clear();
-         strcpy(cnam,"path_");
-         strcat(cnam,type);
-         strcat(cnam,"_");
-         strcat(cnam,name);
-         fin.open(cnam,ios::in);
-         if(!fin.is_open()){
-	    fin.clear();
-            strcpy(cnam,"path_");
-            strcat(cnam,type);
-            fin.open(cnam,ios::in);
-            if(!fin.is_open()){
-               fin.clear();
-               strcpy(cnam,"path");
-               fin.open(cnam,ios::in);
-               if(!fin.is_open()){
-                  cout << "Path file for type " << type << " does not exist!" << endl;
-                  exit(0);
-               }
-            }
-         }
-      }
-      fin.getline(nam,256);
-      fin.close();
-   }
-   else {
-      strcpy(nam,path);
-   }
-
-   if(tpnm<0)strcat(nam,type);
-   else cat_num(type,tpnm,nam);
-   strcat(nam,"_");
-   if(nmnm<0)strcat(nam,name);
-   else cat_num(name,nmnm,nam);
-   strcat(nam,".");
-   strcat(nam,ch);
-}
-
-void FBase::get_pathx(char *nam,long n,const char *ch){
-   char cnam[256],bnam[256];
-   ifstream fin;
-
-   if(eflag==2){
-      strcpy(cnam,"path_");
-      strcat(cnam,pnam);
-      fin.open(cnam,ios::in);
-      if(!fin.is_open()){
-         fin.clear();
-         strcpy(cnam,"path");
-         fin.open(cnam,ios::in);
-         if(!fin.is_open()){
-            cout << "Path file for type " << type << " does not exist!" << endl;
-            exit(0);
-         }
-      }
-      fin.getline(nam,256);
-      fin.close();
-   }
-   else if(eflag){
-      strcpy(cnam,"path_");
-      strcat(cnam,type);
-      strcat(cnam,"_");
-      strcat(cnam,name);
-      strcat(cnam,".");
-      strcat(cnam,ch);
-      fin.open(cnam,ios::in);
-      if(!fin.is_open()){
-         fin.clear();
-         strcpy(cnam,"path_");
-         strcat(cnam,type);
-         strcat(cnam,"_");
-         strcat(cnam,name);
-         fin.open(cnam,ios::in);
-         if(!fin.is_open()){
-            fin.clear();
-            strcpy(cnam,"path_");
-            strcat(cnam,type);
-            fin.open(cnam,ios::in);
-            if(!fin.is_open()){
-               fin.clear();
-               strcpy(cnam,"path");
-               fin.open(cnam,ios::in);
-               if(!fin.is_open()){
-                  cout << "Path file for type " << type << " does not exist!" << endl;
-                  exit(0);
-               }
-            }
-         }
-      }
-      fin.getline(nam,256);
-      fin.close();
-   }
-   else {
-      strcpy(nam,path);
-   }
-
-   if(tpnm<0)strcat(nam,type);
-   else cat_num(type,tpnm,nam);
-   strcat(nam,"_");
-   strcat(nam,add_num(name,n,bnam));
-   strcat(nam,".");
-   strcat(nam,ch);
-}
-
-char *FBase::add_num(const char *ptr,long n,char *buf){
-   char cnam[100];
-   long_str(cnam,n);
-   strcpy(buf,ptr);
-   strcat(buf,cnam);
-   return(buf);
-}
-
-char *FBase::cat_num(const char *ptr,long n,char *buf){
-   char cnam[100];
-   long_str(cnam,n);
-   strcat(buf,ptr);
-   strcat(buf,cnam);
-   return(buf);
-}
-
-int FBase::Gcom(int sflag){
-   if((cflag&sflag)&&!(oflag&sflag)){
-      oflag=oflag|sflag;
-      return(1);
-   }
-   else return(0);
-}
-
-int FBase::Rcom(int sflag){
-   if((cflag&sflag)&&(oflag&sflag)){
-      oflag=oflag&(~sflag);
-      return(1);
-   }
-   else return(0);
-}
-
-ifstream *FBase::get_Istr(const char *a,ios::openmode mode){
-   char cnam[max_str];
-   get_pathx(cnam,a);
-   ifstream *pfin=new ifstream(cnam,mode);
-   if(pfin->is_open())return(pfin);
-   else {
-      cout << "Error: " << cnam << " failed to open!" << endl;
-      exit(0);
-   }
-}
-
-ofstream *FBase::get_Ostr(const char *a,ios::openmode mode){
-   char cnam[max_str];
-   get_pathx(cnam,a);
-   ofstream *pfout=new ofstream(cnam,mode);
-   if(pfout->is_open())return(pfout);
-   else {
-      cout << "Error: " << cnam << " failed to open!" << endl;
-      exit(0);
-   }
-}
-
-fstream *FBase::get_Fstr(const char *a,ios::openmode mode){
-   char cnam[max_str];
-   get_pathx(cnam,a);
-   fstream *pfstr=new fstream(cnam,mode);
-   if(pfstr->is_open())return(pfstr);
-   else {
-      cout << "Error: " << cnam << " failed to open!" << endl;
-      exit(0);
-   }
-}
-
-ifstream *FBase::get_Istr(long n,const char *a,ios::openmode mode){
-   char cnam[max_str];
-   get_pathx(cnam,n,a);
-   ifstream *pfin=new ifstream(cnam,mode);
-   if(pfin->is_open())return(pfin);
-   else {
-      cout << "Error: " << cnam << " failed to open!" << endl;
-      exit(0);
-   }
-}
-
-ofstream *FBase::get_Ostr(long n,const char *a,ios::openmode mode){
-   char cnam[max_str];
-   get_pathx(cnam,n,a);
-   ofstream *pfout=new ofstream(cnam,mode);
-   if(pfout->is_open())return(pfout);
-   else {
-      cout << "Error: " << cnam << " failed to open!" << endl;
-      exit(0);
-   }
-}
-
-fstream *FBase::get_Fstr(long n,const char *a,ios::openmode mode){
-   char cnam[max_str];
-   get_pathx(cnam,n,a);
-   fstream *pfstr=new fstream(cnam,mode);
-   if(pfstr->is_open())return(pfstr);
-   else {
-      cout << "Error: " << cnam << " failed to open!" << endl;
-      exit(0);
-   }
-}
-
-void FBase::dst_Istr(ifstream *pfin){
-   if(!pfin)return;
-   if(!pfin->is_open()){
-      cout << "File not open!" << endl;
-      exit(0);
-   }
-   delete pfin;
-}
-
-void FBase::dst_Ostr(ofstream *pfout){
-   if(!pfout)return;
-   if(!pfout->is_open()){
-      cout << "File not open!" << endl;
-      exit(0);
-   }
-   delete pfout;
-}
-
-void FBase::dst_Fstr(fstream *pfstr){
-   if(!pfstr)return;
-   if(!pfstr->is_open()){
-      cout << "File not open!" << endl;
-      exit(0);
-   }
-   delete pfstr;
-}
-
-long FBase::get_Fsiz(const char *a){
-   if(!Exists(a))return(0);
-   int fld;
-   struct stat datf;
-   char cnam[max_str];
-   get_pathx(cnam,a);
-   fld=::open(cnam,O_RDONLY);
-   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
-   if(fstat(fld,&datf)){cout << cnam << " failed on size \
-      determination" << endl;exit(0);}
-   ::close(fld);
-   return(datf.st_size);
-}
-
-long FBase::get_Fsiz(long n,const char *a){
-   if(!Exists(n,a))return(0);
-   int fld;
-   struct stat datf;
-   char cnam[max_str];
-   get_pathx(cnam,n,a);
-   fld=::open(cnam,O_RDONLY);
-   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
-   if(fstat(fld,&datf)){cout << cnam << " failed on size \
-      determination" << endl;exit(0);}
-   ::close(fld);
-   return(datf.st_size);
-}
-
-char *FBase::get_Read(const char *a){
-   int fld;
-   struct stat datf;
-   char cnam[max_str];
-   get_pathx(cnam,a);
-   fld=::open(cnam,O_RDONLY);
-   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
-   if(fstat(fld,&datf)){cout << cnam << " failed on size \
-      determination" << endl;exit(0);}
-   ::close(fld);
-   char *ptr=new char[datf.st_size];
-   ifstream fin(cnam,ios::in);
-   if(!fin.is_open()){
-      cout << "Error: " << cnam << " failed to open!" << endl;
-      exit(0);
-   }
-   fin.read(ptr,datf.st_size);
-   return(ptr);
-}
-
-char *FBase::get_Read(long n,const char *a){
-   int fld;
-   struct stat datf;
-   char cnam[max_str];
-   get_pathx(cnam,n,a);
-   fld=::open(cnam,O_RDONLY);
-   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
-   if(fstat(fld,&datf)){cout << cnam << " failed on size \
-      determination" << endl;exit(0);}
-   ::close(fld);
-   char *ptr=new char[datf.st_size];
-   ifstream fin(cnam,ios::in);
-   if(!fin.is_open()){
-      cout << "Error: " << cnam << " failed to open!" << endl;
-      exit(0);
-   }
-   fin.read(ptr,datf.st_size);
-   return(ptr);
-}
-
-char *FBase::get_Mmap(const char *a){
-   int fld;
-   struct stat datf;
-   char cnam[max_str];
-   get_pathx(cnam,a);
-   fld=::open(cnam,O_RDONLY);
-   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
-   if(fstat(fld,&datf)){cout << cnam << " failed on size determination" << endl;exit(0);}
-   char *ptr=(char*)mmap(0,datf.st_size,PROT_READ,MAP_PRIVATE|MAP_NORESERVE,fld,0);
-   if(ptr==MAP_FAILED){cout << cnam << " failed to map" << endl;exit(0);}
-   ::close(fld);
-   return(ptr);
-}
-
-char *FBase::get_Mmap(long n,const char *a){
-   int fld;
-   struct stat datf;
-   char cnam[max_str];
-   get_pathx(cnam,n,a);
-   fld=::open(cnam,O_RDONLY);
-   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
-   if(fstat(fld,&datf)){cout << cnam << " failed on size determination" << endl;exit(0);}
-   char *ptr=(char*)mmap(0,datf.st_size,PROT_READ,MAP_PRIVATE|MAP_NORESERVE,fld,0);
-   if(ptr==MAP_FAILED){cout << cnam << " failed to map" << endl;exit(0);}
-   ::close(fld);
-   return(ptr);
-}
-
-char *FBase::get_Wmap(const char *a){
-   int fld;
-   struct stat datf;
-   char cnam[max_str];
-   get_pathx(cnam,a);
-   fld=::open(cnam,O_RDWR);
-   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
-   if(fstat(fld,&datf)){cout << cnam << " failed on size determination" << endl;exit(0);}
-   char *ptr=(char*)mmap(0,datf.st_size,PROT_READ|PROT_WRITE,MAP_SHARED,fld,0);
-   if(ptr==MAP_FAILED){cout << cnam << " failed to map" << endl;exit(0);}
-   ::close(fld);
-   return(ptr);
-}  
-   
-char *FBase::get_Wmap(long n,const char *a){
-   int fld;
-   struct stat datf;
-   char cnam[max_str];
-   get_pathx(cnam,n,a);
-   fld=::open(cnam,O_RDWR);
-   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
-   if(fstat(fld,&datf)){cout << cnam << " failed on size determination" << endl;exit(0);}
-   char *ptr=(char*)mmap(0,datf.st_size,PROT_READ|PROT_WRITE,MAP_SHARED,fld,0);
-   if(ptr==MAP_FAILED){cout << cnam << " failed to map" << endl;exit(0);}
-   ::close(fld);
-   return(ptr);
-}
-
-void FBase::dst_Mmap(const char *a,char *ptr){
-   struct stat datf;
-   char cnam[max_str];
-   if(ptr==NULL){cout << "NULL pointer" << endl;return;}
-   get_pathx(cnam,a);
-   if(stat(cnam,&datf)){cout << cnam << " failed on size determination" << endl;exit(0);}
-   if(munmap(ptr,datf.st_size)){cout << cnam << " failed to unmap" << endl;exit(0);}
-   ptr=NULL;
-}
-
-void FBase::dst_Mmap(long n,const char *a,char *ptr){
-   struct stat datf;
-   char cnam[max_str];
-   if(ptr==NULL){cout << "NULL pointer" << endl;return;}
-   get_pathx(cnam,n,a);
-   if(stat(cnam,&datf)){cout << cnam << " failed on size determination" << endl;exit(0);}
-   if(munmap(ptr,datf.st_size)){cout << cnam << " failed to unmap" << endl;exit(0);}
-   ptr=NULL;
-}
-
-void FBase::bin_Writ(const char *a,long nm,char *ptr){
-   ofstream *pfout=get_Ostr(a,ios::out);
-   long k=100000,i=0;
-   while(i+k<nm){
-      pfout->write((char*)ptr,k);
-      i+=k;
-      ptr=ptr+k;
-   }
-   pfout->write((char*)ptr,nm-i);
-   pfout->close();
-   delete pfout;
-}
-
-void FBase::bin_Writ(long n,const char *a,long nm,char *ptr){
-   ofstream *pfout=get_Ostr(n,a,ios::out);
-   long k=100000,i=0;
-   while(i+k<nm){
-      pfout->write((char*)ptr,k);
-      i+=k;
-      ptr=ptr+k;
-   }
-   pfout->write((char*)ptr,nm-i);
-   pfout->close();
-   delete pfout;
-}
-
-int FBase::Exists(const char *a){
-   char cnam[max_str];
-   get_pathx(cnam,a);
-   ifstream fin(cnam,ios::in);
-   if(fin.is_open()){
-      fin.close();
-      return(1);
-   }
-   else return(0);
-}
-
-int FBase::Exists(long n,const char *a){
-   char cnam[max_str];
-   get_pathx(cnam,n,a);
-   ifstream fin(cnam,ios::in);
-   if(fin.is_open()){
-      fin.close();
-      return(1);
-   }
-   else return(0);
-}
-
-void FBase::mark(long ct, int ivl, const char *what){
-   if(pflag&&(ct%ivl==0)){
-      cout << what << " count=" << ct << endl;
-   }
-}
-   
-}
+#include <iostream>
+#include <fstream>
+#include <cstdlib>
+#include <iomanip>
+#include <cstring>
+#include <cmath>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include "runn.h"
+#include "FBase.h"
+
+using namespace std;
+namespace iret {
+
+FBase::FBase(const char *typ,const char *nam){
+   int lxn=strlen(typ);
+   type=new char[lxn+1];
+   tpnm=-1;
+   nmnm=-1;
+   strcpy(type,typ);
+   lxn=strlen(nam);
+   name=new char[lxn+1];
+   strcpy(name,nam);
+   cflag=0;
+   oflag=0;
+   pflag=get_qflag();
+   eflag=1;
+}
+
+FBase::FBase(const char *typ,int tpn,const char *nam){
+   int lxn=strlen(typ);
+   type=new char[lxn+1];
+   tpnm=tpn;
+   nmnm=-1;
+   strcpy(type,typ);
+   lxn=strlen(nam);
+   name=new char[lxn+1];
+   strcpy(name,nam);
+   cflag=0;
+   oflag=0;
+   pflag=get_qflag();
+   eflag=1;
+}
+
+FBase::FBase(const char *typ,const char *nam,const char *pt){
+   int lxn=strlen(typ);
+   type=new char[lxn+1];
+   tpnm=-1;
+   nmnm=-1;
+   strcpy(type,typ);
+   lxn=strlen(nam);
+   name=new char[lxn+1];
+   strcpy(name,nam);
+   cflag=0;
+   oflag=0;
+   pflag=get_qflag();
+   if(*pt!=':')set_path_name(pt);
+   else set_path_internal(pt+1);
+}
+
+FBase::~FBase(void){
+   delete [] type;
+   delete [] name;
+}
+
+void FBase::set_type_num(int tn){tpnm=tn;}
+
+void FBase::set_name_num(int nn){nmnm=nn;}
+
+void FBase::change_type(const char *typ){
+   if(type!=NULL)delete [] type;
+   int lxn=strlen(typ);
+   type=new char[lxn+1];
+   strcpy(type,typ);
+}
+
+void FBase::change_name(const char *nam){
+   if(name!=NULL)delete [] name;
+   int lxn=strlen(nam);
+   name=new char[lxn+1];
+   strcpy(name,nam);
+}
+
+void FBase::set_name(const char *nam){
+   if(name!=NULL)delete [] name;
+   int lxn=strlen(nam);
+   name=new char[lxn+1];
+   strcpy(name,nam);
+}
+
+void FBase::subname(const char *tph,const char *tpl,const char *nm){
+   char cnam[max_str];
+   long i=strlen(tpl);
+   strcpy(cnam,tpl);
+   cnam[i]='_';
+   cnam[i+1]='\0';
+   strcat(cnam,nm);
+   change_type(tph);
+   change_name(cnam);
+}
+
+void FBase::set_path_internal(const char *pt){
+   long len;
+   if(pt&&(len=strlen(pt))){
+      eflag=0;
+      path=new char[len+1];
+      strcpy(path,pt);
+   }
+   else eflag=1;
+}
+
+void FBase::set_path_name(const char *pa){
+   long len;
+   if(pa&&(len=strlen(pa))){
+      eflag=2;
+      pnam=new char[len+1];
+      strcpy(pnam,pa);
+   }
+   else eflag=1;
+}
+
+void FBase::map_down(FBase *pFb){
+   pFb->change_type(type);
+   pFb->change_name(name);
+   pFb->set_type_num(tpnm);
+   pFb->set_name_num(nmnm);
+   pFb->pflag=pflag;
+   if(eflag==2)pFb->set_path_name(pnam);
+   else if(!eflag)pFb->set_path_internal(path);
+}
+
+void FBase::map_down_sub(FBase *pFb,const char *subtype){
+   pFb->subname(type,name,subtype);
+   pFb->set_type_num(tpnm);
+   pFb->set_name_num(nmnm);
+   pFb->pflag=pflag;
+   if(eflag==2)pFb->set_path_name(pnam);
+   else if(!eflag)pFb->set_path_internal(path);
+}
+
+void FBase::get_pathx(char *nam,const char *ch){
+   char cnam[256];
+   ifstream fin;
+
+   if(eflag==2){
+      strcpy(cnam,"path_");
+      strcat(cnam,pnam);
+      fin.open(cnam,ios::in);
+      if(!fin.is_open()){
+         fin.clear();
+         strcpy(cnam,"path");
+         fin.open(cnam,ios::in);
+         if(!fin.is_open()){
+            cout << "Path file for type " << type << " does not exist!" << endl;
+            exit(0);
+         }
+      }
+      fin.getline(nam,256);
+      fin.close();
+   }
+   else if(eflag){
+      strcpy(cnam,"path_");
+      strcat(cnam,type);
+      strcat(cnam,"_");
+      strcat(cnam,name);
+      strcat(cnam,".");
+      strcat(cnam,ch);
+      fin.open(cnam,ios::in);
+      if(!fin.is_open()){
+         fin.clear();
+         strcpy(cnam,"path_");
+         strcat(cnam,type);
+         strcat(cnam,"_");
+         strcat(cnam,name);
+         fin.open(cnam,ios::in);
+         if(!fin.is_open()){
+	    fin.clear();
+            strcpy(cnam,"path_");
+            strcat(cnam,type);
+            fin.open(cnam,ios::in);
+            if(!fin.is_open()){
+               fin.clear();
+               strcpy(cnam,"path");
+               fin.open(cnam,ios::in);
+               if(!fin.is_open()){
+                  cout << "Path file for type " << type << " does not exist!" << endl;
+                  exit(0);
+               }
+            }
+         }
+      }
+      fin.getline(nam,256);
+      fin.close();
+   }
+   else {
+      strcpy(nam,path);
+   }
+
+   if(tpnm<0)strcat(nam,type);
+   else cat_num(type,tpnm,nam);
+   strcat(nam,"_");
+   if(nmnm<0)strcat(nam,name);
+   else cat_num(name,nmnm,nam);
+   strcat(nam,".");
+   strcat(nam,ch);
+}
+
+void FBase::get_pathx(char *nam,long n,const char *ch){
+   char cnam[256],bnam[256];
+   ifstream fin;
+
+   if(eflag==2){
+      strcpy(cnam,"path_");
+      strcat(cnam,pnam);
+      fin.open(cnam,ios::in);
+      if(!fin.is_open()){
+         fin.clear();
+         strcpy(cnam,"path");
+         fin.open(cnam,ios::in);
+         if(!fin.is_open()){
+            cout << "Path file for type " << type << " does not exist!" << endl;
+            exit(0);
+         }
+      }
+      fin.getline(nam,256);
+      fin.close();
+   }
+   else if(eflag){
+      strcpy(cnam,"path_");
+      strcat(cnam,type);
+      strcat(cnam,"_");
+      strcat(cnam,name);
+      strcat(cnam,".");
+      strcat(cnam,ch);
+      fin.open(cnam,ios::in);
+      if(!fin.is_open()){
+         fin.clear();
+         strcpy(cnam,"path_");
+         strcat(cnam,type);
+         strcat(cnam,"_");
+         strcat(cnam,name);
+         fin.open(cnam,ios::in);
+         if(!fin.is_open()){
+            fin.clear();
+            strcpy(cnam,"path_");
+            strcat(cnam,type);
+            fin.open(cnam,ios::in);
+            if(!fin.is_open()){
+               fin.clear();
+               strcpy(cnam,"path");
+               fin.open(cnam,ios::in);
+               if(!fin.is_open()){
+                  cout << "Path file for type " << type << " does not exist!" << endl;
+                  exit(0);
+               }
+            }
+         }
+      }
+      fin.getline(nam,256);
+      fin.close();
+   }
+   else {
+      strcpy(nam,path);
+   }
+
+   if(tpnm<0)strcat(nam,type);
+   else cat_num(type,tpnm,nam);
+   strcat(nam,"_");
+   strcat(nam,add_num(name,n,bnam));
+   strcat(nam,".");
+   strcat(nam,ch);
+}
+
+char *FBase::add_num(const char *ptr,long n,char *buf){
+   char cnam[100];
+   long_str(cnam,n);
+   strcpy(buf,ptr);
+   strcat(buf,cnam);
+   return(buf);
+}
+
+char *FBase::cat_num(const char *ptr,long n,char *buf){
+   char cnam[100];
+   long_str(cnam,n);
+   strcat(buf,ptr);
+   strcat(buf,cnam);
+   return(buf);
+}
+
+int FBase::Gcom(int sflag){
+   if((cflag&sflag)&&!(oflag&sflag)){
+      oflag=oflag|sflag;
+      return(1);
+   }
+   else return(0);
+}
+
+int FBase::Rcom(int sflag){
+   if((cflag&sflag)&&(oflag&sflag)){
+      oflag=oflag&(~sflag);
+      return(1);
+   }
+   else return(0);
+}
+
+ifstream *FBase::get_Istr(const char *a,ios::openmode mode){
+   char cnam[max_str];
+   get_pathx(cnam,a);
+   ifstream *pfin=new ifstream(cnam,mode);
+   if(pfin->is_open())return(pfin);
+   else {
+      cout << "Error: " << cnam << " failed to open!" << endl;
+      exit(0);
+   }
+}
+
+ofstream *FBase::get_Ostr(const char *a,ios::openmode mode){
+   char cnam[max_str];
+   get_pathx(cnam,a);
+   ofstream *pfout=new ofstream(cnam,mode);
+   if(pfout->is_open())return(pfout);
+   else {
+      cout << "Error: " << cnam << " failed to open!" << endl;
+      exit(0);
+   }
+}
+
+fstream *FBase::get_Fstr(const char *a,ios::openmode mode){
+   char cnam[max_str];
+   get_pathx(cnam,a);
+   fstream *pfstr=new fstream(cnam,mode);
+   if(pfstr->is_open())return(pfstr);
+   else {
+      cout << "Error: " << cnam << " failed to open!" << endl;
+      exit(0);
+   }
+}
+
+ifstream *FBase::get_Istr(long n,const char *a,ios::openmode mode){
+   char cnam[max_str];
+   get_pathx(cnam,n,a);
+   ifstream *pfin=new ifstream(cnam,mode);
+   if(pfin->is_open())return(pfin);
+   else {
+      cout << "Error: " << cnam << " failed to open!" << endl;
+      exit(0);
+   }
+}
+
+ofstream *FBase::get_Ostr(long n,const char *a,ios::openmode mode){
+   char cnam[max_str];
+   get_pathx(cnam,n,a);
+   ofstream *pfout=new ofstream(cnam,mode);
+   if(pfout->is_open())return(pfout);
+   else {
+      cout << "Error: " << cnam << " failed to open!" << endl;
+      exit(0);
+   }
+}
+
+fstream *FBase::get_Fstr(long n,const char *a,ios::openmode mode){
+   char cnam[max_str];
+   get_pathx(cnam,n,a);
+   fstream *pfstr=new fstream(cnam,mode);
+   if(pfstr->is_open())return(pfstr);
+   else {
+      cout << "Error: " << cnam << " failed to open!" << endl;
+      exit(0);
+   }
+}
+
+void FBase::dst_Istr(ifstream *pfin){
+   if(!pfin)return;
+   if(!pfin->is_open()){
+      cout << "File not open!" << endl;
+      exit(0);
+   }
+   delete pfin;
+}
+
+void FBase::dst_Ostr(ofstream *pfout){
+   if(!pfout)return;
+   if(!pfout->is_open()){
+      cout << "File not open!" << endl;
+      exit(0);
+   }
+   delete pfout;
+}
+
+void FBase::dst_Fstr(fstream *pfstr){
+   if(!pfstr)return;
+   if(!pfstr->is_open()){
+      cout << "File not open!" << endl;
+      exit(0);
+   }
+   delete pfstr;
+}
+
+long FBase::get_Fsiz(const char *a){
+   if(!Exists(a))return(0);
+   int fld;
+   struct stat datf;
+   char cnam[max_str];
+   get_pathx(cnam,a);
+   fld=::open(cnam,O_RDONLY);
+   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
+   if(fstat(fld,&datf)){cout << cnam << " failed on size \
+      determination" << endl;exit(0);}
+   ::close(fld);
+   return(datf.st_size);
+}
+
+long FBase::get_Fsiz(long n,const char *a){
+   if(!Exists(n,a))return(0);
+   int fld;
+   struct stat datf;
+   char cnam[max_str];
+   get_pathx(cnam,n,a);
+   fld=::open(cnam,O_RDONLY);
+   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
+   if(fstat(fld,&datf)){cout << cnam << " failed on size \
+      determination" << endl;exit(0);}
+   ::close(fld);
+   return(datf.st_size);
+}
+
+char *FBase::get_Read(const char *a){
+   int fld;
+   struct stat datf;
+   char cnam[max_str];
+   get_pathx(cnam,a);
+   fld=::open(cnam,O_RDONLY);
+   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
+   if(fstat(fld,&datf)){cout << cnam << " failed on size \
+      determination" << endl;exit(0);}
+   ::close(fld);
+   char *ptr=new char[datf.st_size];
+   ifstream fin(cnam,ios::in);
+   if(!fin.is_open()){
+      cout << "Error: " << cnam << " failed to open!" << endl;
+      exit(0);
+   }
+   fin.read(ptr,datf.st_size);
+   return(ptr);
+}
+
+char *FBase::get_Read(long n,const char *a){
+   int fld;
+   struct stat datf;
+   char cnam[max_str];
+   get_pathx(cnam,n,a);
+   fld=::open(cnam,O_RDONLY);
+   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
+   if(fstat(fld,&datf)){cout << cnam << " failed on size \
+      determination" << endl;exit(0);}
+   ::close(fld);
+   char *ptr=new char[datf.st_size];
+   ifstream fin(cnam,ios::in);
+   if(!fin.is_open()){
+      cout << "Error: " << cnam << " failed to open!" << endl;
+      exit(0);
+   }
+   fin.read(ptr,datf.st_size);
+   return(ptr);
+}
+
+char *FBase::get_Mmap(const char *a){
+   int fld;
+   struct stat datf;
+   char cnam[max_str];
+   get_pathx(cnam,a);
+   fld=::open(cnam,O_RDONLY);
+   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
+   if(fstat(fld,&datf)){cout << cnam << " failed on size determination" << endl;exit(0);}
+   char *ptr=(char*)mmap(0,datf.st_size,PROT_READ,MAP_PRIVATE|MAP_NORESERVE,fld,0);
+   if(ptr==MAP_FAILED){cout << cnam << " failed to map" << endl;exit(0);}
+   ::close(fld);
+   return(ptr);
+}
+
+char *FBase::get_Mmap(long n,const char *a){
+   int fld;
+   struct stat datf;
+   char cnam[max_str];
+   get_pathx(cnam,n,a);
+   fld=::open(cnam,O_RDONLY);
+   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
+   if(fstat(fld,&datf)){cout << cnam << " failed on size determination" << endl;exit(0);}
+   char *ptr=(char*)mmap(0,datf.st_size,PROT_READ,MAP_PRIVATE|MAP_NORESERVE,fld,0);
+   if(ptr==MAP_FAILED){cout << cnam << " failed to map" << endl;exit(0);}
+   ::close(fld);
+   return(ptr);
+}
+
+char *FBase::get_Wmap(const char *a){
+   int fld;
+   struct stat datf;
+   char cnam[max_str];
+   get_pathx(cnam,a);
+   fld=::open(cnam,O_RDWR);
+   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
+   if(fstat(fld,&datf)){cout << cnam << " failed on size determination" << endl;exit(0);}
+   char *ptr=(char*)mmap(0,datf.st_size,PROT_READ|PROT_WRITE,MAP_SHARED,fld,0);
+   if(ptr==MAP_FAILED){cout << cnam << " failed to map" << endl;exit(0);}
+   ::close(fld);
+   return(ptr);
+}  
+   
+char *FBase::get_Wmap(long n,const char *a){
+   int fld;
+   struct stat datf;
+   char cnam[max_str];
+   get_pathx(cnam,n,a);
+   fld=::open(cnam,O_RDWR);
+   if(fld<=0){cout << cnam << " failed to open" << endl;exit(0);}
+   if(fstat(fld,&datf)){cout << cnam << " failed on size determination" << endl;exit(0);}
+   char *ptr=(char*)mmap(0,datf.st_size,PROT_READ|PROT_WRITE,MAP_SHARED,fld,0);
+   if(ptr==MAP_FAILED){cout << cnam << " failed to map" << endl;exit(0);}
+   ::close(fld);
+   return(ptr);
+}
+
+void FBase::dst_Mmap(const char *a,char *ptr){
+   struct stat datf;
+   char cnam[max_str];
+   if(ptr==NULL){cout << "NULL pointer" << endl;return;}
+   get_pathx(cnam,a);
+   if(stat(cnam,&datf)){cout << cnam << " failed on size determination" << endl;exit(0);}
+   if(munmap(ptr,datf.st_size)){cout << cnam << " failed to unmap" << endl;exit(0);}
+   ptr=NULL;
+}
+
+void FBase::dst_Mmap(long n,const char *a,char *ptr){
+   struct stat datf;
+   char cnam[max_str];
+   if(ptr==NULL){cout << "NULL pointer" << endl;return;}
+   get_pathx(cnam,n,a);
+   if(stat(cnam,&datf)){cout << cnam << " failed on size determination" << endl;exit(0);}
+   if(munmap(ptr,datf.st_size)){cout << cnam << " failed to unmap" << endl;exit(0);}
+   ptr=NULL;
+}
+
+void FBase::bin_Writ(const char *a,long nm,char *ptr){
+   ofstream *pfout=get_Ostr(a,ios::out);
+   long k=100000,i=0;
+   while(i+k<nm){
+      pfout->write((char*)ptr,k);
+      i+=k;
+      ptr=ptr+k;
+   }
+   pfout->write((char*)ptr,nm-i);
+   pfout->close();
+   delete pfout;
+}
+
+void FBase::bin_Writ(long n,const char *a,long nm,char *ptr){
+   ofstream *pfout=get_Ostr(n,a,ios::out);
+   long k=100000,i=0;
+   while(i+k<nm){
+      pfout->write((char*)ptr,k);
+      i+=k;
+      ptr=ptr+k;
+   }
+   pfout->write((char*)ptr,nm-i);
+   pfout->close();
+   delete pfout;
+}
+
+int FBase::Exists(const char *a){
+   char cnam[max_str];
+   get_pathx(cnam,a);
+   ifstream fin(cnam,ios::in);
+   if(fin.is_open()){
+      fin.close();
+      return(1);
+   }
+   else return(0);
+}
+
+int FBase::Exists(long n,const char *a){
+   char cnam[max_str];
+   get_pathx(cnam,n,a);
+   ifstream fin(cnam,ios::in);
+   if(fin.is_open()){
+      fin.close();
+      return(1);
+   }
+   else return(0);
+}
+
+void FBase::mark(long ct, int ivl, const char *what){
+   if(pflag&&(ct%ivl==0)){
+      cout << what << " count=" << ct << endl;
+   }
+}
+   
+}

Library/FBase.h

@@ -1,248 +1,248 @@
-#ifndef FBASE_H
-#define FBASE_H
-
-#include <iostream>
-#include <fstream>
-
-using namespace std;
-namespace iret {
-
-typedef char *pChr;
-
-class FBase {
-   public:
-      FBase(const char *tp,const char *nm); //tp is type name, nm is name 
-      FBase(const char *tp,int tn,const char *nm); //tp is type name, if 
-         //nonnegative tn is appended to end of tp, nm is name 
-      FBase(const char *tp,const char *nm,const char *pt); //tp is type name, nm is name 
-         //pt is pointer at a string sss and reads the path from file path_sss in 
-         //current directory. But if sss begins with ':' then skips this character 
-         //and remaining string is the path string itself.
-      ~FBase();
-      void set_type_num(int tn); //Sets tpnm and uses if nonnegative: appended
-         //to end of type name
-      void set_name_num(int nn); //Sets nmnm and uses if nonnegative: appended
-         //to end of name
-      void change_type(const char *nm); //Allows change of type string for class.
-      void change_name(const char *nm); //Allows change of name string for class.
-      void set_name(const char *nm); //Allows change of name string for class.
-         //Included for compatibility
-      void subname(const char *tph,const char *tpl,const char *nm); //Uses the 
-         //higher level type tph as type and combines lower level tpl_nm with
-         //name to allow one to keep track of file types.
-      void set_path_internal(const char *pt); //Path is by default external with
-         //eflag=1. But if this function called with nonempty string, then eflag=0
-         //and pt stored in path and used for access to data.
-      void set_path_name(const char *pa); //path will be extracted from path_pa
-         //and eflag=2. Naming conventions for files are unchanged
-      void map_down(FBase *pFb); //Maps naming parameters to class instance pFb
-      void map_down_sub(FBase *pFb,const char *subtype); //Maps naming parameters to class instance pFb
-         //combines subtype with name to make a new name for pFb and type becomes its type
-
-      //Path access functions
-      void get_pathx(char *cn,const char *a);
-         //Reads the path from a file "path_(*name)" and constructs the
-         //file name from as "(*type)_(*name).(*a)". Cats path and file
-         //name and returns the full info in cn.
-      void get_pathx(char *cn,long n,const char *a);
-      char *add_num(const char *ptr,long n,char *buf); //converts long to ascii
-         //and cats to end of string and returns pointer to new string
-         //that results. Does not change input string. The new string is
-         //held in buffer space and this is overwritten at each call.
-      char *cat_num(const char *ptr,long n,char *buf); //converts long to ascii
-         //and cats to end of ptr string and then cats result to end of
-         //whatever is in buffer. Does not change input string. The new string is
-         //held in buffer space.
-
-      //Stream object pointers
-      ifstream *get_Istr(const char *a,ios::openmode m=ios::in);
-         //Opens input file stream by path and name composition.
-      ofstream *get_Ostr(const char *a,ios::openmode m=ios::out);
-         //Opens output file stream by path and name composition.
-      fstream *get_Fstr(const char *a,ios::openmode m=ios::in|ios::out);
-         //Opens output file stream by path and name composition.
-      ifstream *get_Istr(long n,const char *a,ios::openmode m=ios::in);
-      ofstream *get_Ostr(long n,const char *a,ios::openmode m=ios::out);
-      fstream *get_Fstr(long n,const char *a,ios::openmode m=ios::in|ios::out);
-      void dst_Istr(ifstream *pfin);
-      void dst_Ostr(ofstream *pfout);
-      void dst_Fstr(fstream *pfstr);
-
-      //Get file size in bytes
-      long get_Fsiz(const char *a);
-      long get_Fsiz(long n,const char *a);
-
-      //File existence
-      int Exists(const char *a); //returns 1 if file exists
-      int Exists(long n,const char *a); //returns 1 if file exists
-
-      //Read in array pointers
-      char *get_Read(const char *a);
-         //Reads in a file into an char array and returns pointer
-      char *get_Read(long n,const char *a);
-
-      //Memory map pointers
-      char *get_Mmap(const char *a);
-         //Memory maps file by path and name composition.
-      char *get_Mmap(long n,const char *a);
-      char *get_Wmap(const char *a);
-         //Memory maps file by path and name composition.
-         //Allows to modify contents and is written out when dst_Mmap called
-      char *get_Wmap(long n,const char *a);
-         //Allows to modify contents and is written out when dst_Mmap called
-      void dst_Mmap(const char *a,char *ptr);
-         //Removes the memory map for ptr based on path and name composition.
-      void dst_Mmap(long n,const char *a,char *ptr);
-
-      //Array of chars and binary write
-      void bin_Writ(const char *a,long nm,char *ptr); 
-         //Writes out nm bytes binary
-      void bin_Writ(long n,const char *a,long nm,char *ptr); 
-
-      //Write and read 1, 2, or 3 long integers to or from a file
-      template <typename X>
-      void get_Nnum(const char *a,X &m1);
-      template <typename X,typename Y>
-      void get_Nnum(const char *a,X &m1,Y &m2);
-      template <typename X,typename Y,typename Z>
-      void get_Nnum(const char *a,X &m1,Y &m2,Z &m3);
-      template <typename X>
-      void get_Nnum(long n,const char *a,X &m1);
-      template <typename X,typename Y>
-      void get_Nnum(long n,const char *a,X &m1,Y &m2);
-      template <typename X,typename Y,typename Z>
-      void get_Nnum(long n,const char *a,X &m1,Y &m2,Z &m3);
-      template <typename X>
-      void put_Nnum(const char *a,X &m1);
-      template <typename X,typename Y>
-      void put_Nnum(const char *a,X &m1,Y &m2);
-      template <typename X,typename Y,typename Z>
-      void put_Nnum(const char *a,X &m1,Y &m2,Z &m3);
-      template <typename X>
-      void put_Nnum(long n,const char *a,X &m1);
-      template <typename X,typename Y>
-      void put_Nnum(long n,const char *a,X &m1,Y &m2);
-      template <typename X,typename Y,typename Z>
-      void put_Nnum(long n,const char *a,X &m1,Y &m2,Z &m3);
-
-      //Logical accounting functions 
-      int Gcom(int sflag); //sflag is bit marker such as READ_W, etc.
-         //This returns 1 if sflag bit not set in oflag and is in cflag
-         //If this is the case then it sets sflag in oflag.
-      int Rcom(int sflag);
-         //This returns 1 if sflag bit set in oflag and in cflag
-         //If this is the case then it turns off sflag in oflag.
-      void mark(long,int,const char*);
-         //This function prints out string in 3rd argument and count
-         //if first argument is multiple of the second
-
-//Data
-     int cflag; //Command, what should happen to resources.
-     int oflag; //Bit string status of resources, 1 open, 0 closed.
-     int open1; //flags to mark whether a resource is open or not
-     int open2; //0 means closed, 1 means open
-     int open3; //Used for those resources that are either completely
-     int open4; //closed or completely open.
-     int open5;
-     char *type;
-     int tpnm; //If nonnegative integer it is appended to end of type
-        //in constructing file name
-     char *name;
-     int nmnm; //If nonnegative integer it is appended to end of name
-        //in constructing file name
-     int pflag; //Usual print flag, 1 for verbose output, 0 for none
-        //Print flag set to 1 by default. 
-     int eflag; //Flag set to 1 for external path from path file, 0
-        //for internal path
-     char *path; //Path stored here if eflag=0.
-     char *pnam; //Path extension stored here if eflag=2.
-};
-
-//Template functions
-
-template <typename X>
-void FBase::get_Nnum(const char *a,X &m1){
-   ifstream *pfin=get_Istr(a,ios::in);
-   *pfin >> m1;
-   dst_Istr(pfin);
-}
-
-template <typename X,typename Y>
-void FBase::get_Nnum(const char *a,X &m1,Y &m2){
-   ifstream *pfin=get_Istr(a,ios::in);
-   *pfin >> m1 >> m2;
-   dst_Istr(pfin);
-}
-
-template <typename X,typename Y,typename Z>
-void FBase::get_Nnum(const char *a,X &m1,Y &m2,Z &m3){
-   ifstream *pfin=get_Istr(a,ios::in);
-   *pfin >> m1 >> m2 >> m3;
-   dst_Istr(pfin);
-}
-
-template <typename X>
-void FBase::get_Nnum(long n,const char *a,X &m1){
-   ifstream *pfin=get_Istr(n,a,ios::in);
-   *pfin >> m1;
-   dst_Istr(pfin);
-}  
-   
-template <typename X,typename Y>
-void FBase::get_Nnum(long n,const char *a,X &m1,Y &m2){
-   ifstream *pfin=get_Istr(n,a,ios::in);
-   *pfin >> m1 >> m2;
-   dst_Istr(pfin);
-}  
-   
-template <typename X,typename Y,typename Z>
-void FBase::get_Nnum(long n,const char *a,X &m1,Y &m2,Z &m3){
-   ifstream *pfin=get_Istr(n,a,ios::in);
-   *pfin >> m1 >> m2 >> m3;
-   dst_Istr(pfin);
-}
-
-template <typename X>
-void FBase::put_Nnum(const char *a,X &m1){
-   ofstream *pfout=get_Ostr(a,ios::out);
-   *pfout << m1 << endl;
-   dst_Ostr(pfout);
-}
-
-template <typename X,typename Y>
-void FBase::put_Nnum(const char *a,X &m1,Y &m2){
-   ofstream *pfout=get_Ostr(a,ios::out);
-   *pfout << m1 << " " << m2 << endl;
-   dst_Ostr(pfout);
-}
-
-template <typename X,typename Y,typename Z>
-void FBase::put_Nnum(const char *a,X &m1,Y &m2,Z &m3){
-   ofstream *pfout=get_Ostr(a,ios::out);
-   *pfout << m1 << " " << m2 << " " << m3 << endl;
-   dst_Ostr(pfout);
-}
-
-template <typename X>
-void FBase::put_Nnum(long n,const char *a,X &m1){
-   ofstream *pfout=get_Ostr(n,a,ios::out);
-   *pfout << m1 << endl;
-   dst_Ostr(pfout);
-}
-
-template <typename X,typename Y>
-void FBase::put_Nnum(long n,const char *a,X &m1,Y &m2){
-   ofstream *pfout=get_Ostr(n,a,ios::out);
-   *pfout << m1 << " " << m2 << endl;
-   dst_Ostr(pfout);
-}
-
-template <typename X,typename Y,typename Z>
-void FBase::put_Nnum(long n,const char *a,X &m1,Y &m2,Z &m3){
-   ofstream *pfout=get_Ostr(n,a,ios::out);
-   *pfout << m1 << " " << m2 << " " << m3 << endl;
-   dst_Ostr(pfout);
-}
-
-}
-#endif
+#ifndef FBASE_H
+#define FBASE_H
+
+#include <iostream>
+#include <fstream>
+
+using namespace std;
+namespace iret {
+
+typedef char *pChr;
+
+class FBase {
+   public:
+      FBase(const char *tp,const char *nm); //tp is type name, nm is name 
+      FBase(const char *tp,int tn,const char *nm); //tp is type name, if 
+         //nonnegative tn is appended to end of tp, nm is name 
+      FBase(const char *tp,const char *nm,const char *pt); //tp is type name, nm is name 
+         //pt is pointer at a string sss and reads the path from file path_sss in 
+         //current directory. But if sss begins with ':' then skips this character 
+         //and remaining string is the path string itself.
+      ~FBase();
+      void set_type_num(int tn); //Sets tpnm and uses if nonnegative: appended
+         //to end of type name
+      void set_name_num(int nn); //Sets nmnm and uses if nonnegative: appended
+         //to end of name
+      void change_type(const char *nm); //Allows change of type string for class.
+      void change_name(const char *nm); //Allows change of name string for class.
+      void set_name(const char *nm); //Allows change of name string for class.
+         //Included for compatibility
+      void subname(const char *tph,const char *tpl,const char *nm); //Uses the 
+         //higher level type tph as type and combines lower level tpl_nm with
+         //name to allow one to keep track of file types.
+      void set_path_internal(const char *pt); //Path is by default external with
+         //eflag=1. But if this function called with nonempty string, then eflag=0
+         //and pt stored in path and used for access to data.
+      void set_path_name(const char *pa); //path will be extracted from path_pa
+         //and eflag=2. Naming conventions for files are unchanged
+      void map_down(FBase *pFb); //Maps naming parameters to class instance pFb
+      void map_down_sub(FBase *pFb,const char *subtype); //Maps naming parameters to class instance pFb
+         //combines subtype with name to make a new name for pFb and type becomes its type
+
+      //Path access functions
+      void get_pathx(char *cn,const char *a);
+         //Reads the path from a file "path_(*name)" and constructs the
+         //file name from as "(*type)_(*name).(*a)". Cats path and file
+         //name and returns the full info in cn.
+      void get_pathx(char *cn,long n,const char *a);
+      char *add_num(const char *ptr,long n,char *buf); //converts long to ascii
+         //and cats to end of string and returns pointer to new string
+         //that results. Does not change input string. The new string is
+         //held in buffer space and this is overwritten at each call.
+      char *cat_num(const char *ptr,long n,char *buf); //converts long to ascii
+         //and cats to end of ptr string and then cats result to end of
+         //whatever is in buffer. Does not change input string. The new string is
+         //held in buffer space.
+
+      //Stream object pointers
+      ifstream *get_Istr(const char *a,ios::openmode m=ios::in);
+         //Opens input file stream by path and name composition.
+      ofstream *get_Ostr(const char *a,ios::openmode m=ios::out);
+         //Opens output file stream by path and name composition.
+      fstream *get_Fstr(const char *a,ios::openmode m=ios::in|ios::out);
+         //Opens output file stream by path and name composition.
+      ifstream *get_Istr(long n,const char *a,ios::openmode m=ios::in);
+      ofstream *get_Ostr(long n,const char *a,ios::openmode m=ios::out);
+      fstream *get_Fstr(long n,const char *a,ios::openmode m=ios::in|ios::out);
+      void dst_Istr(ifstream *pfin);
+      void dst_Ostr(ofstream *pfout);
+      void dst_Fstr(fstream *pfstr);
+
+      //Get file size in bytes
+      long get_Fsiz(const char *a);
+      long get_Fsiz(long n,const char *a);
+
+      //File existence
+      int Exists(const char *a); //returns 1 if file exists
+      int Exists(long n,const char *a); //returns 1 if file exists
+
+      //Read in array pointers
+      char *get_Read(const char *a);
+         //Reads in a file into an char array and returns pointer
+      char *get_Read(long n,const char *a);
+
+      //Memory map pointers
+      char *get_Mmap(const char *a);
+         //Memory maps file by path and name composition.
+      char *get_Mmap(long n,const char *a);
+      char *get_Wmap(const char *a);
+         //Memory maps file by path and name composition.
+         //Allows to modify contents and is written out when dst_Mmap called
+      char *get_Wmap(long n,const char *a);
+         //Allows to modify contents and is written out when dst_Mmap called
+      void dst_Mmap(const char *a,char *ptr);
+         //Removes the memory map for ptr based on path and name composition.
+      void dst_Mmap(long n,const char *a,char *ptr);
+
+      //Array of chars and binary write
+      void bin_Writ(const char *a,long nm,char *ptr); 
+         //Writes out nm bytes binary
+      void bin_Writ(long n,const char *a,long nm,char *ptr); 
+
+      //Write and read 1, 2, or 3 long integers to or from a file
+      template <typename X>
+      void get_Nnum(const char *a,X &m1);
+      template <typename X,typename Y>
+      void get_Nnum(const char *a,X &m1,Y &m2);
+      template <typename X,typename Y,typename Z>
+      void get_Nnum(const char *a,X &m1,Y &m2,Z &m3);
+      template <typename X>
+      void get_Nnum(long n,const char *a,X &m1);
+      template <typename X,typename Y>
+      void get_Nnum(long n,const char *a,X &m1,Y &m2);
+      template <typename X,typename Y,typename Z>
+      void get_Nnum(long n,const char *a,X &m1,Y &m2,Z &m3);
+      template <typename X>
+      void put_Nnum(const char *a,X &m1);
+      template <typename X,typename Y>
+      void put_Nnum(const char *a,X &m1,Y &m2);
+      template <typename X,typename Y,typename Z>
+      void put_Nnum(const char *a,X &m1,Y &m2,Z &m3);
+      template <typename X>
+      void put_Nnum(long n,const char *a,X &m1);
+      template <typename X,typename Y>
+      void put_Nnum(long n,const char *a,X &m1,Y &m2);
+      template <typename X,typename Y,typename Z>
+      void put_Nnum(long n,const char *a,X &m1,Y &m2,Z &m3);
+
+      //Logical accounting functions 
+      int Gcom(int sflag); //sflag is bit marker such as READ_W, etc.
+         //This returns 1 if sflag bit not set in oflag and is in cflag
+         //If this is the case then it sets sflag in oflag.
+      int Rcom(int sflag);
+         //This returns 1 if sflag bit set in oflag and in cflag
+         //If this is the case then it turns off sflag in oflag.
+      void mark(long,int,const char*);
+         //This function prints out string in 3rd argument and count
+         //if first argument is multiple of the second
+
+//Data
+     int cflag; //Command, what should happen to resources.
+     int oflag; //Bit string status of resources, 1 open, 0 closed.
+     int open1; //flags to mark whether a resource is open or not
+     int open2; //0 means closed, 1 means open
+     int open3; //Used for those resources that are either completely
+     int open4; //closed or completely open.
+     int open5;
+     char *type;
+     int tpnm; //If nonnegative integer it is appended to end of type
+        //in constructing file name
+     char *name;
+     int nmnm; //If nonnegative integer it is appended to end of name
+        //in constructing file name
+     int pflag; //Usual print flag, 1 for verbose output, 0 for none
+        //Print flag set to 1 by default. 
+     int eflag; //Flag set to 1 for external path from path file, 0
+        //for internal path
+     char *path; //Path stored here if eflag=0.
+     char *pnam; //Path extension stored here if eflag=2.
+};
+
+//Template functions
+
+template <typename X>
+void FBase::get_Nnum(const char *a,X &m1){
+   ifstream *pfin=get_Istr(a,ios::in);
+   *pfin >> m1;
+   dst_Istr(pfin);
+}
+
+template <typename X,typename Y>
+void FBase::get_Nnum(const char *a,X &m1,Y &m2){
+   ifstream *pfin=get_Istr(a,ios::in);
+   *pfin >> m1 >> m2;
+   dst_Istr(pfin);
+}
+
+template <typename X,typename Y,typename Z>
+void FBase::get_Nnum(const char *a,X &m1,Y &m2,Z &m3){
+   ifstream *pfin=get_Istr(a,ios::in);
+   *pfin >> m1 >> m2 >> m3;
+   dst_Istr(pfin);
+}
+
+template <typename X>
+void FBase::get_Nnum(long n,const char *a,X &m1){
+   ifstream *pfin=get_Istr(n,a,ios::in);
+   *pfin >> m1;
+   dst_Istr(pfin);
+}  
+   
+template <typename X,typename Y>
+void FBase::get_Nnum(long n,const char *a,X &m1,Y &m2){
+   ifstream *pfin=get_Istr(n,a,ios::in);
+   *pfin >> m1 >> m2;
+   dst_Istr(pfin);
+}  
+   
+template <typename X,typename Y,typename Z>
+void FBase::get_Nnum(long n,const char *a,X &m1,Y &m2,Z &m3){
+   ifstream *pfin=get_Istr(n,a,ios::in);
+   *pfin >> m1 >> m2 >> m3;
+   dst_Istr(pfin);
+}
+
+template <typename X>
+void FBase::put_Nnum(const char *a,X &m1){
+   ofstream *pfout=get_Ostr(a,ios::out);
+   *pfout << m1 << endl;
+   dst_Ostr(pfout);
+}
+
+template <typename X,typename Y>
+void FBase::put_Nnum(const char *a,X &m1,Y &m2){
+   ofstream *pfout=get_Ostr(a,ios::out);
+   *pfout << m1 << " " << m2 << endl;
+   dst_Ostr(pfout);
+}
+
+template <typename X,typename Y,typename Z>
+void FBase::put_Nnum(const char *a,X &m1,Y &m2,Z &m3){
+   ofstream *pfout=get_Ostr(a,ios::out);
+   *pfout << m1 << " " << m2 << " " << m3 << endl;
+   dst_Ostr(pfout);
+}
+
+template <typename X>
+void FBase::put_Nnum(long n,const char *a,X &m1){
+   ofstream *pfout=get_Ostr(n,a,ios::out);
+   *pfout << m1 << endl;
+   dst_Ostr(pfout);
+}
+
+template <typename X,typename Y>
+void FBase::put_Nnum(long n,const char *a,X &m1,Y &m2){
+   ofstream *pfout=get_Ostr(n,a,ios::out);
+   *pfout << m1 << " " << m2 << endl;
+   dst_Ostr(pfout);
+}
+
+template <typename X,typename Y,typename Z>
+void FBase::put_Nnum(long n,const char *a,X &m1,Y &m2,Z &m3){
+   ofstream *pfout=get_Ostr(n,a,ios::out);
+   *pfout << m1 << " " << m2 << " " << m3 << endl;
+   dst_Ostr(pfout);
+}
+
+}
+#endif

Library/Hash.C

@@ -1,733 +1,733 @@
-#include <iostream> 
-#include <fstream> 
-#include <cstdlib>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <sys/mman.h>
-#include <cmath>
-#include <cstring>
-#include <cassert>
-#include "runn.h"
-#include "Hash.h"
-
-using namespace std;
-namespace iret {
-
-Hash::Hash(void) : FBase("hshset","null"){
-}
-
-Hash::Hash(const char *nam) : FBase("hshset",nam){
-}
-
-Hash::Hash(int n,const char *nam) : FBase("hshset",n,nam){
-}
-
-Hash::~Hash(){
-}
-
-void Hash::create_htable(List &Lst,int excess){
-   char cnam[max_str],*cptr,*uptr;
-   int u,len;
-   long ct,i,j,k;
-   ofstream *pfout;
-
-   nwrds=Lst.cnt_key;
-   ct=nwrds;
-   tnum=1;
-   u=0;
-   while(ct=ct/2){tnum*=2;u++;}
-   if(u>30){cout << "Error in size, " << u << endl;exit(0);}
-   i=0;
-   while((u<32)&&(i<excess)){tnum*=2;u++;i++;}
-   tnum--;
-   harr=new long[tnum+2];
-   for(ct=0;ct<tnum+2;ct++)harr[ct]=0;
-
-   farr=new long[1536];
-   ct=1;
-   for(i=0;i<1536;i++){
-      farr[i]=ct=(ct*331)&tnum;
-   }
-  
-   long *pc0=farr,*pc1=farr+128,*pc2=farr+256;
-   long *pc3=farr+384,*pc4=farr+512,*pc5=farr+640;
-   long *pc6=farr+768,*pc7=farr+896,*pc8=farr+1024;
-   long *pc9=farr+1152,*pc10=farr+1280,*pc11=farr+1408;
-   
-   Lst.node_first();
-   while(Lst.node_next()){
-      cptr=Lst.show_str();
-      ct=0;
-      i=0;
-      while(u=*(cptr++)){
-         switch(i){
-            case 0: ct+=*(pc0+u);
-                    break;
-            case 1: ct+=*(pc1+u);
-                    break;
-            case 2: ct+=*(pc2+u);
-                    break;
-            case 3: ct+=*(pc3+u);
-                    break;
-            case 4: ct+=*(pc4+u);
-                    break;
-            case 5: ct+=*(pc5+u);
-                    break;
-            case 6: ct+=*(pc6+u);
-                    break;
-            case 7: ct+=*(pc7+u);
-                    break;
-            case 8: ct+=*(pc8+u);
-                    break;
-            case 9: ct+=*(pc9+u);
-                    break;
-            case 10: ct+=*(pc10+u);
-                    break;
-            case 11: ct+=*(pc11+u);
-                     i-=12;
-                    break;
-         }
-         i++;
-      }
-      (harr[ct&tnum])++;
-   }
-
-   //Set start points in harr.
-   k=0;
-   for(i=0;i<tnum+2;i++){
-      j=harr[i];
-      harr[i]=k;
-      k+=j;
-   }
-   if(k!=nwrds){cout << "Error in summing!" << endl;exit(0);}
-
-   //Write out harr.
-   bin_Writ("ha",(tnum+2)*sizeof(long),(char*)harr);
-
-   //Set addresses
-   char **addt=new char*[nwrds];
-   Lst.node_first();
-   while(Lst.node_next()){
-      uptr=cptr=Lst.show_str();
-      ct=0;
-      i=0;
-      while(u=*(cptr++)){
-         switch(i){
-            case 0: ct+=*(pc0+u);
-                    break;
-            case 1: ct+=*(pc1+u);
-                    break;
-            case 2: ct+=*(pc2+u);
-                    break;
-            case 3: ct+=*(pc3+u);
-                    break;
-            case 4: ct+=*(pc4+u);
-                    break;
-            case 5: ct+=*(pc5+u);
-                    break;
-            case 6: ct+=*(pc6+u);
-                    break;
-            case 7: ct+=*(pc7+u);
-                    break;
-            case 8: ct+=*(pc8+u);
-                    break;
-            case 9: ct+=*(pc9+u);
-                    break;
-            case 10: ct+=*(pc10+u);
-                    break;
-            case 11: ct+=*(pc11+u);
-                     i-=12;
-                    break;
-         }
-         i++;
-      }
-      k=ct&tnum;
-      addt[harr[k]]=uptr;
-      (harr[k])++;
-   }
-
-   //Write out string file
-   pfout=get_Ostr("str");
-   k=0;
-   for(i=0;i<nwrds;i++){
-      *pfout << addt[i] << ends;
-      len=strlen((char*)addt[i])+1;
-      addt[i]=(char*)k;
-      k+=len;
-   }
-   dst_Ostr(pfout);
-
-   //Write out addr file
-   bin_Writ("ad",nwrds*sizeof(long),(char*)addt);
-   delete [] addt;
-
-   //Write out counts
-   pfout=get_Ostr("nm");
-   *pfout << nwrds << " " << tnum << " " << k << endl;
-   dst_Ostr(pfout);
-   delete [] harr;
-   delete [] farr;
-}
-
-//In memory model intended for small sets
-void Hash::create_htableM(List &Lst,int excess){
-   char cnam[max_str],*cptr,*uptr;
-   int u,len;
-   long ct,i,j,k,*barr;
-   ofstream *pfout;
-
-   nwrds=Lst.cnt_key;
-   ct=nwrds;
-   tnum=1;
-   u=0;
-   while(ct=ct/2){tnum*=2;u++;}
-   if(u>30){cout << "Error in size, " << u << endl;exit(0);}
-   i=0;
-   while((u<32)&&(i<excess)){tnum*=2;u++;i++;}
-   tnum--;
-   harr=new long[tnum+2];
-   barr=new long[tnum+2];
-   for(ct=0;ct<tnum+2;ct++)harr[ct]=0;
-
-   farr=new long[1536];
-   ct=1;
-   for(i=0;i<1536;i++){
-      farr[i]=ct=(ct*331)&tnum;
-   }
-  
-   px0=farr,px1=farr+128,px2=farr+256;
-   px3=farr+384,px4=farr+512,px5=farr+640;
-   px6=farr+768,px7=farr+896,px8=farr+1024;
-   px9=farr+1152,px10=farr+1280,px11=farr+1408;
-   
-   Lst.node_first();
-   while(Lst.node_next()){
-      cptr=Lst.show_str();
-      ct=0;
-      i=0;
-      while(u=*(cptr++)){
-         switch(i){
-            case 0: ct+=*(px0+u);
-                    break;
-            case 1: ct+=*(px1+u);
-                    break;
-            case 2: ct+=*(px2+u);
-                    break;
-            case 3: ct+=*(px3+u);
-                    break;
-            case 4: ct+=*(px4+u);
-                    break;
-            case 5: ct+=*(px5+u);
-                    break;
-            case 6: ct+=*(px6+u);
-                    break;
-            case 7: ct+=*(px7+u);
-                    break;
-            case 8: ct+=*(px8+u);
-                    break;
-            case 9: ct+=*(px9+u);
-                    break;
-            case 10: ct+=*(px10+u);
-                    break;
-            case 11: ct+=*(px11+u);
-                     i-=12;
-                    break;
-         }
-         i++;
-      }
-      (harr[ct&tnum])++;
-   }
-
-   //Set start points in harr.
-   k=0;
-   for(i=0;i<tnum+2;i++){
-      j=harr[i];
-      barr[i]=harr[i]=k;
-      k+=j;
-   }
-   if(k!=nwrds){cout << "Error in summing!" << endl;exit(0);}
-
-   //Set addresses
-   len=0;
-   char **addt=new char*[nwrds];
-   Lst.node_first();
-   while(Lst.node_next()){
-      uptr=cptr=Lst.show_str();
-      len+=strlen(uptr)+1;
-      ct=0;
-      i=0;
-      while(u=*(cptr++)){
-         switch(i){
-            case 0: ct+=*(px0+u);
-                    break;
-            case 1: ct+=*(px1+u);
-                    break;
-            case 2: ct+=*(px2+u);
-                    break;
-            case 3: ct+=*(px3+u);
-                    break;
-            case 4: ct+=*(px4+u);
-                    break;
-            case 5: ct+=*(px5+u);
-                    break;
-            case 6: ct+=*(px6+u);
-                    break;
-            case 7: ct+=*(px7+u);
-                    break;
-            case 8: ct+=*(px8+u);
-                    break;
-            case 9: ct+=*(px9+u);
-                    break;
-            case 10: ct+=*(px10+u);
-                    break;
-            case 11: ct+=*(px11+u);
-                     i-=12;
-                    break;
-         }
-         i++;
-      }
-      k=ct&tnum;
-      addt[barr[k]]=uptr;
-      (barr[k])++;
-   }
-   strmap=new char[len];
-
-   //Set up string array
-   k=0;
-   for(i=0;i<nwrds;i++){
-      len=strlen((char*)addt[i])+1;
-      strcpy(strmap+k,addt[i]);
-      addt[i]=(char*)k;
-      k+=len;
-   }
-   addr=(long*)addt;
-   delete [] barr;
-}
-
-void Hash::create_htable(int mz,List &Lst,int excess){
-   char cnam[max_str],*cptr,*uptr;
-   int u,len;
-   long ct,i,j,k;
-   ofstream *pfout;
-
-   nwrds=Lst.cnt_key;
-   ct=nwrds;
-   tnum=1;
-   u=0;
-   while(ct=ct/2){tnum*=2;u++;}
-   if(u>30){cout << "Error in size, " << u << endl;exit(0);}
-   i=0;
-   while((u<32)&&(i<excess)){tnum*=2;u++;i++;}
-   tnum--;
-   harr=new long[tnum+2];
-   for(ct=0;ct<tnum+2;ct++)harr[ct]=0;
-
-   farr=new long[1536];
-   ct=1;
-   for(i=0;i<1536;i++){
-      farr[i]=ct=(ct*331)&tnum;
-   }
-  
-   long *pc0=farr,*pc1=farr+128,*pc2=farr+256;
-   long *pc3=farr+384,*pc4=farr+512,*pc5=farr+640;
-   long *pc6=farr+768,*pc7=farr+896,*pc8=farr+1024;
-   long *pc9=farr+1152,*pc10=farr+1280,*pc11=farr+1408;
-   
-   Lst.node_first();
-   while(Lst.node_next()){
-      cptr=Lst.show_str();
-      ct=0;
-      i=0;
-      while(u=*(cptr++)){
-         switch(i){
-            case 0: ct+=*(pc0+u);
-                    break;
-            case 1: ct+=*(pc1+u);
-                    break;
-            case 2: ct+=*(pc2+u);
-                    break;
-            case 3: ct+=*(pc3+u);
-                    break;
-            case 4: ct+=*(pc4+u);
-                    break;
-            case 5: ct+=*(pc5+u);
-                    break;
-            case 6: ct+=*(pc6+u);
-                    break;
-            case 7: ct+=*(pc7+u);
-                    break;
-            case 8: ct+=*(pc8+u);
-                    break;
-            case 9: ct+=*(pc9+u);
-                    break;
-            case 10: ct+=*(pc10+u);
-                    break;
-            case 11: ct+=*(pc11+u);
-                     i-=12;
-                    break;
-         }
-         i++;
-      }
-      (harr[ct&tnum])++;
-   }
-
-   //Set start points in harr.
-   k=0;
-   for(i=0;i<tnum+2;i++){
-      j=harr[i];
-      harr[i]=k;
-      k+=j;
-   }
-   if(k!=nwrds){cout << "Error in summing!" << endl;exit(0);}
-
-   //Write out harr.
-   bin_Writ(mz,"ha",(tnum+2)*sizeof(long),(char*)harr);
-
-   //Set addresses
-   char **addt=new char*[nwrds];
-   Lst.node_first();
-   while(Lst.node_next()){
-      uptr=cptr=Lst.show_str();
-      ct=0;
-      i=0;
-      while(u=*(cptr++)){
-         switch(i){
-            case 0: ct+=*(pc0+u);
-                    break;
-            case 1: ct+=*(pc1+u);
-                    break;
-            case 2: ct+=*(pc2+u);
-                    break;
-            case 3: ct+=*(pc3+u);
-                    break;
-            case 4: ct+=*(pc4+u);
-                    break;
-            case 5: ct+=*(pc5+u);
-                    break;
-            case 6: ct+=*(pc6+u);
-                    break;
-            case 7: ct+=*(pc7+u);
-                    break;
-            case 8: ct+=*(pc8+u);
-                    break;
-            case 9: ct+=*(pc9+u);
-                    break;
-            case 10: ct+=*(pc10+u);
-                    break;
-            case 11: ct+=*(pc11+u);
-                     i-=12;
-                    break;
-         }
-         i++;
-      }
-      k=ct&tnum;
-      addt[harr[k]]=uptr;
-      (harr[k])++;
-   }
-
-   //Write out string file
-   pfout=get_Ostr(mz,"str");
-   k=0;
-   for(i=0;i<nwrds;i++){
-      *pfout << addt[i] << ends;
-      len=strlen((char*)addt[i])+1;
-      addt[i]=(char*)k;
-      k+=len;
-   }
-   dst_Ostr(pfout);
-
-   //Write out addr file
-   bin_Writ(mz,"ad",nwrds*sizeof(long),(char*)addt);
-   delete [] addt;
-
-   //Write out counts
-   pfout=get_Ostr(mz,"nm");
-   *pfout << nwrds << " " << tnum << " " << k << endl;
-   dst_Ostr(pfout);
-   delete [] harr;
-   delete [] farr;
-}
-
-void Hash::gopen_htable_map(void){
-   char cnam[max_str],*cptr;
-   int fld;
-   long ct,asize,i;
-   
-   ifstream *pfin=get_Istr("nm");
-   *pfin >> nwrds >> tnum >> asize;
-   dst_Istr(pfin);
-
-   harr=(long*)get_Mmap("ha");
-   addr=(long*)get_Mmap("ad");
-   strmap=get_Mmap("str");
-
-   farr=new long[1536];
-   ct=1;
-   for(i=0;i<1536;i++){
-      farr[i]=ct=(ct*331)&tnum;
-   }
-
-   px0=farr,px1=farr+128,px2=farr+256;
-   px3=farr+384,px4=farr+512,px5=farr+640;
-   px6=farr+768,px7=farr+896,px8=farr+1024;
-   px9=farr+1152,px10=farr+1280,px11=farr+1408;
-}
-
-void Hash::gopen_htable_map(int mz){
-   char cnam[max_str],*cptr;
-   int fld;
-   long ct,asize,i;
-   
-   ifstream *pfin=get_Istr(mz,"nm");
-   *pfin >> nwrds >> tnum >> asize;
-   dst_Istr(pfin);
-
-   harr=(long*)get_Mmap(mz,"ha");
-   addr=(long*)get_Mmap(mz,"ad");
-   strmap=get_Mmap(mz,"str");
-
-   farr=new long[1536];
-   ct=1;
-   for(i=0;i<1536;i++){
-      farr[i]=ct=(ct*331)&tnum;
-   }
-
-   px0=farr,px1=farr+128,px2=farr+256;
-   px3=farr+384,px4=farr+512,px5=farr+640;
-   px6=farr+768,px7=farr+896,px8=farr+1024;
-   px9=farr+1152,px10=farr+1280,px11=farr+1408;
-}
-
-void Hash::gopen_htable_copy(Hash *pH){
-   char cnam[max_str],*cptr;
-   int fld;
-   long ct,asize,i;
-
-   nwrds=pH->nwrds;
-   tnum=pH->tnum;
-
-   harr=pH->harr;
-   addr=pH->addr;
-   strmap=pH->strmap;
-
-   farr=pH->farr;
-
-   px0=farr,px1=farr+128,px2=farr+256;
-   px3=farr+384,px4=farr+512,px5=farr+640;
-   px6=farr+768,px7=farr+896,px8=farr+1024;
-   px9=farr+1152,px10=farr+1280,px11=farr+1408;
-}
-
-long Hash::find(const char *str){
-   register long ct=0,i=0,k;
-   register int ic;
-   register const char *utr=str;
-   while(ic=*(utr++)){
-      switch(i){
-         case 0: ct+=*(px0+ic);
-                 break;
-         case 1: ct+=*(px1+ic);
-                 break;
-         case 2: ct+=*(px2+ic);
-                 break;
-         case 3: ct+=*(px3+ic);
-                 break;
-         case 4: ct+=*(px4+ic);
-                 break;
-         case 5: ct+=*(px5+ic);
-                 break;
-         case 6: ct+=*(px6+ic);
-                 break;
-         case 7: ct+=*(px7+ic);
-                 break;
-         case 8: ct+=*(px8+ic);
-                 break;
-         case 9: ct+=*(px9+ic);
-                 break;
-         case 10: ct+=*(px10+ic);
-                 break;
-         case 11: ct+=*(px11+ic);
-                  i-=12;
-                 break;
-      }
-      i++;
-   }
-   k=ct&tnum;
-   ct=harr[k+1];
-   i=harr[k];
-//cout << k << " " << i << " " << addr[i] << " " << ct << " " << addr[ct] << endl;
-   switch(ct-i){
-      case 0: return(0);
-              break;
-      case 1: if(!strcmp(str,strmap+addr[i]))return(i+1);
-              else return(0);
-              break;
-      case 2: ic=strcmp(str,strmap+addr[i]);
-              if(ic>0){
-                 if(!strcmp(str,strmap+addr[i+1]))return(i+2);
-                 else return(0);
-              }
-              else if(ic<0)return(0);
-              else return(i+1);
-              break;
-      default: ic=strcmp(str,strmap+addr[i]);
-               if(ic<0)return(0);
-               else if(!ic)return(i+1);
-               ct--;
-               ic=strcmp(str,strmap+addr[ct]);
-               if(ic>0)return(0);
-               else if(!ic)return(ct+1);
-               while(ct-i>1){
-                  k=(ct+i)/2;
-                  ic=strcmp(str,strmap+addr[k]);
-                  if(ic>0)i=k;
-                  else if(ic<0)ct=k;
-                  else return(k+1);
-               }
-               return(0);
-   }
-}
-
-void Hash::gclose_htable_map(void){
-   dst_Mmap("ha",(char*)harr);
-   dst_Mmap("ad",(char*)addr);
-   dst_Mmap("str",strmap);
-   delete [] farr;
-}
-
-void Hash::gclose_htable_map(int mz){
-   dst_Mmap(mz,"ha",(char*)harr);
-   dst_Mmap(mz,"ad",(char*)addr);
-   dst_Mmap(mz,"str",strmap);
-   delete [] farr;
-}
-
-//Chash code
-
-Chash::Chash() : Hash(){
-   change_type("cshset");
-}
-
-Chash::Chash(const char *str) : Hash(str){
-   change_type("cshset");
-}
-
-Chash::Chash(int n,const char *str) : Hash(n,str){
-   change_type("cshset");
-}
-
-Chash::~Chash(void){}
-
-void Chash::create_ctable(Count &Ct,int excess){
-   create_htable(Ct,excess);
-   gopen_htable_map();
-   long n,i=0;
-   long *pct=new long[Ct.cnt_key];
-   Ct.node_first();
-   while(Ct.node_next()){
-      if(n=find(Ct.show_str())){
-         pct[n-1]=Ct.count();
-      }        
-      else {
-         cout << "Error in Count tree!" << endl;exit(0);
-      }
-      mark(++i,10000,"count terms");
-   }
-   bin_Writ("ct",Ct.cnt_key*sizeof(long),(char*)pct);
-   delete [] pct;
-   cnt=(long*)get_Mmap("ct");
-   gclose_htable_map();
-}
-
-void Chash::create_ctable(List &Lt,int excess){
-   create_htable(Lt,excess);
-   gopen_htable_map();
-   long n,i=1;
-   long *pct=new long[Lt.cnt_key];
-   Lt.node_first();
-   while(Lt.node_next()){
-      if(n=find(Lt.show_str())){
-         pct[n-1]=i;
-      }
-      else {
-         cout << "Error in List tree!" << endl;exit(0);
-      }
-      mark(++i,10000,"count terms");
-   }
-   bin_Writ("ct",Lt.cnt_key*sizeof(long),(char*)pct);
-   delete [] pct;
-   cnt=(long*)get_Mmap("ct");
-   gclose_htable_map();
-}
-
-void Chash::create_ctable(int mz,Count &Ct,int excess){
-   create_htable(mz,Ct,excess);
-   gopen_htable_map(mz);
-   long n,i=0;
-   long *pct=new long[Ct.cnt_key];
-   Ct.node_first();
-   while(Ct.node_next()){
-      if(n=find(Ct.show_str())){
-         pct[n-1]=Ct.count();
-      }        
-      else {
-         cout << "Error in Count tree!" << endl;exit(0);
-      }
-      mark(++i,10000,"count terms");
-   }
-   bin_Writ(mz,"ct",Ct.cnt_key*sizeof(long),(char*)pct);
-   delete [] pct;
-   cnt=(long*)get_Mmap(mz,"ct");
-   gclose_htable_map(mz);
-}
-
-void Chash::create_ctable(int mz,List &Lt,int excess){
-   create_htable(mz,Lt,excess);
-   gopen_htable_map(mz);
-   long n,i=1;
-   long *pct=new long[Lt.cnt_key];
-   Lt.node_first();
-   while(Lt.node_next()){
-      if(n=find(Lt.show_str())){
-         pct[n-1]=i;
-      }
-      else {
-         cout << "Error in List tree!" << endl;exit(0);
-      }
-      mark(++i,10000,"count terms");
-   }
-   bin_Writ(mz,"ct",Lt.cnt_key*sizeof(long),(char*)pct);
-   delete [] pct;
-   cnt=(long*)get_Mmap(mz,"ct");
-   gclose_htable_map(mz);
-}
-
-void Chash::gopen_ctable_map(void){
-   gopen_htable_map();
-   cnt=(long*)get_Mmap("ct");
-}   
-
-void Chash::gopen_ctable_map(int mz){
-   gopen_htable_map(mz);
-   cnt=(long*)get_Mmap(mz,"ct");
-}   
-
-void Chash::gclose_ctable_map(void){
-   gclose_htable_map();
-   dst_Mmap("ct",(char*)cnt);
-}   
-
-void Chash::gclose_ctable_map(int mz){
-   gclose_htable_map(mz);
-   dst_Mmap(mz,"ct",(char*)cnt);
-}   
-
-long Chash::count(const char *str){
-   long n=find(str);
-   if(n)return(cnt[n-1]);
-   else return(0);
-}
-
-}
-
+#include <iostream> 
+#include <fstream> 
+#include <cstdlib>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <cmath>
+#include <cstring>
+#include <cassert>
+#include "runn.h"
+#include "Hash.h"
+
+using namespace std;
+namespace iret {
+
+Hash::Hash(void) : FBase("hshset","null"){
+}
+
+Hash::Hash(const char *nam) : FBase("hshset",nam){
+}
+
+Hash::Hash(int n,const char *nam) : FBase("hshset",n,nam){
+}
+
+Hash::~Hash(){
+}
+
+void Hash::create_htable(List &Lst,int excess){
+   char cnam[max_str],*cptr,*uptr;
+   int u,len;
+   long ct,i,j,k;
+   ofstream *pfout;
+
+   nwrds=Lst.cnt_key;
+   ct=nwrds;
+   tnum=1;
+   u=0;
+   while(ct=ct/2){tnum*=2;u++;}
+   if(u>30){cout << "Error in size, " << u << endl;exit(0);}
+   i=0;
+   while((u<32)&&(i<excess)){tnum*=2;u++;i++;}
+   tnum--;
+   harr=new long[tnum+2];
+   for(ct=0;ct<tnum+2;ct++)harr[ct]=0;
+
+   farr=new long[1536];
+   ct=1;
+   for(i=0;i<1536;i++){
+      farr[i]=ct=(ct*331)&tnum;
+   }
+  
+   long *pc0=farr,*pc1=farr+128,*pc2=farr+256;
+   long *pc3=farr+384,*pc4=farr+512,*pc5=farr+640;
+   long *pc6=farr+768,*pc7=farr+896,*pc8=farr+1024;
+   long *pc9=farr+1152,*pc10=farr+1280,*pc11=farr+1408;
+   
+   Lst.node_first();
+   while(Lst.node_next()){
+      cptr=Lst.show_str();
+      ct=0;
+      i=0;
+      while(u=*(cptr++)){
+         switch(i){
+            case 0: ct+=*(pc0+u);
+                    break;
+            case 1: ct+=*(pc1+u);
+                    break;
+            case 2: ct+=*(pc2+u);
+                    break;
+            case 3: ct+=*(pc3+u);
+                    break;
+            case 4: ct+=*(pc4+u);
+                    break;
+            case 5: ct+=*(pc5+u);
+                    break;
+            case 6: ct+=*(pc6+u);
+                    break;
+            case 7: ct+=*(pc7+u);
+                    break;
+            case 8: ct+=*(pc8+u);
+                    break;
+            case 9: ct+=*(pc9+u);
+                    break;
+            case 10: ct+=*(pc10+u);
+                    break;
+            case 11: ct+=*(pc11+u);
+                     i-=12;
+                    break;
+         }
+         i++;
+      }
+      (harr[ct&tnum])++;
+   }
+
+   //Set start points in harr.
+   k=0;
+   for(i=0;i<tnum+2;i++){
+      j=harr[i];
+      harr[i]=k;
+      k+=j;
+   }
+   if(k!=nwrds){cout << "Error in summing!" << endl;exit(0);}
+
+   //Write out harr.
+   bin_Writ("ha",(tnum+2)*sizeof(long),(char*)harr);
+
+   //Set addresses
+   char **addt=new char*[nwrds];
+   Lst.node_first();
+   while(Lst.node_next()){
+      uptr=cptr=Lst.show_str();
+      ct=0;
+      i=0;
+      while(u=*(cptr++)){
+         switch(i){
+            case 0: ct+=*(pc0+u);
+                    break;
+            case 1: ct+=*(pc1+u);
+                    break;
+            case 2: ct+=*(pc2+u);
+                    break;
+            case 3: ct+=*(pc3+u);
+                    break;
+            case 4: ct+=*(pc4+u);
+                    break;
+            case 5: ct+=*(pc5+u);
+                    break;
+            case 6: ct+=*(pc6+u);
+                    break;
+            case 7: ct+=*(pc7+u);
+                    break;
+            case 8: ct+=*(pc8+u);
+                    break;
+            case 9: ct+=*(pc9+u);
+                    break;
+            case 10: ct+=*(pc10+u);
+                    break;
+            case 11: ct+=*(pc11+u);
+                     i-=12;
+                    break;
+         }
+         i++;
+      }
+      k=ct&tnum;
+      addt[harr[k]]=uptr;
+      (harr[k])++;
+   }
+
+   //Write out string file
+   pfout=get_Ostr("str");
+   k=0;
+   for(i=0;i<nwrds;i++){
+      *pfout << addt[i] << ends;
+      len=strlen((char*)addt[i])+1;
+      addt[i]=(char*)k;
+      k+=len;
+   }
+   dst_Ostr(pfout);
+
+   //Write out addr file
+   bin_Writ("ad",nwrds*sizeof(long),(char*)addt);
+   delete [] addt;
+
+   //Write out counts
+   pfout=get_Ostr("nm");
+   *pfout << nwrds << " " << tnum << " " << k << endl;
+   dst_Ostr(pfout);
+   delete [] harr;
+   delete [] farr;
+}
+
+//In memory model intended for small sets
+void Hash::create_htableM(List &Lst,int excess){
+   char cnam[max_str],*cptr,*uptr;
+   int u,len;
+   long ct,i,j,k,*barr;
+   ofstream *pfout;
+
+   nwrds=Lst.cnt_key;
+   ct=nwrds;
+   tnum=1;
+   u=0;
+   while(ct=ct/2){tnum*=2;u++;}
+   if(u>30){cout << "Error in size, " << u << endl;exit(0);}
+   i=0;
+   while((u<32)&&(i<excess)){tnum*=2;u++;i++;}
+   tnum--;
+   harr=new long[tnum+2];
+   barr=new long[tnum+2];
+   for(ct=0;ct<tnum+2;ct++)harr[ct]=0;
+
+   farr=new long[1536];
+   ct=1;
+   for(i=0;i<1536;i++){
+      farr[i]=ct=(ct*331)&tnum;
+   }
+  
+   px0=farr,px1=farr+128,px2=farr+256;
+   px3=farr+384,px4=farr+512,px5=farr+640;
+   px6=farr+768,px7=farr+896,px8=farr+1024;
+   px9=farr+1152,px10=farr+1280,px11=farr+1408;
+   
+   Lst.node_first();
+   while(Lst.node_next()){
+      cptr=Lst.show_str();
+      ct=0;
+      i=0;
+      while(u=*(cptr++)){
+         switch(i){
+            case 0: ct+=*(px0+u);
+                    break;
+            case 1: ct+=*(px1+u);
+                    break;
+            case 2: ct+=*(px2+u);
+                    break;
+            case 3: ct+=*(px3+u);
+                    break;
+            case 4: ct+=*(px4+u);
+                    break;
+            case 5: ct+=*(px5+u);
+                    break;
+            case 6: ct+=*(px6+u);
+                    break;
+            case 7: ct+=*(px7+u);
+                    break;
+            case 8: ct+=*(px8+u);
+                    break;
+            case 9: ct+=*(px9+u);
+                    break;
+            case 10: ct+=*(px10+u);
+                    break;
+            case 11: ct+=*(px11+u);
+                     i-=12;
+                    break;
+         }
+         i++;
+      }
+      (harr[ct&tnum])++;
+   }
+
+   //Set start points in harr.
+   k=0;
+   for(i=0;i<tnum+2;i++){
+      j=harr[i];
+      barr[i]=harr[i]=k;
+      k+=j;
+   }
+   if(k!=nwrds){cout << "Error in summing!" << endl;exit(0);}
+
+   //Set addresses
+   len=0;
+   char **addt=new char*[nwrds];
+   Lst.node_first();
+   while(Lst.node_next()){
+      uptr=cptr=Lst.show_str();
+      len+=strlen(uptr)+1;
+      ct=0;
+      i=0;
+      while(u=*(cptr++)){
+         switch(i){
+            case 0: ct+=*(px0+u);
+                    break;
+            case 1: ct+=*(px1+u);
+                    break;
+            case 2: ct+=*(px2+u);
+                    break;
+            case 3: ct+=*(px3+u);
+                    break;
+            case 4: ct+=*(px4+u);
+                    break;
+            case 5: ct+=*(px5+u);
+                    break;
+            case 6: ct+=*(px6+u);
+                    break;
+            case 7: ct+=*(px7+u);
+                    break;
+            case 8: ct+=*(px8+u);
+                    break;
+            case 9: ct+=*(px9+u);
+                    break;
+            case 10: ct+=*(px10+u);
+                    break;
+            case 11: ct+=*(px11+u);
+                     i-=12;
+                    break;
+         }
+         i++;
+      }
+      k=ct&tnum;
+      addt[barr[k]]=uptr;
+      (barr[k])++;
+   }
+   strmap=new char[len];
+
+   //Set up string array
+   k=0;
+   for(i=0;i<nwrds;i++){
+      len=strlen((char*)addt[i])+1;
+      strcpy(strmap+k,addt[i]);
+      addt[i]=(char*)k;
+      k+=len;
+   }
+   addr=(long*)addt;
+   delete [] barr;
+}
+
+void Hash::create_htable(int mz,List &Lst,int excess){
+   char cnam[max_str],*cptr,*uptr;
+   int u,len;
+   long ct,i,j,k;
+   ofstream *pfout;
+
+   nwrds=Lst.cnt_key;
+   ct=nwrds;
+   tnum=1;
+   u=0;
+   while(ct=ct/2){tnum*=2;u++;}
+   if(u>30){cout << "Error in size, " << u << endl;exit(0);}
+   i=0;
+   while((u<32)&&(i<excess)){tnum*=2;u++;i++;}
+   tnum--;
+   harr=new long[tnum+2];
+   for(ct=0;ct<tnum+2;ct++)harr[ct]=0;
+
+   farr=new long[1536];
+   ct=1;
+   for(i=0;i<1536;i++){
+      farr[i]=ct=(ct*331)&tnum;
+   }
+  
+   long *pc0=farr,*pc1=farr+128,*pc2=farr+256;
+   long *pc3=farr+384,*pc4=farr+512,*pc5=farr+640;
+   long *pc6=farr+768,*pc7=farr+896,*pc8=farr+1024;
+   long *pc9=farr+1152,*pc10=farr+1280,*pc11=farr+1408;
+   
+   Lst.node_first();
+   while(Lst.node_next()){
+      cptr=Lst.show_str();
+      ct=0;
+      i=0;
+      while(u=*(cptr++)){
+         switch(i){
+            case 0: ct+=*(pc0+u);
+                    break;
+            case 1: ct+=*(pc1+u);
+                    break;
+            case 2: ct+=*(pc2+u);
+                    break;
+            case 3: ct+=*(pc3+u);
+                    break;
+            case 4: ct+=*(pc4+u);
+                    break;
+            case 5: ct+=*(pc5+u);
+                    break;
+            case 6: ct+=*(pc6+u);
+                    break;
+            case 7: ct+=*(pc7+u);
+                    break;
+            case 8: ct+=*(pc8+u);
+                    break;
+            case 9: ct+=*(pc9+u);
+                    break;
+            case 10: ct+=*(pc10+u);
+                    break;
+            case 11: ct+=*(pc11+u);
+                     i-=12;
+                    break;
+         }
+         i++;
+      }
+      (harr[ct&tnum])++;
+   }
+
+   //Set start points in harr.
+   k=0;
+   for(i=0;i<tnum+2;i++){
+      j=harr[i];
+      harr[i]=k;
+      k+=j;
+   }
+   if(k!=nwrds){cout << "Error in summing!" << endl;exit(0);}
+
+   //Write out harr.
+   bin_Writ(mz,"ha",(tnum+2)*sizeof(long),(char*)harr);
+
+   //Set addresses
+   char **addt=new char*[nwrds];
+   Lst.node_first();
+   while(Lst.node_next()){
+      uptr=cptr=Lst.show_str();
+      ct=0;
+      i=0;
+      while(u=*(cptr++)){
+         switch(i){
+            case 0: ct+=*(pc0+u);
+                    break;
+            case 1: ct+=*(pc1+u);
+                    break;
+            case 2: ct+=*(pc2+u);
+                    break;
+            case 3: ct+=*(pc3+u);
+                    break;
+            case 4: ct+=*(pc4+u);
+                    break;
+            case 5: ct+=*(pc5+u);
+                    break;
+            case 6: ct+=*(pc6+u);
+                    break;
+            case 7: ct+=*(pc7+u);
+                    break;
+            case 8: ct+=*(pc8+u);
+                    break;
+            case 9: ct+=*(pc9+u);
+                    break;
+            case 10: ct+=*(pc10+u);
+                    break;
+            case 11: ct+=*(pc11+u);
+                     i-=12;
+                    break;
+         }
+         i++;
+      }
+      k=ct&tnum;
+      addt[harr[k]]=uptr;
+      (harr[k])++;
+   }
+
+   //Write out string file
+   pfout=get_Ostr(mz,"str");
+   k=0;
+   for(i=0;i<nwrds;i++){
+      *pfout << addt[i] << ends;
+      len=strlen((char*)addt[i])+1;
+      addt[i]=(char*)k;
+      k+=len;
+   }
+   dst_Ostr(pfout);
+
+   //Write out addr file
+   bin_Writ(mz,"ad",nwrds*sizeof(long),(char*)addt);
+   delete [] addt;
+
+   //Write out counts
+   pfout=get_Ostr(mz,"nm");
+   *pfout << nwrds << " " << tnum << " " << k << endl;
+   dst_Ostr(pfout);
+   delete [] harr;
+   delete [] farr;
+}
+
+void Hash::gopen_htable_map(void){
+   char cnam[max_str],*cptr;
+   int fld;
+   long ct,asize,i;
+   
+   ifstream *pfin=get_Istr("nm");
+   *pfin >> nwrds >> tnum >> asize;
+   dst_Istr(pfin);
+
+   harr=(long*)get_Mmap("ha");
+   addr=(long*)get_Mmap("ad");
+   strmap=get_Mmap("str");
+
+   farr=new long[1536];
+   ct=1;
+   for(i=0;i<1536;i++){
+      farr[i]=ct=(ct*331)&tnum;
+   }
+
+   px0=farr,px1=farr+128,px2=farr+256;
+   px3=farr+384,px4=farr+512,px5=farr+640;
+   px6=farr+768,px7=farr+896,px8=farr+1024;
+   px9=farr+1152,px10=farr+1280,px11=farr+1408;
+}
+
+void Hash::gopen_htable_map(int mz){
+   char cnam[max_str],*cptr;
+   int fld;
+   long ct,asize,i;
+   
+   ifstream *pfin=get_Istr(mz,"nm");
+   *pfin >> nwrds >> tnum >> asize;
+   dst_Istr(pfin);
+
+   harr=(long*)get_Mmap(mz,"ha");
+   addr=(long*)get_Mmap(mz,"ad");
+   strmap=get_Mmap(mz,"str");
+
+   farr=new long[1536];
+   ct=1;
+   for(i=0;i<1536;i++){
+      farr[i]=ct=(ct*331)&tnum;
+   }
+
+   px0=farr,px1=farr+128,px2=farr+256;
+   px3=farr+384,px4=farr+512,px5=farr+640;
+   px6=farr+768,px7=farr+896,px8=farr+1024;
+   px9=farr+1152,px10=farr+1280,px11=farr+1408;
+}
+
+void Hash::gopen_htable_copy(Hash *pH){
+   char cnam[max_str],*cptr;
+   int fld;
+   long ct,asize,i;
+
+   nwrds=pH->nwrds;
+   tnum=pH->tnum;
+
+   harr=pH->harr;
+   addr=pH->addr;
+   strmap=pH->strmap;
+
+   farr=pH->farr;
+
+   px0=farr,px1=farr+128,px2=farr+256;
+   px3=farr+384,px4=farr+512,px5=farr+640;
+   px6=farr+768,px7=farr+896,px8=farr+1024;
+   px9=farr+1152,px10=farr+1280,px11=farr+1408;
+}
+
+long Hash::find(const char *str){
+   register long ct=0,i=0,k;
+   register int ic;
+   register const char *utr=str;
+   while(ic=*(utr++)){
+      switch(i){
+         case 0: ct+=*(px0+ic);
+                 break;
+         case 1: ct+=*(px1+ic);
+                 break;
+         case 2: ct+=*(px2+ic);
+                 break;
+         case 3: ct+=*(px3+ic);
+                 break;
+         case 4: ct+=*(px4+ic);
+                 break;
+         case 5: ct+=*(px5+ic);
+                 break;
+         case 6: ct+=*(px6+ic);
+                 break;
+         case 7: ct+=*(px7+ic);
+                 break;
+         case 8: ct+=*(px8+ic);
+                 break;
+         case 9: ct+=*(px9+ic);
+                 break;
+         case 10: ct+=*(px10+ic);
+                 break;
+         case 11: ct+=*(px11+ic);
+                  i-=12;
+                 break;
+      }
+      i++;
+   }
+   k=ct&tnum;
+   ct=harr[k+1];
+   i=harr[k];
+//cout << k << " " << i << " " << addr[i] << " " << ct << " " << addr[ct] << endl;
+   switch(ct-i){
+      case 0: return(0);
+              break;
+      case 1: if(!strcmp(str,strmap+addr[i]))return(i+1);
+              else return(0);
+              break;
+      case 2: ic=strcmp(str,strmap+addr[i]);
+              if(ic>0){
+                 if(!strcmp(str,strmap+addr[i+1]))return(i+2);
+                 else return(0);
+              }
+              else if(ic<0)return(0);
+              else return(i+1);
+              break;
+      default: ic=strcmp(str,strmap+addr[i]);
+               if(ic<0)return(0);
+               else if(!ic)return(i+1);
+               ct--;
+               ic=strcmp(str,strmap+addr[ct]);
+               if(ic>0)return(0);
+               else if(!ic)return(ct+1);
+               while(ct-i>1){
+                  k=(ct+i)/2;
+                  ic=strcmp(str,strmap+addr[k]);
+                  if(ic>0)i=k;
+                  else if(ic<0)ct=k;
+                  else return(k+1);
+               }
+               return(0);
+   }
+}
+
+void Hash::gclose_htable_map(void){
+   dst_Mmap("ha",(char*)harr);
+   dst_Mmap("ad",(char*)addr);
+   dst_Mmap("str",strmap);
+   delete [] farr;
+}
+
+void Hash::gclose_htable_map(int mz){
+   dst_Mmap(mz,"ha",(char*)harr);
+   dst_Mmap(mz,"ad",(char*)addr);
+   dst_Mmap(mz,"str",strmap);
+   delete [] farr;
+}
+
+//Chash code
+
+Chash::Chash() : Hash(){
+   change_type("cshset");
+}
+
+Chash::Chash(const char *str) : Hash(str){
+   change_type("cshset");
+}
+
+Chash::Chash(int n,const char *str) : Hash(n,str){
+   change_type("cshset");
+}
+
+Chash::~Chash(void){}
+
+void Chash::create_ctable(Count &Ct,int excess){
+   create_htable(Ct,excess);
+   gopen_htable_map();
+   long n,i=0;
+   long *pct=new long[Ct.cnt_key];
+   Ct.node_first();
+   while(Ct.node_next()){
+      if(n=find(Ct.show_str())){
+         pct[n-1]=Ct.count();
+      }        
+      else {
+         cout << "Error in Count tree!" << endl;exit(0);
+      }
+      mark(++i,10000,"count terms");
+   }
+   bin_Writ("ct",Ct.cnt_key*sizeof(long),(char*)pct);
+   delete [] pct;
+   cnt=(long*)get_Mmap("ct");
+   gclose_htable_map();
+}
+
+void Chash::create_ctable(List &Lt,int excess){
+   create_htable(Lt,excess);
+   gopen_htable_map();
+   long n,i=1;
+   long *pct=new long[Lt.cnt_key];
+   Lt.node_first();
+   while(Lt.node_next()){
+      if(n=find(Lt.show_str())){
+         pct[n-1]=i;
+      }
+      else {
+         cout << "Error in List tree!" << endl;exit(0);
+      }
+      mark(++i,10000,"count terms");
+   }
+   bin_Writ("ct",Lt.cnt_key*sizeof(long),(char*)pct);
+   delete [] pct;
+   cnt=(long*)get_Mmap("ct");
+   gclose_htable_map();
+}
+
+void Chash::create_ctable(int mz,Count &Ct,int excess){
+   create_htable(mz,Ct,excess);
+   gopen_htable_map(mz);
+   long n,i=0;
+   long *pct=new long[Ct.cnt_key];
+   Ct.node_first();
+   while(Ct.node_next()){
+      if(n=find(Ct.show_str())){
+         pct[n-1]=Ct.count();
+      }        
+      else {
+         cout << "Error in Count tree!" << endl;exit(0);
+      }
+      mark(++i,10000,"count terms");
+   }
+   bin_Writ(mz,"ct",Ct.cnt_key*sizeof(long),(char*)pct);
+   delete [] pct;
+   cnt=(long*)get_Mmap(mz,"ct");
+   gclose_htable_map(mz);
+}
+
+void Chash::create_ctable(int mz,List &Lt,int excess){
+   create_htable(mz,Lt,excess);
+   gopen_htable_map(mz);
+   long n,i=1;
+   long *pct=new long[Lt.cnt_key];
+   Lt.node_first();
+   while(Lt.node_next()){
+      if(n=find(Lt.show_str())){
+         pct[n-1]=i;
+      }
+      else {
+         cout << "Error in List tree!" << endl;exit(0);
+      }
+      mark(++i,10000,"count terms");
+   }
+   bin_Writ(mz,"ct",Lt.cnt_key*sizeof(long),(char*)pct);
+   delete [] pct;
+   cnt=(long*)get_Mmap(mz,"ct");
+   gclose_htable_map(mz);
+}
+
+void Chash::gopen_ctable_map(void){
+   gopen_htable_map();
+   cnt=(long*)get_Mmap("ct");
+}   
+
+void Chash::gopen_ctable_map(int mz){
+   gopen_htable_map(mz);
+   cnt=(long*)get_Mmap(mz,"ct");
+}   
+
+void Chash::gclose_ctable_map(void){
+   gclose_htable_map();
+   dst_Mmap("ct",(char*)cnt);
+}   
+
+void Chash::gclose_ctable_map(int mz){
+   gclose_htable_map(mz);
+   dst_Mmap(mz,"ct",(char*)cnt);
+}   
+
+long Chash::count(const char *str){
+   long n=find(str);
+   if(n)return(cnt[n-1]);
+   else return(0);
+}
+
+}
+

Library/Hash.h

@@ -1,92 +1,92 @@
-#ifndef HASH_H
-#define HASH_H
-
-#include <iostream>
-#include <fstream>
-#include <Btree.h>
-#include <FBase.h>
-
-namespace iret {
-
-class Hash : public FBase {
-public:
-  Hash(void);
-  Hash(const char *nm);
-  Hash(int n,const char *nm); //n gets appended to type if >-1
-  ~Hash();
-
-  void create_htable(List &Lst,int excess); //"str" for file of strings, 
-      //"ad" for address file, "nm" numbers, 
-      //"ha" hash array. Excess is # powers of 2 above size.
-  void create_htableM(List &Lst,int excess); //creates in memory ready for use
-      //and no need to call gopen or gclose functions
-  void create_htable(int mz,List &Lst,int excess); //"str" for file of strings, 
-      //Creates a numbered version of above
-
-  void gopen_htable_map(void); //Creates memory maps
-  void gopen_htable_map(int mz); //Creates memory maps
-  void gclose_htable_map(void); //Destroys memory maps
-     //and deletes memory
-  void gclose_htable_map(int mz); //Destroys memory maps
-     //and deletes memory
-  void gopen_htable_copy(Hash *pH); //Copies memory maps
-
-  long find(const char *str); //Return number+1 if present, else 0.
-      //Number is not lexical order but hash order and then lexical
-      //within collesion groups.
-
-  //Data
-  char *strmap; //Holds the bit map.
-  long *addr; //Holds the offsets to strmap.
-  long nwrds; //Number of words.
-  long tnum; //Truncation number, size of har.
-  long *harr; //Holds hash array.
-  long *farr; //Holds the hash coefficients.
-  long *px0;
-  long *px1;
-  long *px2;
-  long *px3;
-  long *px4;
-  long *px5;
-  long *px6;
-  long *px7;
-  long *px8;
-  long *px9;
-  long *px10;
-  long *px11;
-};
-
-class Chash : public Hash {
-public:
-  Chash(void);
-  Chash(const char *nm);
-  Chash(int n,const char *nm); //n gets appended to type if >-1
-  ~Chash(void);
-
-  void create_ctable(Count &Ct,int excess); //Adds "ct" for counts
-     //Calls create_htable and then prodoces the array of counts.
-  void create_ctable(int mz,Count &Ct,int excess); //Adds "ct" for counts
-     //Creates a numbered version of above
-  void create_ctable(List &Lt,int excess); //Adds "ct" for term # 
-     //and starts the count at 1 and in lexical order. count() will
-     //return 0 if term not in list.
-  void create_ctable(int mz,List &Lt,int excess); //Adds "ct" for term # 
-     //Creates a numbered version of above
- 
-  void gopen_ctable_map(void); //Calls gopen_htable_map and also
-     //maps "ct" file.
-  void gopen_ctable_map(int mz); //Calls gopen_htable_map and also
-     //maps "ct" file.
-  void gclose_ctable_map(void); //Calls gclose_htable_map and also
-     //Unmaps "ct" file.
-  void gclose_ctable_map(int mz); //Calls gclose_htable_map and also
-     //Unmaps "ct" file.
-
-  long count(const char *str); //Returns count if present, else 0.
-
-  //Data
-  long *cnt;
-};
-
-}
-#endif
+#ifndef HASH_H
+#define HASH_H
+
+#include <iostream>
+#include <fstream>
+#include <Btree.h>
+#include <FBase.h>
+
+namespace iret {
+
+class Hash : public FBase {
+public:
+  Hash(void);
+  Hash(const char *nm);
+  Hash(int n,const char *nm); //n gets appended to type if >-1
+  ~Hash();
+
+  void create_htable(List &Lst,int excess); //"str" for file of strings, 
+      //"ad" for address file, "nm" numbers, 
+      //"ha" hash array. Excess is # powers of 2 above size.
+  void create_htableM(List &Lst,int excess); //creates in memory ready for use
+      //and no need to call gopen or gclose functions
+  void create_htable(int mz,List &Lst,int excess); //"str" for file of strings, 
+      //Creates a numbered version of above
+
+  void gopen_htable_map(void); //Creates memory maps
+  void gopen_htable_map(int mz); //Creates memory maps
+  void gclose_htable_map(void); //Destroys memory maps
+     //and deletes memory
+  void gclose_htable_map(int mz); //Destroys memory maps
+     //and deletes memory
+  void gopen_htable_copy(Hash *pH); //Copies memory maps
+
+  long find(const char *str); //Return number+1 if present, else 0.
+      //Number is not lexical order but hash order and then lexical
+      //within collesion groups.
+
+  //Data
+  char *strmap; //Holds the bit map.
+  long *addr; //Holds the offsets to strmap.
+  long nwrds; //Number of words.
+  long tnum; //Truncation number, size of har.
+  long *harr; //Holds hash array.
+  long *farr; //Holds the hash coefficients.
+  long *px0;
+  long *px1;
+  long *px2;
+  long *px3;
+  long *px4;
+  long *px5;
+  long *px6;
+  long *px7;
+  long *px8;
+  long *px9;
+  long *px10;
+  long *px11;
+};
+
+class Chash : public Hash {
+public:
+  Chash(void);
+  Chash(const char *nm);
+  Chash(int n,const char *nm); //n gets appended to type if >-1
+  ~Chash(void);
+
+  void create_ctable(Count &Ct,int excess); //Adds "ct" for counts
+     //Calls create_htable and then prodoces the array of counts.
+  void create_ctable(int mz,Count &Ct,int excess); //Adds "ct" for counts
+     //Creates a numbered version of above
+  void create_ctable(List &Lt,int excess); //Adds "ct" for term # 
+     //and starts the count at 1 and in lexical order. count() will
+     //return 0 if term not in list.
+  void create_ctable(int mz,List &Lt,int excess); //Adds "ct" for term # 
+     //Creates a numbered version of above
+ 
+  void gopen_ctable_map(void); //Calls gopen_htable_map and also
+     //maps "ct" file.
+  void gopen_ctable_map(int mz); //Calls gopen_htable_map and also
+     //maps "ct" file.
+  void gclose_ctable_map(void); //Calls gclose_htable_map and also
+     //Unmaps "ct" file.
+  void gclose_ctable_map(int mz); //Calls gclose_htable_map and also
+     //Unmaps "ct" file.
+
+  long count(const char *str); //Returns count if present, else 0.
+
+  //Data
+  long *cnt;
+};
+
+}
+#endif

Library/MPtok.C

@@ -1,2036 +1,2036 @@
-#include <stdio.h>
-#include <ctype.h>
-#include <string.h>
-#include <stdlib.h>
-
-#include <string>
-#include <iostream>
-#include <fstream>
-#include <sstream>
-
-#include "MPtok.h"
-
-// These options are probably compile time constants
-
-static char option_tagsep = '_';	// The tagsep character
-static char option_replacesep = '-';	// Replace tagsep with this
-
-static void chomp(char *line)
-{
-	int     i;
-
-	i = strlen(line) - 1;
-	while (i >= 0 && line[i] == '\n' || line[i] == '\r')
-		line[i--] = '\0';
-}
-
-// Data structure and algorithm for finding common pairs.
-
-// read a file of pairs into a data structure,
-// the file must be sorted first
-
-void MPtok::init_pair(const string& file_name)
-{
-	filebuf fb;
-	fb.open(file_name.c_str(), ios::in);
-	istream is(&fb);
-	string pair;
-
-	while (1)
-	{
-		getline(is, pair);
-		if (is.fail()) break;
-		if (pair.size() > 0) common_pair.insert(pair);
-	}
-
-	fb.close();
-}
-
-// List of abbreviations in 3 categories
-// ABB = can occur mid sentence
-// EOS = can occur at end of sentence
-// NUM = only used before numbers
-
-void MPtok::init_abbr(const string& file_name)
-{
-	filebuf fb;
-	fb.open(file_name.c_str(), ios::in);
-	istream is(&fb);
-	string typ, abb;
-	map<string,int> val;
-	val["ABB"] = ABB_ABB; val["EOS"] = ABB_EOS; val["NUM"] = ABB_NUM;
-
-	while (is.good())
-	{
-		is >> typ;
-		if (val.count(typ))
-		{
-			is >> abb;
-			if (abb.size() > 0) common_abbr[abb] = val[typ];
-		}
-	}
-	fb.close();
-}
-
-static char nextchar(const char *t, int i)
-{
-	while (isspace(t[i])) i++;
-	return t[i];
-}
-
-// Look for a token at or prior to the text position
-
-static int lookbehind(const char *t, int i, const char *s, int *tokflag)
-{
-	int	k = (int) strlen(s) - 1;
-
-	while (i > 0 && isspace(t[i])) i--;
-
-	while (k >= 0 && i >= 0)
-	{
-		if (k > 0 && tokflag[i]) break;
-
-		if (tolower(s[k]) != tolower(t[i]))
-			return -1;
-		k--;
-		i--;
-	}
-
-	return (k < 0 && tokflag[i+1]) ? i + 1 : -1;
-}
-
-// Look for a token at or following the text position
-
-static int lookahead(const char *t, int i, const char *s, int *tokflag)
-{
-	int	k = 0;
-
-	while (isspace(t[i])) i++;
-
-	while (k < strlen(s) && i < strlen(t))
-	{
-		if (k > 0 && tokflag[i]) break;
-
-		if (tolower(s[k]) != tolower(t[i]))
-			return -1;
-		k++;
-		i++;
-	}
-
-	return (k == strlen(s) && tokflag[i]) ? i - (int) strlen(s) : -1;
-}
-
-// Set the initial tokens at spaces
-
-void MPtok::tok_0()
-{
-	int i;
-
-	tokflag[0] = 1;
-	for (i = 1; i < text_len; i++)
-	{
-		tokflag[i] = isspace(text[i]) || (i > 0 && isspace(text[i - 1])) ? 1 : 0;
-	}
-	tokflag[i] = 1;
-}
-
-// Get quotes preceded by open parens
-//
-// A double quote, preceded by a space or open bracket is a separate token
-//
-
-void MPtok::tok_1()
-{
-	for (int i = 1; i < text_len; i++)
-	{
-		if (text[i] == '"' && strchr("([{<", text[i-1]))
-		{
-			tokflag[i] = 1;
-			if (i + 1 < text_len) tokflag[i+1] = 1;
-		}
-	}
-}
-
-// Look for ellipses
-//
-// Three dots in a row is a separate token
-
-void MPtok::tok_2()
-{
-	for (int i = 1; i + 2 < text_len; i++)
-	{
-		if (strncmp(&text[i], "...", 3) == 0)
-		{
-			tokflag[i] = 1;
-			if (i + 3 < text_len) tokflag[i+3] = 1;
-		}
-	}
-}
-
-// Non-sentence-ending punctuation
-//
-// Certain punctuation characters are separate tokens
-
-void MPtok::tok_3()
-{
-	for (int i = 0; i < text_len; i++)
-	{
-		// If it is a comma and the next char is not a space and option_comma = 0
-
-		if (option_comma == 0 && text[i] == ',' && isspace(text[i + 1]) == 0)
-		{
-			// do nothing
-		} else if (strchr(",;:@#$%&", text[i]))
-		{
-			tokflag[i] = 1;
-			tokflag[i + 1] = 1;
-		}
-	}
-}
-
-// Separate the slashes
-//
-// Slashes are a separate token
-// except for +/-, +/+, -/-, -/+, and and/or.
-
-void MPtok::tok_5_6_7()
-{
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '/')
-		{
-			tokflag[i] = 1;
-			if (i+1 < text_len) tokflag[i+1] = 1;
-
-			// Put back +/-, etc, unless option_hyphen is 1
-
-			if (i - 1 >= 0
-			&& i + 1 < text_len
-			&& ((option_new < 9
-				&& text[i - 1] == '+' || (text[i - 1] == '-' && option_hyphen == 0)
-				&& text[i + 1] == '+' || (text[i + 1] == '-' && option_hyphen == 0))
-			|| (option_new >= 9
-				&& (text[i - 1] == '+' || text[i - 1] == '-')
-				&& (text[i + 1] == '+' || text[i + 1] == '-'))))
-			{
-				tokflag[i - 1] = 1;
-				tokflag[i] = tokflag[i+1] = 0;
-				tokflag[i + 2] = 1;
-			}
-
-			// Put back and/or, etc
-
-			if (option_new <= 7)
-			{
-				if (i > 5 && strncmp(text + i - 5, " and/or ", 8) == 0)
-				{
-					for (int j = 1; j < 5; j++)
-						tokflag[i - 2 + j] = 0;
-				}
-			} else
-			{
-				if (i > 4 && strncmp(text + i - 4, " and/or ", 8) == 0)
-				{
-					for (int j = 1; j < 6; j++)
-						tokflag[i - 3 + j] = 0;
-				}
-			}
-		}
-	}
-}
-
-// All brackets
-//
-// Any open or closed bracket is a separate token
-//
-// Exclamation and question mark
-//
-// Any question or exclamation mark is a separate token
-
-void MPtok::tok_8_9()
-{
-	for (int i = 0; i < text_len; i++)
-	{
-		if (strchr("[](){}<>", text[i])
-		|| strchr("?!", text[i]))
-		{
-			tokflag[i] = 1;
-			if (i + 1 < text_len) tokflag[i+1] = 1;
-		}
-	}
-}
-
-// Period at the end of a string may be followed by closed-bracket or quote
-//
-// A period that is preceded by a non-period
-// and optionally followed by a close paren
-// and any amount of space at the end of the string
-// is a separate token.
-
-void MPtok::tok_10()
-{
-	for (int i = text_len - 1; i >= 0; i--)
-	{
-		if (isspace(text[i])) continue;
-		if (strchr("])}>\"'", text[i])) continue;
-		if (text[i] != '.') break;
-		if (text[i] == '.' && (i - 1 < 0 || text[i-1] != '.'))
-		{
-			tokflag[i] = 1;
-			if (i + 1 < text_len) tokflag[i+1] = 1;
-		}
-	}
-}
-
-// Period followed by a capitalized word
-//
-// A period preceded by a character that is not another period and not a space
-// and followed by a space then an upper case letter is a separate token
-
-void MPtok::tok_11()
-{
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '.'
-		&& (i + 1 < text_len && isspace(text[i+1]))
-		&& (i - 1 < 0 || text[i - 1] != '.' || isspace(text[i-1]) == 0)
-		&& isupper(nextchar(text, i + 1)))
-			tokflag[i] = 1;
-	}
-}
-
-// A normal word followed by a period
-//
-// A period followed by a space
-// and preceded by 2 or more alphabetic characters or hyphens
-// is a separate token
-
-void MPtok::tok_12()
-{
-	int wcnt = 0;
-
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '.'
-		&& tokflag[i + 1]
-		&& wcnt >= 2)
-			tokflag[i] = 1;
-
-		if (isalpha(text[i]) || text[i] == '-')
-			++wcnt;
-		else
-			wcnt = 0;
-	}
-}
-
-// A non-normal token (that has no lower case letters) followed by a period
-//
-// A period at the end of a token made of characters excluding lower case
-// is a separate token
-
-void MPtok::tok_13()
-{
-	int	stok = 0;
-	int	wcnt = 0;
-
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '.'
-		&& tokflag[i + 1]
-		&& wcnt >= 2)
-			tokflag[i] = 1;
-
-		if (tokflag[i] == 1) stok = 1;
-
-		if (islower(text[i]) || text[i] == '.')
-		{
-			stok = 0;
-			wcnt = 0;
-		}
-
-		if (stok)
-			wcnt++;
-	}
-}
-
-// put some periods with single-letter abbreviations
-//
-// A single alphabetic token followed by a period followed
-// by a token that does not begin with an upper case letter
-// or number is taken to be an abbreviation and the period
-// does not start a new token.
-//
-// NOTE: This does not recognize initials in people's names,
-//	 that problem is not simply solved.
-
-void MPtok::tok_14()
-{
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '.'
-		&& i - 1 >= 0 && isalpha(text[i - 1]) && tokflag[i - 1]
-		&& tokflag[i + 1]
-		&& isupper(nextchar(text, i + 1)) == 0
-		&& isdigit(nextchar(text, i + 1)) == 0
-		&& nextchar(text, i + 1) != '('
-		)
-		{
-			tokflag[i] = 0;
-		}
-	}
-}
-
-void MPtok::tok_15()
-{
-	int	i, j, k, a;
-	char	buff[MAX_ABB + 1];
-
-	for (i = 0; i < text_len; i++)
-	{
-		// only start at a current token
-
-		if (! tokflag[i]) continue;
-
-		// find alphabetic followed by period
-
-		buff[0] = '\0';
-		for (k = 0; i + k < text_len && k < MAX_ABB; k++)
-		{
-			buff[k] = text[i+k]; buff[k+1] = '\0';
-			if (k > 0 && buff[k] == '.') break; // this is good
-			if (! isalpha(buff[k])) { buff[0] = '\0'; break; } // this is not good
-		}
-
-		if (buff[0] == '\0' || i + k == text_len || k == MAX_ABB) continue;
-
-		// at this point, buff[k] == '.' add 1 to make it the length
-
-		k++;
-
-		// if not found, try finding a concatenated abbrev
-
-		if (! common_abbr.count(buff))
-		{
-			for (; i + k < text_len && k < MAX_ABB; k++)
-			{
-				buff[k] = text[i+k]; buff[k+1] = '\0';
-				if (k > 0 && buff[k] == '.') break; // this is good
-				if (! isalpha(buff[k])) { buff[0] = '\0'; break; } // this is not good
-			}
-
-			if (buff[0] == '\0' || i + k == text_len || k == MAX_ABB) continue;
-
-			// at this point, buff[k] == '.' add 1 to make it the length
-
-			k++;
-		}
-
-		// if not found, give up
-
-		if (! common_abbr.count(buff)) continue;
-
-		if (common_abbr[buff] == ABB_NUM)
-		{
-			for (j = i + k; j < text_len && isspace(text[j]); j++) ;	// next must be a number
-			if (! isdigit(text[j])) continue;				// go to next abbreviation
-		} else if (common_abbr[buff] == ABB_EOS)
-		{
-			for (j = i + k; j < text_len && isspace(text[j]); j++) ;	// if next token is upper case letter
-			if (isupper(text[j])) tokflag[i + (--k)] = 1;			// tokenize the final period of this abbreviation
-		}
-
-		// clear all token flags
-
-		for (j = 1; j < k; j++) tokflag[i + j] = 0;
-	}
-}
-
-// Check for common pairs that should not be considered sentence breaks
-
-void MPtok::tok_15_1()
-{
-	int	i, j, k, tnum, p;
-	char	buff[MAX_ABB + 1];
-
-	for (i = 0; i < text_len; i++)
-	{
-		if (! tokflag[i]) continue;
-
-		// must be alphanumeric token followed by period token followed by space followed by alphanumeric token
-
-		tnum = 0;
-		buff[0] = '\0';
-		for (p = k = 0; i + k < text_len && k < MAX_ABB; k++)
-		{
-			buff[k] = text[i+k]; buff[k+1] = '\0';
-
-			if (isspace(buff[k]))
-			{
-				if (tnum == 2) break;					// this is good
-				else if (tnum == 1) continue;				// ok
-				else { buff[0] = '\0'; break; }				// this shouldn't happen
-			}
-
-			if (tokflag[i+k])
-			{
-				if (tnum > 2) break;					// done
-				else tnum++;
-			}
-
-			if (tnum == 1 && buff[k] == '.') p = k;
-			if (tnum == 1 && buff[k] != '.') { buff[0] = '\0'; break; }	// nope
-			if (! isalnum(buff[k])) { buff[0] = '\0'; break; }		// nope
-		}
-
-		if (buff[0] == '\0' || i + k == text_len || k == MAX_ABB) continue;
-
-		// at this point buff is a potential pair, so untokenize the period, that's all
-
-		if (common_pair.count(buff))
-			tokflag[p] = 0;
-	}
-}
-
-// Get cases where a space after a sentence has been omitted
-//
-// A period that occurs in a token consisting of alphabetic
-// letters with a vowel to the left and the right is a
-// separate token.
-
-void MPtok::tok_16()
-{
-	int	j;
-	int	has_vowel;
-
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '.' && tokflag[i] == 0)
-		{
-			has_vowel = 0;
-			for (j = i - 1; j >= 0; --j)
-			{
-				if (isalpha(text[j]) == 0)
-					break;
-				if (strchr("aeiouAEIOU", text[j]))
-					has_vowel = 1;
-				if (tokflag[j])
-					break;
-			}
-			if ((j >= 0 && tokflag[j] == 0) || has_vowel == 0)
-				continue;
-
-			j = i + 1;
-
-			has_vowel = 0;
-			for (; j < text_len && tokflag[j] == 0; ++j)
-			{
-				if (isalpha(text[j]) == 0)
-					break;
-				if (strchr("aeiouAEIOU", text[j]))
-					has_vowel = 1;
-			}
-
-			if ((j < text_len && tokflag[j] == 0) || has_vowel == 0)
-				continue;
-
-			tokflag[i] = 1;
-			tokflag[i + 1] = 1;
-		}
-	}
-}
-
-// Correction to tok_16,
-// Don't count if the token before is a single letter
-// or the token following is a single letter other than 'a'.
-// Also, don't count if the token to the right is gov, com, edu, etc.
-// because those are web addresses!
-
-#define COMPLEX_WINDOW 40
-
-enum {COMPLEX_NOT = 0, COMPLEX_YES, COMPLEX_DONE};
-
-struct _complex {
-	int	flag;
-	int	offset;
-	const char *str;
-	int	len;
-} complex[] = {
-	COMPLEX_YES, 0, "complex", 7,
-	COMPLEX_NOT, 0, "complexi", 8,
-	COMPLEX_NOT, 0, "complexed", 9,
-	COMPLEX_NOT, 0, "complexa", 8,
-	COMPLEX_NOT, 0, "complex-", 8,
-	COMPLEX_NOT, 0, "complexl", 8,
-	COMPLEX_NOT, 0, "complexu", 8,
-	COMPLEX_NOT, -1, "-complex", 7,
-	COMPLEX_NOT, -2, "nocomplex", 9,
-	COMPLEX_NOT, -3, "subcomplex", 10,
-	COMPLEX_YES, 0, "hybrid", 6,
-	COMPLEX_NOT, 0, "hybridi", 7,
-	COMPLEX_NOT, 0, "hybrido", 7,
-	COMPLEX_NOT, 0, "hybrida", 7,
-	COMPLEX_NOT, 0, "hybrid-", 7,
-	COMPLEX_NOT, -1, "-hybrid", 7,
-	COMPLEX_YES, 0, "duplex", 6,
-	COMPLEX_NOT, -1, "oduplex", 7,
-	COMPLEX_DONE, 0, NULL, 0,
-};
-
-int MPtok::complex_check()
-{
-	int	last_period = -2*COMPLEX_WINDOW;
-	int	last_complex = -2*COMPLEX_WINDOW;
-	int	i, j;
-	int	complex_match;
-
-	for (i = 0; i < text_len; i++)
-	{
-		if (text[i] == '.')
-		{
-			if (i - last_complex <= COMPLEX_WINDOW)
-				return 1;
-			last_period = i;
-		}
-
-		complex_match = 0;
-		for (j = 0; complex[j].str; j++)
-		{
-			if (complex[j].flag == COMPLEX_NOT)
-			{
-				if (i + complex[j].offset >= 0
-				&& strncmp(text+i+complex[j].offset, complex[j].str, complex[j].len) == 0)
-				{
-					// don't match here
-					complex_match = 0;
-				}
-			} else if (complex[j].flag == COMPLEX_YES)
-			{
-				if (i + complex[j].offset >= 0
-				&& strncmp(text+i+complex[j].offset, complex[j].str, complex[j].len) == 0)
-				{
-					// match here
-					complex_match = 1;
-				}
-			}
-		}
-
-		if (complex_match)
-		{
-			if (i - last_period <= COMPLEX_WINDOW)
-				return 1;
-			last_complex = i;
-		}
-	}
-	return 0;
-}
-
-void MPtok::tok_16_1()
-{
-	int	i, j;
-	char	v1, v2;
-	int	c1, c2;
-
-	if (option_new == 3 && strstr(text, "complex"))
-		return;
-
-	if (option_new >= 4 && complex_check())
-		return;
-
-	for (i = 0; i < text_len; i++)
-	{
-		if (text[i] == '.' && tokflag[i] == 0)
-		{
-			char	suffix[10];
-			int	s_i;
-
-			v1 = '\0';
-			c1 = 0;
-			for (j = i - 1; j >= 0; --j)
-			{
-				if (isalpha(text[j]) == 0)
-					break;
-				if (strchr("aeiouAEIOU", text[j]))
-					v1 = tolower(text[j]);
-				c1++;
-				if (tokflag[j])
-					break;
-			}
-			if ((j >= 0 && tokflag[j] == 0)
-			|| v1 == '\0'
-			|| c1 == 1)
-				continue;
-
-			j = i + 1;
-
-			v2 = '\0';
-			c2 = 0;
-			s_i = 0;
-			for (; j < text_len && tokflag[j] == 0; ++j)
-			{
-				if (isalpha(text[j]) == 0)
-					break;
-				if (strchr("aeiouAEIOU", text[j]))
-					v2 = tolower(text[j]);
-				if (s_i < 3)
-					suffix[s_i++] = tolower(text[j]); suffix[s_i] = '\0';
-				c2++;
-			}
-
-			if ((j < text_len && tokflag[j] == 0)
-			|| v2 == '\0'
-			|| (c2 == 1 && v2 != 'a')
-			|| (c2 == 3 && tokflag[j] == 1 && s_i == 3
-				&& (strcmp(suffix, "gov") == 0
-					|| strcmp(suffix, "edu") == 0
-					|| strcmp(suffix, "org") == 0
-					|| strcmp(suffix, "com") == 0)))
-				continue;
-
-			tokflag[i] = 1;
-			tokflag[i + 1] = 1;
-		}
-	}
-}
-
-
-// Numeric endings of sentences
-//
-// A period after a numeric token followed by a token that starts
-// with an alphabetic character, is a separate token.
-//
-// This should be covered already by tok_13
-
-void MPtok::tok_17()
-{
-	int	j;
-
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '.'
-		&& tokflag[i] == 0
-		&& tokflag[i + 1])
-		{
-			for (j = i - 1; j >= 0 && isdigit(text[j]) && tokflag[j] == 0; --j)
-				;
-			if (j >= 0 && j < i - 1 && tokflag[j] && isalpha(nextchar(text, i + 1)))
-				tokflag[i] = 1;
-		}
-	}
-}
-
-// period at end of string is a token
-
-void MPtok::tok_20()
-{
-	for (int i = text_len - 1; i >= 0; --i)
-	{
-		if (isspace(text[i]))
-			continue;
-
-		if (strchr(".!?", text[i]))
-			tokflag[i] = 1;
-
-		break;
-	}
-}
-
-// a period that follows a non-common word, and that is
-// followed by a lower case common word is probably not a token
-
-void MPtok::tok_20_1()
-{
-	int	j;
-
-	for (int i = 0; i < text_len; ++i)
-	{
-		if (text[i] == '.' && tokflag[i] == 1)
-		{
-			int tcnt, lcnt, ocnt;
-			tcnt = lcnt = ocnt = 0;
-
-			// make sure the previous word was *not* common
-
-			for (j = i - 1; j >= 0; j--)
-			{
-				if (isspace(text[j])) continue;
-				if (option_new >= 2)
-				{
-					if (islower(text[j]) == 0 && text[j] != '-') ocnt++;
-				} else
-				{
-					if (! islower(text[j])) ocnt++;
-				}
-
-				if (tokflag[j] || j == 0)
-				{
-					if (ocnt == 0)
-					{
-						goto nexti;
-					}
-					break;
-				}
-			}
-
-			tcnt = lcnt = ocnt = 0;
-
-			// make sure the next word is common
-
-			for (j = i + 1; j < text_len; j++)
-			{
-				if (isspace(text[j])) continue;
-				if (tokflag[j]) tcnt++;
-
-				if (tcnt == 2 || j == text_len - 1)
-				{
-					if (lcnt > 0 && ocnt == 0) tokflag[i] = 0;
-					break;
-				}
-
-				if (islower(text[j])) lcnt++;
-				else ocnt++;
-			}
-		}
-nexti:		;
-	}
-}
-
-// tokenized period followed by non-space other than close paren
-// is not a token
-
-void MPtok::tok_20_2()
-{
-	int	j;
-
-	for (int i = 0; i < text_len - 1; ++i)
-	{
-		if (text[i] == '.' && tokflag[i] == 1
-		&& strchr(" ()[]\"\'\n\t\r", text[i+1]) == 0)
-		{
-			tokflag[i] = 0;
-		}
-	}
-}
-
-
-// long dash
-//
-// A pair of hyphens is a complete token
-
-void MPtok::tok_21()
-{
-	for (int i = 0; i + 1 < text_len; i++)
-	{
-		if (strncmp(&text[i], "--", 2) == 0)
-		{
-			tokflag[i] = 1;
-			if (i + 2 < text_len)
-			{
-				i += 2;
-				tokflag[i] = 1;
-			}
-		}
-	}
-}
-
-// hyphens
-//
-// If specified as an option, a hyphen between letters is a complete token
-
-void MPtok::tok_21a()
-{
-	if (option_hyphen == 0) return;
-
-	for (int i = 0; i + 1 < text_len; i++)
-	{
-		if (text[i] == '-'
-		&& (i == 0 || text[i-1] != '-')
-		&& text[i+1] != '-')
-		{
-			tokflag[i] = 1;
-			tokflag[i+1] = 1;
-		}
-	}
-}
-
-
-// quote
-//
-// Any double quote is a separate token
-
-void MPtok::tok_22()
-{
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '"')
-		{
-			tokflag[i] = 1;
-			if (i + 1 < text_len)
-			{
-				i += 1;
-				tokflag[i] = 1;
-			}
-		}
-	}
-}
-
-// possessive
-//
-// Any single quote at the end of a token that is not
-// preceded by a single quote is a separate token
-
-void MPtok::tok_23()
-{
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '\''
-		&& (i - 1 >= 0 && text[i - 1] != '\'')
-		&& tokflag[i + 1])
-		{
-			tokflag[i] = 1;
-		}
-	}
-}
-
-
-// quote
-//
-// If a single quote starts a token, or is preceded by a
-// single quote, and followed by a character
-// that is not a single quote, then
-// the character to it's right is the start of a new token
-
-void MPtok::tok_24()
-{
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '\''
-		&& (tokflag[i] == 1 || (i - 1 >= 0 && text[i - 1] == '\''))
-		&& (i + 1 < text_len && text[i + 1] != '\''))
-		{
-			tokflag[i + 1] = 1;
-		}
-	}
-}
-
-// put back possessive
-//
-// A single quote that is a whole token followed by a lower case s
-// that is also a whole token (without space between them)
-// should be merged into a single token
-
-void MPtok::tok_25()
-{
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '\''
-		&& tokflag[i] == 1
-		&& i + 1 < text_len && text[i + 1] == 's'
-		&& tokflag[i+1] == 1
-		&& (i + 2 >= text_len || isspace(text[i + 2]) || tokflag[i + 2] == 1))
-		{
-			tokflag[i + 1] = 0;
-		}
-	}
-}
-
-// quote
-//
-// A pair of single quotes is a separate token
-
-void MPtok::tok_26()
-{
-	for (int i = 0; i < text_len; i++)
-	{
-		if (strncmp(&text[i], "''", 2) == 0
-		|| strncmp(&text[i], "``", 2) == 0)
-		{
-			tokflag[i] = 1;
-			if (i + 2 < text_len) tokflag[i + 2] = 1;
-		}
-	}
-}
-
-// possessive
-//
-// A single quote followed by a letter s is a possessive
-
-void MPtok::tok_27()
-{
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '\''
-		&& i + 1 < text_len
-		&& tolower(text[i + 1]) == 's'
-		&& (i + 2 >= text_len || tokflag[i + 2]))
-		{
-			tokflag[i] = 1;
-		}
-	}
-}
-
-// split "cannot" to "can not"
-//
-// A single token that is the word cannot (in any case)
-// is split into two words
-
-void MPtok::tok_28()
-{
-	for (int i = 0; i < text_len; i++)
-	{
-		if ((strncmp(&text[i], "cannot", 6) == 0
-		|| strncmp(&text[i], "Cannot", 6) == 0)
-		&& tokflag[i + 6])
-		{
-			tokflag[i + 3] = 1;
-		}
-	}
-}
-
-// put list item elements back at sentence end
-//
-// A period that is preceded by an alphanumeric (no space)
-// and any amount of preceding space and an end-mark
-// stays with the alphanumeric.
-
-void MPtok::tok_29()
-{
-	int	j;
-
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '.'
-		&& tokflag[i] && tokflag[i + 1]
-		&& i - 1 >= 0 && isalnum(text[i - 1])
-		&& tokflag[i - 1]
-		&& ((j = lookbehind(text, i-2, ".", tokflag)) >= 0
-		||  (j = lookbehind(text, i-2, "?", tokflag)) >= 0
-		||  (j = lookbehind(text, i-2, "!", tokflag)) >= 0)
-		&& tokflag[j])
-		{
-			tokflag[i] = 0;
-		}
-	}
-}
-
-// attach list elements to the beginnings of their sentences
-// this means, attach the period to the list element
-//
-// a list element is a single letter or a one or two digits
-// which is preceded by an end of sentence ".!?;"
-// or colon (provided it doesn't belong to a proportion construct)
-
-void MPtok::tok_29a()
-{
-	int	i, j;
-
-	for (i = 0; i < text_len; i++)
-	{
-		if (text[i] == '.' && tokflag[i])
-		{
-			// Look back, make sure the token before the period
-			// is either single alphanumeric, or at most a two digit number
-			// and the character before that is a punctuation ".?!:,"
-
-			int tcnt, acnt, dcnt, pcnt, ocnt, scnt;
-			tcnt = acnt = dcnt = pcnt = ocnt = scnt = 0;
-			char p;
-
-			for (j = i - 1; j >= 0; j--)
-			{
-				if (isspace(text[j])) { scnt++; continue; }
-				else if (tcnt == 0 && isalpha(text[j])) ++acnt;
-				else if (tcnt == 0 && isdigit(text[j])) ++dcnt;
-				else if (tcnt == 1 && strchr(".!?:;,", text[j])) { pcnt++; p = text[j]; }
-				else ocnt++;
-
-				if (tokflag[j] || j == 0)
-				{
-					tcnt++;
-					if (tcnt == 1 && ocnt == 0 && scnt == 0
-					&& ((acnt == 1 && dcnt == 0) || (acnt == 0 && dcnt > 0 && dcnt <= 2)))
-					{
-						// This is acceptable
-					} else if (tcnt == 2 && pcnt <= 1 && ocnt == 0 && scnt > 0)
-					{
-						if (p == ':')
-						{
-							while (--j >= 0 && isspace(text[j]))
-								;
-							if (j >= 0 && isdigit(text[j]))
-							{
-								// It's probably a proportion
-								break;
-							}
-						}
-						// Jackpot
-						tokflag[i] = 0;
-					} else
-					{
-						// This is not
-						break;
-					}
-					scnt = 0;
-				}
-			}
-		}
-	}
-}
-
-// list elements at the beginning of a string
-//
-// An alphanumeric token followed by a period
-// at the beginning of the line stays with the
-// alphanumeric
-
-void MPtok::tok_30()
-{
-	int	i = 0;
-
-	while (isspace(text[i])) i++;
-
-	if (isalnum(text[i])
-	&& tokflag[i]
-	&& i + 1 < text_len
-	&& text[i + 1] == '.'
-	&& tokflag[i + 1])
-	{
-		tokflag[i + 1] = 0;
-	}
-}
-
-// process American style numbers
-
-void MPtok::tok_31()
-{
-	int	j;
-
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == ','
-		&& i + 3 < text_len
-		&& tokflag[i] && tokflag[i + 1]
-		&& isdigit(text[i + 1])
-		&& isdigit(text[i + 2])
-		&& isdigit(text[i + 3])
-		&& i - 1 >= 0 && isdigit(text[i - 1])
-		)
-		{
-			tokflag[i] = 0;
-			tokflag[i + 1] = 0;
-		}
-	}
-}
-
-// process British style numbers
-
-void MPtok::tok_32()
-{
-	int	j;
-
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == ' '
-		&& i + 3 < text_len
-		&& tokflag[i] && tokflag[i + 1]
-		&& isdigit(text[i + 1])
-		&& isdigit(text[i + 2])
-		&& isdigit(text[i + 3])
-		&& i - 1 >= 0 && isdigit(text[i - 1])
-		)
-		{
-			tokflag[i] = 0;
-			tokflag[i + 1] = 0;
-		}
-	}
-}
-
-// tokenize unicode escapes
-//
-// Added
-
-void MPtok::tok_33()
-{
-	int	j;
-
-	for (int i = 0; i < text_len; i++)
-	{
-		if (text[i] == '&')
-		{
-			if (text[i + 1] == '#')
-			{
-				for (j = i + 2; isdigit(text[j]); j++)
-					;
-			} else
-			{
-				for (j = i + 1; isalpha(text[j]); j++)
-					;
-			}
-
-			if (text[j] == ';')
-			{
-				// Tokenize the escape, untokenize everything inside
-
-				tokflag[i] = 1;
-				for (i++; i <= j; i++) tokflag[i] = 0;
-				tokflag[i] = 1;
-			}
-		}
-	}
-}
-
-// Remove tags if they are present
-
-void MPtok::tok_un()
-{
-	int untok = 0;
-	for (int i = 0; text[i]; ++i)
-	{
-		if (isspace(text[i])) untok = 0;
-		if (text[i] == option_tagsep) untok = 1;
-		if (untok) text[i] = ' ';
-	}
-}
-
-
-void MPtok::set_tokflag()
-{
-	int	i;
-
-	tok_0();
-	tok_1();
-	tok_2();
-	tok_3();
-
-	// step 4 replaces tag char, this is done at output
-
-	tok_5_6_7();
-	tok_8_9();
-
-	tok_10();
-	tok_11();
-	if (option_new >= 1)
-	{
-		tok_21();
-		tok_21a();
-		tok_22();
-		tok_23();
-		tok_24();
-		tok_25();
-		tok_26();
-		tok_27();
-	}
-	tok_12();
-	tok_13();
-	tok_14();
-	if (option_new <= 5)
-		tok_15();
-	if (option_new < 2)
-		tok_16();
-	tok_17();
-
-	// steps 18 and 19 recognize periods within parens,
-	// and this is moved to the segmentation section
-
-	tok_20();
-	if (option_new >= 1)
-	{
-		tok_20_1();
-		tok_20_2();
-		if (option_new >= 2)
-			tok_16_1();
-		if (option_new >= 6)
-			tok_15();
-		if (option_new >= 7)
-			tok_15_1();
-	}
-	if (option_new < 1)
-	{
-		tok_21();
-		tok_21a();
-		tok_22();
-		tok_23();
-		tok_24();
-		tok_25();
-		tok_26();
-		tok_27();
-	}
-	tok_28();
-	if (option_new >= 1)
-		tok_29a();
-	else
-		tok_29();
-	tok_30();
-	tok_31();
-	tok_32();
-
-	tok_33();
-}
-
-/* set_endflag
-** 
-** After tokflag has been set, find the possible sentence endings.
-*/
-
-void MPtok::set_endflag()
-{
-	int	i;
-
-	// The following tests look for end-stops and label them.
-	// They include steps 18 and 19
-
-	for (i = 0; i <= text_len; i++)
-		endflag[i] = 0;
-
-	// Count the number of unmatched parens
-
-	int up = 0;	// unmatched round parens
-	int ub = 0;	// unmatched brackets
-
-	for (i = 0; i < text_len; i++)
-	{
-		if (text[i] == '(') ++up;
-		if (text[i] == ')') --up;
-		if (text[i] == '[') ++ub;
-		if (text[i] == ']') --ub;
-		if (up < 0) up = 0;
-		if (ub < 0) ub = 0;
-	}
-
-	// Now find the end-of-sentence marks
-
-	// tok_18: periods within parentheses, allow for nesting
-	// tok_19: periods within brackets, allow for nesting
-	//	the perl version solves this by putting the period
-	//	back with the previous token, but a better solution
-	//	is to allow it to be tokenized but just don't
-	// 	allow it to be an end-of-sentence.
-	//	Therefore, these are moved to the segmentation
-	//	section
-
-	int p = 0;	// round parens
-	int b = 0;	// brackets
-
-	for (i = 0; i < text_len; i++)
-	{
-		if (text[i] == '(') ++p;
-		if (text[i] == ')') --p;
-		if (text[i] == '[') ++b;
-		if (text[i] == ']') --b;
-		if (p < 0) p = 0;
-		if (b < 0) b = 0;
-
-		if (strchr(".!?", text[i])
-		&& tokflag[i]
-		&& tokflag[i + 1])
-		{
-			if (option_segment && p <= up && b <= ub)
-				endflag[i] = 1;
-
-			// This is optional to join periods with
-			// probable abbreviations
-
-			if (p > up || b > ub)
-				tokflag[i] = 0;
-		}
-	}
-
-	// endtokens followed by a single or double quote, which matches
-	// a single or double quote in the previous sentence
-
-	if (option_new >= 1)
-	{
-		int	dquo, squo;
-		dquo = squo = 0;
-
-		for (i = 0; i < text_len; i++)
-		{
-			if (text[i] == '"') dquo = ! dquo;
-			else if (text[i] == '\'') squo = ! squo;
-			else if (endflag[i])
-			{
-				if ((text[i+1] == '"' && dquo) || (text[i+1] == '\'' && squo))
-				{
-					endflag[i] = 0;
-
-					// But don't end at all if the next token is something
-					// other than an upper case letter.
-
-					if (option_new >= 2)
-					{
-						int	j;
-						int	ok = 0;
-
-						for (j = i + 2; j < text_len; j++)
-						{
-							if (isspace(text[j])) continue;
-							// if (isupper(text[j]))
-							if (isupper(text[j]) || text[j] == '(')
-							{
-								ok = 1;
-								break;
-							}
-							if (tokflag[j]) break;
-						}
-
-						if (ok)
-							endflag[i+1] = 1;
-					} else
-					{
-						endflag[i+1] = 1;
-					}
-				}
-				dquo = squo = 0;
-			}
-		}
-	}
-}
-
-
-/* set_endflag_01
-** 
-** After tokflag has been set, find the possible sentence endings.
-** This has improved paren matching.
-*/
-
-#define MAX_MATCH 500		// Maximum length to get a paren match
-
-void MPtok::set_endflag_01()
-{
-	int	match[text_len];
-	int	i, j;
-
-	// The following tests look for end-stops and label them.
-	// They include steps 18 and 19
-
-	for (i = 0; i <= text_len; i++)
-		endflag[i] = 0;
-
-	for (i = 0; i < text_len; i++)
-		match[i] = 0;
-
-	for (i = text_len - 1; i >= 0; i--)
-	{
-		if (text[i] == '(' || text[i] == '[')
-		{
-			for (j = i + 1; text[j] && j - i <= MAX_MATCH; j++)
-			{
-				// Skip parens that are already matched
-
-				if (match[j] > j)
-				{
-					j = match[j];
-					continue;
-				}
-
-				// Look for a matching close paren
-
-				if (match[j] == 0
-				&& ((text[i] == '(' && text[j] == ')')
-				|| (text[i] == '[' && text[j] == ']')))
-				{
-					match[i] = j;
-					match[j] = i;
-					break;
-				}
-			}
-		}
-	}
-
-	int next_match = 0;
-	for (i = 0; i < text_len; i++)
-	{
-		if (match[i] > next_match)
-			next_match = match[i];
-
-		if (strchr(".!?", text[i])
-		&& tokflag[i]
-		&& tokflag[i + 1]
-		&& (option_new <= 4 || option_doteos == 1 || (i > 0 && isspace(text[i-1]) == 0)))
-		{
-			if (i <= next_match)
-				tokflag[i] = 0;
-			else if (option_segment)
-				endflag[i] = 1;
-		}
-	}
-
-	// endtokens followed by a single or double quote, which matches
-	// a single or double quote in the previous sentence
-
-	int	dquo, squo;
-	dquo = squo = 0;
-
-	for (i = 0; i < text_len; i++)
-	{
-		if (option_new <= 7 && text[i] == '"') dquo = ! dquo;
-		else if (option_new >= 8 && text[i] == '"' && tokflag[i] && tokflag[i+1]) dquo = ! dquo;
-		else if (option_new <= 7 && text[i] == '\'') squo = ! squo;
-		else if (option_new >= 8 && text[i] == '\''
-		&& tokflag[i] && (tokflag[i+1] || (text[i+1] == '\'' && tokflag[i+2]))) squo = ! squo;
-		else if (endflag[i])
-		{
-			if ((text[i+1] == '"' && dquo) || (text[i+1] == '\'' && squo))
-			{
-				endflag[i] = 0;
-
-				// But don't end at all if the next token is something
-				// other than an upper case letter.
-
-				if (option_new >= 2)
-				{
-					int	j;
-					int	ok = 0;
-
-					for (j = i + 2; j < text_len; j++)
-					{
-						if (isspace(text[j])) continue;
-						// if (isupper(text[j]))
-						if (isupper(text[j]) || text[j] == '(')
-						{
-							ok = 1;
-							break;
-						}
-						if (tokflag[j]) break;
-					}
-
-					if (ok)
-						endflag[i+1] = 1;
-				} else
-				{
-					endflag[i+1] = 1;
-				}
-			}
-			dquo = squo = 0;
-		}
-	}
-}
-
-
-// Size buffer: return the size of the buffer required to hold all of the tokenized text.
-// It can be simply estimated by a formula that depends only on the length of text and number of tokens.
-
-int MPtok::size_buff()
-{
-	int	size = 1;			// Start with null terminator
-	int	t = option_pretag.size();	// for each tag, the length of the UNTAG string
-
-	if (t <= 0) t = 1;			// Make sure there is at least one
-	t += 2;					// Add one for underscore and one for space
-
-	for (int i = 0; i < text_len; i++)
-	{
-		size++;				// Count all characters
-		if (tokflag[i]) size += t;	// Count token delimiters (may overcount)
-		if (endflag[i]) size++;		// Add one for newline
-	}
-	return size;
-}
-
-
-/* append_token
-** 
-** Save a single token to a buffer.
-*/
-
-void MPtok::append_token(string& buff, int& sp, char *tok, int ef)
-{
-	// Convert tag separator chars and back quotes (?)
-
-	for (int i = 0; tok[i]; i++)
-	{
-		if (tok[i] == option_tagsep) tok[i] = option_replacesep;
-		if (tok[i] == '`') tok[i] = '\'';
-	}
-
-	// Skip whitespace if tokens are being output
-	// Otherwise, skip whitespace at the start of a sentence
-
-	if (option_token || ! sp) while (isspace(*tok)) ++tok;
-
-	// Save the token
-
-	if (strlen(tok) > 0)
-	{
-		// Add delimiter if needed
-
-		if (option_token && sp) buff += ' ';
-
-		// Append token to output
-
-		if (option_new < 9)
-		{
-			while (*tok && (! option_token || ! isspace(*tok)))
-				buff += *(tok++);
-		} else
-		{
-			while (*tok)
-				buff += *(tok++);
-		}
-
-		sp = 1;
-
-		// Add tag holders
-
-		if (option_token && option_pretag.size() > 0)
-		{
-			buff += option_tagsep;
-			buff += option_pretag;
-		}
-
-		// If it was end of sentence, then add newline
-
-		if (ef)
-		{
-			buff += '\n';
-			sp = 0;
-		}
-	}
-}
-
-// Strip whitespace after sentences
-
-static void adjust_space(string& buff)
-{
-	while (buff.size() > 0 && isspace(buff[0])) buff.erase(0, 1);
-
-	// delete two spaces in a row, but keep newlines
-
-	for (int i = 1; i < buff.size(); i++)
-	{
-		if (isspace(buff[i]) && isspace(buff[i-1]))
-			buff.erase((buff[i] == '\n')?(--i):(i--), 1);
-	}
-
-	for (int i = buff.size() -  1; i >= 0 && isspace(buff[i]); i--)
-		buff.erase(i, 1);
-}
-
-/* token_string
-**
-** After the tokflag and endflag have been set, copy the tokens to the buffer.
-*/
-
-string MPtok::token_string()
-{
-	string buff;
-
-	int	i;
-
-	// Move token starts to non-whitespace chars
-
-	int last_tok = 0;
-	for (i = 0; i < text_len; i++)
-	{
-		if (tokflag[i] == 1 && isspace(text[i]))
-		{
-			tokflag[i] = 0;
-			last_tok = 1;
-		} else if (isspace(text[i]) == 0 && last_tok)
-		{
-			tokflag[i] = 1;
-			last_tok = 0;
-		}
-	}
-
-	// Extract the tokens and print them out now
-
-	char	*tok = new char[text_len + 1];
-	int	pos = 0;
-	int	sp = 0;
-	int	ef = 0;
-
-	tok[pos] = '\0';
-
-	for (i = 0; i <= text_len; i++)
-	{
-		// The start of a new token
-
-		if (tokflag[i])
-		{
-			// Print the current token
-
-			append_token(buff, sp, tok, ef);
-
-			// Start a new token
-
-			pos = 0;
-			tok[pos] = '\0';
-
-			ef = 0;
-		}
-
-		// Append to the current token
-
-		tok[pos++] = text[i];
-		tok[pos] = '\0';
-
-		// If any of the characters in the token are endflagged,
-		// Then pass this information along for end-of-sentence
-
-		if (endflag[i]) ef = 1;
-	}
-
-	// Print the last token
-
-	append_token(buff, sp, tok, ef);
-
-	delete[] tok;
-
-	// Adjust the end of sentence boundaries
-
-	adjust_space(buff);
-
-	return buff;
-}
-
-void MPtok::map_escapes()
-{
-	char	*s;
-	int	j, k, ch;
-	char	buff[10];
-
-	k = 0;
-	for (int i = 0; text[i]; i++)
-	{
-		if (text[i] == '&' && text[i + 1] == '#')
-		{
-			for (s = &buff[0], j = 2; j <= 4 && i + j < text_len && isdigit(text[i + j]); j++)
-				*s++ = text[i + j];
-			*s = '\0';
-			ch = atoi(buff);
-			if (strlen(buff) > 0 && text[i + j] == ';' && ch > 0 && ch <= 256)
-			{
-				text[k] = ch;
-				if (! text[k]) text[k] = ' ';
-				k++;
-				i = i + j;
-				continue;
-			}
-		}
-		text[k++] = text[i];
-	}
-	text[k] = '\0';
-	text_len = k;
-}
-
-MPtok::MPtok(string idir, const string& cnam)
-{
-	tok_initialized = 0;
-
-	if (idir.size() == 0)
-	{
-		char	*p = getenv("MEDPOST_HOME");
-		if (p && strlen(p))
-		{
-			idir = p;
-
-			int found = idir.find("=");
-			if (found != string::npos)
-				idir = idir.substr(found + 1);
-		}
-	}
-
-
-	if (idir.size() == 0)
-	{
-		char    buff[1000];
-		FILE *fp = fopen("path_medpost", "r");
-		if (fp)
-		{
-			if (fgets(buff, 1000, fp))
-			{
-				chomp(buff);
-				idir = &buff[0];
-			}
-			fclose(fp);
-		}
-	}
-
-	if (idir.size() == 0)
-		idir = "/home/natxie/CPP64/lib/FIXED_DATA/";
-
-	option_dir = idir;
-
-	option_token = 1;
-	option_segment = 1;
-	option_hyphen = 0;
-	option_comma = 1;
-	option_pretok = 0;
-	option_new = MPTOK_VERSION;
-	option_doteos = 0;
-
-	if (cnam.size() > 0)
-	{
-		option_cnam = "_";
-		option_cnam += cnam;
-	}
-
-	init();
-}
-
-void MPtok::init(void)
-{
-	if (tok_initialized) return;
-
-	string fname;
-
-	fname = option_dir + "/medpost" + option_cnam + ".pairs";
-	init_pair(fname);
-
-	fname = option_dir + "/medpost" + option_cnam + ".abbr";
-	init_abbr(fname);
-
-	tok_initialized = 1;
-}
-
-MPtok::~MPtok()
-{
-}
-
-// Global tokenizer
-
-string MPtok::tokenize(const string& txt, int mt)
-{
-	if (option_pretok) return save_string(txt);
-
-	option_token = mt;
-	text_len = txt.size();
-	if (text_len == 0) return string("");
-
-	text = new char[text_len + 1];
-	strcpy(text, txt.c_str());
-
-	map_escapes();
-
-	if (text_len == 0) return NULL;
-
-	tokflag = new int[text_len + 1];
-	endflag = new int[text_len + 1];
-
-	set_tokflag();
-	if (option_new < 3)
-		set_endflag();
-	else
-		set_endflag_01();
-
-	string buff = token_string();
-	save_string(buff);
-
-	delete[] text; text = NULL;
-	delete[] tokflag; tokflag = NULL;
-	delete[] endflag; endflag = NULL;
-
-	return buff;
-}
-
-string MPtok::tokenize(const string& text)
-{
-	return tokenize(text, 1);
-}
-
-string MPtok::segment(const string& text)
-{
-	sent.clear();
-
-	// tokenize the text
-
-	int save_option_segment = option_segment;
-	option_segment = 1;
-	string buff = tokenize(text, 0);
-	option_segment = save_option_segment;
-
-	if (buff.size() == 0) return text;
-
-	int found = 0;
-	int pos = 0;
-
-	while (pos < buff.size())
-	{
-		found = buff.find('\n', pos);
-		if (found == string::npos)
-		{
-			sent.push_back(buff.substr(pos));
-			pos = buff.size();
-		} else
-		{
-			sent.push_back(buff.substr(pos, found - pos));
-			pos = found + 1;
-		}
-	}
-
-	return buff;
-}
-
-string MPtok::save_string(const string& s)
-{
-	stringstream ss (stringstream::in | stringstream::out);
-	string	w, t;
-	int	found;
-	string	ret;
-
-	word.clear();
-	tag.clear();
-
-	ss << s;
-	while (ss.good())
-	{
-		ss >> w;
-		if (w.size() == 0) break;
-
-		found = w.find('_');
-
-		if (found != string::npos)
-		{
-			t = w.substr(found + 1);
-			w.resize(found);
-			word.push_back(w);
-			tag.push_back(t);
-		} else
-		{
-			word.push_back(w);
-			tag.push_back(option_pretag);
-
-		}
-		if (ret.size() > 0) ret += " ";
-		ret += w;
-	}
-
-	// now look for continuation tags...
-
-	for (int i = 0; i < word.size(); i++)
-	{
-		int j = tag[i].size() - 1;
-		if (j >= 0 && tag[i][j] == '+' && i < tag.size() - 1)
-		{
-			word[i] = word[i] + " " + word[i + 1];
-			tag[i] = tag[i + 1];
-			word.erase(word.begin() + i + 1, word.begin() + i + 2);
-			tag.erase(tag.begin() + i + 1, tag.begin() + i + 2);
-			i--;
-		}
-	}
-
-	return ret;
-}
-
-
-static int count_words(const char *s)
-{
-	int     i;
-
-	i = 1;
-	for (; *s; ++s)
-	{
-		if (*s == ' ') ++i;
-	}
-	return i;
-}
-
-static void print_word(const char *s, int i)
-{
-	for (; i > 0 && *s; ++s) { if (*s == ' ') --i; }
-	while (*s && *s != ' ') { printf("%c", *s); ++s; }
-}
-
-void MPtok::print(int how)
-{
-	int     i, j, w;
-
-	if (how != 0 && how != 2)
-	{
-		printf("print(%d) not defined\n", how);
-		return;
-	}
-
-	for (i = 0; i < word.size(); ++i)
-	{
-		// Get the words from an idiom
-
-		for (w = 0; w < count_words(word[i].c_str()); ++w)
-		{
-			if (how == 2 && i + w > 0) printf(" ");
-
-			print_word(word[i].c_str(), w);
-
-			if (how == 0)
-			{
-				printf(" tagged %s", tag[i].c_str());
-				if (w < count_words(word[i].c_str()) - 1) printf("+");
-				printf("\n");
-			} else if (how == 2)
-			{
-				printf("%s%s", "_", tag[i].c_str());
-				if (w < count_words(word[i].c_str()) - 1) printf("+");
-			}
-		}
-	}
-	if (how == 2)
-		printf("\n");
-}
-
-void MPtok::merge_words(int s, int n)
-{
-	string	tmp = word[s];
-
-	for (int i = s + 1; i < s + n; i++)
-	{
-		tmp += " ";
-		tmp += word[i];
-	}
-
-	// printf("merging words : '%s' n = %d\n", tmp.c_str(), n);
-
-	for (int k = s; k + n < word.size(); k++)
-	{
-		word[k+1] = word[k+n];
-		tag[k+1] = tag[k+n];
-	}
-
-	// Fixup the remaining array
-
-	word.resize(word.size() - n + 1); 
-	tag.resize(word.size());
-
-	word[s] = tmp;
-}
-
-void MPtok::split_words()
-{
-	for (int i = 0; i < word.size(); i++)
-	{
-		int found = word[i].find(' ');
-
-		if (found != string::npos)
-		{
-			string tmp1(word[i], 0, found);
-			string tmp2(word[i], found + 1, string::npos);
-
-			// Move all the words and tags down
-
-			word.resize(word.size() + 1);
-			tag.resize(tag.size() + 1);
-
-			for (int j = word.size() - 1; j > i; j--)
-			{
-				word[j] = word[j - 1];
-				tag[j] = tag[j - 1];
-			}
-
-			word[i] = tmp1;
-			tag[i] = tag[i+1];
-			tag[i] += "+";
-
-			word[i+1] = tmp2;
-		}
-	}
-}
-
-// Callable functions to set internal options
-
-void MPtok::set_segment(int i) { option_segment = i; }
-void MPtok::set_hyphen(int i) { option_hyphen = i; }
-void MPtok::set_comma(int i) { option_comma = i; }
-void MPtok::set_pretag(char *a) { option_pretag = a; }
-void MPtok::set_pretok(int i) { option_pretok = i; }
-void MPtok::set_new(int i) { option_new = i; }
-void MPtok::set_doteos(int i) { option_doteos = i; }
+#include <stdio.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include <string>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+
+#include "MPtok.h"
+
+// These options are probably compile time constants
+
+static char option_tagsep = '_';	// The tagsep character
+static char option_replacesep = '-';	// Replace tagsep with this
+
+static void chomp(char *line)
+{
+	int     i;
+
+	i = strlen(line) - 1;
+	while (i >= 0 && line[i] == '\n' || line[i] == '\r')
+		line[i--] = '\0';
+}
+
+// Data structure and algorithm for finding common pairs.
+
+// read a file of pairs into a data structure,
+// the file must be sorted first
+
+void MPtok::init_pair(const string& file_name)
+{
+	filebuf fb;
+	fb.open(file_name.c_str(), ios::in);
+	istream is(&fb);
+	string pair;
+
+	while (1)
+	{
+		getline(is, pair);
+		if (is.fail()) break;
+		if (pair.size() > 0) common_pair.insert(pair);
+	}
+
+	fb.close();
+}
+
+// List of abbreviations in 3 categories
+// ABB = can occur mid sentence
+// EOS = can occur at end of sentence
+// NUM = only used before numbers
+
+void MPtok::init_abbr(const string& file_name)
+{
+	filebuf fb;
+	fb.open(file_name.c_str(), ios::in);
+	istream is(&fb);
+	string typ, abb;
+	map<string,int> val;
+	val["ABB"] = ABB_ABB; val["EOS"] = ABB_EOS; val["NUM"] = ABB_NUM;
+
+	while (is.good())
+	{
+		is >> typ;
+		if (val.count(typ))
+		{
+			is >> abb;
+			if (abb.size() > 0) common_abbr[abb] = val[typ];
+		}
+	}
+	fb.close();
+}
+
+static char nextchar(const char *t, int i)
+{
+	while (isspace(t[i])) i++;
+	return t[i];
+}
+
+// Look for a token at or prior to the text position
+
+static int lookbehind(const char *t, int i, const char *s, int *tokflag)
+{
+	int	k = (int) strlen(s) - 1;
+
+	while (i > 0 && isspace(t[i])) i--;
+
+	while (k >= 0 && i >= 0)
+	{
+		if (k > 0 && tokflag[i]) break;
+
+		if (tolower(s[k]) != tolower(t[i]))
+			return -1;
+		k--;
+		i--;
+	}
+
+	return (k < 0 && tokflag[i+1]) ? i + 1 : -1;
+}
+
+// Look for a token at or following the text position
+
+static int lookahead(const char *t, int i, const char *s, int *tokflag)
+{
+	int	k = 0;
+
+	while (isspace(t[i])) i++;
+
+	while (k < strlen(s) && i < strlen(t))
+	{
+		if (k > 0 && tokflag[i]) break;
+
+		if (tolower(s[k]) != tolower(t[i]))
+			return -1;
+		k++;
+		i++;
+	}
+
+	return (k == strlen(s) && tokflag[i]) ? i - (int) strlen(s) : -1;
+}
+
+// Set the initial tokens at spaces
+
+void MPtok::tok_0()
+{
+	int i;
+
+	tokflag[0] = 1;
+	for (i = 1; i < text_len; i++)
+	{
+		tokflag[i] = isspace(text[i]) || (i > 0 && isspace(text[i - 1])) ? 1 : 0;
+	}
+	tokflag[i] = 1;
+}
+
+// Get quotes preceded by open parens
+//
+// A double quote, preceded by a space or open bracket is a separate token
+//
+
+void MPtok::tok_1()
+{
+	for (int i = 1; i < text_len; i++)
+	{
+		if (text[i] == '"' && strchr("([{<", text[i-1]))
+		{
+			tokflag[i] = 1;
+			if (i + 1 < text_len) tokflag[i+1] = 1;
+		}
+	}
+}
+
+// Look for ellipses
+//
+// Three dots in a row is a separate token
+
+void MPtok::tok_2()
+{
+	for (int i = 1; i + 2 < text_len; i++)
+	{
+		if (strncmp(&text[i], "...", 3) == 0)
+		{
+			tokflag[i] = 1;
+			if (i + 3 < text_len) tokflag[i+3] = 1;
+		}
+	}
+}
+
+// Non-sentence-ending punctuation
+//
+// Certain punctuation characters are separate tokens
+
+void MPtok::tok_3()
+{
+	for (int i = 0; i < text_len; i++)
+	{
+		// If it is a comma and the next char is not a space and option_comma = 0
+
+		if (option_comma == 0 && text[i] == ',' && isspace(text[i + 1]) == 0)
+		{
+			// do nothing
+		} else if (strchr(",;:@#$%&", text[i]))
+		{
+			tokflag[i] = 1;
+			tokflag[i + 1] = 1;
+		}
+	}
+}
+
+// Separate the slashes
+//
+// Slashes are a separate token
+// except for +/-, +/+, -/-, -/+, and and/or.
+
+void MPtok::tok_5_6_7()
+{
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '/')
+		{
+			tokflag[i] = 1;
+			if (i+1 < text_len) tokflag[i+1] = 1;
+
+			// Put back +/-, etc, unless option_hyphen is 1
+
+			if (i - 1 >= 0
+			&& i + 1 < text_len
+			&& ((option_new < 9
+				&& text[i - 1] == '+' || (text[i - 1] == '-' && option_hyphen == 0)
+				&& text[i + 1] == '+' || (text[i + 1] == '-' && option_hyphen == 0))
+			|| (option_new >= 9
+				&& (text[i - 1] == '+' || text[i - 1] == '-')
+				&& (text[i + 1] == '+' || text[i + 1] == '-'))))
+			{
+				tokflag[i - 1] = 1;
+				tokflag[i] = tokflag[i+1] = 0;
+				tokflag[i + 2] = 1;
+			}
+
+			// Put back and/or, etc
+
+			if (option_new <= 7)
+			{
+				if (i > 5 && strncmp(text + i - 5, " and/or ", 8) == 0)
+				{
+					for (int j = 1; j < 5; j++)
+						tokflag[i - 2 + j] = 0;
+				}
+			} else
+			{
+				if (i > 4 && strncmp(text + i - 4, " and/or ", 8) == 0)
+				{
+					for (int j = 1; j < 6; j++)
+						tokflag[i - 3 + j] = 0;
+				}
+			}
+		}
+	}
+}
+
+// All brackets
+//
+// Any open or closed bracket is a separate token
+//
+// Exclamation and question mark
+//
+// Any question or exclamation mark is a separate token
+
+void MPtok::tok_8_9()
+{
+	for (int i = 0; i < text_len; i++)
+	{
+		if (strchr("[](){}<>", text[i])
+		|| strchr("?!", text[i]))
+		{
+			tokflag[i] = 1;
+			if (i + 1 < text_len) tokflag[i+1] = 1;
+		}
+	}
+}
+
+// Period at the end of a string may be followed by closed-bracket or quote
+//
+// A period that is preceded by a non-period
+// and optionally followed by a close paren
+// and any amount of space at the end of the string
+// is a separate token.
+
+void MPtok::tok_10()
+{
+	for (int i = text_len - 1; i >= 0; i--)
+	{
+		if (isspace(text[i])) continue;
+		if (strchr("])}>\"'", text[i])) continue;
+		if (text[i] != '.') break;
+		if (text[i] == '.' && (i - 1 < 0 || text[i-1] != '.'))
+		{
+			tokflag[i] = 1;
+			if (i + 1 < text_len) tokflag[i+1] = 1;
+		}
+	}
+}
+
+// Period followed by a capitalized word
+//
+// A period preceded by a character that is not another period and not a space
+// and followed by a space then an upper case letter is a separate token
+
+void MPtok::tok_11()
+{
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '.'
+		&& (i + 1 < text_len && isspace(text[i+1]))
+		&& (i - 1 < 0 || text[i - 1] != '.' || isspace(text[i-1]) == 0)
+		&& isupper(nextchar(text, i + 1)))
+			tokflag[i] = 1;
+	}
+}
+
+// A normal word followed by a period
+//
+// A period followed by a space
+// and preceded by 2 or more alphabetic characters or hyphens
+// is a separate token
+
+void MPtok::tok_12()
+{
+	int wcnt = 0;
+
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '.'
+		&& tokflag[i + 1]
+		&& wcnt >= 2)
+			tokflag[i] = 1;
+
+		if (isalpha(text[i]) || text[i] == '-')
+			++wcnt;
+		else
+			wcnt = 0;
+	}
+}
+
+// A non-normal token (that has no lower case letters) followed by a period
+//
+// A period at the end of a token made of characters excluding lower case
+// is a separate token
+
+void MPtok::tok_13()
+{
+	int	stok = 0;
+	int	wcnt = 0;
+
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '.'
+		&& tokflag[i + 1]
+		&& wcnt >= 2)
+			tokflag[i] = 1;
+
+		if (tokflag[i] == 1) stok = 1;
+
+		if (islower(text[i]) || text[i] == '.')
+		{
+			stok = 0;
+			wcnt = 0;
+		}
+
+		if (stok)
+			wcnt++;
+	}
+}
+
+// put some periods with single-letter abbreviations
+//
+// A single alphabetic token followed by a period followed
+// by a token that does not begin with an upper case letter
+// or number is taken to be an abbreviation and the period
+// does not start a new token.
+//
+// NOTE: This does not recognize initials in people's names,
+//	 that problem is not simply solved.
+
+void MPtok::tok_14()
+{
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '.'
+		&& i - 1 >= 0 && isalpha(text[i - 1]) && tokflag[i - 1]
+		&& tokflag[i + 1]
+		&& isupper(nextchar(text, i + 1)) == 0
+		&& isdigit(nextchar(text, i + 1)) == 0
+		&& nextchar(text, i + 1) != '('
+		)
+		{
+			tokflag[i] = 0;
+		}
+	}
+}
+
+void MPtok::tok_15()
+{
+	int	i, j, k, a;
+	char	buff[MAX_ABB + 1];
+
+	for (i = 0; i < text_len; i++)
+	{
+		// only start at a current token
+
+		if (! tokflag[i]) continue;
+
+		// find alphabetic followed by period
+
+		buff[0] = '\0';
+		for (k = 0; i + k < text_len && k < MAX_ABB; k++)
+		{
+			buff[k] = text[i+k]; buff[k+1] = '\0';
+			if (k > 0 && buff[k] == '.') break; // this is good
+			if (! isalpha(buff[k])) { buff[0] = '\0'; break; } // this is not good
+		}
+
+		if (buff[0] == '\0' || i + k == text_len || k == MAX_ABB) continue;
+
+		// at this point, buff[k] == '.' add 1 to make it the length
+
+		k++;
+
+		// if not found, try finding a concatenated abbrev
+
+		if (! common_abbr.count(buff))
+		{
+			for (; i + k < text_len && k < MAX_ABB; k++)
+			{
+				buff[k] = text[i+k]; buff[k+1] = '\0';
+				if (k > 0 && buff[k] == '.') break; // this is good
+				if (! isalpha(buff[k])) { buff[0] = '\0'; break; } // this is not good
+			}
+
+			if (buff[0] == '\0' || i + k == text_len || k == MAX_ABB) continue;
+
+			// at this point, buff[k] == '.' add 1 to make it the length
+
+			k++;
+		}
+
+		// if not found, give up
+
+		if (! common_abbr.count(buff)) continue;
+
+		if (common_abbr[buff] == ABB_NUM)
+		{
+			for (j = i + k; j < text_len && isspace(text[j]); j++) ;	// next must be a number
+			if (! isdigit(text[j])) continue;				// go to next abbreviation
+		} else if (common_abbr[buff] == ABB_EOS)
+		{
+			for (j = i + k; j < text_len && isspace(text[j]); j++) ;	// if next token is upper case letter
+			if (isupper(text[j])) tokflag[i + (--k)] = 1;			// tokenize the final period of this abbreviation
+		}
+
+		// clear all token flags
+
+		for (j = 1; j < k; j++) tokflag[i + j] = 0;
+	}
+}
+
+// Check for common pairs that should not be considered sentence breaks
+
+void MPtok::tok_15_1()
+{
+	int	i, j, k, tnum, p;
+	char	buff[MAX_ABB + 1];
+
+	for (i = 0; i < text_len; i++)
+	{
+		if (! tokflag[i]) continue;
+
+		// must be alphanumeric token followed by period token followed by space followed by alphanumeric token
+
+		tnum = 0;
+		buff[0] = '\0';
+		for (p = k = 0; i + k < text_len && k < MAX_ABB; k++)
+		{
+			buff[k] = text[i+k]; buff[k+1] = '\0';
+
+			if (isspace(buff[k]))
+			{
+				if (tnum == 2) break;					// this is good
+				else if (tnum == 1) continue;				// ok
+				else { buff[0] = '\0'; break; }				// this shouldn't happen
+			}
+
+			if (tokflag[i+k])
+			{
+				if (tnum > 2) break;					// done
+				else tnum++;
+			}
+
+			if (tnum == 1 && buff[k] == '.') p = k;
+			if (tnum == 1 && buff[k] != '.') { buff[0] = '\0'; break; }	// nope
+			if (! isalnum(buff[k])) { buff[0] = '\0'; break; }		// nope
+		}
+
+		if (buff[0] == '\0' || i + k == text_len || k == MAX_ABB) continue;
+
+		// at this point buff is a potential pair, so untokenize the period, that's all
+
+		if (common_pair.count(buff))
+			tokflag[p] = 0;
+	}
+}
+
+// Get cases where a space after a sentence has been omitted
+//
+// A period that occurs in a token consisting of alphabetic
+// letters with a vowel to the left and the right is a
+// separate token.
+
+void MPtok::tok_16()
+{
+	int	j;
+	int	has_vowel;
+
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '.' && tokflag[i] == 0)
+		{
+			has_vowel = 0;
+			for (j = i - 1; j >= 0; --j)
+			{
+				if (isalpha(text[j]) == 0)
+					break;
+				if (strchr("aeiouAEIOU", text[j]))
+					has_vowel = 1;
+				if (tokflag[j])
+					break;
+			}
+			if ((j >= 0 && tokflag[j] == 0) || has_vowel == 0)
+				continue;
+
+			j = i + 1;
+
+			has_vowel = 0;
+			for (; j < text_len && tokflag[j] == 0; ++j)
+			{
+				if (isalpha(text[j]) == 0)
+					break;
+				if (strchr("aeiouAEIOU", text[j]))
+					has_vowel = 1;
+			}
+
+			if ((j < text_len && tokflag[j] == 0) || has_vowel == 0)
+				continue;
+
+			tokflag[i] = 1;
+			tokflag[i + 1] = 1;
+		}
+	}
+}
+
+// Correction to tok_16,
+// Don't count if the token before is a single letter
+// or the token following is a single letter other than 'a'.
+// Also, don't count if the token to the right is gov, com, edu, etc.
+// because those are web addresses!
+
+#define COMPLEX_WINDOW 40
+
+enum {COMPLEX_NOT = 0, COMPLEX_YES, COMPLEX_DONE};
+
+struct _complex {
+	int	flag;
+	int	offset;
+	const char *str;
+	int	len;
+} complex[] = {
+	COMPLEX_YES, 0, "complex", 7,
+	COMPLEX_NOT, 0, "complexi", 8,
+	COMPLEX_NOT, 0, "complexed", 9,
+	COMPLEX_NOT, 0, "complexa", 8,
+	COMPLEX_NOT, 0, "complex-", 8,
+	COMPLEX_NOT, 0, "complexl", 8,
+	COMPLEX_NOT, 0, "complexu", 8,
+	COMPLEX_NOT, -1, "-complex", 7,
+	COMPLEX_NOT, -2, "nocomplex", 9,
+	COMPLEX_NOT, -3, "subcomplex", 10,
+	COMPLEX_YES, 0, "hybrid", 6,
+	COMPLEX_NOT, 0, "hybridi", 7,
+	COMPLEX_NOT, 0, "hybrido", 7,
+	COMPLEX_NOT, 0, "hybrida", 7,
+	COMPLEX_NOT, 0, "hybrid-", 7,
+	COMPLEX_NOT, -1, "-hybrid", 7,
+	COMPLEX_YES, 0, "duplex", 6,
+	COMPLEX_NOT, -1, "oduplex", 7,
+	COMPLEX_DONE, 0, NULL, 0,
+};
+
+int MPtok::complex_check()
+{
+	int	last_period = -2*COMPLEX_WINDOW;
+	int	last_complex = -2*COMPLEX_WINDOW;
+	int	i, j;
+	int	complex_match;
+
+	for (i = 0; i < text_len; i++)
+	{
+		if (text[i] == '.')
+		{
+			if (i - last_complex <= COMPLEX_WINDOW)
+				return 1;
+			last_period = i;
+		}
+
+		complex_match = 0;
+		for (j = 0; complex[j].str; j++)
+		{
+			if (complex[j].flag == COMPLEX_NOT)
+			{
+				if (i + complex[j].offset >= 0
+				&& strncmp(text+i+complex[j].offset, complex[j].str, complex[j].len) == 0)
+				{
+					// don't match here
+					complex_match = 0;
+				}
+			} else if (complex[j].flag == COMPLEX_YES)
+			{
+				if (i + complex[j].offset >= 0
+				&& strncmp(text+i+complex[j].offset, complex[j].str, complex[j].len) == 0)
+				{
+					// match here
+					complex_match = 1;
+				}
+			}
+		}
+
+		if (complex_match)
+		{
+			if (i - last_period <= COMPLEX_WINDOW)
+				return 1;
+			last_complex = i;
+		}
+	}
+	return 0;
+}
+
+void MPtok::tok_16_1()
+{
+	int	i, j;
+	char	v1, v2;
+	int	c1, c2;
+
+	if (option_new == 3 && strstr(text, "complex"))
+		return;
+
+	if (option_new >= 4 && complex_check())
+		return;
+
+	for (i = 0; i < text_len; i++)
+	{
+		if (text[i] == '.' && tokflag[i] == 0)
+		{
+			char	suffix[10];
+			int	s_i;
+
+			v1 = '\0';
+			c1 = 0;
+			for (j = i - 1; j >= 0; --j)
+			{
+				if (isalpha(text[j]) == 0)
+					break;
+				if (strchr("aeiouAEIOU", text[j]))
+					v1 = tolower(text[j]);
+				c1++;
+				if (tokflag[j])
+					break;
+			}
+			if ((j >= 0 && tokflag[j] == 0)
+			|| v1 == '\0'
+			|| c1 == 1)
+				continue;
+
+			j = i + 1;
+
+			v2 = '\0';
+			c2 = 0;
+			s_i = 0;
+			for (; j < text_len && tokflag[j] == 0; ++j)
+			{
+				if (isalpha(text[j]) == 0)
+					break;
+				if (strchr("aeiouAEIOU", text[j]))
+					v2 = tolower(text[j]);
+				if (s_i < 3)
+					suffix[s_i++] = tolower(text[j]); suffix[s_i] = '\0';
+				c2++;
+			}
+
+			if ((j < text_len && tokflag[j] == 0)
+			|| v2 == '\0'
+			|| (c2 == 1 && v2 != 'a')
+			|| (c2 == 3 && tokflag[j] == 1 && s_i == 3
+				&& (strcmp(suffix, "gov") == 0
+					|| strcmp(suffix, "edu") == 0
+					|| strcmp(suffix, "org") == 0
+					|| strcmp(suffix, "com") == 0)))
+				continue;
+
+			tokflag[i] = 1;
+			tokflag[i + 1] = 1;
+		}
+	}
+}
+
+
+// Numeric endings of sentences
+//
+// A period after a numeric token followed by a token that starts
+// with an alphabetic character, is a separate token.
+//
+// This should be covered already by tok_13
+
+void MPtok::tok_17()
+{
+	int	j;
+
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '.'
+		&& tokflag[i] == 0
+		&& tokflag[i + 1])
+		{
+			for (j = i - 1; j >= 0 && isdigit(text[j]) && tokflag[j] == 0; --j)
+				;
+			if (j >= 0 && j < i - 1 && tokflag[j] && isalpha(nextchar(text, i + 1)))
+				tokflag[i] = 1;
+		}
+	}
+}
+
+// period at end of string is a token
+
+void MPtok::tok_20()
+{
+	for (int i = text_len - 1; i >= 0; --i)
+	{
+		if (isspace(text[i]))
+			continue;
+
+		if (strchr(".!?", text[i]))
+			tokflag[i] = 1;
+
+		break;
+	}
+}
+
+// a period that follows a non-common word, and that is
+// followed by a lower case common word is probably not a token
+
+void MPtok::tok_20_1()
+{
+	int	j;
+
+	for (int i = 0; i < text_len; ++i)
+	{
+		if (text[i] == '.' && tokflag[i] == 1)
+		{
+			int tcnt, lcnt, ocnt;
+			tcnt = lcnt = ocnt = 0;
+
+			// make sure the previous word was *not* common
+
+			for (j = i - 1; j >= 0; j--)
+			{
+				if (isspace(text[j])) continue;
+				if (option_new >= 2)
+				{
+					if (islower(text[j]) == 0 && text[j] != '-') ocnt++;
+				} else
+				{
+					if (! islower(text[j])) ocnt++;
+				}
+
+				if (tokflag[j] || j == 0)
+				{
+					if (ocnt == 0)
+					{
+						goto nexti;
+					}
+					break;
+				}
+			}
+
+			tcnt = lcnt = ocnt = 0;
+
+			// make sure the next word is common
+
+			for (j = i + 1; j < text_len; j++)
+			{
+				if (isspace(text[j])) continue;
+				if (tokflag[j]) tcnt++;
+
+				if (tcnt == 2 || j == text_len - 1)
+				{
+					if (lcnt > 0 && ocnt == 0) tokflag[i] = 0;
+					break;
+				}
+
+				if (islower(text[j])) lcnt++;
+				else ocnt++;
+			}
+		}
+nexti:		;
+	}
+}
+
+// tokenized period followed by non-space other than close paren
+// is not a token
+
+void MPtok::tok_20_2()
+{
+	int	j;
+
+	for (int i = 0; i < text_len - 1; ++i)
+	{
+		if (text[i] == '.' && tokflag[i] == 1
+		&& strchr(" ()[]\"\'\n\t\r", text[i+1]) == 0)
+		{
+			tokflag[i] = 0;
+		}
+	}
+}
+
+
+// long dash
+//
+// A pair of hyphens is a complete token
+
+void MPtok::tok_21()
+{
+	for (int i = 0; i + 1 < text_len; i++)
+	{
+		if (strncmp(&text[i], "--", 2) == 0)
+		{
+			tokflag[i] = 1;
+			if (i + 2 < text_len)
+			{
+				i += 2;
+				tokflag[i] = 1;
+			}
+		}
+	}
+}
+
+// hyphens
+//
+// If specified as an option, a hyphen between letters is a complete token
+
+void MPtok::tok_21a()
+{
+	if (option_hyphen == 0) return;
+
+	for (int i = 0; i + 1 < text_len; i++)
+	{
+		if (text[i] == '-'
+		&& (i == 0 || text[i-1] != '-')
+		&& text[i+1] != '-')
+		{
+			tokflag[i] = 1;
+			tokflag[i+1] = 1;
+		}
+	}
+}
+
+
+// quote
+//
+// Any double quote is a separate token
+
+void MPtok::tok_22()
+{
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '"')
+		{
+			tokflag[i] = 1;
+			if (i + 1 < text_len)
+			{
+				i += 1;
+				tokflag[i] = 1;
+			}
+		}
+	}
+}
+
+// possessive
+//
+// Any single quote at the end of a token that is not
+// preceded by a single quote is a separate token
+
+void MPtok::tok_23()
+{
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '\''
+		&& (i - 1 >= 0 && text[i - 1] != '\'')
+		&& tokflag[i + 1])
+		{
+			tokflag[i] = 1;
+		}
+	}
+}
+
+
+// quote
+//
+// If a single quote starts a token, or is preceded by a
+// single quote, and followed by a character
+// that is not a single quote, then
+// the character to it's right is the start of a new token
+
+void MPtok::tok_24()
+{
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '\''
+		&& (tokflag[i] == 1 || (i - 1 >= 0 && text[i - 1] == '\''))
+		&& (i + 1 < text_len && text[i + 1] != '\''))
+		{
+			tokflag[i + 1] = 1;
+		}
+	}
+}
+
+// put back possessive
+//
+// A single quote that is a whole token followed by a lower case s
+// that is also a whole token (without space between them)
+// should be merged into a single token
+
+void MPtok::tok_25()
+{
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '\''
+		&& tokflag[i] == 1
+		&& i + 1 < text_len && text[i + 1] == 's'
+		&& tokflag[i+1] == 1
+		&& (i + 2 >= text_len || isspace(text[i + 2]) || tokflag[i + 2] == 1))
+		{
+			tokflag[i + 1] = 0;
+		}
+	}
+}
+
+// quote
+//
+// A pair of single quotes is a separate token
+
+void MPtok::tok_26()
+{
+	for (int i = 0; i < text_len; i++)
+	{
+		if (strncmp(&text[i], "''", 2) == 0
+		|| strncmp(&text[i], "``", 2) == 0)
+		{
+			tokflag[i] = 1;
+			if (i + 2 < text_len) tokflag[i + 2] = 1;
+		}
+	}
+}
+
+// possessive
+//
+// A single quote followed by a letter s is a possessive
+
+void MPtok::tok_27()
+{
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '\''
+		&& i + 1 < text_len
+		&& tolower(text[i + 1]) == 's'
+		&& (i + 2 >= text_len || tokflag[i + 2]))
+		{
+			tokflag[i] = 1;
+		}
+	}
+}
+
+// split "cannot" to "can not"
+//
+// A single token that is the word cannot (in any case)
+// is split into two words
+
+void MPtok::tok_28()
+{
+	for (int i = 0; i < text_len; i++)
+	{
+		if ((strncmp(&text[i], "cannot", 6) == 0
+		|| strncmp(&text[i], "Cannot", 6) == 0)
+		&& tokflag[i + 6])
+		{
+			tokflag[i + 3] = 1;
+		}
+	}
+}
+
+// put list item elements back at sentence end
+//
+// A period that is preceded by an alphanumeric (no space)
+// and any amount of preceding space and an end-mark
+// stays with the alphanumeric.
+
+void MPtok::tok_29()
+{
+	int	j;
+
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '.'
+		&& tokflag[i] && tokflag[i + 1]
+		&& i - 1 >= 0 && isalnum(text[i - 1])
+		&& tokflag[i - 1]
+		&& ((j = lookbehind(text, i-2, ".", tokflag)) >= 0
+		||  (j = lookbehind(text, i-2, "?", tokflag)) >= 0
+		||  (j = lookbehind(text, i-2, "!", tokflag)) >= 0)
+		&& tokflag[j])
+		{
+			tokflag[i] = 0;
+		}
+	}
+}
+
+// attach list elements to the beginnings of their sentences
+// this means, attach the period to the list element
+//
+// a list element is a single letter or a one or two digits
+// which is preceded by an end of sentence ".!?;"
+// or colon (provided it doesn't belong to a proportion construct)
+
+void MPtok::tok_29a()
+{
+	int	i, j;
+
+	for (i = 0; i < text_len; i++)
+	{
+		if (text[i] == '.' && tokflag[i])
+		{
+			// Look back, make sure the token before the period
+			// is either single alphanumeric, or at most a two digit number
+			// and the character before that is a punctuation ".?!:,"
+
+			int tcnt, acnt, dcnt, pcnt, ocnt, scnt;
+			tcnt = acnt = dcnt = pcnt = ocnt = scnt = 0;
+			char p;
+
+			for (j = i - 1; j >= 0; j--)
+			{
+				if (isspace(text[j])) { scnt++; continue; }
+				else if (tcnt == 0 && isalpha(text[j])) ++acnt;
+				else if (tcnt == 0 && isdigit(text[j])) ++dcnt;
+				else if (tcnt == 1 && strchr(".!?:;,", text[j])) { pcnt++; p = text[j]; }
+				else ocnt++;
+
+				if (tokflag[j] || j == 0)
+				{
+					tcnt++;
+					if (tcnt == 1 && ocnt == 0 && scnt == 0
+					&& ((acnt == 1 && dcnt == 0) || (acnt == 0 && dcnt > 0 && dcnt <= 2)))
+					{
+						// This is acceptable
+					} else if (tcnt == 2 && pcnt <= 1 && ocnt == 0 && scnt > 0)
+					{
+						if (p == ':')
+						{
+							while (--j >= 0 && isspace(text[j]))
+								;
+							if (j >= 0 && isdigit(text[j]))
+							{
+								// It's probably a proportion
+								break;
+							}
+						}
+						// Jackpot
+						tokflag[i] = 0;
+					} else
+					{
+						// This is not
+						break;
+					}
+					scnt = 0;
+				}
+			}
+		}
+	}
+}
+
+// list elements at the beginning of a string
+//
+// An alphanumeric token followed by a period
+// at the beginning of the line stays with the
+// alphanumeric
+
+void MPtok::tok_30()
+{
+	int	i = 0;
+
+	while (isspace(text[i])) i++;
+
+	if (isalnum(text[i])
+	&& tokflag[i]
+	&& i + 1 < text_len
+	&& text[i + 1] == '.'
+	&& tokflag[i + 1])
+	{
+		tokflag[i + 1] = 0;
+	}
+}
+
+// process American style numbers
+
+void MPtok::tok_31()
+{
+	int	j;
+
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == ','
+		&& i + 3 < text_len
+		&& tokflag[i] && tokflag[i + 1]
+		&& isdigit(text[i + 1])
+		&& isdigit(text[i + 2])
+		&& isdigit(text[i + 3])
+		&& i - 1 >= 0 && isdigit(text[i - 1])
+		)
+		{
+			tokflag[i] = 0;
+			tokflag[i + 1] = 0;
+		}
+	}
+}
+
+// process British style numbers
+
+void MPtok::tok_32()
+{
+	int	j;
+
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == ' '
+		&& i + 3 < text_len
+		&& tokflag[i] && tokflag[i + 1]
+		&& isdigit(text[i + 1])
+		&& isdigit(text[i + 2])
+		&& isdigit(text[i + 3])
+		&& i - 1 >= 0 && isdigit(text[i - 1])
+		)
+		{
+			tokflag[i] = 0;
+			tokflag[i + 1] = 0;
+		}
+	}
+}
+
+// tokenize unicode escapes
+//
+// Added
+
+void MPtok::tok_33()
+{
+	int	j;
+
+	for (int i = 0; i < text_len; i++)
+	{
+		if (text[i] == '&')
+		{
+			if (text[i + 1] == '#')
+			{
+				for (j = i + 2; isdigit(text[j]); j++)
+					;
+			} else
+			{
+				for (j = i + 1; isalpha(text[j]); j++)
+					;
+			}
+
+			if (text[j] == ';')
+			{
+				// Tokenize the escape, untokenize everything inside
+
+				tokflag[i] = 1;
+				for (i++; i <= j; i++) tokflag[i] = 0;
+				tokflag[i] = 1;
+			}
+		}
+	}
+}
+
+// Remove tags if they are present
+
+void MPtok::tok_un()
+{
+	int untok = 0;
+	for (int i = 0; text[i]; ++i)
+	{
+		if (isspace(text[i])) untok = 0;
+		if (text[i] == option_tagsep) untok = 1;
+		if (untok) text[i] = ' ';
+	}
+}
+
+
+void MPtok::set_tokflag()
+{
+	int	i;
+
+	tok_0();
+	tok_1();
+	tok_2();
+	tok_3();
+
+	// step 4 replaces tag char, this is done at output
+
+	tok_5_6_7();
+	tok_8_9();
+
+	tok_10();
+	tok_11();
+	if (option_new >= 1)
+	{
+		tok_21();
+		tok_21a();
+		tok_22();
+		tok_23();
+		tok_24();
+		tok_25();
+		tok_26();
+		tok_27();
+	}
+	tok_12();
+	tok_13();
+	tok_14();
+	if (option_new <= 5)
+		tok_15();
+	if (option_new < 2)
+		tok_16();
+	tok_17();
+
+	// steps 18 and 19 recognize periods within parens,
+	// and this is moved to the segmentation section
+
+	tok_20();
+	if (option_new >= 1)
+	{
+		tok_20_1();
+		tok_20_2();
+		if (option_new >= 2)
+			tok_16_1();
+		if (option_new >= 6)
+			tok_15();
+		if (option_new >= 7)
+			tok_15_1();
+	}
+	if (option_new < 1)
+	{
+		tok_21();
+		tok_21a();
+		tok_22();
+		tok_23();
+		tok_24();
+		tok_25();
+		tok_26();
+		tok_27();
+	}
+	tok_28();
+	if (option_new >= 1)
+		tok_29a();
+	else
+		tok_29();
+	tok_30();
+	tok_31();
+	tok_32();
+
+	tok_33();
+}
+
+/* set_endflag
+** 
+** After tokflag has been set, find the possible sentence endings.
+*/
+
+void MPtok::set_endflag()
+{
+	int	i;
+
+	// The following tests look for end-stops and label them.
+	// They include steps 18 and 19
+
+	for (i = 0; i <= text_len; i++)
+		endflag[i] = 0;
+
+	// Count the number of unmatched parens
+
+	int up = 0;	// unmatched round parens
+	int ub = 0;	// unmatched brackets
+
+	for (i = 0; i < text_len; i++)
+	{
+		if (text[i] == '(') ++up;
+		if (text[i] == ')') --up;
+		if (text[i] == '[') ++ub;
+		if (text[i] == ']') --ub;
+		if (up < 0) up = 0;
+		if (ub < 0) ub = 0;
+	}
+
+	// Now find the end-of-sentence marks
+
+	// tok_18: periods within parentheses, allow for nesting
+	// tok_19: periods within brackets, allow for nesting
+	//	the perl version solves this by putting the period
+	//	back with the previous token, but a better solution
+	//	is to allow it to be tokenized but just don't
+	// 	allow it to be an end-of-sentence.
+	//	Therefore, these are moved to the segmentation
+	//	section
+
+	int p = 0;	// round parens
+	int b = 0;	// brackets
+
+	for (i = 0; i < text_len; i++)
+	{
+		if (text[i] == '(') ++p;
+		if (text[i] == ')') --p;
+		if (text[i] == '[') ++b;
+		if (text[i] == ']') --b;
+		if (p < 0) p = 0;
+		if (b < 0) b = 0;
+
+		if (strchr(".!?", text[i])
+		&& tokflag[i]
+		&& tokflag[i + 1])
+		{
+			if (option_segment && p <= up && b <= ub)
+				endflag[i] = 1;
+
+			// This is optional to join periods with
+			// probable abbreviations
+
+			if (p > up || b > ub)
+				tokflag[i] = 0;
+		}
+	}
+
+	// endtokens followed by a single or double quote, which matches
+	// a single or double quote in the previous sentence
+
+	if (option_new >= 1)
+	{
+		int	dquo, squo;
+		dquo = squo = 0;
+
+		for (i = 0; i < text_len; i++)
+		{
+			if (text[i] == '"') dquo = ! dquo;
+			else if (text[i] == '\'') squo = ! squo;
+			else if (endflag[i])
+			{
+				if ((text[i+1] == '"' && dquo) || (text[i+1] == '\'' && squo))
+				{
+					endflag[i] = 0;
+
+					// But don't end at all if the next token is something
+					// other than an upper case letter.
+
+					if (option_new >= 2)
+					{
+						int	j;
+						int	ok = 0;
+
+						for (j = i + 2; j < text_len; j++)
+						{
+							if (isspace(text[j])) continue;
+							// if (isupper(text[j]))
+							if (isupper(text[j]) || text[j] == '(')
+							{
+								ok = 1;
+								break;
+							}
+							if (tokflag[j]) break;
+						}
+
+						if (ok)
+							endflag[i+1] = 1;
+					} else
+					{
+						endflag[i+1] = 1;
+					}
+				}
+				dquo = squo = 0;
+			}
+		}
+	}
+}
+
+
+/* set_endflag_01
+** 
+** After tokflag has been set, find the possible sentence endings.
+** This has improved paren matching.
+*/
+
+#define MAX_MATCH 500		// Maximum length to get a paren match
+
+void MPtok::set_endflag_01()
+{
+	int	match[text_len];
+	int	i, j;
+
+	// The following tests look for end-stops and label them.
+	// They include steps 18 and 19
+
+	for (i = 0; i <= text_len; i++)
+		endflag[i] = 0;
+
+	for (i = 0; i < text_len; i++)
+		match[i] = 0;
+
+	for (i = text_len - 1; i >= 0; i--)
+	{
+		if (text[i] == '(' || text[i] == '[')
+		{
+			for (j = i + 1; text[j] && j - i <= MAX_MATCH; j++)
+			{
+				// Skip parens that are already matched
+
+				if (match[j] > j)
+				{
+					j = match[j];
+					continue;
+				}
+
+				// Look for a matching close paren
+
+				if (match[j] == 0
+				&& ((text[i] == '(' && text[j] == ')')
+				|| (text[i] == '[' && text[j] == ']')))
+				{
+					match[i] = j;
+					match[j] = i;
+					break;
+				}
+			}
+		}
+	}
+
+	int next_match = 0;
+	for (i = 0; i < text_len; i++)
+	{
+		if (match[i] > next_match)
+			next_match = match[i];
+
+		if (strchr(".!?", text[i])
+		&& tokflag[i]
+		&& tokflag[i + 1]
+		&& (option_new <= 4 || option_doteos == 1 || (i > 0 && isspace(text[i-1]) == 0)))
+		{
+			if (i <= next_match)
+				tokflag[i] = 0;
+			else if (option_segment)
+				endflag[i] = 1;
+		}
+	}
+
+	// endtokens followed by a single or double quote, which matches
+	// a single or double quote in the previous sentence
+
+	int	dquo, squo;
+	dquo = squo = 0;
+
+	for (i = 0; i < text_len; i++)
+	{
+		if (option_new <= 7 && text[i] == '"') dquo = ! dquo;
+		else if (option_new >= 8 && text[i] == '"' && tokflag[i] && tokflag[i+1]) dquo = ! dquo;
+		else if (option_new <= 7 && text[i] == '\'') squo = ! squo;
+		else if (option_new >= 8 && text[i] == '\''
+		&& tokflag[i] && (tokflag[i+1] || (text[i+1] == '\'' && tokflag[i+2]))) squo = ! squo;
+		else if (endflag[i])
+		{
+			if ((text[i+1] == '"' && dquo) || (text[i+1] == '\'' && squo))
+			{
+				endflag[i] = 0;
+
+				// But don't end at all if the next token is something
+				// other than an upper case letter.
+
+				if (option_new >= 2)
+				{
+					int	j;
+					int	ok = 0;
+
+					for (j = i + 2; j < text_len; j++)
+					{
+						if (isspace(text[j])) continue;
+						// if (isupper(text[j]))
+						if (isupper(text[j]) || text[j] == '(')
+						{
+							ok = 1;
+							break;
+						}
+						if (tokflag[j]) break;
+					}
+
+					if (ok)
+						endflag[i+1] = 1;
+				} else
+				{
+					endflag[i+1] = 1;
+				}
+			}
+			dquo = squo = 0;
+		}
+	}
+}
+
+
+// Size buffer: return the size of the buffer required to hold all of the tokenized text.
+// It can be simply estimated by a formula that depends only on the length of text and number of tokens.
+
+int MPtok::size_buff()
+{
+	int	size = 1;			// Start with null terminator
+	int	t = option_pretag.size();	// for each tag, the length of the UNTAG string
+
+	if (t <= 0) t = 1;			// Make sure there is at least one
+	t += 2;					// Add one for underscore and one for space
+
+	for (int i = 0; i < text_len; i++)
+	{
+		size++;				// Count all characters
+		if (tokflag[i]) size += t;	// Count token delimiters (may overcount)
+		if (endflag[i]) size++;		// Add one for newline
+	}
+	return size;
+}
+
+
+/* append_token
+** 
+** Save a single token to a buffer.
+*/
+
+void MPtok::append_token(string& buff, int& sp, char *tok, int ef)
+{
+	// Convert tag separator chars and back quotes (?)
+
+	for (int i = 0; tok[i]; i++)
+	{
+		if (tok[i] == option_tagsep) tok[i] = option_replacesep;
+		if (tok[i] == '`') tok[i] = '\'';
+	}
+
+	// Skip whitespace if tokens are being output
+	// Otherwise, skip whitespace at the start of a sentence
+
+	if (option_token || ! sp) while (isspace(*tok)) ++tok;
+
+	// Save the token
+
+	if (strlen(tok) > 0)
+	{
+		// Add delimiter if needed
+
+		if (option_token && sp) buff += ' ';
+
+		// Append token to output
+
+		if (option_new < 9)
+		{
+			while (*tok && (! option_token || ! isspace(*tok)))
+				buff += *(tok++);
+		} else
+		{
+			while (*tok)
+				buff += *(tok++);
+		}
+
+		sp = 1;
+
+		// Add tag holders
+
+		if (option_token && option_pretag.size() > 0)
+		{
+			buff += option_tagsep;
+			buff += option_pretag;
+		}
+
+		// If it was end of sentence, then add newline
+
+		if (ef)
+		{
+			buff += '\n';
+			sp = 0;
+		}
+	}
+}
+
+// Strip whitespace after sentences
+
+static void adjust_space(string& buff)
+{
+	while (buff.size() > 0 && isspace(buff[0])) buff.erase(0, 1);
+
+	// delete two spaces in a row, but keep newlines
+
+	for (int i = 1; i < buff.size(); i++)
+	{
+		if (isspace(buff[i]) && isspace(buff[i-1]))
+			buff.erase((buff[i] == '\n')?(--i):(i--), 1);
+	}
+
+	for (int i = buff.size() -  1; i >= 0 && isspace(buff[i]); i--)
+		buff.erase(i, 1);
+}
+
+/* token_string
+**
+** After the tokflag and endflag have been set, copy the tokens to the buffer.
+*/
+
+string MPtok::token_string()
+{
+	string buff;
+
+	int	i;
+
+	// Move token starts to non-whitespace chars
+
+	int last_tok = 0;
+	for (i = 0; i < text_len; i++)
+	{
+		if (tokflag[i] == 1 && isspace(text[i]))
+		{
+			tokflag[i] = 0;
+			last_tok = 1;
+		} else if (isspace(text[i]) == 0 && last_tok)
+		{
+			tokflag[i] = 1;
+			last_tok = 0;
+		}
+	}
+
+	// Extract the tokens and print them out now
+
+	char	*tok = new char[text_len + 1];
+	int	pos = 0;
+	int	sp = 0;
+	int	ef = 0;
+
+	tok[pos] = '\0';
+
+	for (i = 0; i <= text_len; i++)
+	{
+		// The start of a new token
+
+		if (tokflag[i])
+		{
+			// Print the current token
+
+			append_token(buff, sp, tok, ef);
+
+			// Start a new token
+
+			pos = 0;
+			tok[pos] = '\0';
+
+			ef = 0;
+		}
+
+		// Append to the current token
+
+		tok[pos++] = text[i];
+		tok[pos] = '\0';
+
+		// If any of the characters in the token are endflagged,
+		// Then pass this information along for end-of-sentence
+
+		if (endflag[i]) ef = 1;
+	}
+
+	// Print the last token
+
+	append_token(buff, sp, tok, ef);
+
+	delete[] tok;
+
+	// Adjust the end of sentence boundaries
+
+	adjust_space(buff);
+
+	return buff;
+}
+
+void MPtok::map_escapes()
+{
+	char	*s;
+	int	j, k, ch;
+	char	buff[10];
+
+	k = 0;
+	for (int i = 0; text[i]; i++)
+	{
+		if (text[i] == '&' && text[i + 1] == '#')
+		{
+			for (s = &buff[0], j = 2; j <= 4 && i + j < text_len && isdigit(text[i + j]); j++)
+				*s++ = text[i + j];
+			*s = '\0';
+			ch = atoi(buff);
+			if (strlen(buff) > 0 && text[i + j] == ';' && ch > 0 && ch <= 256)
+			{
+				text[k] = ch;
+				if (! text[k]) text[k] = ' ';
+				k++;
+				i = i + j;
+				continue;
+			}
+		}
+		text[k++] = text[i];
+	}
+	text[k] = '\0';
+	text_len = k;
+}
+
+MPtok::MPtok(string idir, const string& cnam)
+{
+	tok_initialized = 0;
+
+	if (idir.size() == 0)
+	{
+		char	*p = getenv("MEDPOST_HOME");
+		if (p && strlen(p))
+		{
+			idir = p;
+
+			int found = idir.find("=");
+			if (found != string::npos)
+				idir = idir.substr(found + 1);
+		}
+	}
+
+
+	if (idir.size() == 0)
+	{
+		char    buff[1000];
+		FILE *fp = fopen("path_medpost", "r");
+		if (fp)
+		{
+			if (fgets(buff, 1000, fp))
+			{
+				chomp(buff);
+				idir = &buff[0];
+			}
+			fclose(fp);
+		}
+	}
+
+	if (idir.size() == 0)
+		idir = "/home/natxie/CPP64/lib/FIXED_DATA/";
+
+	option_dir = idir;
+
+	option_token = 1;
+	option_segment = 1;
+	option_hyphen = 0;
+	option_comma = 1;
+	option_pretok = 0;
+	option_new = MPTOK_VERSION;
+	option_doteos = 0;
+
+	if (cnam.size() > 0)
+	{
+		option_cnam = "_";
+		option_cnam += cnam;
+	}
+
+	init();
+}
+
+void MPtok::init(void)
+{
+	if (tok_initialized) return;
+
+	string fname;
+
+	fname = option_dir + "/medpost" + option_cnam + ".pairs";
+	init_pair(fname);
+
+	fname = option_dir + "/medpost" + option_cnam + ".abbr";
+	init_abbr(fname);
+
+	tok_initialized = 1;
+}
+
+MPtok::~MPtok()
+{
+}
+
+// Global tokenizer
+
+string MPtok::tokenize(const string& txt, int mt)
+{
+	if (option_pretok) return save_string(txt);
+
+	option_token = mt;
+	text_len = txt.size();
+	if (text_len == 0) return string("");
+
+	text = new char[text_len + 1];
+	strcpy(text, txt.c_str());
+
+	map_escapes();
+
+	if (text_len == 0) return NULL;
+
+	tokflag = new int[text_len + 1];
+	endflag = new int[text_len + 1];
+
+	set_tokflag();
+	if (option_new < 3)
+		set_endflag();
+	else
+		set_endflag_01();
+
+	string buff = token_string();
+	save_string(buff);
+
+	delete[] text; text = NULL;
+	delete[] tokflag; tokflag = NULL;
+	delete[] endflag; endflag = NULL;
+
+	return buff;
+}
+
+string MPtok::tokenize(const string& text)
+{
+	return tokenize(text, 1);
+}
+
+string MPtok::segment(const string& text)
+{
+	sent.clear();
+
+	// tokenize the text
+
+	int save_option_segment = option_segment;
+	option_segment = 1;
+	string buff = tokenize(text, 0);
+	option_segment = save_option_segment;
+
+	if (buff.size() == 0) return text;
+
+	int found = 0;
+	int pos = 0;
+
+	while (pos < buff.size())
+	{
+		found = buff.find('\n', pos);
+		if (found == string::npos)
+		{
+			sent.push_back(buff.substr(pos));
+			pos = buff.size();
+		} else
+		{
+			sent.push_back(buff.substr(pos, found - pos));
+			pos = found + 1;
+		}
+	}
+
+	return buff;
+}
+
+string MPtok::save_string(const string& s)
+{
+	stringstream ss (stringstream::in | stringstream::out);
+	string	w, t;
+	int	found;
+	string	ret;
+
+	word.clear();
+	tag.clear();
+
+	ss << s;
+	while (ss.good())
+	{
+		ss >> w;
+		if (w.size() == 0) break;
+
+		found = w.find('_');
+
+		if (found != string::npos)
+		{
+			t = w.substr(found + 1);
+			w.resize(found);
+			word.push_back(w);
+			tag.push_back(t);
+		} else
+		{
+			word.push_back(w);
+			tag.push_back(option_pretag);
+
+		}
+		if (ret.size() > 0) ret += " ";
+		ret += w;
+	}
+
+	// now look for continuation tags...
+
+	for (int i = 0; i < word.size(); i++)
+	{
+		int j = tag[i].size() - 1;
+		if (j >= 0 && tag[i][j] == '+' && i < tag.size() - 1)
+		{
+			word[i] = word[i] + " " + word[i + 1];
+			tag[i] = tag[i + 1];
+			word.erase(word.begin() + i + 1, word.begin() + i + 2);
+			tag.erase(tag.begin() + i + 1, tag.begin() + i + 2);
+			i--;
+		}
+	}
+
+	return ret;
+}
+
+
+static int count_words(const char *s)
+{
+	int     i;
+
+	i = 1;
+	for (; *s; ++s)
+	{
+		if (*s == ' ') ++i;
+	}
+	return i;
+}
+
+static void print_word(const char *s, int i)
+{
+	for (; i > 0 && *s; ++s) { if (*s == ' ') --i; }
+	while (*s && *s != ' ') { printf("%c", *s); ++s; }
+}
+
+void MPtok::print(int how)
+{
+	int     i, j, w;
+
+	if (how != 0 && how != 2)
+	{
+		printf("print(%d) not defined\n", how);
+		return;
+	}
+
+	for (i = 0; i < word.size(); ++i)
+	{
+		// Get the words from an idiom
+
+		for (w = 0; w < count_words(word[i].c_str()); ++w)
+		{
+			if (how == 2 && i + w > 0) printf(" ");
+
+			print_word(word[i].c_str(), w);
+
+			if (how == 0)
+			{
+				printf(" tagged %s", tag[i].c_str());
+				if (w < count_words(word[i].c_str()) - 1) printf("+");
+				printf("\n");
+			} else if (how == 2)
+			{
+				printf("%s%s", "_", tag[i].c_str());
+				if (w < count_words(word[i].c_str()) - 1) printf("+");
+			}
+		}
+	}
+	if (how == 2)
+		printf("\n");
+}
+
+void MPtok::merge_words(int s, int n)
+{
+	string	tmp = word[s];
+
+	for (int i = s + 1; i < s + n; i++)
+	{
+		tmp += " ";
+		tmp += word[i];
+	}
+
+	// printf("merging words : '%s' n = %d\n", tmp.c_str(), n);
+
+	for (int k = s; k + n < word.size(); k++)
+	{
+		word[k+1] = word[k+n];
+		tag[k+1] = tag[k+n];
+	}
+
+	// Fixup the remaining array
+
+	word.resize(word.size() - n + 1); 
+	tag.resize(word.size());
+
+	word[s] = tmp;
+}
+
+void MPtok::split_words()
+{
+	for (int i = 0; i < word.size(); i++)
+	{
+		int found = word[i].find(' ');
+
+		if (found != string::npos)
+		{
+			string tmp1(word[i], 0, found);
+			string tmp2(word[i], found + 1, string::npos);
+
+			// Move all the words and tags down
+
+			word.resize(word.size() + 1);
+			tag.resize(tag.size() + 1);
+
+			for (int j = word.size() - 1; j > i; j--)
+			{
+				word[j] = word[j - 1];
+				tag[j] = tag[j - 1];
+			}
+
+			word[i] = tmp1;
+			tag[i] = tag[i+1];
+			tag[i] += "+";
+
+			word[i+1] = tmp2;
+		}
+	}
+}
+
+// Callable functions to set internal options
+
+void MPtok::set_segment(int i) { option_segment = i; }
+void MPtok::set_hyphen(int i) { option_hyphen = i; }
+void MPtok::set_comma(int i) { option_comma = i; }
+void MPtok::set_pretag(char *a) { option_pretag = a; }
+void MPtok::set_pretok(int i) { option_pretok = i; }
+void MPtok::set_new(int i) { option_new = i; }
+void MPtok::set_doteos(int i) { option_doteos = i; }

Library/MPtok.h

@@ -1,141 +1,141 @@
-#ifndef _MPTOK_H
-#define _MPTOK_H
-
-#include <stdio.h>
-
-#include <string>
-#include <vector>
-#include <map>
-#include <set>
-
-using namespace std;
-
-#define MPTOK_VERSION 11			// The latest version
-
-// Maximum number of words in a sentence
-
-#define MAX_WORDS 10000
-
-enum { ABB_ABB, ABB_EOS, ABB_NUM };
-#define MAX_ABB 100
-
-/*! \brief A class to perform tokenization.
- *
- * The MPtag class can be used to perform tokenization and segmentation
- * of strings into tokens or sentences. It is inherited and used by MPtag
- * so if the user is only interested in tagging, this class does not
- * need to be referenced.
- */
-
-class MPtok
-{
-public:
-	/// \brief A MPtok object, giving the install directory \p idir where data files can be found
-	MPtok(string idir = "", const string& cnam = "");
-	~MPtok();
-
-	void init();				// Initialize (call only once)
-	void init(const string& idir) { option_dir = idir; init(); } // Initialize using specified install directory
-
-	string option_pretag;			// The tag to use on tokens
-	int option_segment;			// Segment into sentences
-	int option_hyphen;			// Hyphens are separate tokens
-	int option_comma;			// Commas are always tokenized
-	int option_pretok;			// The text is pre-tokenized
-	int option_new;				// Use new algorithms, used in development only
-	int option_doteos;			// If " . " occurs, it's an end EOS (new >= 5)
-
-	void set_segment(int i);		///< \brief Sentences are broken up during tokenization (default 1)
-	void set_token(int i);			///< \brief Break tokens apart with white space (default 1)
-	void set_hyphen(int i);			///< \brief Hyphens are separate tokens (default 0)
-	void set_comma(int i);			///< \brief Commas are separate tokens (default 1)
-	void set_pretag(char *a);		///< \brief Use this tag on all tokens (default empty string)
-	void set_pretok(int i);			///< \brief Assume string is already tokenized using spaces (default 0)
-	void set_new(int i);			///< \brief Use a previous algorithm (defaults to most recent)
-	void set_doteos(int i);			///< \brief Ignore abbreviations, and always assume a period ends a sentence (default 0)
-
-	void merge_words(int s, int e);		// merge words between s and e (idiom)
-	void split_words(void);			// split all merged words
-
-	string tokenize(const string&);		///< \brief Tokenize, save (in \p word), and return space delimited tokens
-	string segment(const string&);		///< \brief Segment, save (in \p sent), and return newline delimited sentences
-
-	string save_string(const string&);	// save a buffer
-	string tokenize_nosave(const string&);	// tokenize without saving
-	string tokenize(const string&,int);	// do tokenization with or without inserting spaces between them
-
-	void print(int);			///< \brief Print tokens/tags with given verbosity
-
-	vector<string> word;			///< \brief Vector of words (tokens) of most recently tagged (or tokenized) text
-	vector<string> tag;			///< \brief Vector of tags of most recently tagged (or tokenized) text
-	vector<string> sent;			///< \brief Vector of sentences of most recently sentence-segmented text
-
-	char	*text;				// Input text arg
-	int	text_len;			// It's length
-	int	*tokflag;			// token flags
-	int	*endflag;			// end-sentence flags
-
-	string option_cnam;			// A suffix, for opening variant support files
-	string option_dir;			// Directory to find things
-
-protected:
-
-	void set_tokflag();
-	void set_endflag();
-	void set_endflag_01();
-	int size_buff();
-
-	void init_pair(const string& file_name);	// read a file of common pairs
-	void init_abbr(const string& file_name);	// read a file of abbreviations
-
-	void tok_0();
-	void tok_1();
-	void tok_2();
-	void tok_3();
-	void tok_5_6_7();
-	void tok_8_9();
-	void tok_10();
-	void tok_11();
-	void tok_12();
-	void tok_13();
-	void tok_14();
-	void tok_15();
-	void tok_15_1();
-	void tok_16();
-	void tok_16_1();
-	void tok_17();
-	void tok_20();
-	void tok_20_1();
-	void tok_20_2();
-	void tok_21();
-	void tok_21a();
-	void tok_22();
-	void tok_23();
-	void tok_24();
-	void tok_25();
-	void tok_26();
-	void tok_27();
-	void tok_28();
-	void tok_29();
-	void tok_29a();
-	void tok_30();
-	void tok_31();
-	void tok_32();
-	void tok_33();
-	int complex_check();
-	void map_escapes();
-	void tok_un();
-
-	void append_token(string&, int&, char*, int);
-	string token_string();
-
-	set<string> common_pair;
-	map<string,int> common_abbr;
-
-private:
-	int option_token;			// Output tokenized text (only use internally)
-	int tok_initialized;			// is it inited?
-};
-
-#endif
-
+#ifndef _MPTOK_H
+#define _MPTOK_H
+
+#include <stdio.h>
+
+#include <string>
+#include <vector>
+#include <map>
+#include <set>
+
+using namespace std;
+
+#define MPTOK_VERSION 11			// The latest version
+
+// Maximum number of words in a sentence
+
+#define MAX_WORDS 10000
+
+enum { ABB_ABB, ABB_EOS, ABB_NUM };
+#define MAX_ABB 100
+
+/*! \brief A class to perform tokenization.
+ *
+ * The MPtag class can be used to perform tokenization and segmentation
+ * of strings into tokens or sentences. It is inherited and used by MPtag
+ * so if the user is only interested in tagging, this class does not
+ * need to be referenced.
+ */
+
+class MPtok
+{
+public:
+	/// \brief A MPtok object, giving the install directory \p idir where data files can be found
+	MPtok(string idir = "", const string& cnam = "");
+	~MPtok();
+
+	void init();				// Initialize (call only once)
+	void init(const string& idir) { option_dir = idir; init(); } // Initialize using specified install directory
+
+	string option_pretag;			// The tag to use on tokens
+	int option_segment;			// Segment into sentences
+	int option_hyphen;			// Hyphens are separate tokens
+	int option_comma;			// Commas are always tokenized
+	int option_pretok;			// The text is pre-tokenized
+	int option_new;				// Use new algorithms, used in development only
+	int option_doteos;			// If " . " occurs, it's an end EOS (new >= 5)
+
+	void set_segment(int i);		///< \brief Sentences are broken up during tokenization (default 1)
+	void set_token(int i);			///< \brief Break tokens apart with white space (default 1)
+	void set_hyphen(int i);			///< \brief Hyphens are separate tokens (default 0)
+	void set_comma(int i);			///< \brief Commas are separate tokens (default 1)
+	void set_pretag(char *a);		///< \brief Use this tag on all tokens (default empty string)
+	void set_pretok(int i);			///< \brief Assume string is already tokenized using spaces (default 0)
+	void set_new(int i);			///< \brief Use a previous algorithm (defaults to most recent)
+	void set_doteos(int i);			///< \brief Ignore abbreviations, and always assume a period ends a sentence (default 0)
+
+	void merge_words(int s, int e);		// merge words between s and e (idiom)
+	void split_words(void);			// split all merged words
+
+	string tokenize(const string&);		///< \brief Tokenize, save (in \p word), and return space delimited tokens
+	string segment(const string&);		///< \brief Segment, save (in \p sent), and return newline delimited sentences
+
+	string save_string(const string&);	// save a buffer
+	string tokenize_nosave(const string&);	// tokenize without saving
+	string tokenize(const string&,int);	// do tokenization with or without inserting spaces between them
+
+	void print(int);			///< \brief Print tokens/tags with given verbosity
+
+	vector<string> word;			///< \brief Vector of words (tokens) of most recently tagged (or tokenized) text
+	vector<string> tag;			///< \brief Vector of tags of most recently tagged (or tokenized) text
+	vector<string> sent;			///< \brief Vector of sentences of most recently sentence-segmented text
+
+	char	*text;				// Input text arg
+	int	text_len;			// It's length
+	int	*tokflag;			// token flags
+	int	*endflag;			// end-sentence flags
+
+	string option_cnam;			// A suffix, for opening variant support files
+	string option_dir;			// Directory to find things
+
+protected:
+
+	void set_tokflag();
+	void set_endflag();
+	void set_endflag_01();
+	int size_buff();
+
+	void init_pair(const string& file_name);	// read a file of common pairs
+	void init_abbr(const string& file_name);	// read a file of abbreviations
+
+	void tok_0();
+	void tok_1();
+	void tok_2();
+	void tok_3();
+	void tok_5_6_7();
+	void tok_8_9();
+	void tok_10();
+	void tok_11();
+	void tok_12();
+	void tok_13();
+	void tok_14();
+	void tok_15();
+	void tok_15_1();
+	void tok_16();
+	void tok_16_1();
+	void tok_17();
+	void tok_20();
+	void tok_20_1();
+	void tok_20_2();
+	void tok_21();
+	void tok_21a();
+	void tok_22();
+	void tok_23();
+	void tok_24();
+	void tok_25();
+	void tok_26();
+	void tok_27();
+	void tok_28();
+	void tok_29();
+	void tok_29a();
+	void tok_30();
+	void tok_31();
+	void tok_32();
+	void tok_33();
+	int complex_check();
+	void map_escapes();
+	void tok_un();
+
+	void append_token(string&, int&, char*, int);
+	string token_string();
+
+	set<string> common_pair;
+	map<string,int> common_abbr;
+
+private:
+	int option_token;			// Output tokenized text (only use internally)
+	int tok_initialized;			// is it inited?
+};
+
+#endif
+

Library/Makefile

@@ -1,13 +1,13 @@
-SRC_DIR=./
-TRASHFILES = *.o *~ *.bak core	
-LIB_INC=-I./
-#.KEEP_STATE:
-libops.a: runn.o Btree.o FBase.o Hash.o MPtok.o \
-	AbbrStra.o AbbrvE.o Ab3P.o
-	ar rus $@ $?
-OS=-g
-%.o: $(SRC_DIR)/%.C
-	g++ -c $(OS) $< -o $@ $(LIB_INC)
-
-clean: rm -f $(TRASHFILES)
-
+SRC_DIR=./
+TRASHFILES = *.o *~ *.bak core	
+LIB_INC=-I./
+#.KEEP_STATE:
+libops.a: runn.o Btree.o FBase.o Hash.o MPtok.o \
+	AbbrStra.o AbbrvE.o Ab3P.o
+	ar rus $@ $?
+OS=-g
+%.o: $(SRC_DIR)/%.C
+	g++ -c $(OS) $< -o $@ $(LIB_INC)
+
+clean: rm -f $(TRASHFILES)
+

Library/WordData/Ab3P_prec.dat

@@ -1,145 +1,145 @@
-Al 1 FirstLetOneChSF 0.967224
-Al 2 FirstLet 0.99818
-Al 2 FirstLetGen 0.994292
-Al 2 WithinWrdFWrd 0.989054
-Al 2 FirstLetGenStp 0.970019
-Al 2 ContLet 0.96935
-Al 2 WithinWrdFLet 0.941981
-Al 2 FirstLetGenSkp 0.949988
-Al 2 WithinWrdFWrdSkp 0.947364
-Al 2 ContLetSkp 0.877216
-Al 2 WithinWrdWrd 0.74768
-Al 2 WithinWrdFLetSkp 0.640805
-Num 2 ContLet 0.975372
-Num 2 ContLetSkp 0.96617
-Num 2 WithinWrdFWrdSkp 0.988426
-Num 2 FirstLetGen2 0.909995
-Num 2 FirstLetGenStp 0.856401
-Num 2 FirstLetGenSkp 0.858132
-Num 2 WithinWrdFWrd 0.726155
-Num 2 WithinWrdFLetSkp 0.607829
-Num 2 WithinWrdFLet 0.493922
-Spec 2 FirstLetGen2 0.854368
-Spec 2 FirstLetGenStp 0.664622
-Spec 2 FirstLetGenSkp 0.657475
-Al 3 FirstLet 0.999808
-Al 3 FirstLetGen 0.999408
-Al 3 FirstLetGenS 0.998732
-Al 3 WithinWrdFWrd 0.997824
-Al 3 FirstLetGenStp 0.997839
-Al 3 FirstLetGenStp2 0.997264
-Al 3 FirstLetGenSkp 0.988583
-Al 3 ContLet 0.987697
-Al 3 WithinWrdFWrdSkp 0.981107
-Al 3 WithinWrdFLet 0.981322
-Al 3 ContLetSkp 0.968185
-Al 3 WithinWrdWrd 0.9437
-Al 3 WithinWrdFLetSkp 0.904799
-Al 3 WithinWrdLet 0.663735
-Al 3 AnyLet 0.303503
-Num 3 FirstLetGen2 0.998497
-Num 3 WithinWrdFWrd 0.99964
-Num 3 FirstLetGenStp 0.998807
-Num 3 FirstLetGenStp2 0.991256
-Num 3 FirstLetGenSkp 0.991202
-Num 3 ContLet 0.996938
-Num 3 WithinWrdFWrdSkp 0.998821
-Num 3 WithinWrdFLet 0.985676
-Num 3 ContLetSkp 0.995076
-Num 3 WithinWrdWrd 0.999245
-Num 3 WithinWrdFLetSkp 0.971123
-Num 3 WithinWrdLet 0.819989
-Num 3 AnyLet 0.797932
-Spec 3 FirstLetGen2 0.978311
-Spec 3 FirstLetGenStp 0.977779
-Spec 3 FirstLetGenStp2 0.929197
-Spec 3 WithinWrdFWrd 0.930654
-Spec 3 ContLet 0.923911
-Spec 3 FirstLetGenSkp 0.904086
-Spec 3 WithinWrdFWrdSkp 0.893989
-Spec 3 ContLetSkp 0.851583
-Spec 3 WithinWrdFLet 0.712331
-Spec 3 WithinWrdFLetSkp 0.64667
-Spec 3 WithinWrdWrd 0.428
-Al 4 FirstLet 0.999964
-Al 4 FirstLetGen 0.99993
-Al 4 FirstLetGenS 0.999811
-Al 4 WithinWrdFWrd 0.999616
-Al 4 FirstLetGenStp 0.999868
-Al 4 FirstLetGenStp2 0.999948
-Al 4 FirstLetGenSkp 0.998534
-Al 4 ContLet 0.992792
-Al 4 WithinWrdFWrdSkp 0.997097
-Al 4 WithinWrdFLet 0.992955
-Al 4 ContLetSkp 0.985568
-Al 4 WithinWrdWrd 0.995823
-Al 4 WithinWrdFLetSkp 0.976873
-Al 4 WithinWrdLet 0.917863
-Al 4 AnyLet 0.696532
-Num 4 FirstLetGen2 0.99992
-Num 4 WithinWrdFWrd 0.999835
-Num 4 FirstLetGenStp 0.999903
-Num 4 FirstLetGenStp2 0.999936
-Num 4 FirstLetGenSkp 0.999577
-Num 4 ContLet 0.999555
-Num 4 WithinWrdFWrdSkp 0.999885
-Num 4 WithinWrdFLet 0.9975
-Num 4 ContLetSkp 0.998578
-Num 4 WithinWrdWrd 0.997703
-Num 4 WithinWrdFLetSkp 0.996501
-Num 4 WithinWrdLet 0.986326
-Num 4 AnyLet 0.953126
-Spec 4 FirstLetGen2 0.99278
-Spec 4 FirstLetGenStp 0.98597
-Spec 4 FirstLetGenStp2 0.982127
-Spec 4 WithinWrdFWrd 0.997649
-Spec 4 ContLet 0.980869
-Spec 4 FirstLetGenSkp 0.944843
-Spec 4 WithinWrdFWrdSkp 0.985685
-Spec 4 ContLetSkp 0.973983
-Spec 4 WithinWrdFLet 0.992773
-Spec 4 WithinWrdFLetSkp 0.863247
-Spec 4 WithinWrdWrd 0.931745
-Spec 4 WithinWrdLet 0.418068
-Spec 4 AnyLet 0.223562
-Al 5 FirstLet 0.999979
-Al 5 FirstLetGen 0.999979
-Al 5 FirstLetGenS 0.999913
-Al 5 WithinWrdFWrd 0.999928
-Al 5 FirstLetGenStp 0.999989
-Al 5 FirstLetGenStp2 0.999887
-Al 5 FirstLetGenSkp 0.999852
-Al 5 ContLet 0.997596
-Al 5 WithinWrdFWrdSkp 0.999602
-Al 5 WithinWrdFLet 0.997473
-Al 5 ContLetSkp 0.989703
-Al 5 WithinWrdWrd 0.999812
-Al 5 WithinWrdFLetSkp 0.986066
-Al 5 WithinWrdLet 0.889324
-Al 5 AnyLet 0.73859
-Num 5 FirstLetGen2 0.999987
-Num 5 WithinWrdFWrd 0.999922
-Num 5 FirstLetGenStp 0.99998
-Num 5 FirstLetGenStp2 1
-Num 5 FirstLetGenSkp 0.999901
-Num 5 ContLet 0.999613
-Num 5 WithinWrdFWrdSkp 0.999937
-Num 5 WithinWrdFLet 0.999386
-Num 5 ContLetSkp 0.999312
-Num 5 WithinWrdWrd 1
-Num 5 WithinWrdFLetSkp 0.998939
-Num 5 WithinWrdLet 0.996068
-Num 5 AnyLet 0.986193
-Spec 5 FirstLetGen2 0.999701
-Spec 5 FirstLetGenStp 0.9999
-Spec 5 FirstLetGenStp2 0.999757
-Spec 5 WithinWrdFWrd 0.999517
-Spec 5 ContLet 0.994648
-Spec 5 FirstLetGenSkp 0.997065
-Spec 5 WithinWrdFWrdSkp 0.998513
-Spec 5 ContLetSkp 0.992445
-Spec 5 WithinWrdFLet 0.996623
-Spec 5 WithinWrdFLetSkp 0.978026
-Spec 5 WithinWrdWrd 0.996879
-Spec 5 WithinWrdLet 0.862993
+Al 1 FirstLetOneChSF 0.967224
+Al 2 FirstLet 0.99818
+Al 2 FirstLetGen 0.994292
+Al 2 WithinWrdFWrd 0.989054
+Al 2 FirstLetGenStp 0.970019
+Al 2 ContLet 0.96935
+Al 2 WithinWrdFLet 0.941981
+Al 2 FirstLetGenSkp 0.949988
+Al 2 WithinWrdFWrdSkp 0.947364
+Al 2 ContLetSkp 0.877216
+Al 2 WithinWrdWrd 0.74768
+Al 2 WithinWrdFLetSkp 0.640805
+Num 2 ContLet 0.975372
+Num 2 ContLetSkp 0.96617
+Num 2 WithinWrdFWrdSkp 0.988426
+Num 2 FirstLetGen2 0.909995
+Num 2 FirstLetGenStp 0.856401
+Num 2 FirstLetGenSkp 0.858132
+Num 2 WithinWrdFWrd 0.726155
+Num 2 WithinWrdFLetSkp 0.607829
+Num 2 WithinWrdFLet 0.493922
+Spec 2 FirstLetGen2 0.854368
+Spec 2 FirstLetGenStp 0.664622
+Spec 2 FirstLetGenSkp 0.657475
+Al 3 FirstLet 0.999808
+Al 3 FirstLetGen 0.999408
+Al 3 FirstLetGenS 0.998732
+Al 3 WithinWrdFWrd 0.997824
+Al 3 FirstLetGenStp 0.997839
+Al 3 FirstLetGenStp2 0.997264
+Al 3 FirstLetGenSkp 0.988583
+Al 3 ContLet 0.987697
+Al 3 WithinWrdFWrdSkp 0.981107
+Al 3 WithinWrdFLet 0.981322
+Al 3 ContLetSkp 0.968185
+Al 3 WithinWrdWrd 0.9437
+Al 3 WithinWrdFLetSkp 0.904799
+Al 3 WithinWrdLet 0.663735
+Al 3 AnyLet 0.303503
+Num 3 FirstLetGen2 0.998497
+Num 3 WithinWrdFWrd 0.99964
+Num 3 FirstLetGenStp 0.998807
+Num 3 FirstLetGenStp2 0.991256
+Num 3 FirstLetGenSkp 0.991202
+Num 3 ContLet 0.996938
+Num 3 WithinWrdFWrdSkp 0.998821
+Num 3 WithinWrdFLet 0.985676
+Num 3 ContLetSkp 0.995076
+Num 3 WithinWrdWrd 0.999245
+Num 3 WithinWrdFLetSkp 0.971123
+Num 3 WithinWrdLet 0.819989
+Num 3 AnyLet 0.797932
+Spec 3 FirstLetGen2 0.978311
+Spec 3 FirstLetGenStp 0.977779
+Spec 3 FirstLetGenStp2 0.929197
+Spec 3 WithinWrdFWrd 0.930654
+Spec 3 ContLet 0.923911
+Spec 3 FirstLetGenSkp 0.904086
+Spec 3 WithinWrdFWrdSkp 0.893989
+Spec 3 ContLetSkp 0.851583
+Spec 3 WithinWrdFLet 0.712331
+Spec 3 WithinWrdFLetSkp 0.64667
+Spec 3 WithinWrdWrd 0.428
+Al 4 FirstLet 0.999964
+Al 4 FirstLetGen 0.99993
+Al 4 FirstLetGenS 0.999811
+Al 4 WithinWrdFWrd 0.999616
+Al 4 FirstLetGenStp 0.999868
+Al 4 FirstLetGenStp2 0.999948
+Al 4 FirstLetGenSkp 0.998534
+Al 4 ContLet 0.992792
+Al 4 WithinWrdFWrdSkp 0.997097
+Al 4 WithinWrdFLet 0.992955
+Al 4 ContLetSkp 0.985568
+Al 4 WithinWrdWrd 0.995823
+Al 4 WithinWrdFLetSkp 0.976873
+Al 4 WithinWrdLet 0.917863
+Al 4 AnyLet 0.696532
+Num 4 FirstLetGen2 0.99992
+Num 4 WithinWrdFWrd 0.999835
+Num 4 FirstLetGenStp 0.999903
+Num 4 FirstLetGenStp2 0.999936
+Num 4 FirstLetGenSkp 0.999577
+Num 4 ContLet 0.999555
+Num 4 WithinWrdFWrdSkp 0.999885
+Num 4 WithinWrdFLet 0.9975
+Num 4 ContLetSkp 0.998578
+Num 4 WithinWrdWrd 0.997703
+Num 4 WithinWrdFLetSkp 0.996501
+Num 4 WithinWrdLet 0.986326
+Num 4 AnyLet 0.953126
+Spec 4 FirstLetGen2 0.99278
+Spec 4 FirstLetGenStp 0.98597
+Spec 4 FirstLetGenStp2 0.982127
+Spec 4 WithinWrdFWrd 0.997649
+Spec 4 ContLet 0.980869
+Spec 4 FirstLetGenSkp 0.944843
+Spec 4 WithinWrdFWrdSkp 0.985685
+Spec 4 ContLetSkp 0.973983
+Spec 4 WithinWrdFLet 0.992773
+Spec 4 WithinWrdFLetSkp 0.863247
+Spec 4 WithinWrdWrd 0.931745
+Spec 4 WithinWrdLet 0.418068
+Spec 4 AnyLet 0.223562
+Al 5 FirstLet 0.999979
+Al 5 FirstLetGen 0.999979
+Al 5 FirstLetGenS 0.999913
+Al 5 WithinWrdFWrd 0.999928
+Al 5 FirstLetGenStp 0.999989
+Al 5 FirstLetGenStp2 0.999887
+Al 5 FirstLetGenSkp 0.999852
+Al 5 ContLet 0.997596
+Al 5 WithinWrdFWrdSkp 0.999602
+Al 5 WithinWrdFLet 0.997473
+Al 5 ContLetSkp 0.989703
+Al 5 WithinWrdWrd 0.999812
+Al 5 WithinWrdFLetSkp 0.986066
+Al 5 WithinWrdLet 0.889324
+Al 5 AnyLet 0.73859
+Num 5 FirstLetGen2 0.999987
+Num 5 WithinWrdFWrd 0.999922
+Num 5 FirstLetGenStp 0.99998
+Num 5 FirstLetGenStp2 1
+Num 5 FirstLetGenSkp 0.999901
+Num 5 ContLet 0.999613
+Num 5 WithinWrdFWrdSkp 0.999937
+Num 5 WithinWrdFLet 0.999386
+Num 5 ContLetSkp 0.999312
+Num 5 WithinWrdWrd 1
+Num 5 WithinWrdFLetSkp 0.998939
+Num 5 WithinWrdLet 0.996068
+Num 5 AnyLet 0.986193
+Spec 5 FirstLetGen2 0.999701
+Spec 5 FirstLetGenStp 0.9999
+Spec 5 FirstLetGenStp2 0.999757
+Spec 5 WithinWrdFWrd 0.999517
+Spec 5 ContLet 0.994648
+Spec 5 FirstLetGenSkp 0.997065
+Spec 5 WithinWrdFWrdSkp 0.998513
+Spec 5 ContLetSkp 0.992445
+Spec 5 WithinWrdFLet 0.996623
+Spec 5 WithinWrdFLetSkp 0.978026
+Spec 5 WithinWrdWrd 0.996879
+Spec 5 WithinWrdLet 0.862993
 Spec 5 AnyLet 0.745608

Library/WordData/stop

@@ -1,313 +1,313 @@
-a
-about
-above
-across
-after
-afterwards
-again
-against
-al
-all
-almost
-alone
-along
-already
-also
-although
-always
-am
-among
-amongst
-an
-analyze
-and
-another
-any
-anyhow
-anyone
-anything
-anywhere
-applicable
-apply
-are
-around
-as
-assume
-at
-be
-became
-because
-become
-becomes
-becoming
-been
-before
-beforehand
-being
-below
-beside
-besides
-between
-beyond
-both
-but
-by
-came
-cannot
-cc
-cm
-come
-compare
-could
-de
-dealing
-department
-depend
-did
-discover
-dl
-do
-does
-during
-each
-ec
-ed
-effected
-eg
-either
-else
-elsewhere
-enough
-et
-etc
-ever
-every
-everyone
-everything
-everywhere
-except
-find
-for
-found
-from
-further
-get
-give
-go
-gov
-had
-has
-have
-he
-hence
-her
-here
-hereafter
-hereby
-herein
-hereupon
-hers
-herself
-him
-himself
-his
-how
-however
-hr
-ie
-if
-ii
-iii
-in
-inc
-incl
-indeed
-into
-investigate
-is
-it
-its
-itself
-j
-jour
-journal
-just
-kg
-last
-latter
-latterly
-lb
-ld
-letter
-like
-ltd
-made
-make
-many
-may
-me
-meanwhile
-mg
-might
-ml
-mm
-mo
-more
-moreover
-most
-mostly
-mr
-much
-must
-my
-myself
-namely
-neither
-never
-nevertheless
-next
-no
-nobody
-noone
-nor
-not
-nothing
-now
-nowhere
-of
-off
-often
-on
-only
-onto
-or
-other
-others
-otherwise
-our
-ours
-ourselves
-out
-over
-own
-oz
-per
-perhaps
-pm
-precede
-presently
-previously
-pt
-rather
-regarding
-relate
-said
-same
-seem
-seemed
-seeming
-seems
-seriously
-several
-she
-should
-show
-showed
-shown
-since
-so
-some
-somehow
-someone
-something
-sometime
-sometimes
-somewhere
-still
-studied
-sub
-such
-take
-tell
-th
-than
-that
-the
-their
-them
-themselves
-then
-thence
-there
-thereafter
-thereby
-therefore
-therein
-thereupon
-these
-they
-this
-thorough
-those
-though
-through
-throughout
-thru
-thus
-to
-together
-too
-toward
-towards
-try
-type
-ug
-under
-unless
-until
-up
-upon
-us
-used
-using
-various
-very
-via
-was
-we
-were
-what
-whatever
-when
-whence
-whenever
-where
-whereafter
-whereas
-whereby
-wherein
-whereupon
-wherever
-whether
-which
-while
-whither
-who
-whoever
-whom
-whose
-why
-will
-with
-within
-without
-wk
-would
-wt
-yet
-you
-your
-yours
-yourself
-yourselves
-yr
+a
+about
+above
+across
+after
+afterwards
+again
+against
+al
+all
+almost
+alone
+along
+already
+also
+although
+always
+am
+among
+amongst
+an
+analyze
+and
+another
+any
+anyhow
+anyone
+anything
+anywhere
+applicable
+apply
+are
+around
+as
+assume
+at
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+being
+below
+beside
+besides
+between
+beyond
+both
+but
+by
+came
+cannot
+cc
+cm
+come
+compare
+could
+de
+dealing
+department
+depend
+did
+discover
+dl
+do
+does
+during
+each
+ec
+ed
+effected
+eg
+either
+else
+elsewhere
+enough
+et
+etc
+ever
+every
+everyone
+everything
+everywhere
+except
+find
+for
+found
+from
+further
+get
+give
+go
+gov
+had
+has
+have
+he
+hence
+her
+here
+hereafter
+hereby
+herein
+hereupon
+hers
+herself
+him
+himself
+his
+how
+however
+hr
+ie
+if
+ii
+iii
+in
+inc
+incl
+indeed
+into
+investigate
+is
+it
+its
+itself
+j
+jour
+journal
+just
+kg
+last
+latter
+latterly
+lb
+ld
+letter
+like
+ltd
+made
+make
+many
+may
+me
+meanwhile
+mg
+might
+ml
+mm
+mo
+more
+moreover
+most
+mostly
+mr
+much
+must
+my
+myself
+namely
+neither
+never
+nevertheless
+next
+no
+nobody
+noone
+nor
+not
+nothing
+now
+nowhere
+of
+off
+often
+on
+only
+onto
+or
+other
+others
+otherwise
+our
+ours
+ourselves
+out
+over
+own
+oz
+per
+perhaps
+pm
+precede
+presently
+previously
+pt
+rather
+regarding
+relate
+said
+same
+seem
+seemed
+seeming
+seems
+seriously
+several
+she
+should
+show
+showed
+shown
+since
+so
+some
+somehow
+someone
+something
+sometime
+sometimes
+somewhere
+still
+studied
+sub
+such
+take
+tell
+th
+than
+that
+the
+their
+them
+themselves
+then
+thence
+there
+thereafter
+thereby
+therefore
+therein
+thereupon
+these
+they
+this
+thorough
+those
+though
+through
+throughout
+thru
+thus
+to
+together
+too
+toward
+towards
+try
+type
+ug
+under
+unless
+until
+up
+upon
+us
+used
+using
+various
+very
+via
+was
+we
+were
+what
+whatever
+when
+whence
+whenever
+where
+whereafter
+whereas
+whereby
+wherein
+whereupon
+wherever
+whether
+which
+while
+whither
+who
+whoever
+whom
+whose
+why
+will
+with
+within
+without
+wk
+would
+wt
+yet
+you
+your
+yours
+yourself
+yourselves
+yr

Library/runn.C

@@ -1,216 +1,216 @@
-#include <iostream>
-#include <fstream>
-#include <cstdlib>
-#include <sstream>
-#include <iomanip>
-#include <cstring>
-#include <cmath>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/mman.h>
-#include "runn.h"
-using namespace std;
-namespace iret {
-
-int mark(int pflag, long ct, int ivl, const char *what){
-if(pflag&&((ct%ivl)==0)){cout << what << " count=" << ct << endl;
-                                        return(1);}
-else return(0);
-}
-
-int get_qflag(){
-   int pflag=1;
-   ifstream fin("quiet.flag",ios::in);
-   if(fin.is_open()){
-      fin >> pflag;
-      fin.close();
-      fin.clear();
-   }
-   return(pflag);
-}
-
-int get_pathw(char *nam,const char *pfl,const char *pex,const char *ch){
-   char cnam[256];
-
-   strcpy(cnam,"path_");
-   strcat(cnam,pfl);
-   strcat(cnam,"_");
-   strcat(cnam,pex);
-   strcat(cnam,".");
-   strcat(cnam,ch);
-   ifstream fin(cnam,ios::in);
-   if(!fin.is_open()){
-      fin.clear();
-      strcpy(cnam,"path_");
-      strcat(cnam,pfl);
-      strcat(cnam,"_");
-      strcat(cnam,pex);
-      fin.open(cnam,ios::in);
-      if(!fin.is_open()){
-	 fin.clear();
-         strcpy(cnam,"path_");
-         strcat(cnam,pfl);
-         fin.open(cnam,ios::in);
-         if(!fin.is_open()){
-           fin.clear();
-           strcpy(cnam,"path");
-           fin.open(cnam,ios::in);
-           if(!fin.is_open()){
-             cout << "Path file for type " << pfl
-                  << " does not exist!" << endl;
-             exit(1);
-           }
-         }
-      }
-   }
-
-   fin.getline(nam,256);
-   fin.close();
-   strcat(nam,pfl);
-   strcat(nam,"_");
-   strcat(nam,pex);
-   strcat(nam,".");
-   strcat(nam,ch);
-   return(1);
-}
-
-char *add_num(const char *ptr,long n,char *buf){
-   char cnam[100];
-   long_str(cnam,n);
-   strcpy(buf,ptr);
-   strcat(buf,cnam);
-   return(buf);
-}
-
-long gseed(int x, char **v, const char *c){
-   long seed;
-
-   seed=clnga(x,v,c,"seed for random number generator");
-   srandom((unsigned int)seed);
-   return seed;
-}
-
-long zrand(long p){
-   return(((long)random())%p);
-}
-
-void shuffle(long n,long *idx){
-   long i,j,k;
-   for(i=0;i<n;i++){
-      k=zrand(n);
-      j=*(idx+i);
-      *(idx+i)=*(idx+k);
-      *(idx+k)=j;
-   }
-}
-
-void dshuffle(long n,long *idx){
-   long i,j,k;
-   for(i=n-1;i>0;i--){
-      k=zrand(i+1);
-      j=idx[i];
-      idx[i]=idx[k];
-      idx[k]=j;
-   }
-}
-
-long clnga(int x, char **v, const char *c, const char *name){
-   int i,flag=1;
-   long num;
-
-   for(i=1;i<x-1;i++)
-      if(strcmp(c,*(v+i))==0){
-         flag=0;
-         istringstream oss(*(v+i+1) );
-         oss >> num;
-         if(oss.fail()){
-            cout << "Enter " << name << ":" << endl;
-            cin >> num;
-         }
-      }
-   if(flag==1){
-      cout << "Enter " << name << ":" << endl; 
-      cin >> num;
-      cin.get();
-   }
-   return(num);
-}
-
-long rnd(double p)
-{
-return((long)floor(p+.5));
-}
-
-double cdbla(int x, char **v, const char *c, const char *name){
-   int i,flag=1;
-   double num;
-
-   for(i=1;i<x-1;i++)
-      if(strcmp(c,*(v+i))==0){
-         flag=0;
-         istringstream oss(*(v+i+1));
-         oss >> num;
-         if(oss.fail()){
-            cout << "Enter " << name << ":" << endl;
-            cin >> num;
-         }
-      }
-   if(flag==1){
-      cout << "Enter " << name << ":" << endl; 
-      cin >> num;
-      cin.get();
-   }
-   return(num);
-}
-
-char *cstra(int x, char **v, const char *c, const char *name){
-   int i;
-   char cnam[max_str];
-
-   for(i=1;i<x-1;i++){
-      if(strcmp(c,*(v+i))==0){
-         return(*(v+i+1));
-      }
-   }
-
-   restart:
-   cout << "Enter " << name << ":" << endl;
-   cin.getline(cnam,max_str);
-   if(i=cin.gcount()){
-      char *pch=new char[i+1];
-      strcpy(pch,cnam);
-      return(pch);
-   }
-   else {
-      cin.clear();
-      goto restart;
-   }
-}
-
-//Function to convert a long to a null terminated string.
-void long_str(char *cnam,long n){
-  ostringstream oss;
-  oss << n;
-  const string & str = oss.str();
-  str.copy(cnam,20);
-  cnam[str.length()]='\0';
-}
-
-//Function to convert a string with null termination
-//to a long.
-void str_long(char *cnam,long &n){
-   istringstream(cnam) >> n;
-}
-
-//Function to convert first two char of string to an
-//integer. Should be an ASCII null terminated string
-int trac(const char *str){
-   if(!(*str))return(0);
-   else {
-      return((int)(*(str+1))+128*((int)(*str)));
-   }
-}
-
-}
+#include <iostream>
+#include <fstream>
+#include <cstdlib>
+#include <sstream>
+#include <iomanip>
+#include <cstring>
+#include <cmath>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include "runn.h"
+using namespace std;
+namespace iret {
+
+int mark(int pflag, long ct, int ivl, const char *what){
+if(pflag&&((ct%ivl)==0)){cout << what << " count=" << ct << endl;
+                                        return(1);}
+else return(0);
+}
+
+int get_qflag(){
+   int pflag=1;
+   ifstream fin("quiet.flag",ios::in);
+   if(fin.is_open()){
+      fin >> pflag;
+      fin.close();
+      fin.clear();
+   }
+   return(pflag);
+}
+
+int get_pathw(char *nam,const char *pfl,const char *pex,const char *ch){
+   char cnam[256];
+
+   strcpy(cnam,"path_");
+   strcat(cnam,pfl);
+   strcat(cnam,"_");
+   strcat(cnam,pex);
+   strcat(cnam,".");
+   strcat(cnam,ch);
+   ifstream fin(cnam,ios::in);
+   if(!fin.is_open()){
+      fin.clear();
+      strcpy(cnam,"path_");
+      strcat(cnam,pfl);
+      strcat(cnam,"_");
+      strcat(cnam,pex);
+      fin.open(cnam,ios::in);
+      if(!fin.is_open()){
+	 fin.clear();
+         strcpy(cnam,"path_");
+         strcat(cnam,pfl);
+         fin.open(cnam,ios::in);
+         if(!fin.is_open()){
+           fin.clear();
+           strcpy(cnam,"path");
+           fin.open(cnam,ios::in);
+           if(!fin.is_open()){
+             cout << "Path file for type " << pfl
+                  << " does not exist!" << endl;
+             exit(1);
+           }
+         }
+      }
+   }
+
+   fin.getline(nam,256);
+   fin.close();
+   strcat(nam,pfl);
+   strcat(nam,"_");
+   strcat(nam,pex);
+   strcat(nam,".");
+   strcat(nam,ch);
+   return(1);
+}
+
+char *add_num(const char *ptr,long n,char *buf){
+   char cnam[100];
+   long_str(cnam,n);
+   strcpy(buf,ptr);
+   strcat(buf,cnam);
+   return(buf);
+}
+
+long gseed(int x, char **v, const char *c){
+   long seed;
+
+   seed=clnga(x,v,c,"seed for random number generator");
+   srandom((unsigned int)seed);
+   return seed;
+}
+
+long zrand(long p){
+   return(((long)random())%p);
+}
+
+void shuffle(long n,long *idx){
+   long i,j,k;
+   for(i=0;i<n;i++){
+      k=zrand(n);
+      j=*(idx+i);
+      *(idx+i)=*(idx+k);
+      *(idx+k)=j;
+   }
+}
+
+void dshuffle(long n,long *idx){
+   long i,j,k;
+   for(i=n-1;i>0;i--){
+      k=zrand(i+1);
+      j=idx[i];
+      idx[i]=idx[k];
+      idx[k]=j;
+   }
+}
+
+long clnga(int x, char **v, const char *c, const char *name){
+   int i,flag=1;
+   long num;
+
+   for(i=1;i<x-1;i++)
+      if(strcmp(c,*(v+i))==0){
+         flag=0;
+         istringstream oss(*(v+i+1) );
+         oss >> num;
+         if(oss.fail()){
+            cout << "Enter " << name << ":" << endl;
+            cin >> num;
+         }
+      }
+   if(flag==1){
+      cout << "Enter " << name << ":" << endl; 
+      cin >> num;
+      cin.get();
+   }
+   return(num);
+}
+
+long rnd(double p)
+{
+return((long)floor(p+.5));
+}
+
+double cdbla(int x, char **v, const char *c, const char *name){
+   int i,flag=1;
+   double num;
+
+   for(i=1;i<x-1;i++)
+      if(strcmp(c,*(v+i))==0){
+         flag=0;
+         istringstream oss(*(v+i+1));
+         oss >> num;
+         if(oss.fail()){
+            cout << "Enter " << name << ":" << endl;
+            cin >> num;
+         }
+      }
+   if(flag==1){
+      cout << "Enter " << name << ":" << endl; 
+      cin >> num;
+      cin.get();
+   }
+   return(num);
+}
+
+char *cstra(int x, char **v, const char *c, const char *name){
+   int i;
+   char cnam[max_str];
+
+   for(i=1;i<x-1;i++){
+      if(strcmp(c,*(v+i))==0){
+         return(*(v+i+1));
+      }
+   }
+
+   restart:
+   cout << "Enter " << name << ":" << endl;
+   cin.getline(cnam,max_str);
+   if(i=cin.gcount()){
+      char *pch=new char[i+1];
+      strcpy(pch,cnam);
+      return(pch);
+   }
+   else {
+      cin.clear();
+      goto restart;
+   }
+}
+
+//Function to convert a long to a null terminated string.
+void long_str(char *cnam,long n){
+  ostringstream oss;
+  oss << n;
+  const string & str = oss.str();
+  str.copy(cnam,20);
+  cnam[str.length()]='\0';
+}
+
+//Function to convert a string with null termination
+//to a long.
+void str_long(char *cnam,long &n){
+   istringstream(cnam) >> n;
+}
+
+//Function to convert first two char of string to an
+//integer. Should be an ASCII null terminated string
+int trac(const char *str){
+   if(!(*str))return(0);
+   else {
+      return((int)(*(str+1))+128*((int)(*str)));
+   }
+}
+
+}

Library/runn.h

@@ -1,392 +1,392 @@
-#ifndef RUNN_H
-#define RUNN_H
-
-#include <fstream>
-#include <iostream>
-#include <cctype>
-#include <cstring>
-#include <cstdlib>
-using namespace std;
-namespace iret {
-
-const int word_cnt = 5000; //Maximum number of words in a document.
-const int word_len = 1500; //Maximum word length.
-const long max_str=1500;  //Maximum string length.
-
-int get_pathw(char *cn,const char *dfl,const char *dex,const char *a);
-   //Reads the path from a file "path_(*dfl)" and constructs the
-   //file name from as "(*dfl)_(*dex).(*a)". Cats path and file
-   //name and returns the full info in cn.
-char *add_num(const char *ptr,long n,char *buf); //converts long to ascii 
-   //and cats to end of string and returns pointer to new string 
-   //that results. Does not change input string. The new string is
-   //held in buffer space and this is overwritten at each call.
-
-int get_qflag();
-   //This function gets the value of the print flag pflag that is 
-   //used to control output.
-int mark(int,long,int,const char*);
-   //This function is used to print out information that indicates 
-   //how a function is progressing. It is dependent on the value of 
-   //pflag.
-long gseed(int,char**,const char*);
-   //This function is called to allow the input of a seed value for 
-   //the random number generator. It must be called in main or the 
-   //arguments of main must be passed down to it if it is to allow 
-   //command line entry. Otherwise the first argument may be set to 
-   //zero and it may be used to enter the seed at run time from the 
-   //console.
-long clnga(int,char**,const char*,const char*);
-   //Allows a long to be entered from the console at run time if the 
-   //first argument is set to zero. If the first two arguments are 
-   //the arguments of main, then it allows command line entry with
-   //the flag that is the third argument and with a statement about 
-   //the input that is the fourth argument.
-double cdbla(int,char**,const char*,const char*);
-char *cstra(int,char**,const char*,const char*);
-long zrand(long);
-   //Produces a random long integer that is in the range [0,argument).
-   //Machinery of the random number generator.
-void shuffle(long n,long *idx); //Randomly shuffles an array of longs.
-void dshuffle(long n,long *idx); //Randomly shuffles an array of longs.
-   //Improved version suggested by Don Comeau
-long rnd(double);
-   //Rounds off a double and returns the integer that results.
-
-   //Reads in a string including white space and ends the string 
-   //just before the character a.
-inline int get_string(char *cnam,ifstream &ifile,char a){
-   char *pch = cnam;
-   long j=1;
-
-   start:
-   if((*(pch++)=ifile.get())!=EOF){
-      if(*(pch-1)==a){pch--;goto start;}
-      while(((*(pch++)=ifile.get())!=a)&&(j<max_str))j++;
-      if(j<max_str){
-         *(--pch)='\0';
-         return(j);
-      }
-      else return(0);
-   }
-   return(0);
-}
-
-inline int get_strinf(char *cnam,fstream &ifile,char a){
-   char *pch = cnam;
-   long j=1;
-   if((*(pch++)=ifile.get())!=EOF){
-      while(((*(pch++)=ifile.get())!=a)&&(j<max_str))j++;
-      if(j<max_str){
-         *(--pch)='\0';
-         return(j);
-      }
-      else return(0);
-   }
-   return(0);
-}
-
-//Function to lower case a string.
-inline void lower_case(char *cnam){
-   int i=0;
-   char ch;
-
-   while((ch=cnam[i])!='\0'){
-       cnam[i++]=tolower(ch);
-   }
-}
-
-//Note that ordering functions beginning with sS or hS
-//produce an order that is increasing with increasing
-//index, while sR or hR produces the reverse order.
-
-template <class X>
-void sSort(const long ix, X *idx){
-  long k, j, ir, i;
-  X rra;
-
-  if(ix<=1)return;
-
-  k=(ix>>1);
-  ir=ix-1;
-  for(;;) {
-    if(k>0) {
-      rra=idx[--k];
-    }
-    else {
-      rra=idx[ir];
-      idx[ir] = idx[0];
-      if(--ir ==0) {
-        idx[0]=rra;
-        return;
-      }
-    }
-    i=k;
-    j=((k+1)<<1)-1;
-    while(j<=ir) {
-      if(j<ir && (idx[j]<idx[j+1])) ++j;
-      if(rra<idx[j]) {
-        idx[i]=idx[j];
-        j +=(i=j)+1;
-      }
-      else j=ir+1;
-    }
-    idx[i]=rra;
-  }
-}
-
-template <class X> 
-void sRort(const long ix, X *idx){
-  long k, j, ir, i;
-  X rra;
-   
-  if(ix<=1)return;
-   
-  k=(ix>>1); 
-  ir=ix-1;
-  for(;;) {
-    if(k>0) {
-      rra=idx[--k];
-    }
-    else {
-      rra=idx[ir]; 
-      idx[ir] = idx[0];
-      if(--ir ==0) {
-        idx[0]=rra;
-        return;
-      }
-    }
-    i=k;
-    j=((k+1)<<1)-1;
-    while(j<=ir) {
-      if(j<ir && (idx[j]>idx[j+1])) ++j;
-      if(rra>idx[j]) {
-        idx[i]=idx[j];
-        j +=(i=j)+1;
-      }
-      else j=ir+1;
-    }
-    idx[i]=rra;
-  }
-}
-
-template <class X, class Y>
-void hSort(const long n, X *ra, Y *rb) {
-  long k, j, ir, i;
-  X rra;
-  Y rrb;
-
-  if(n<=1)return;
-
-  k=(n>>1);
-  ir=n-1;
-  for(;;) {
-    if(k>0) {
-      rra=ra[--k];
-      rrb=rb[k];
-    }
-    else {
-      rra=ra[ir];
-      rrb=rb[ir];
-      ra[ir] = ra[0];
-      rb[ir] = rb[0];
-      if(--ir ==0) {
-        ra[0]=rra;
-        rb[0]=rrb;
-        return;
-      }
-    }
-    i=k;
-    j=((k+1)<<1)-1;
-    while(j<=ir) {
-      if(j<ir && ra[j] < ra[j+1]) ++j;
-      if(rra<ra[j]) {
-        ra[i]=ra[j];
-        rb[i]=rb[j];
-        j +=(i=j)+1;
-      }
-      else j=ir+1;
-    }
-    ra[i]=rra;
-    rb[i]=rrb;
-  }
-}
-
-template <class X, class Y, class Z>
-void hSort(const long n, X *ra, Y *rb, Z *rc) {
-  long k, j, ir, i;
-  X rra;
-  Y rrb;
-  Z rrc;
-
-  if(n<=1)return;
-
-  k=(n>>1);
-  ir=n-1;
-  for(;;) {
-    if(k>0) {
-      rra=ra[--k];
-      rrb=rb[k];
-      rrc=rc[k];
-    }
-    else {
-      rra=ra[ir];
-      rrb=rb[ir];
-      rrc=rc[ir];
-      ra[ir] = ra[0];
-      rb[ir] = rb[0];
-      rc[ir] = rc[0];
-      if(--ir ==0) {
-        ra[0]=rra;
-        rb[0]=rrb;
-        rc[0]=rrc;
-        return;
-      }
-    }
-    i=k;
-    j=((k+1)<<1)-1;
-    while(j<=ir) {
-      if(j<ir && ra[j] < ra[j+1]) ++j;
-      if(rra<ra[j]) {
-        ra[i]=ra[j];
-        rb[i]=rb[j];
-        rc[i]=rc[j];
-        j +=(i=j)+1;
-      }
-      else j=ir+1;
-    }
-    ra[i]=rra;
-    rb[i]=rrb;
-    rc[i]=rrc;
-  }
-}
-
-template <class X, class Y>
-void hRort(const long n, X *ra, Y *rb) {
-  long k, j, ir, i;
-  X rra;
-  Y rrb;
-
-  if(n<=1)return;
-
-  k=(n>>1);
-  ir=n-1;
-  for(;;) {
-    if(k>0) {
-      rra=ra[--k];
-      rrb=rb[k];
-    }
-    else {
-      rra=ra[ir];
-      rrb=rb[ir];
-      ra[ir] = ra[0];
-      rb[ir] = rb[0];
-      if(--ir ==0) {
-        ra[0]=rra;
-        rb[0]=rrb;
-        return;
-      }
-    }
-    i=k;
-    j=((k+1)<<1)-1;
-    while(j<=ir) {
-      if(j<ir && ra[j] > ra[j+1]) ++j;
-      if(rra>ra[j]) {
-        ra[i]=ra[j];
-        rb[i]=rb[j];
-        j +=(i=j)+1;
-      }
-      else j=ir+1;
-    }
-    ra[i]=rra;
-    rb[i]=rrb;
-  }
-}
-
-template <class X, class Y, class Z>
-void hRort(const long n, X *ra, Y *rb, Z *rc) {
-  long k, j, ir, i;
-  X rra;
-  Y rrb;
-  Z rrc;
-
-  if(n<=1)return;
-
-  k=(n>>1);
-  ir=n-1;
-  for(;;) {
-    if(k>0) {
-      rra=ra[--k];
-      rrb=rb[k];
-      rrc=rc[k];
-    }
-    else {
-      rra=ra[ir];
-      rrb=rb[ir];
-      rrc=rc[ir];
-      ra[ir] = ra[0];
-      rb[ir] = rb[0];
-      rc[ir] = rc[0];
-      if(--ir ==0) {
-        ra[0]=rra;
-        rb[0]=rrb;
-        rc[0]=rrc;
-        return;
-      }
-    }
-    i=k;
-    j=((k+1)<<1)-1;
-    while(j<=ir) {
-      if(j<ir && ra[j] > ra[j+1]) ++j;
-      if(rra>ra[j]) {
-        ra[i]=ra[j];
-        rb[i]=rb[j];
-        rc[i]=rc[j];
-        j +=(i=j)+1;
-      }
-      else j=ir+1;
-    }
-    ra[i]=rra;
-    rb[i]=rrb;
-    rc[i]=rrc;
-  }
-}
-
-
-//Function to convert a long to a null terminated string.
-void long_str(char *cnam,long n);
-
-//Function to convert a string with null termination
-//to a long.
-void str_long(char *cnam,long &n);
-
-//Function to convert first two char of string to an
-//integer. Should be an ASCII null terminated string
-int trac(const char *str);
-
-template<typename Y,typename Z>
-void xshuffle(Y n,Z *idx){ //Randomly shuffles an array of longs.
-   Y i,k;
-   Z u;
-   for(i=n-1;i>0;i--){
-      k=(Y)zrand((long)i+1);
-      u=idx[i];
-      idx[i]=idx[k];
-      idx[k]=u;
-   }
-}
-
-template<class Z>
-void dxhuffle(long n,Z *idx){ //Randomly shuffles an array type Z*.
-   long i,k;
-   Z xx;
-   for(i=n-1;i>0;i--){
-      k=zrand(i+1);
-      xx=idx[i];
-      idx[i]=idx[k];
-      idx[k]=xx;
-   }
-}
-
-}
-#endif
+#ifndef RUNN_H
+#define RUNN_H
+
+#include <fstream>
+#include <iostream>
+#include <cctype>
+#include <cstring>
+#include <cstdlib>
+using namespace std;
+namespace iret {
+
+const int word_cnt = 5000; //Maximum number of words in a document.
+const int word_len = 1500; //Maximum word length.
+const long max_str=1500;  //Maximum string length.
+
+int get_pathw(char *cn,const char *dfl,const char *dex,const char *a);
+   //Reads the path from a file "path_(*dfl)" and constructs the
+   //file name from as "(*dfl)_(*dex).(*a)". Cats path and file
+   //name and returns the full info in cn.
+char *add_num(const char *ptr,long n,char *buf); //converts long to ascii 
+   //and cats to end of string and returns pointer to new string 
+   //that results. Does not change input string. The new string is
+   //held in buffer space and this is overwritten at each call.
+
+int get_qflag();
+   //This function gets the value of the print flag pflag that is 
+   //used to control output.
+int mark(int,long,int,const char*);
+   //This function is used to print out information that indicates 
+   //how a function is progressing. It is dependent on the value of 
+   //pflag.
+long gseed(int,char**,const char*);
+   //This function is called to allow the input of a seed value for 
+   //the random number generator. It must be called in main or the 
+   //arguments of main must be passed down to it if it is to allow 
+   //command line entry. Otherwise the first argument may be set to 
+   //zero and it may be used to enter the seed at run time from the 
+   //console.
+long clnga(int,char**,const char*,const char*);
+   //Allows a long to be entered from the console at run time if the 
+   //first argument is set to zero. If the first two arguments are 
+   //the arguments of main, then it allows command line entry with
+   //the flag that is the third argument and with a statement about 
+   //the input that is the fourth argument.
+double cdbla(int,char**,const char*,const char*);
+char *cstra(int,char**,const char*,const char*);
+long zrand(long);
+   //Produces a random long integer that is in the range [0,argument).
+   //Machinery of the random number generator.
+void shuffle(long n,long *idx); //Randomly shuffles an array of longs.
+void dshuffle(long n,long *idx); //Randomly shuffles an array of longs.
+   //Improved version suggested by Don Comeau
+long rnd(double);
+   //Rounds off a double and returns the integer that results.
+
+   //Reads in a string including white space and ends the string 
+   //just before the character a.
+inline int get_string(char *cnam,ifstream &ifile,char a){
+   char *pch = cnam;
+   long j=1;
+
+   start:
+   if((*(pch++)=ifile.get())!=EOF){
+      if(*(pch-1)==a){pch--;goto start;}
+      while(((*(pch++)=ifile.get())!=a)&&(j<max_str))j++;
+      if(j<max_str){
+         *(--pch)='\0';
+         return(j);
+      }
+      else return(0);
+   }
+   return(0);
+}
+
+inline int get_strinf(char *cnam,fstream &ifile,char a){
+   char *pch = cnam;
+   long j=1;
+   if((*(pch++)=ifile.get())!=EOF){
+      while(((*(pch++)=ifile.get())!=a)&&(j<max_str))j++;
+      if(j<max_str){
+         *(--pch)='\0';
+         return(j);
+      }
+      else return(0);
+   }
+   return(0);
+}
+
+//Function to lower case a string.
+inline void lower_case(char *cnam){
+   int i=0;
+   char ch;
+
+   while((ch=cnam[i])!='\0'){
+       cnam[i++]=tolower(ch);
+   }
+}
+
+//Note that ordering functions beginning with sS or hS
+//produce an order that is increasing with increasing
+//index, while sR or hR produces the reverse order.
+
+template <class X>
+void sSort(const long ix, X *idx){
+  long k, j, ir, i;
+  X rra;
+
+  if(ix<=1)return;
+
+  k=(ix>>1);
+  ir=ix-1;
+  for(;;) {
+    if(k>0) {
+      rra=idx[--k];
+    }
+    else {
+      rra=idx[ir];
+      idx[ir] = idx[0];
+      if(--ir ==0) {
+        idx[0]=rra;
+        return;
+      }
+    }
+    i=k;
+    j=((k+1)<<1)-1;
+    while(j<=ir) {
+      if(j<ir && (idx[j]<idx[j+1])) ++j;
+      if(rra<idx[j]) {
+        idx[i]=idx[j];
+        j +=(i=j)+1;
+      }
+      else j=ir+1;
+    }
+    idx[i]=rra;
+  }
+}
+
+template <class X> 
+void sRort(const long ix, X *idx){
+  long k, j, ir, i;
+  X rra;
+   
+  if(ix<=1)return;
+   
+  k=(ix>>1); 
+  ir=ix-1;
+  for(;;) {
+    if(k>0) {
+      rra=idx[--k];
+    }
+    else {
+      rra=idx[ir]; 
+      idx[ir] = idx[0];
+      if(--ir ==0) {
+        idx[0]=rra;
+        return;
+      }
+    }
+    i=k;
+    j=((k+1)<<1)-1;
+    while(j<=ir) {
+      if(j<ir && (idx[j]>idx[j+1])) ++j;
+      if(rra>idx[j]) {
+        idx[i]=idx[j];
+        j +=(i=j)+1;
+      }
+      else j=ir+1;
+    }
+    idx[i]=rra;
+  }
+}
+
+template <class X, class Y>
+void hSort(const long n, X *ra, Y *rb) {
+  long k, j, ir, i;
+  X rra;
+  Y rrb;
+
+  if(n<=1)return;
+
+  k=(n>>1);
+  ir=n-1;
+  for(;;) {
+    if(k>0) {
+      rra=ra[--k];
+      rrb=rb[k];
+    }
+    else {
+      rra=ra[ir];
+      rrb=rb[ir];
+      ra[ir] = ra[0];
+      rb[ir] = rb[0];
+      if(--ir ==0) {
+        ra[0]=rra;
+        rb[0]=rrb;
+        return;
+      }
+    }
+    i=k;
+    j=((k+1)<<1)-1;
+    while(j<=ir) {
+      if(j<ir && ra[j] < ra[j+1]) ++j;
+      if(rra<ra[j]) {
+        ra[i]=ra[j];
+        rb[i]=rb[j];
+        j +=(i=j)+1;
+      }
+      else j=ir+1;
+    }
+    ra[i]=rra;
+    rb[i]=rrb;
+  }
+}
+
+template <class X, class Y, class Z>
+void hSort(const long n, X *ra, Y *rb, Z *rc) {
+  long k, j, ir, i;
+  X rra;
+  Y rrb;
+  Z rrc;
+
+  if(n<=1)return;
+
+  k=(n>>1);
+  ir=n-1;
+  for(;;) {
+    if(k>0) {
+      rra=ra[--k];
+      rrb=rb[k];
+      rrc=rc[k];
+    }
+    else {
+      rra=ra[ir];
+      rrb=rb[ir];
+      rrc=rc[ir];
+      ra[ir] = ra[0];
+      rb[ir] = rb[0];
+      rc[ir] = rc[0];
+      if(--ir ==0) {
+        ra[0]=rra;
+        rb[0]=rrb;
+        rc[0]=rrc;
+        return;
+      }
+    }
+    i=k;
+    j=((k+1)<<1)-1;
+    while(j<=ir) {
+      if(j<ir && ra[j] < ra[j+1]) ++j;
+      if(rra<ra[j]) {
+        ra[i]=ra[j];
+        rb[i]=rb[j];
+        rc[i]=rc[j];
+        j +=(i=j)+1;
+      }
+      else j=ir+1;
+    }
+    ra[i]=rra;
+    rb[i]=rrb;
+    rc[i]=rrc;
+  }
+}
+
+template <class X, class Y>
+void hRort(const long n, X *ra, Y *rb) {
+  long k, j, ir, i;
+  X rra;
+  Y rrb;
+
+  if(n<=1)return;
+
+  k=(n>>1);
+  ir=n-1;
+  for(;;) {
+    if(k>0) {
+      rra=ra[--k];
+      rrb=rb[k];
+    }
+    else {
+      rra=ra[ir];
+      rrb=rb[ir];
+      ra[ir] = ra[0];
+      rb[ir] = rb[0];
+      if(--ir ==0) {
+        ra[0]=rra;
+        rb[0]=rrb;
+        return;
+      }
+    }
+    i=k;
+    j=((k+1)<<1)-1;
+    while(j<=ir) {
+      if(j<ir && ra[j] > ra[j+1]) ++j;
+      if(rra>ra[j]) {
+        ra[i]=ra[j];
+        rb[i]=rb[j];
+        j +=(i=j)+1;
+      }
+      else j=ir+1;
+    }
+    ra[i]=rra;
+    rb[i]=rrb;
+  }
+}
+
+template <class X, class Y, class Z>
+void hRort(const long n, X *ra, Y *rb, Z *rc) {
+  long k, j, ir, i;
+  X rra;
+  Y rrb;
+  Z rrc;
+
+  if(n<=1)return;
+
+  k=(n>>1);
+  ir=n-1;
+  for(;;) {
+    if(k>0) {
+      rra=ra[--k];
+      rrb=rb[k];
+      rrc=rc[k];
+    }
+    else {
+      rra=ra[ir];
+      rrb=rb[ir];
+      rrc=rc[ir];
+      ra[ir] = ra[0];
+      rb[ir] = rb[0];
+      rc[ir] = rc[0];
+      if(--ir ==0) {
+        ra[0]=rra;
+        rb[0]=rrb;
+        rc[0]=rrc;
+        return;
+      }
+    }
+    i=k;
+    j=((k+1)<<1)-1;
+    while(j<=ir) {
+      if(j<ir && ra[j] > ra[j+1]) ++j;
+      if(rra>ra[j]) {
+        ra[i]=ra[j];
+        rb[i]=rb[j];
+        rc[i]=rc[j];
+        j +=(i=j)+1;
+      }
+      else j=ir+1;
+    }
+    ra[i]=rra;
+    rb[i]=rrb;
+    rc[i]=rrc;
+  }
+}
+
+
+//Function to convert a long to a null terminated string.
+void long_str(char *cnam,long n);
+
+//Function to convert a string with null termination
+//to a long.
+void str_long(char *cnam,long &n);
+
+//Function to convert first two char of string to an
+//integer. Should be an ASCII null terminated string
+int trac(const char *str);
+
+template<typename Y,typename Z>
+void xshuffle(Y n,Z *idx){ //Randomly shuffles an array of longs.
+   Y i,k;
+   Z u;
+   for(i=n-1;i>0;i--){
+      k=(Y)zrand((long)i+1);
+      u=idx[i];
+      idx[i]=idx[k];
+      idx[k]=u;
+   }
+}
+
+template<class Z>
+void dxhuffle(long n,Z *idx){ //Randomly shuffles an array type Z*.
+   long i,k;
+   Z xx;
+   for(i=n-1;i>0;i--){
+      k=zrand(i+1);
+      xx=idx[i];
+      idx[i]=idx[k];
+      idx[k]=xx;
+   }
+}
+
+}
+#endif

requirements-py310.txt

@@ -1,7 +1,7 @@
-tensorflow==2.8
-transformers==4.37.2
-stanza==1.4.0
-spacy==3.2.4
-bioc==2.0.post4
-spacy==3.2.4
+tensorflow==2.8
+transformers==4.37.2
+stanza==1.4.0
+spacy==3.2.4
+bioc==2.0.post4
+spacy==3.2.4
 protobuf==3.20.1

requirements.txt

@@ -1,76 +1,76 @@
-absl-py
-astunparse
-attrs
-bioc
-blis
-cachetools
-catalogue
-certifi
-charset-normalizer
-click
-cymem
-emoji
-filelock
-gast
-google-auth
-google-auth-oauthlib
-google-pasta
-grpcio
-h5py
-huggingface-hub
-idna
-importlib-metadata
-intervaltree
-Jinja2
-joblib
-jsonlines
-Keras-Preprocessing
-langcodes
-lxml
-Markdown
-MarkupSafe
-murmurhash
-numpy
-oauthlib
-opt-einsum
-packaging
-pathy
-preshed
-protobuf
-pyasn1
-pyasn1-modules
-pydantic
-pyparsing
-PyYAML
-regex
-requests
-requests-oauthlib
-rsa
-sacremoses
-scipy
-six
-smart-open
-sortedcontainers
-spacy
-spacy-legacy
-spacy-loggers
-srsly
-stanza
-tensorboard
-tensorboard-data-server
-tensorboard-plugin-wit
-tensorflow
-tensorflow-estimator
-termcolor
-thinc
-tokenizers
-torch
-tqdm
-transformers
-typer
-typing_extensions
-urllib3
-wasabi
-Werkzeug
-wrapt
-zipp
+absl-py
+astunparse
+attrs
+bioc
+blis
+cachetools
+catalogue
+certifi
+charset-normalizer
+click
+cymem
+emoji
+filelock
+gast
+google-auth
+google-auth-oauthlib
+google-pasta
+grpcio
+h5py
+huggingface-hub
+idna
+importlib-metadata
+intervaltree
+Jinja2
+joblib
+jsonlines
+Keras-Preprocessing
+langcodes
+lxml
+Markdown
+MarkupSafe
+murmurhash
+numpy
+oauthlib
+opt-einsum
+packaging
+pathy
+preshed
+protobuf
+pyasn1
+pyasn1-modules
+pydantic
+pyparsing
+PyYAML
+regex
+requests
+requests-oauthlib
+rsa
+sacremoses
+scipy
+six
+smart-open
+sortedcontainers
+spacy
+spacy-legacy
+spacy-loggers
+srsly
+stanza
+tensorboard
+tensorboard-data-server
+tensorboard-plugin-wit
+tensorflow
+tensorflow-estimator
+termcolor
+thinc
+tokenizers
+torch
+tqdm
+transformers
+typer
+typing_extensions
+urllib3
+wasabi
+Werkzeug
+wrapt
+zipp

run_batches.py

@@ -1,12 +1,19 @@
 import argparse
 import logging
+import os
 import shutil
 import subprocess
-import time
-from datetime import timedelta
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
+from tqdm.contrib.concurrent import process_map
+
+
+def batch(iterable, n=1):
+    l = len(iterable)
+    for ndx in range(0, l, n):
+        yield iterable[ndx : min(ndx + n, l)]
+
 
 def main():
     logging.basicConfig(level=logging.INFO)
@@ -14,7 +21,8 @@ def main():
     parser.add_argument("--mode", type=str, default="gnorm2", help="mode to run in (gnorm2, gnormplus)")
     parser.add_argument("input_dir", type=str, help="directory containing files to process")
     parser.add_argument("output_dir", type=str, help="directory to write processed files to")
-    parser.add_argument("--batch_size", type=int, default=64)
+    parser.add_argument("--batch_size", type=int, default=8)
+    parser.add_argument("--max_workers", type=int, default=os.cpu_count() - 4)
     args = parser.parse_args()
 
     input_dir = Path(args.input_dir)
@@ -32,51 +40,57 @@ def main():
 
     logging.info(f"Processing {len(input_files)} files")
 
-
-    while input_files:
-        start = time.time()
-        logging.info(f"{len(input_files)} remaining files")
-        input_files_batch = list(input_files)[: args.batch_size]
-
-        with TemporaryDirectory() as temp_dir_SR, TemporaryDirectory() as temp_dir_GNR, TemporaryDirectory() as temp_dir_SA, TemporaryDirectory() as input_temp_dir, TemporaryDirectory() as output_temp_dir:
-            input_temp_dir = Path(input_temp_dir)
-            output_temp_dir = Path(output_temp_dir)
-            for file in input_files_batch:
-                shutil.copy(input_dir / file, input_temp_dir)
-
-            if args.mode == "gnorm2":
-                command_SR = f"java -Xmx60G -Xms30G -jar GNormPlus.jar {str(input_temp_dir)} {str(temp_dir_SR)} setup.SR.txt"
-                command_GNR_SA = f"python GeneNER_SpeAss_run.py -i {str(temp_dir_SR)} -r {str(temp_dir_GNR)} -a {str(temp_dir_SA)} -n gnorm_trained_models/geneNER/GeneNER-Bioformer.h5 -s gnorm_trained_models/SpeAss/SpeAss-Bioformer.h5"
-                command_GN = f"java -Xmx60G -Xms30G -jar GNormPlus.jar {str(temp_dir_SA)} {str(output_temp_dir)} setup.GN.txt"
-                commands = [command_SR, command_GNR_SA, command_GN]
-            elif args.mode == "gnormplus":
-                commands = [f"java -Xmx60G -Xms30G -jar GNormPlus.jar {str(input_temp_dir)} {str(output_temp_dir)} setup.txt"]
-            else:
-                raise ValueError(f"Invalid mode: {args.mode}")
-
-            for command in commands:
-                try:
-                    logging.info(command)
-                    subprocess.run([command], check=True, shell=True)
-                except subprocess.CalledProcessError as e:
-                    logging.exception(f"Error running command: {command}")
-                    raise e
-
-            output_paths = output_temp_dir.rglob("*")
-            output_files = set(file.name for file in output_paths)
-            for output_path, output_file in zip(output_paths, output_files):
-                shutil.copy(output_path, output_dir)
-                input_files.remove(output_file)
-            end = time.time()
-            logging.info(f"Processed {len(output_files)} files in {timedelta(seconds=end - start)}")
-
-            if not len(output_files):
-                raise Exception("No files were output")
-
-            if output_files:
-                logging.info(
-                    f"Estimated time remaining: {timedelta(seconds=(end - start) * len(input_files) / output_files)}"
-                )
+    input_files = sorted(input_files, key=lambda file: (input_dir / file).stat().st_size)
+
+    input_files_batches = list(batch(list(input_files), args.batch_size))
+    process_map(
+        run_batch,
+        input_files_batches,
+        [input_dir] * len(input_files_batches),
+        [output_dir] * len(input_files_batches),
+        [args.mode] * len(input_files_batches),
+        max_workers=args.max_workers,
+        chunksize=1,
+    )
+
+
+def run_batch(input_files_batch, input_dir, output_dir, mode):
+    with TemporaryDirectory() as temp_dir_SR, TemporaryDirectory() as temp_dir_GNR, TemporaryDirectory() as temp_dir_SA, TemporaryDirectory() as input_temp_dir, TemporaryDirectory() as output_temp_dir:
+        input_temp_dir = Path(input_temp_dir)
+        output_temp_dir = Path(output_temp_dir)
+        for file in input_files_batch:
+            logging.info(f"cp {input_dir / file} {input_temp_dir}")
+            shutil.copy(input_dir / file, input_temp_dir)
+
+        if mode == "gnorm2":
+            command_SR = (
+                f"java -Xmx32G -Xms16G -jar GNormPlus.jar {str(input_temp_dir)} {str(temp_dir_SR)} setup.SR.txt"
+            )
+            command_GNR_SA = f"python GeneNER_SpeAss_run.py -i {str(temp_dir_SR)} -r {str(temp_dir_GNR)} -a {str(temp_dir_SA)} -n gnorm_trained_models/geneNER/GeneNER-Bioformer.h5 -s gnorm_trained_models/SpeAss/SpeAss-Bioformer.h5"
+            command_GN = (
+                f"java -Xmx32G -Xms16G -jar GNormPlus.jar {str(temp_dir_SA)} {str(output_temp_dir)} setup.GN.txt"
+            )
+            commands = [command_SR, command_GNR_SA, command_GN]
+        elif mode == "gnormplus":
+            commands = [
+                f"java -Xmx32G -Xms16G -jar GNormPlus.jar {str(input_temp_dir)} {str(output_temp_dir)} setup.txt"
+            ]
+        else:
+            raise ValueError(f"Invalid mode: {mode}")
+
+        for command in commands:
+            try:
+                logging.info(command)
+                subprocess.run([command], check=True, shell=True)
+            except subprocess.CalledProcessError as e:
+                logging.exception(f"Error running command: {command}")
+                raise e
+
+        output_paths = list(output_temp_dir.rglob("*"))
+        for output_path in output_paths:
+            logging.info(f"cp {output_path} {output_dir}")
+            shutil.copy(output_path, output_dir)
+            output_file = output_path.name
 
 
 if __name__ == "__main__":

src_Java/GNormPluslib/BioCDoc.java

@@ -1,1344 +1,1344 @@
-/**
- * Project: GNormPlus
- * Function: Data storage in BioC format
- */
-
-package GNormPluslib;
-
-import bioc.BioCAnnotation;
-import bioc.BioCCollection;
-import bioc.BioCDocument;
-import bioc.BioCLocation;
-import bioc.BioCPassage;
-
-import bioc.io.BioCDocumentWriter;
-import bioc.io.BioCFactory;
-import bioc.io.woodstox.ConnectorWoodstox;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.UnsupportedEncodingException;
-import java.time.LocalDate;
-import java.time.ZoneId;
-
-import javax.xml.stream.XMLStreamException;
-
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-public class BioCDoc 
-{
-	/*
-	 * Contexts in BioC file
-	 */
-	public ArrayList<String> PMIDs=new ArrayList<String>(); // Type: PMIDs
-	public ArrayList<ArrayList<String>> PassageNames = new ArrayList(); // PassageName
-	public ArrayList<ArrayList<Integer>> PassageOffsets = new ArrayList(); // PassageOffset
-	public ArrayList<ArrayList<String>> PassageContexts = new ArrayList(); // PassageContext
-	public ArrayList<ArrayList<ArrayList<String>>> Annotations = new ArrayList(); // Annotation - GNormPlus
-	
-	public String BioCFormatCheck(String InputFile) throws IOException
-	{
-		
-		ConnectorWoodstox connector = new ConnectorWoodstox();
-		BioCCollection collection = new BioCCollection();
-		try
-		{
-			collection = connector.startRead(new InputStreamReader(new FileInputStream(InputFile), "UTF-8"));
-		}
-		catch (UnsupportedEncodingException | FileNotFoundException | XMLStreamException e) 
-		{
-			BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(InputFile), "UTF-8"));
-			String line="";
-			String status="";
-			String Pmid = "";
-			boolean tiabs=false;
-			Pattern patt = Pattern.compile("^([^\\|\\t]+)\\|([^\\|\\t]+)\\|(.*)$");
-			while ((line = br.readLine()) != null)  
-			{
-				Matcher mat = patt.matcher(line);
-				if(mat.find()) //Title|Abstract
-	        	{
-					if(Pmid.equals(""))
-					{
-						Pmid = mat.group(1);
-					}
-					else if(!Pmid.equals(mat.group(1)))
-					{
-						return "[Error]: "+InputFile+" - A blank is needed between "+Pmid+" and "+mat.group(1)+".";
-					}
-					status = "tiabs";
-					tiabs = true;
-	        	}
-				else if (line.contains("\t")) //Annotation
-	        	{
-	        	}
-				else if(line.length()==0) //Processing
-				{
-					if(status.equals(""))
-					{
-						if(Pmid.equals(""))
-						{
-							return "[Error]: "+InputFile+" - It's neither BioC nor PubTator format. PMID is empty.";
-						}
-						else
-						{
-							return "[Error]: "+InputFile+" - A redundant blank is after "+Pmid+".";
-						}
-					}
-					Pmid="";
-					status="";
-				}
-			}
-			br.close();
-			if(tiabs == false)
-			{
-				return "[Error]: "+InputFile+" - It's neither BioC nor PubTator format.";
-			}
-			if(status.equals(""))
-			{
-				return "PubTator";
-			}
-			else
-			{
-				return "[Error]: "+InputFile+" - The last column missed a blank.";
-			}
-		}
-		return "BioC";
-	}
-	public void PubTator2BioC(String input,String output) throws IOException, XMLStreamException // Input
-	{
-		/*
-		 *  PubTator2BioC
-		 */
-		String parser = BioCFactory.WOODSTOX;
-		BioCFactory factory = BioCFactory.newFactory(parser);
-		BioCDocumentWriter BioCOutputFormat = factory.createBioCDocumentWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8"));
-		BioCCollection biocCollection = new BioCCollection();
-		
-		//time
-		ZoneId zonedId = ZoneId.of( "America/Montreal" );
-		LocalDate today = LocalDate.now( zonedId );
-		biocCollection.setDate(today.toString());
-		
-		biocCollection.setKey("BioC.key");//key
-		biocCollection.setSource("GNormPlus");//source
-		
-		BioCOutputFormat.writeCollectionInfo(biocCollection);
-		BufferedReader inputfile = new BufferedReader(new InputStreamReader(new FileInputStream(input), "UTF-8"));
-		ArrayList<String> ParagraphType=new ArrayList<String>(); // Type: Title|Abstract
-		ArrayList<String> ParagraphContent = new ArrayList<String>(); // Text
-		ArrayList<String> annotations = new ArrayList<String>(); // Annotation
-		String line;
-		String Pmid="";
-		while ((line = inputfile.readLine()) != null)  
-		{
-			if(line.contains("|") && !line.contains("\t")) //Title|Abstract
-        	{
-				String str[]=line.split("\\|",-1);
-				Pmid=str[0];
-				if(str[1].equals("t"))
-				{
-					str[1]="title";
-				}
-				if(str[1].equals("a"))
-				{
-					str[1]="abstract";
-				}
-				ParagraphType.add(str[1]);
-				if(str.length==3)
-				{
-					String txt = str[2];
-					txt = txt.replaceAll("ω","w");
-					txt = txt.replaceAll("μ","u");
-					txt = txt.replaceAll("κ","k");
-					txt = txt.replaceAll("α","a");
-					txt = txt.replaceAll("γ","g");
-					txt = txt.replaceAll("ɣ","g");
-					txt = txt.replaceAll("β","b");
-					txt = txt.replaceAll("×","x");
-					txt = txt.replaceAll("‑","-");
-					txt = txt.replaceAll("¹","1");
-					txt = txt.replaceAll("²","2");
-					txt = txt.replaceAll("°","o");
-					txt = txt.replaceAll("ö","o");
-					txt = txt.replaceAll("é","e");
-					txt = txt.replaceAll("à","a");
-					txt = txt.replaceAll("Á","A");
-					txt = txt.replaceAll("ε","e");
-					txt = txt.replaceAll("θ","O");
-					txt = txt.replaceAll("•",".");
-					txt = txt.replaceAll("µ","u");
-					txt = txt.replaceAll("λ","r");
-					txt = txt.replaceAll("⁺","+");
-					txt = txt.replaceAll("ν","v");
-					txt = txt.replaceAll("ï","i");
-					txt = txt.replaceAll("ã","a");
-					txt = txt.replaceAll("≡","=");
-					txt = txt.replaceAll("ó","o");
-					txt = txt.replaceAll("³","3");
-					txt = txt.replaceAll("〖","[");
-					txt = txt.replaceAll("〗","]");
-					txt = txt.replaceAll("Å","A");
-					txt = txt.replaceAll("ρ","p");
-					txt = txt.replaceAll("ü","u");
-					txt = txt.replaceAll("ɛ","e");
-					txt = txt.replaceAll("č","c");
-					txt = txt.replaceAll("š","s");
-					txt = txt.replaceAll("ß","b");
-					txt = txt.replaceAll("═","=");
-					txt = txt.replaceAll("£","L");
-					txt = txt.replaceAll("Ł","L");
-					txt = txt.replaceAll("ƒ","f");
-					txt = txt.replaceAll("ä","a");
-					txt = txt.replaceAll("–","-");
-					txt = txt.replaceAll("⁻","-");
-					txt = txt.replaceAll("〈","<");
-					txt = txt.replaceAll("〉",">");
-					txt = txt.replaceAll("χ","X");
-					txt = txt.replaceAll("Đ","D");
-					txt = txt.replaceAll("‰","%");
-					txt = txt.replaceAll("·",".");
-					txt = txt.replaceAll("→",">");
-					txt = txt.replaceAll("←","<");
-					txt = txt.replaceAll("ζ","z");
-					txt = txt.replaceAll("π","p");
-					txt = txt.replaceAll("τ","t");
-					txt = txt.replaceAll("ξ","X");
-					txt = txt.replaceAll("η","h");
-					txt = txt.replaceAll("ø","0");
-					txt = txt.replaceAll("Δ","D");
-					txt = txt.replaceAll("∆","D");
-					txt = txt.replaceAll("∑","S");
-					txt = txt.replaceAll("Ω","O");
-					txt = txt.replaceAll("δ","d");
-					txt = txt.replaceAll("σ","s");
-					txt = txt.replaceAll("Φ","F");
-					txt = txt.replaceAll("[^\\~\\!\\@\\#\\$\\%\\^\\&\\*\\(\\)\\_\\+\\{\\}\\|\\:\"\\<\\>\\?\\`\\-\\=\\[\\]\\;\\'\\,\\.\\/\\r\\n0-9a-zA-Z ]"," ");
-					ParagraphContent.add(txt);
-				}
-				else
-				{
-					ParagraphContent.add("- No text -");
-				}
-        	}
-			else if (line.contains("\t")) //Annotation
-        	{
-				String anno[]=line.split("\t");
-				if(anno.length==6)
-				{
-					annotations.add(anno[1]+"\t"+anno[2]+"\t"+anno[3]+"\t"+anno[4]+"\t"+anno[5]);
-				}
-				else if(anno.length==5)
-				{
-					annotations.add(anno[1]+"\t"+anno[2]+"\t"+anno[3]+"\t"+anno[4]);
-				}
-        	}
-			else if(line.length()==0) //Processing
-			{
-				BioCDocument biocDocument = new BioCDocument();
-				biocDocument.setID(Pmid);
-				int startoffset=0;
-				for(int i=0;i<ParagraphType.size();i++)
-				{
-					BioCPassage biocPassage = new BioCPassage();
-					Map<String, String> Infons = new HashMap<String, String>();
-					Infons.put("type", ParagraphType.get(i));
-					biocPassage.setInfons(Infons);
-					biocPassage.setText(ParagraphContent.get(i));
-					biocPassage.setOffset(startoffset);
-					startoffset=startoffset+ParagraphContent.get(i).length()+1;
-					for(int j=0;j<annotations.size();j++)
-					{
-						String anno[]=annotations.get(j).split("\t");
-						if(Integer.parseInt(anno[0])<startoffset && Integer.parseInt(anno[0])>=startoffset-ParagraphContent.get(i).length()-1)
-						{
-							BioCAnnotation biocAnnotation = new BioCAnnotation();
-							Map<String, String> AnnoInfons = new HashMap<String, String>();
-							if(anno.length==5)
-							{
-								AnnoInfons.put("Identifier", anno[4]);
-							}
-							AnnoInfons.put("type", anno[3]);
-							biocAnnotation.setInfons(AnnoInfons);
-							BioCLocation location = new BioCLocation();
-							location.setOffset(Integer.parseInt(anno[0]));
-							location.setLength(Integer.parseInt(anno[1])-Integer.parseInt(anno[0]));
-							biocAnnotation.setLocation(location);
-							biocAnnotation.setText(anno[2]);
-							biocPassage.addAnnotation(biocAnnotation);
-						}
-					}
-					biocDocument.addPassage(biocPassage);
-				}
-				biocCollection.addDocument(biocDocument);
-				ParagraphType.clear();
-				ParagraphContent.clear();
-				annotations.clear();
-				BioCOutputFormat.writeDocument(biocDocument);
-			}
-		}
-		BioCOutputFormat.close();
-		inputfile.close();
-	}
-	public void BioC2PubTator(String input,String output) throws IOException, XMLStreamException //Output
-	{
-		/*
-		 * BioC2PubTator
-		 */
-		HashMap<String, String> pmidlist = new HashMap<String, String>(); // check if appear duplicate pmids
-		boolean duplicate = false;
-		BufferedWriter PubTatorOutputFormat = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8"));
-		ConnectorWoodstox connector = new ConnectorWoodstox();
-		BioCCollection collection = new BioCCollection();
-		collection = connector.startRead(new InputStreamReader(new FileInputStream(input), "UTF-8"));
-		while (connector.hasNext()) 
-		{
-			BioCDocument document = connector.next();
-			String PMID = document.getID();
-			if(pmidlist.containsKey(PMID)){System.out.println("\nError: duplicate pmid-"+PMID);duplicate = true;}
-			else{pmidlist.put(PMID,"");}
-			String Anno="";
-			for (BioCPassage passage : document.getPassages()) 
-			{
-				if(passage.getInfon("type").equals("title"))
-				{
-					PubTatorOutputFormat.write(PMID+"|t|"+passage.getText()+"\n");
-				}
-				else if(passage.getInfon("type").equals("abstract"))
-				{
-					PubTatorOutputFormat.write(PMID+"|a|"+passage.getText()+"\n");
-				}
-				else
-				{
-					PubTatorOutputFormat.write(PMID+"|"+passage.getInfon("type")+"|"+passage.getText()+"\n");
-				}
-				
-				for (BioCAnnotation annotation : passage.getAnnotations()) 
-				{
-					String Annotype = annotation.getInfon("type");
-					String Annoid="";
-					String Proteinid="";
-					if(Annotype.matches("(Gene|FamilyName|DomainMotif)"))
-					{
-						if(annotation.getInfons().containsKey("NCBI Gene"))
-						{
-							Annoid = annotation.getInfon("NCBI Gene");
-							String Annoidlist[]=Annoid.split(";");
-							Annoid="";
-							for(int x=0;x<Annoidlist.length;x++)
-							{
-								//Normalization2Protein
-								String proteinid="";
-								String homoid="";
-								
-								if(GNormPlus.Normalization2Protein_hash.containsKey(Annoidlist[x]))
-								{
-									proteinid=GNormPlus.Normalization2Protein_hash.get(Annoidlist[x]);
-								}
-								if(GNormPlus.HomologeneID_hash.containsKey(Annoidlist[x]))
-								{
-									homoid=GNormPlus.HomologeneID_hash.get(Annoidlist[x]);
-								}
-								
-								if((!proteinid.equals("")) || (!homoid.equals("")))
-								{
-									if(Annoid.equals(""))
-									{
-										Annoid=Annoidlist[x]+"(";
-										if(!proteinid.equals(""))
-										{
-											Annoid=Annoid+"UniProt:"+proteinid;
-										}
-										if(!homoid.equals(""))
-										{
-											if(!proteinid.equals(""))
-											{
-												Annoid=Annoid+";";
-											}
-											Annoid=Annoid+"Homoid:"+homoid;
-										}
-										Annoid=Annoid+")";
-									}
-									else
-									{
-										Annoid=Annoid+";"+Annoidlist[x]+"(";
-										if(!proteinid.equals(""))
-										{
-											Annoid=Annoid+"UniProt:"+proteinid;
-										}
-										if(!homoid.equals(""))
-										{
-											if(!proteinid.equals(""))
-											{
-												Annoid=Annoid+";";
-											}
-											Annoid=Annoid+"Homoid:"+homoid;
-										}
-										Annoid=Annoid+")";
-									}
-								}
-								else
-								{
-									if(Annoid.equals(""))
-									{
-										Annoid=Annoidlist[x];
-									}
-									else
-									{
-										Annoid=Annoid+";"+Annoidlist[x];
-									}
-								}
-							}
-						}
-						//else if(annotation.getInfons().containsKey("NCBI Homologene"))
-						//{
-						//	Annoid = annotation.getInfon("NCBI Homologene");
-						//}
-						//else if(!annotation.getInfons().containsKey("FocusSpecies"))
-						//{
-						//	Annoid = annotation.getInfon("FocusSpecies");
-						//}
-						else
-						{
-							Annoid = annotation.getInfon("Identifier");
-						}
-					}
-					else if(Annotype.equals("Species") || Annotype.equals("Genus") || Annotype.equals("Strain"))
-					{
-						if(annotation.getInfons().containsKey("NCBI Taxonomy"))
-						{
-							Annoid = annotation.getInfon("NCBI Taxonomy");
-						}
-						else
-						{
-							Annoid = annotation.getInfon("Identifier");
-						}
-					}
-					else if(Annotype.equals("CellLine"))
-					{
-						if(annotation.getInfons().containsKey("NCBI Taxonomy"))
-						{
-							Annoid = annotation.getInfon("NCBI Taxonomy");
-						}
-						else
-						{
-							Annoid = annotation.getInfon("Identifier");
-						}
-					}
-					else
-					{
-						Annoid = annotation.getInfon("Identifier");
-					}
-					int start = annotation.getLocations().get(0).getOffset();
-					int last = start + annotation.getLocations().get(0).getLength();
-					String AnnoMention=annotation.getText();
-					if(Annoid != null && !Annoid.equals(null) && !Annoid.equals(""))
-					{
-						Anno=Anno+PMID+"\t"+start+"\t"+last+"\t"+AnnoMention+"\t"+Annotype+"\t"+Annoid+"\n";
-					}
-					else
-					{
-						Anno=Anno+PMID+"\t"+start+"\t"+last+"\t"+AnnoMention+"\t"+Annotype+"\n";
-					}
-				}
-			}
-			PubTatorOutputFormat.write(Anno+"\n");
-		}
-		PubTatorOutputFormat.close();
-		if(duplicate == true){System.exit(0);}
-	}
-	public void BioC2PubTator(String original_input,String input,String output) throws IOException, XMLStreamException //Output
-	{
-		/* original tiabs*/
-		BufferedReader inputfile = new BufferedReader(new InputStreamReader(new FileInputStream(original_input), "UTF-8"));
-		HashMap<String,String> ParagraphContent = new HashMap<String,String>(); // [PMID,0] -> title
-		HashMap<String,String> annotations = new HashMap<String,String>(); // PMID ->Annotation
-		String line;
-		String Pmid="";
-		int count_paragraph=0;
-		while ((line = inputfile.readLine()) != null)  
-		{
-			if(line.contains("|") && !line.contains("\t")) //Title|Abstract
-        	{
-				String str[]=line.split("\\|",-1);
-				Pmid=str[0];
-				ParagraphContent.put(Pmid+"\t"+str[1],str[2]);
-				count_paragraph++;
-        	}
-			else if (line.contains("\t")) //Annotation
-        	{
-				annotations.put(Pmid, annotations.get(Pmid)+line);
-        	}
-			else if(line.length()==0) //Processing
-			{
-				count_paragraph=0;
-			}
-		}
-		inputfile.close();
-		
-		/*
-		 * BioC2PubTator
-		 */
-		HashMap<String, String> pmidlist = new HashMap<String, String>(); // check if appear duplicate pmids
-		boolean duplicate = false;
-		BufferedWriter PubTatorOutputFormat = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8"));
-		ConnectorWoodstox connector = new ConnectorWoodstox();
-		BioCCollection collection = new BioCCollection();
-		collection = connector.startRead(new InputStreamReader(new FileInputStream(input), "UTF-8"));
-		while (connector.hasNext()) 
-		{
-			BioCDocument document = connector.next();
-			String PMID = document.getID();
-			if(pmidlist.containsKey(PMID)){System.out.println("\nError: duplicate pmid-"+PMID);duplicate = true;}
-			else{pmidlist.put(PMID,"");}
-			String Anno="";
-			for (BioCPassage passage : document.getPassages()) 
-			{
-				if(passage.getInfon("type").equals("title") || passage.getInfon("type").equals("t"))
-				{
-					PubTatorOutputFormat.write(PMID+"|t|"+ParagraphContent.get(PMID+"\tt")+"\n");
-				}
-				else if(passage.getInfon("type").equals("abstract") || passage.getInfon("type").equals("a"))
-				{
-					PubTatorOutputFormat.write(PMID+"|a|"+ParagraphContent.get(PMID+"\ta")+"\n");
-				}
-				else
-				{
-					PubTatorOutputFormat.write(PMID+"|"+passage.getInfon("type")+"|"+passage.getText()+"\n");
-				}
-				
-				for (BioCAnnotation annotation : passage.getAnnotations()) 
-				{
-					String Annotype = annotation.getInfon("type");
-					String Annoid="";
-					String Proteinid="";
-					if(Annotype.matches("(Gene|FamilyName|DomainMotif)"))
-					{
-						if(annotation.getInfons().containsKey("NCBI Gene"))
-						{
-							Annoid = annotation.getInfon("NCBI Gene");
-							String Annoidlist[]=Annoid.split(";");
-							Annoid="";
-							for(int x=0;x<Annoidlist.length;x++)
-							{
-								//Normalization2Protein
-								String proteinid="";
-								String homoid="";
-								
-								if(GNormPlus.Normalization2Protein_hash.containsKey(Annoidlist[x]))
-								{
-									proteinid=GNormPlus.Normalization2Protein_hash.get(Annoidlist[x]);
-								}
-								if(GNormPlus.HomologeneID_hash.containsKey(Annoidlist[x]))
-								{
-									homoid=GNormPlus.HomologeneID_hash.get(Annoidlist[x]);
-								}
-								
-								if((!proteinid.equals("")) || (!homoid.equals("")))
-								{
-									if(Annoid.equals(""))
-									{
-										Annoid=Annoidlist[x]+"(";
-										if(!proteinid.equals(""))
-										{
-											Annoid=Annoid+"UniProt:"+proteinid;
-										}
-										if(!homoid.equals(""))
-										{
-											if(!proteinid.equals(""))
-											{
-												Annoid=Annoid+";";
-											}
-											Annoid=Annoid+"Homoid:"+homoid;
-										}
-										Annoid=Annoid+")";
-									}
-									else
-									{
-										Annoid=Annoid+";"+Annoidlist[x]+"(";
-										if(!proteinid.equals(""))
-										{
-											Annoid=Annoid+"UniProt:"+proteinid;
-										}
-										if(!homoid.equals(""))
-										{
-											if(!proteinid.equals(""))
-											{
-												Annoid=Annoid+";";
-											}
-											Annoid=Annoid+"Homoid:"+homoid;
-										}
-										Annoid=Annoid+")";
-									}
-								}
-								else
-								{
-									if(Annoid.equals(""))
-									{
-										Annoid=Annoidlist[x];
-									}
-									else
-									{
-										Annoid=Annoid+";"+Annoidlist[x];
-									}
-								}
-							}
-						}
-						//else if(annotation.getInfons().containsKey("NCBI Homologene"))
-						//{
-						//	Annoid = annotation.getInfon("NCBI Homologene");
-						//}
-						//else if(annotation.getInfons().containsKey("FocusSpecies"))
-						//{
-						//	Annoid = annotation.getInfon("FocusSpecies");
-						//}
-						else
-						{
-							Annoid = annotation.getInfon("Identifier");
-						}
-					}
-					else if(Annotype.equals("Species") || Annotype.equals("Genus") || Annotype.equals("Strain"))
-					{
-						if(annotation.getInfons().containsKey("NCBI Taxonomy"))
-						{
-							Annoid = annotation.getInfon("NCBI Taxonomy");
-						}
-						else
-						{
-							Annoid = annotation.getInfon("Identifier");
-						}
-					}
-					else if(Annotype.equals("CellLine"))
-					{
-						if(annotation.getInfons().containsKey("NCBI Taxonomy"))
-						{
-							Annoid = annotation.getInfon("NCBI Taxonomy");
-						}
-						else
-						{
-							Annoid = annotation.getInfon("Identifier");
-						}
-					}
-					else
-					{
-						if(annotation.getInfons().containsKey("Identifier"))
-						{
-							Annoid = annotation.getInfon("Identifier");
-						}
-						else
-						{
-							Annoid = "";
-						}
-					}
-					int start = annotation.getLocations().get(0).getOffset();
-					int last = start + annotation.getLocations().get(0).getLength();
-					String AnnoMention=annotation.getText();
-					if(Annoid != null && !Annoid.equals(null) && !Annoid.equals(""))
-					{
-						Anno=Anno+PMID+"\t"+start+"\t"+last+"\t"+AnnoMention+"\t"+Annotype+"\t"+Annoid+"\n";
-					}
-					else
-					{
-						Anno=Anno+PMID+"\t"+start+"\t"+last+"\t"+AnnoMention+"\t"+Annotype+"\n";
-					}
-				}
-			}
-			PubTatorOutputFormat.write(Anno+"\n");
-		}
-		PubTatorOutputFormat.close();
-		if(duplicate == true){System.exit(0);}
-	}
-	public void BioCReader(String input) throws IOException, XMLStreamException
-	{
-		ConnectorWoodstox connector = new ConnectorWoodstox();
-		BioCCollection collection = new BioCCollection();
-		collection = connector.startRead(new InputStreamReader(new FileInputStream(input), "UTF-8"));
-		
-		/*
-		 * Per document
-		 */
-		while (connector.hasNext()) 
-		{
-			BioCDocument document = connector.next();
-			PMIDs.add(document.getID());
-			
-			ArrayList<String> PassageName= new ArrayList<String>(); // array of Passage name
-			ArrayList<Integer> PassageOffset= new ArrayList<Integer>(); // array of Passage offset
-			ArrayList<String> PassageContext= new ArrayList<String>(); // array of Passage context
-			ArrayList<ArrayList<String>> AnnotationInPMID= new ArrayList(); // array of Annotations in the PassageName
-			
-			/*
-			 * Per Passage
-			 */
-			for (BioCPassage passage : document.getPassages()) 
-			{
-				PassageName.add(passage.getInfon("type")); //Paragraph
-				String txt = passage.getText();
-				if(txt.matches("[\t ]+"))
-				{
-					txt = txt.replaceAll(".","@");
-				}
-				else
-				{
-					//if(passage.getInfon("type").toLowerCase().equals("table"))
-					//{
-					//	txt=txt.replaceAll(" ", "|");
-					//}
-					txt = txt.replaceAll("ω","w");
-					txt = txt.replaceAll("μ","u");
-					txt = txt.replaceAll("κ","k");
-					txt = txt.replaceAll("α","a");
-					txt = txt.replaceAll("γ","g");
-					txt = txt.replaceAll("ɣ","g");
-					txt = txt.replaceAll("β","b");
-					txt = txt.replaceAll("×","x");
-					txt = txt.replaceAll("‑","-");
-					txt = txt.replaceAll("¹","1");
-					txt = txt.replaceAll("²","2");
-					txt = txt.replaceAll("°","o");
-					txt = txt.replaceAll("ö","o");
-					txt = txt.replaceAll("é","e");
-					txt = txt.replaceAll("à","a");
-					txt = txt.replaceAll("Á","A");
-					txt = txt.replaceAll("ε","e");
-					txt = txt.replaceAll("θ","O");
-					txt = txt.replaceAll("•",".");
-					txt = txt.replaceAll("µ","u");
-					txt = txt.replaceAll("λ","r");
-					txt = txt.replaceAll("⁺","+");
-					txt = txt.replaceAll("ν","v");
-					txt = txt.replaceAll("ï","i");
-					txt = txt.replaceAll("ã","a");
-					txt = txt.replaceAll("≡","=");
-					txt = txt.replaceAll("ó","o");
-					txt = txt.replaceAll("³","3");
-					txt = txt.replaceAll("〖","[");
-					txt = txt.replaceAll("〗","]");
-					txt = txt.replaceAll("Å","A");
-					txt = txt.replaceAll("ρ","p");
-					txt = txt.replaceAll("ü","u");
-					txt = txt.replaceAll("ɛ","e");
-					txt = txt.replaceAll("č","c");
-					txt = txt.replaceAll("š","s");
-					txt = txt.replaceAll("ß","b");
-					txt = txt.replaceAll("═","=");
-					txt = txt.replaceAll("£","L");
-					txt = txt.replaceAll("Ł","L");
-					txt = txt.replaceAll("ƒ","f");
-					txt = txt.replaceAll("ä","a");
-					txt = txt.replaceAll("–","-");
-					txt = txt.replaceAll("⁻","-");
-					txt = txt.replaceAll("〈","<");
-					txt = txt.replaceAll("〉",">");
-					txt = txt.replaceAll("χ","X");
-					txt = txt.replaceAll("Đ","D");
-					txt = txt.replaceAll("‰","%");
-					txt = txt.replaceAll("·",".");
-					txt = txt.replaceAll("→",">");
-					txt = txt.replaceAll("←","<");
-					txt = txt.replaceAll("ζ","z");
-					txt = txt.replaceAll("π","p");
-					txt = txt.replaceAll("τ","t");
-					txt = txt.replaceAll("ξ","X");
-					txt = txt.replaceAll("η","h");
-					txt = txt.replaceAll("ø","0");
-					txt = txt.replaceAll("Δ","D");
-					txt = txt.replaceAll("∆","D");
-					txt = txt.replaceAll("∑","S");
-					txt = txt.replaceAll("Ω","O");
-					txt = txt.replaceAll("δ","d");
-					txt = txt.replaceAll("σ","s");
-					txt = txt.replaceAll("Φ","F");
-					//txt = txt.replaceAll("[^\\~\\!\\@\\#\\$\\%\\^\\&\\*\\(\\)\\_\\+\\{\\}\\|\\:\"\\<\\>\\?\\`\\-\\=\\[\\]\\;\\'\\,\\.\\/\\r\\n0-9a-zA-Z ]"," ");
-				}
-				if(passage.getText().equals("") || passage.getText().matches("[ ]+"))
-				{
-					PassageContext.add("-notext-"); //Context
-				}
-				else
-				{
-					PassageContext.add(txt); //Context
-				}
-				PassageOffset.add(passage.getOffset()); //Offset
-				ArrayList<String> AnnotationInPassage= new ArrayList<String>(); // array of Annotations in the PassageName
-				AnnotationInPMID.add(AnnotationInPassage);
-			}
-			PassageNames.add(PassageName);
-			PassageContexts.add(PassageContext);
-			PassageOffsets.add(PassageOffset);
-			Annotations.add(AnnotationInPMID);
-		}	
-	}
-	public void BioCReaderWithAnnotation(String input) throws IOException, XMLStreamException
-	{
-		ConnectorWoodstox connector = new ConnectorWoodstox();
-		BioCCollection collection = new BioCCollection();
-		collection = connector.startRead(new InputStreamReader(new FileInputStream(input), "UTF-8"));
-		
-		/*
-		 * Per document
-		 */
-		while (connector.hasNext()) 
-		{
-			BioCDocument document = connector.next();
-			PMIDs.add(document.getID());
-			
-			ArrayList<String> PassageName= new ArrayList<String>(); // array of Passage name
-			ArrayList<Integer> PassageOffset= new ArrayList<Integer>(); // array of Passage offset
-			ArrayList<String> PassageContext= new ArrayList<String>(); // array of Passage context
-			ArrayList<ArrayList<String>> AnnotationInPMID= new ArrayList(); // array of Annotations in the PassageName
-			
-			/*
-			 * Per Passage
-			 */
-			for (BioCPassage passage : document.getPassages()) 
-			{
-				PassageName.add(passage.getInfon("type")); //Paragraph
-				
-				String txt = passage.getText();
-				if(txt.matches("[\t ]+"))
-				{
-					txt = txt.replaceAll(".","@");
-				}
-				else
-				{
-					//if(passage.getInfon("type").toLowerCase().equals("table"))
-					//{
-					//	txt=txt.replaceAll(" ", "|");
-					//}
-					txt = txt.replaceAll("ω","w");
-					txt = txt.replaceAll("μ","u");
-					txt = txt.replaceAll("κ","k");
-					txt = txt.replaceAll("α","a");
-					txt = txt.replaceAll("γ","g");
-					txt = txt.replaceAll("ɣ","g");
-					txt = txt.replaceAll("β","b");
-					txt = txt.replaceAll("×","x");
-					txt = txt.replaceAll("‑","-");
-					txt = txt.replaceAll("¹","1");
-					txt = txt.replaceAll("²","2");
-					txt = txt.replaceAll("°","o");
-					txt = txt.replaceAll("ö","o");
-					txt = txt.replaceAll("é","e");
-					txt = txt.replaceAll("à","a");
-					txt = txt.replaceAll("Á","A");
-					txt = txt.replaceAll("ε","e");
-					txt = txt.replaceAll("θ","O");
-					txt = txt.replaceAll("•",".");
-					txt = txt.replaceAll("µ","u");
-					txt = txt.replaceAll("λ","r");
-					txt = txt.replaceAll("⁺","+");
-					txt = txt.replaceAll("ν","v");
-					txt = txt.replaceAll("ï","i");
-					txt = txt.replaceAll("ã","a");
-					txt = txt.replaceAll("≡","=");
-					txt = txt.replaceAll("ó","o");
-					txt = txt.replaceAll("³","3");
-					txt = txt.replaceAll("〖","[");
-					txt = txt.replaceAll("〗","]");
-					txt = txt.replaceAll("Å","A");
-					txt = txt.replaceAll("ρ","p");
-					txt = txt.replaceAll("ü","u");
-					txt = txt.replaceAll("ɛ","e");
-					txt = txt.replaceAll("č","c");
-					txt = txt.replaceAll("š","s");
-					txt = txt.replaceAll("ß","b");
-					txt = txt.replaceAll("═","=");
-					txt = txt.replaceAll("£","L");
-					txt = txt.replaceAll("Ł","L");
-					txt = txt.replaceAll("ƒ","f");
-					txt = txt.replaceAll("ä","a");
-					txt = txt.replaceAll("–","-");
-					txt = txt.replaceAll("⁻","-");
-					txt = txt.replaceAll("〈","<");
-					txt = txt.replaceAll("〉",">");
-					txt = txt.replaceAll("χ","X");
-					txt = txt.replaceAll("Đ","D");
-					txt = txt.replaceAll("‰","%");
-					txt = txt.replaceAll("·",".");
-					txt = txt.replaceAll("→",">");
-					txt = txt.replaceAll("←","<");
-					txt = txt.replaceAll("ζ","z");
-					txt = txt.replaceAll("π","p");
-					txt = txt.replaceAll("τ","t");
-					txt = txt.replaceAll("ξ","X");
-					txt = txt.replaceAll("η","h");
-					txt = txt.replaceAll("ø","0");
-					txt = txt.replaceAll("Δ","D");
-					txt = txt.replaceAll("∆","D");
-					txt = txt.replaceAll("∑","S");
-					txt = txt.replaceAll("Ω","O");
-					txt = txt.replaceAll("δ","d");
-					txt = txt.replaceAll("σ","s");
-					txt = txt.replaceAll("Φ","F");
-					//txt = txt.replaceAll("[^\\~\\!\\@\\#\\$\\%\\^\\&\\*\\(\\)\\_\\+\\{\\}\\|\\:\"\\<\\>\\?\\`\\-\\=\\[\\]\\;\\'\\,\\.\\/\\r\\n0-9a-zA-Z ]"," ");
-				}
-				if(passage.getText().equals("") || passage.getText().matches("[ ]+"))
-				{
-					PassageContext.add("-notext-"); //Context
-				}
-				else
-				{
-					PassageContext.add(txt); //Context
-				}
-				PassageOffset.add(passage.getOffset()); //Offset
-				ArrayList<String> AnnotationInPassage= new ArrayList<String>(); // array of Annotations in the PassageName
-				
-				/*
-				 * Per Annotation :
-				 * start
-				 * last
-				 * mention
-				 * type
-				 * id
-				 */
-				for (BioCAnnotation Anno : passage.getAnnotations()) 
-				{
-					int start = Anno.getLocations().get(0).getOffset()-passage.getOffset(); // start
-					int last = start + Anno.getLocations().get(0).getLength(); // last
-					String AnnoMention=Anno.getText(); // mention
-					String Annotype = Anno.getInfon("type"); // type
-					String Annoid = Anno.getInfon("Identifier"); // identifier | MESH
-					if(Annoid == null)
-					{
-						Annoid = Anno.getInfon("Identifier"); // identifier | MESH
-					}
-					if(Annoid == null || Annoid.equals("null"))
-					{
-						AnnotationInPassage.add(start+"\t"+last+"\t"+AnnoMention+"\t"+Annotype); //paragraph
-					}
-					else
-					{
-						AnnotationInPassage.add(start+"\t"+last+"\t"+AnnoMention+"\t"+Annotype+"\t"+Annoid); //paragraph
-					}
-				}
-				AnnotationInPMID.add(AnnotationInPassage);
-			}
-			PassageNames.add(PassageName);
-			PassageContexts.add(PassageContext);
-			PassageOffsets.add(PassageOffset);
-			Annotations.add(AnnotationInPMID);
-		}	
-	}
-	public void BioCOutput(String input,String output, ArrayList<ArrayList<ArrayList<String>>> Annotations,boolean Final,boolean RemovePreviousAnno) throws IOException, XMLStreamException
-	{
-		boolean ShowUnNormalizedMention = false;
-		if(GNormPlus.setup_hash.containsKey("ShowUnNormalizedMention") && GNormPlus.setup_hash.get("ShowUnNormalizedMention").equals("True"))
-		{
-			ShowUnNormalizedMention = true;
-		}
-		
-		BioCDocumentWriter BioCOutputFormat = BioCFactory.newFactory(BioCFactory.WOODSTOX).createBioCDocumentWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8"));
-		BioCCollection biocCollection_input = new BioCCollection();
-		BioCCollection biocCollection_output = new BioCCollection();
-		
-		//input: BioC
-		ConnectorWoodstox connector = new ConnectorWoodstox();
-		biocCollection_input = connector.startRead(new InputStreamReader(new FileInputStream(input), "UTF-8"));
-		BioCOutputFormat.writeCollectionInfo(biocCollection_input);
-		int i=0; //count for pmid
-		while (connector.hasNext()) 
-		{
-			BioCDocument document_output = new BioCDocument();
-			BioCDocument document_input = connector.next();
-			String PMID=document_input.getID();
-			document_output.setID(PMID);
-			int annotation_count=0;
-			int j=0; //count for paragraph
-			for (BioCPassage passage_input : document_input.getPassages()) 
-			{
-				BioCPassage passage_output = passage_input;
-				
-				if(RemovePreviousAnno == true) //clean the previous annotation, if the NER result is provided
-				{
-					passage_output.clearAnnotations();
-				}
-				else
-				{
-					for (BioCAnnotation annotation : passage_output.getAnnotations()) 
-					{
-						annotation.setID(""+annotation_count);
-						annotation_count++;
-					}
-				}
-				
-				int passage_Offset = passage_input.getOffset();
-				String passage_Text = passage_input.getText();
-				ArrayList<String> AnnotationInPassage = new ArrayList<String>();
-				//ArrayList<String> AnnotationInPassage = Annotations.get(i).get(j);
-				if(Annotations.size()>i && Annotations.get(i).size()>j)
-				{
-					for(int a=0;a<Annotations.get(i).get(j).size();a++)
-					{
-						String Anno[]=Annotations.get(i).get(j).get(a).split("\\t");
-						int start = Integer.parseInt(Anno[0]);
-						int last = Integer.parseInt(Anno[1]);
-						boolean found = false;
-						if(passage_Text.length()>last)
-						{
-							String mention = Anno[2];
-							if(Final == true && passage_Text.length()>=last)
-							{
-								mention = passage_Text.substring(start, last);
-							}
-							if(mention.matches(".*\t.*"))
-							{
-								Anno[3]=Anno[4];
-								if(Anno.length>=6)
-								{
-									Anno[4]=Anno[5];
-								}
-							}
-							String type = Anno[3];
-							String id = ""; // optional
-							if(Anno.length>=5){id = Anno[4];}
-							if(Final == true)
-							{
-								for(int b=0;b<AnnotationInPassage.size();b++)
-								{
-									String Annob[]=AnnotationInPassage.get(b).split("\\t");
-									int startb = Integer.parseInt(Annob[0]);
-									int lastb = Integer.parseInt(Annob[1]);
-									String mentionb = Annob[2];
-									if(Final == true && passage_Text.length()>=lastb)
-									{
-										mentionb = passage_Text.substring(startb, lastb);
-									}
-									if(mentionb.matches(".*\t.*"))
-									{
-										Annob[3]=Annob[4];
-										if(Annob.length>=6)
-										{
-											Annob[4]=Annob[5];
-										}
-									}
-									String typeb = Annob[3];
-									String idb = ""; // optional
-									if(Annob.length>=5){idb = Annob[4];}
-									
-									if(start == startb && last == lastb && type.equals(typeb))
-									{
-										found = true;
-										if(id.matches("(Focus|Right|Left|Prefix|GeneID|Tax):[0-9]+") && (!idb.equals("")))
-										{
-										}
-										else if(idb.matches("(Focus|Right|Left|Prefix|GeneID|Tax):[0-9]+") && (!id.matches("(Focus|Right|Left|Prefix|GeneID|Tax):[0-9]+")) && (!id.equals("")))
-										{
-											AnnotationInPassage.set(b, start+"\t"+last+"\t"+mention+"\t"+type+"\t"+id);
-										}
-										else
-										{
-											if(id.equals(""))
-											{
-											}
-											else
-											{
-												AnnotationInPassage.set(b, start+"\t"+last+"\t"+mention+"\t"+type+"\t"+idb+";"+id);
-											}
-											
-										}
-										break;
-									}
-								}
-							}
-						}
-						if(found == false)
-						{
-							AnnotationInPassage.add(Annotations.get(i).get(j).get(a));
-						}
-					}
-				}
-				for(int a=0;a<AnnotationInPassage.size();a++)
-				{
-					String Anno[]=AnnotationInPassage.get(a).split("\\t");
-					HashMap <String,String> id_hash = new HashMap <String,String>();
-					if(Anno.length>=5)
-					{
-						int start = Integer.parseInt(Anno[0]);
-						int last = Integer.parseInt(Anno[1]);
-						String mention = Anno[2];
-						if(Final == true && passage_Text.length()>=last)
-						{
-							mention = passage_Text.substring(start, last);
-						}
-						if(mention.matches(".*\t.*"))
-						{
-							Anno[3]=Anno[4];
-							if(Anno.length>=6)
-							{
-								Anno[4]=Anno[5];
-							}
-						}
-						String ids = Anno[4];
-						String idlist[]=ids.split(",");
-						for(int b=0;b<idlist.length;b++)
-						{
-							id_hash.put(idlist[b], "");
-						}
-						ids = "";
-						for(String id :id_hash.keySet())
-						{
-							if(ids.equals(""))
-							{
-								ids = id;
-							}
-							else
-							{
-								ids = ids + ";" + id;
-							}
-						}
-						AnnotationInPassage.set(a, Anno[0]+"\t"+Anno[1]+"\t"+Anno[2]+"\t"+Anno[3]+"\t"+ids);
-					}					
-				}
-				
-				for(int a=0;a<AnnotationInPassage.size();a++)
-				{
-					String Anno[]=AnnotationInPassage.get(a).split("\\t");
-					int start = Integer.parseInt(Anno[0]);
-					int last = Integer.parseInt(Anno[1]);
-					if(passage_Text.length()>last)
-					{
-						String mention = Anno[2];
-						if(Final == true && passage_Text.length()>=last)
-						{
-							mention = passage_Text.substring(start, last);
-						}
-						if(mention.matches(".*\t.*"))
-						{
-							Anno[3]=Anno[4];
-							if(Anno.length>=6)
-							{
-								Anno[4]=Anno[5];
-							}
-						}
-						String type = Anno[3];
-						if(type.equals("GeneID")){type="Gene";}
-						BioCAnnotation biocAnnotation = new BioCAnnotation();
-						Map<String, String> AnnoInfons = new HashMap<String, String>();
-						AnnoInfons.put("type", type);
-						if(Anno.length>=5)
-						{
-							String identifier = Anno[4];
-							if(Final == true && ShowUnNormalizedMention==false)
-							{
-								if(type.matches("(FamilyName|Domain|Gene)"))
-								{
-									Pattern ptmp0 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)\\|([0-9\\;]+)$");
-									Matcher mtmp0 = ptmp0.matcher(identifier);
-									Pattern ptmp1 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)\\|([0-9]+)\\-([0-9]+)$");
-									Matcher mtmp1 = ptmp1.matcher(identifier);
-									Pattern ptmp2 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)$");
-									Matcher mtmp2 = ptmp2.matcher(identifier);
-									Pattern ptmp3 = Pattern.compile("^Homo\\:([0-9]+)$");
-									Matcher mtmp3 = ptmp3.matcher(identifier);
-									if(mtmp0.find())
-									{
-										String Method_SA = mtmp0.group(1);
-										String TaxonomyID = mtmp0.group(2);
-										String NCBIGeneID = mtmp0.group(3);
-										if(GNormPlus.Normalization2Protein_hash.containsKey(NCBIGeneID))
-										{
-											AnnoInfons.put("UniProt", GNormPlus.Normalization2Protein_hash.get(NCBIGeneID));
-										}
-										if(GNormPlus.HomologeneID_hash.containsKey(NCBIGeneID))
-										{
-											AnnoInfons.put("NCBI Homologene", GNormPlus.HomologeneID_hash.get(NCBIGeneID));
-										}
-										AnnoInfons.put("NCBI Gene", NCBIGeneID);
-									}
-									else if(mtmp1.find())
-									{
-										String Method_SA = mtmp1.group(1);
-										String TaxonomyID = mtmp1.group(2);
-										String NCBIGeneID = mtmp1.group(3);
-										String HomoID = mtmp1.group(4);
-										if(GNormPlus.Normalization2Protein_hash.containsKey(NCBIGeneID))
-										{
-											AnnoInfons.put("UniProt", GNormPlus.Normalization2Protein_hash.get(NCBIGeneID));
-										}
-										if(GNormPlus.HomologeneID_hash.containsKey(NCBIGeneID))
-										{
-											AnnoInfons.put("NCBI Homologene", GNormPlus.HomologeneID_hash.get(NCBIGeneID));
-										}
-										AnnoInfons.put("NCBI Gene", NCBIGeneID);
-									}
-									else if(mtmp2.find())
-									{
-										String Method_SA = mtmp2.group(1);
-										String TaxonomyID = mtmp2.group(2);
-										AnnoInfons.put("FocusSpecies", "NCBITaxonomyID:"+TaxonomyID);
-									}
-									else if(mtmp3.find())
-									{
-										String Method_SA = mtmp3.group(1);
-										String HomoID = mtmp3.group(2);
-										AnnoInfons.put("NCBI Homologene", HomoID);
-									}
-									else
-									{
-										String identifiers[] =  identifier.split(";");
-										if(identifiers.length>1)
-										{
-											ArrayList<String> identifierSTR = new ArrayList<String>();
-											ArrayList<String> ProteinidSTR = new ArrayList<String>();
-											ArrayList<String> HomoidSTR = new ArrayList<String>();
-											for(int idi=0;idi<identifiers.length;idi++)
-											{
-												Pattern ptmp4 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)\\|([0-9]+)\\-([0-9]+)$");
-												Matcher mtmp4 = ptmp4.matcher(identifiers[idi]);
-												Pattern ptmp5 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)\\|([0-9\\;]+)$");
-												Matcher mtmp5 = ptmp5.matcher(identifiers[idi]);
-												if(mtmp4.find())
-												{
-													String Method_SA = mtmp4.group(1);
-													String TaxonomyID = mtmp4.group(2);
-													String NCBIGeneID = mtmp4.group(3);
-													String HomoID = mtmp4.group(4);
-													if(!identifierSTR.contains(NCBIGeneID))
-													{
-														identifierSTR.add(NCBIGeneID);
-													}
-													if(GNormPlus.Normalization2Protein_hash.containsKey(NCBIGeneID))
-													{
-														if(!ProteinidSTR.contains(GNormPlus.Normalization2Protein_hash.containsKey(NCBIGeneID)))
-														{
-															ProteinidSTR.add(GNormPlus.Normalization2Protein_hash.get(NCBIGeneID));
-														}
-													}
-													if(GNormPlus.HomologeneID_hash.containsKey(NCBIGeneID))
-													{
-														if(!HomoidSTR.contains(GNormPlus.HomologeneID_hash.containsKey(NCBIGeneID)))
-														{
-															HomoidSTR.add(GNormPlus.HomologeneID_hash.get(NCBIGeneID));
-														}
-													}
-													
-												}
-												else if(mtmp5.find())
-												{
-													String Method_SA = mtmp5.group(1);
-													String TaxonomyID = mtmp5.group(2);
-													String NCBIGeneID = mtmp5.group(3);
-													if(!identifierSTR.contains(NCBIGeneID))
-													{
-														identifierSTR.add(NCBIGeneID);
-													}
-												}
-											}
-											String idSTR="";
-											for(int x=0;x<identifierSTR.size();x++)
-											{
-												if(idSTR.equals(""))
-												{
-													idSTR = identifierSTR.get(x);
-												}
-												else
-												{
-													idSTR = idSTR+";"+identifierSTR.get(x);
-												}
-											}
-											AnnoInfons.put("NCBI Gene", idSTR);
-											
-											String pidSTR="";
-											for(int x=0;x<ProteinidSTR.size();x++)
-											{
-												if(pidSTR.equals(""))
-												{
-													pidSTR = ProteinidSTR.get(x);
-												}
-												else
-												{
-													pidSTR = pidSTR+";"+ProteinidSTR.get(x);
-												}
-											}
-											if(!pidSTR.equals(""))
-											{
-												AnnoInfons.put("UniProt", pidSTR);
-											}
-											
-											String hidSTR="";
-											for(int x=0;x<HomoidSTR.size();x++)
-											{
-												if(hidSTR.equals(""))
-												{
-													hidSTR = HomoidSTR.get(x);
-												}
-												else
-												{
-													hidSTR = hidSTR+";"+HomoidSTR.get(x);
-												}
-											}
-											if(!hidSTR.equals(""))
-											{
-												AnnoInfons.put("NCBI Homologene", hidSTR);
-											}
-										}
-										//else
-										//{
-										//	AnnoInfons.put("Identifier", identifier);
-										//}
-									}
-								}
-								else if (type.matches("(Species|Genus|Strain)"))
-								{
-									AnnoInfons.put("type", type);
-									AnnoInfons.put("NCBI Taxonomy", identifier);
-								}
-								else if (type.matches("Cell"))
-								{
-									AnnoInfons.put("type", "CellLine");
-									AnnoInfons.put("NCBI Taxonomy", identifier);
-								}
-								else
-								{
-									AnnoInfons.put("Identifier", identifier);
-								}
-							}
-							else
-							{
-								AnnoInfons.put("Identifier", identifier);
-							}
-						}
-						biocAnnotation.setInfons(AnnoInfons);
-						BioCLocation location = new BioCLocation();
-						location.setOffset(start+passage_Offset);
-						location.setLength(last-start);
-						biocAnnotation.setLocation(location);
-						biocAnnotation.setText(mention);
-						biocAnnotation.setID(""+annotation_count);
-						annotation_count++;
-						if(Final == true)
-						{
-							if(AnnoInfons.containsKey("Identifier") || AnnoInfons.containsKey("NCBI Homologene") || AnnoInfons.containsKey("NCBI Gene") || AnnoInfons.containsKey("NCBI Taxonomy"))
-							{
-								passage_output.addAnnotation(biocAnnotation);
-							}
-						}
-						else
-						{
-							passage_output.addAnnotation(biocAnnotation);
-						}
-					}
-				}
-				document_output.addPassage(passage_output);
-				j++;
-			}
-			biocCollection_output.addDocument(document_output);
-			BioCOutputFormat.writeDocument(document_output);
-			i++;
-		}
-		BioCOutputFormat.close();
-	}	
+/**
+ * Project: GNormPlus
+ * Function: Data storage in BioC format
+ */
+
+package GNormPluslib;
+
+import bioc.BioCAnnotation;
+import bioc.BioCCollection;
+import bioc.BioCDocument;
+import bioc.BioCLocation;
+import bioc.BioCPassage;
+
+import bioc.io.BioCDocumentWriter;
+import bioc.io.BioCFactory;
+import bioc.io.woodstox.ConnectorWoodstox;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.time.LocalDate;
+import java.time.ZoneId;
+
+import javax.xml.stream.XMLStreamException;
+
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+public class BioCDoc 
+{
+	/*
+	 * Contexts in BioC file
+	 */
+	public ArrayList<String> PMIDs=new ArrayList<String>(); // Type: PMIDs
+	public ArrayList<ArrayList<String>> PassageNames = new ArrayList(); // PassageName
+	public ArrayList<ArrayList<Integer>> PassageOffsets = new ArrayList(); // PassageOffset
+	public ArrayList<ArrayList<String>> PassageContexts = new ArrayList(); // PassageContext
+	public ArrayList<ArrayList<ArrayList<String>>> Annotations = new ArrayList(); // Annotation - GNormPlus
+	
+	public String BioCFormatCheck(String InputFile) throws IOException
+	{
+		
+		ConnectorWoodstox connector = new ConnectorWoodstox();
+		BioCCollection collection = new BioCCollection();
+		try
+		{
+			collection = connector.startRead(new InputStreamReader(new FileInputStream(InputFile), "UTF-8"));
+		}
+		catch (UnsupportedEncodingException | FileNotFoundException | XMLStreamException e) 
+		{
+			BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(InputFile), "UTF-8"));
+			String line="";
+			String status="";
+			String Pmid = "";
+			boolean tiabs=false;
+			Pattern patt = Pattern.compile("^([^\\|\\t]+)\\|([^\\|\\t]+)\\|(.*)$");
+			while ((line = br.readLine()) != null)  
+			{
+				Matcher mat = patt.matcher(line);
+				if(mat.find()) //Title|Abstract
+	        	{
+					if(Pmid.equals(""))
+					{
+						Pmid = mat.group(1);
+					}
+					else if(!Pmid.equals(mat.group(1)))
+					{
+						return "[Error]: "+InputFile+" - A blank is needed between "+Pmid+" and "+mat.group(1)+".";
+					}
+					status = "tiabs";
+					tiabs = true;
+	        	}
+				else if (line.contains("\t")) //Annotation
+	        	{
+	        	}
+				else if(line.length()==0) //Processing
+				{
+					if(status.equals(""))
+					{
+						if(Pmid.equals(""))
+						{
+							return "[Error]: "+InputFile+" - It's neither BioC nor PubTator format. PMID is empty.";
+						}
+						else
+						{
+							return "[Error]: "+InputFile+" - A redundant blank is after "+Pmid+".";
+						}
+					}
+					Pmid="";
+					status="";
+				}
+			}
+			br.close();
+			if(tiabs == false)
+			{
+				return "[Error]: "+InputFile+" - It's neither BioC nor PubTator format.";
+			}
+			if(status.equals(""))
+			{
+				return "PubTator";
+			}
+			else
+			{
+				return "[Error]: "+InputFile+" - The last column missed a blank.";
+			}
+		}
+		return "BioC";
+	}
+	public void PubTator2BioC(String input,String output) throws IOException, XMLStreamException // Input
+	{
+		/*
+		 *  PubTator2BioC
+		 */
+		String parser = BioCFactory.WOODSTOX;
+		BioCFactory factory = BioCFactory.newFactory(parser);
+		BioCDocumentWriter BioCOutputFormat = factory.createBioCDocumentWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8"));
+		BioCCollection biocCollection = new BioCCollection();
+		
+		//time
+		ZoneId zonedId = ZoneId.of( "America/Montreal" );
+		LocalDate today = LocalDate.now( zonedId );
+		biocCollection.setDate(today.toString());
+		
+		biocCollection.setKey("BioC.key");//key
+		biocCollection.setSource("GNormPlus");//source
+		
+		BioCOutputFormat.writeCollectionInfo(biocCollection);
+		BufferedReader inputfile = new BufferedReader(new InputStreamReader(new FileInputStream(input), "UTF-8"));
+		ArrayList<String> ParagraphType=new ArrayList<String>(); // Type: Title|Abstract
+		ArrayList<String> ParagraphContent = new ArrayList<String>(); // Text
+		ArrayList<String> annotations = new ArrayList<String>(); // Annotation
+		String line;
+		String Pmid="";
+		while ((line = inputfile.readLine()) != null)  
+		{
+			if(line.contains("|") && !line.contains("\t")) //Title|Abstract
+        	{
+				String str[]=line.split("\\|",-1);
+				Pmid=str[0];
+				if(str[1].equals("t"))
+				{
+					str[1]="title";
+				}
+				if(str[1].equals("a"))
+				{
+					str[1]="abstract";
+				}
+				ParagraphType.add(str[1]);
+				if(str.length==3)
+				{
+					String txt = str[2];
+					txt = txt.replaceAll("ω","w");
+					txt = txt.replaceAll("μ","u");
+					txt = txt.replaceAll("κ","k");
+					txt = txt.replaceAll("α","a");
+					txt = txt.replaceAll("γ","g");
+					txt = txt.replaceAll("ɣ","g");
+					txt = txt.replaceAll("β","b");
+					txt = txt.replaceAll("×","x");
+					txt = txt.replaceAll("‑","-");
+					txt = txt.replaceAll("¹","1");
+					txt = txt.replaceAll("²","2");
+					txt = txt.replaceAll("°","o");
+					txt = txt.replaceAll("ö","o");
+					txt = txt.replaceAll("é","e");
+					txt = txt.replaceAll("à","a");
+					txt = txt.replaceAll("Á","A");
+					txt = txt.replaceAll("ε","e");
+					txt = txt.replaceAll("θ","O");
+					txt = txt.replaceAll("•",".");
+					txt = txt.replaceAll("µ","u");
+					txt = txt.replaceAll("λ","r");
+					txt = txt.replaceAll("⁺","+");
+					txt = txt.replaceAll("ν","v");
+					txt = txt.replaceAll("ï","i");
+					txt = txt.replaceAll("ã","a");
+					txt = txt.replaceAll("≡","=");
+					txt = txt.replaceAll("ó","o");
+					txt = txt.replaceAll("³","3");
+					txt = txt.replaceAll("〖","[");
+					txt = txt.replaceAll("〗","]");
+					txt = txt.replaceAll("Å","A");
+					txt = txt.replaceAll("ρ","p");
+					txt = txt.replaceAll("ü","u");
+					txt = txt.replaceAll("ɛ","e");
+					txt = txt.replaceAll("č","c");
+					txt = txt.replaceAll("š","s");
+					txt = txt.replaceAll("ß","b");
+					txt = txt.replaceAll("═","=");
+					txt = txt.replaceAll("£","L");
+					txt = txt.replaceAll("Ł","L");
+					txt = txt.replaceAll("ƒ","f");
+					txt = txt.replaceAll("ä","a");
+					txt = txt.replaceAll("–","-");
+					txt = txt.replaceAll("⁻","-");
+					txt = txt.replaceAll("〈","<");
+					txt = txt.replaceAll("〉",">");
+					txt = txt.replaceAll("χ","X");
+					txt = txt.replaceAll("Đ","D");
+					txt = txt.replaceAll("‰","%");
+					txt = txt.replaceAll("·",".");
+					txt = txt.replaceAll("→",">");
+					txt = txt.replaceAll("←","<");
+					txt = txt.replaceAll("ζ","z");
+					txt = txt.replaceAll("π","p");
+					txt = txt.replaceAll("τ","t");
+					txt = txt.replaceAll("ξ","X");
+					txt = txt.replaceAll("η","h");
+					txt = txt.replaceAll("ø","0");
+					txt = txt.replaceAll("Δ","D");
+					txt = txt.replaceAll("∆","D");
+					txt = txt.replaceAll("∑","S");
+					txt = txt.replaceAll("Ω","O");
+					txt = txt.replaceAll("δ","d");
+					txt = txt.replaceAll("σ","s");
+					txt = txt.replaceAll("Φ","F");
+					txt = txt.replaceAll("[^\\~\\!\\@\\#\\$\\%\\^\\&\\*\\(\\)\\_\\+\\{\\}\\|\\:\"\\<\\>\\?\\`\\-\\=\\[\\]\\;\\'\\,\\.\\/\\r\\n0-9a-zA-Z ]"," ");
+					ParagraphContent.add(txt);
+				}
+				else
+				{
+					ParagraphContent.add("- No text -");
+				}
+        	}
+			else if (line.contains("\t")) //Annotation
+        	{
+				String anno[]=line.split("\t");
+				if(anno.length==6)
+				{
+					annotations.add(anno[1]+"\t"+anno[2]+"\t"+anno[3]+"\t"+anno[4]+"\t"+anno[5]);
+				}
+				else if(anno.length==5)
+				{
+					annotations.add(anno[1]+"\t"+anno[2]+"\t"+anno[3]+"\t"+anno[4]);
+				}
+        	}
+			else if(line.length()==0) //Processing
+			{
+				BioCDocument biocDocument = new BioCDocument();
+				biocDocument.setID(Pmid);
+				int startoffset=0;
+				for(int i=0;i<ParagraphType.size();i++)
+				{
+					BioCPassage biocPassage = new BioCPassage();
+					Map<String, String> Infons = new HashMap<String, String>();
+					Infons.put("type", ParagraphType.get(i));
+					biocPassage.setInfons(Infons);
+					biocPassage.setText(ParagraphContent.get(i));
+					biocPassage.setOffset(startoffset);
+					startoffset=startoffset+ParagraphContent.get(i).length()+1;
+					for(int j=0;j<annotations.size();j++)
+					{
+						String anno[]=annotations.get(j).split("\t");
+						if(Integer.parseInt(anno[0])<startoffset && Integer.parseInt(anno[0])>=startoffset-ParagraphContent.get(i).length()-1)
+						{
+							BioCAnnotation biocAnnotation = new BioCAnnotation();
+							Map<String, String> AnnoInfons = new HashMap<String, String>();
+							if(anno.length==5)
+							{
+								AnnoInfons.put("Identifier", anno[4]);
+							}
+							AnnoInfons.put("type", anno[3]);
+							biocAnnotation.setInfons(AnnoInfons);
+							BioCLocation location = new BioCLocation();
+							location.setOffset(Integer.parseInt(anno[0]));
+							location.setLength(Integer.parseInt(anno[1])-Integer.parseInt(anno[0]));
+							biocAnnotation.setLocation(location);
+							biocAnnotation.setText(anno[2]);
+							biocPassage.addAnnotation(biocAnnotation);
+						}
+					}
+					biocDocument.addPassage(biocPassage);
+				}
+				biocCollection.addDocument(biocDocument);
+				ParagraphType.clear();
+				ParagraphContent.clear();
+				annotations.clear();
+				BioCOutputFormat.writeDocument(biocDocument);
+			}
+		}
+		BioCOutputFormat.close();
+		inputfile.close();
+	}
+	public void BioC2PubTator(String input,String output) throws IOException, XMLStreamException //Output
+	{
+		/*
+		 * BioC2PubTator
+		 */
+		HashMap<String, String> pmidlist = new HashMap<String, String>(); // check if appear duplicate pmids
+		boolean duplicate = false;
+		BufferedWriter PubTatorOutputFormat = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8"));
+		ConnectorWoodstox connector = new ConnectorWoodstox();
+		BioCCollection collection = new BioCCollection();
+		collection = connector.startRead(new InputStreamReader(new FileInputStream(input), "UTF-8"));
+		while (connector.hasNext()) 
+		{
+			BioCDocument document = connector.next();
+			String PMID = document.getID();
+			if(pmidlist.containsKey(PMID)){System.out.println("\nError: duplicate pmid-"+PMID);duplicate = true;}
+			else{pmidlist.put(PMID,"");}
+			String Anno="";
+			for (BioCPassage passage : document.getPassages()) 
+			{
+				if(passage.getInfon("type").equals("title"))
+				{
+					PubTatorOutputFormat.write(PMID+"|t|"+passage.getText()+"\n");
+				}
+				else if(passage.getInfon("type").equals("abstract"))
+				{
+					PubTatorOutputFormat.write(PMID+"|a|"+passage.getText()+"\n");
+				}
+				else
+				{
+					PubTatorOutputFormat.write(PMID+"|"+passage.getInfon("type")+"|"+passage.getText()+"\n");
+				}
+				
+				for (BioCAnnotation annotation : passage.getAnnotations()) 
+				{
+					String Annotype = annotation.getInfon("type");
+					String Annoid="";
+					String Proteinid="";
+					if(Annotype.matches("(Gene|FamilyName|DomainMotif)"))
+					{
+						if(annotation.getInfons().containsKey("NCBI Gene"))
+						{
+							Annoid = annotation.getInfon("NCBI Gene");
+							String Annoidlist[]=Annoid.split(";");
+							Annoid="";
+							for(int x=0;x<Annoidlist.length;x++)
+							{
+								//Normalization2Protein
+								String proteinid="";
+								String homoid="";
+								
+								if(GNormPlus.Normalization2Protein_hash.containsKey(Annoidlist[x]))
+								{
+									proteinid=GNormPlus.Normalization2Protein_hash.get(Annoidlist[x]);
+								}
+								if(GNormPlus.HomologeneID_hash.containsKey(Annoidlist[x]))
+								{
+									homoid=GNormPlus.HomologeneID_hash.get(Annoidlist[x]);
+								}
+								
+								if((!proteinid.equals("")) || (!homoid.equals("")))
+								{
+									if(Annoid.equals(""))
+									{
+										Annoid=Annoidlist[x]+"(";
+										if(!proteinid.equals(""))
+										{
+											Annoid=Annoid+"UniProt:"+proteinid;
+										}
+										if(!homoid.equals(""))
+										{
+											if(!proteinid.equals(""))
+											{
+												Annoid=Annoid+";";
+											}
+											Annoid=Annoid+"Homoid:"+homoid;
+										}
+										Annoid=Annoid+")";
+									}
+									else
+									{
+										Annoid=Annoid+";"+Annoidlist[x]+"(";
+										if(!proteinid.equals(""))
+										{
+											Annoid=Annoid+"UniProt:"+proteinid;
+										}
+										if(!homoid.equals(""))
+										{
+											if(!proteinid.equals(""))
+											{
+												Annoid=Annoid+";";
+											}
+											Annoid=Annoid+"Homoid:"+homoid;
+										}
+										Annoid=Annoid+")";
+									}
+								}
+								else
+								{
+									if(Annoid.equals(""))
+									{
+										Annoid=Annoidlist[x];
+									}
+									else
+									{
+										Annoid=Annoid+";"+Annoidlist[x];
+									}
+								}
+							}
+						}
+						//else if(annotation.getInfons().containsKey("NCBI Homologene"))
+						//{
+						//	Annoid = annotation.getInfon("NCBI Homologene");
+						//}
+						//else if(!annotation.getInfons().containsKey("FocusSpecies"))
+						//{
+						//	Annoid = annotation.getInfon("FocusSpecies");
+						//}
+						else
+						{
+							Annoid = annotation.getInfon("Identifier");
+						}
+					}
+					else if(Annotype.equals("Species") || Annotype.equals("Genus") || Annotype.equals("Strain"))
+					{
+						if(annotation.getInfons().containsKey("NCBI Taxonomy"))
+						{
+							Annoid = annotation.getInfon("NCBI Taxonomy");
+						}
+						else
+						{
+							Annoid = annotation.getInfon("Identifier");
+						}
+					}
+					else if(Annotype.equals("CellLine"))
+					{
+						if(annotation.getInfons().containsKey("NCBI Taxonomy"))
+						{
+							Annoid = annotation.getInfon("NCBI Taxonomy");
+						}
+						else
+						{
+							Annoid = annotation.getInfon("Identifier");
+						}
+					}
+					else
+					{
+						Annoid = annotation.getInfon("Identifier");
+					}
+					int start = annotation.getLocations().get(0).getOffset();
+					int last = start + annotation.getLocations().get(0).getLength();
+					String AnnoMention=annotation.getText();
+					if(Annoid != null && !Annoid.equals(null) && !Annoid.equals(""))
+					{
+						Anno=Anno+PMID+"\t"+start+"\t"+last+"\t"+AnnoMention+"\t"+Annotype+"\t"+Annoid+"\n";
+					}
+					else
+					{
+						Anno=Anno+PMID+"\t"+start+"\t"+last+"\t"+AnnoMention+"\t"+Annotype+"\n";
+					}
+				}
+			}
+			PubTatorOutputFormat.write(Anno+"\n");
+		}
+		PubTatorOutputFormat.close();
+		if(duplicate == true){System.exit(0);}
+	}
+	public void BioC2PubTator(String original_input,String input,String output) throws IOException, XMLStreamException //Output
+	{
+		/* original tiabs*/
+		BufferedReader inputfile = new BufferedReader(new InputStreamReader(new FileInputStream(original_input), "UTF-8"));
+		HashMap<String,String> ParagraphContent = new HashMap<String,String>(); // [PMID,0] -> title
+		HashMap<String,String> annotations = new HashMap<String,String>(); // PMID ->Annotation
+		String line;
+		String Pmid="";
+		int count_paragraph=0;
+		while ((line = inputfile.readLine()) != null)  
+		{
+			if(line.contains("|") && !line.contains("\t")) //Title|Abstract
+        	{
+				String str[]=line.split("\\|",-1);
+				Pmid=str[0];
+				ParagraphContent.put(Pmid+"\t"+str[1],str[2]);
+				count_paragraph++;
+        	}
+			else if (line.contains("\t")) //Annotation
+        	{
+				annotations.put(Pmid, annotations.get(Pmid)+line);
+        	}
+			else if(line.length()==0) //Processing
+			{
+				count_paragraph=0;
+			}
+		}
+		inputfile.close();
+		
+		/*
+		 * BioC2PubTator
+		 */
+		HashMap<String, String> pmidlist = new HashMap<String, String>(); // check if appear duplicate pmids
+		boolean duplicate = false;
+		BufferedWriter PubTatorOutputFormat = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8"));
+		ConnectorWoodstox connector = new ConnectorWoodstox();
+		BioCCollection collection = new BioCCollection();
+		collection = connector.startRead(new InputStreamReader(new FileInputStream(input), "UTF-8"));
+		while (connector.hasNext()) 
+		{
+			BioCDocument document = connector.next();
+			String PMID = document.getID();
+			if(pmidlist.containsKey(PMID)){System.out.println("\nError: duplicate pmid-"+PMID);duplicate = true;}
+			else{pmidlist.put(PMID,"");}
+			String Anno="";
+			for (BioCPassage passage : document.getPassages()) 
+			{
+				if(passage.getInfon("type").equals("title") || passage.getInfon("type").equals("t"))
+				{
+					PubTatorOutputFormat.write(PMID+"|t|"+ParagraphContent.get(PMID+"\tt")+"\n");
+				}
+				else if(passage.getInfon("type").equals("abstract") || passage.getInfon("type").equals("a"))
+				{
+					PubTatorOutputFormat.write(PMID+"|a|"+ParagraphContent.get(PMID+"\ta")+"\n");
+				}
+				else
+				{
+					PubTatorOutputFormat.write(PMID+"|"+passage.getInfon("type")+"|"+passage.getText()+"\n");
+				}
+				
+				for (BioCAnnotation annotation : passage.getAnnotations()) 
+				{
+					String Annotype = annotation.getInfon("type");
+					String Annoid="";
+					String Proteinid="";
+					if(Annotype.matches("(Gene|FamilyName|DomainMotif)"))
+					{
+						if(annotation.getInfons().containsKey("NCBI Gene"))
+						{
+							Annoid = annotation.getInfon("NCBI Gene");
+							String Annoidlist[]=Annoid.split(";");
+							Annoid="";
+							for(int x=0;x<Annoidlist.length;x++)
+							{
+								//Normalization2Protein
+								String proteinid="";
+								String homoid="";
+								
+								if(GNormPlus.Normalization2Protein_hash.containsKey(Annoidlist[x]))
+								{
+									proteinid=GNormPlus.Normalization2Protein_hash.get(Annoidlist[x]);
+								}
+								if(GNormPlus.HomologeneID_hash.containsKey(Annoidlist[x]))
+								{
+									homoid=GNormPlus.HomologeneID_hash.get(Annoidlist[x]);
+								}
+								
+								if((!proteinid.equals("")) || (!homoid.equals("")))
+								{
+									if(Annoid.equals(""))
+									{
+										Annoid=Annoidlist[x]+"(";
+										if(!proteinid.equals(""))
+										{
+											Annoid=Annoid+"UniProt:"+proteinid;
+										}
+										if(!homoid.equals(""))
+										{
+											if(!proteinid.equals(""))
+											{
+												Annoid=Annoid+";";
+											}
+											Annoid=Annoid+"Homoid:"+homoid;
+										}
+										Annoid=Annoid+")";
+									}
+									else
+									{
+										Annoid=Annoid+";"+Annoidlist[x]+"(";
+										if(!proteinid.equals(""))
+										{
+											Annoid=Annoid+"UniProt:"+proteinid;
+										}
+										if(!homoid.equals(""))
+										{
+											if(!proteinid.equals(""))
+											{
+												Annoid=Annoid+";";
+											}
+											Annoid=Annoid+"Homoid:"+homoid;
+										}
+										Annoid=Annoid+")";
+									}
+								}
+								else
+								{
+									if(Annoid.equals(""))
+									{
+										Annoid=Annoidlist[x];
+									}
+									else
+									{
+										Annoid=Annoid+";"+Annoidlist[x];
+									}
+								}
+							}
+						}
+						//else if(annotation.getInfons().containsKey("NCBI Homologene"))
+						//{
+						//	Annoid = annotation.getInfon("NCBI Homologene");
+						//}
+						//else if(annotation.getInfons().containsKey("FocusSpecies"))
+						//{
+						//	Annoid = annotation.getInfon("FocusSpecies");
+						//}
+						else
+						{
+							Annoid = annotation.getInfon("Identifier");
+						}
+					}
+					else if(Annotype.equals("Species") || Annotype.equals("Genus") || Annotype.equals("Strain"))
+					{
+						if(annotation.getInfons().containsKey("NCBI Taxonomy"))
+						{
+							Annoid = annotation.getInfon("NCBI Taxonomy");
+						}
+						else
+						{
+							Annoid = annotation.getInfon("Identifier");
+						}
+					}
+					else if(Annotype.equals("CellLine"))
+					{
+						if(annotation.getInfons().containsKey("NCBI Taxonomy"))
+						{
+							Annoid = annotation.getInfon("NCBI Taxonomy");
+						}
+						else
+						{
+							Annoid = annotation.getInfon("Identifier");
+						}
+					}
+					else
+					{
+						if(annotation.getInfons().containsKey("Identifier"))
+						{
+							Annoid = annotation.getInfon("Identifier");
+						}
+						else
+						{
+							Annoid = "";
+						}
+					}
+					int start = annotation.getLocations().get(0).getOffset();
+					int last = start + annotation.getLocations().get(0).getLength();
+					String AnnoMention=annotation.getText();
+					if(Annoid != null && !Annoid.equals(null) && !Annoid.equals(""))
+					{
+						Anno=Anno+PMID+"\t"+start+"\t"+last+"\t"+AnnoMention+"\t"+Annotype+"\t"+Annoid+"\n";
+					}
+					else
+					{
+						Anno=Anno+PMID+"\t"+start+"\t"+last+"\t"+AnnoMention+"\t"+Annotype+"\n";
+					}
+				}
+			}
+			PubTatorOutputFormat.write(Anno+"\n");
+		}
+		PubTatorOutputFormat.close();
+		if(duplicate == true){System.exit(0);}
+	}
+	public void BioCReader(String input) throws IOException, XMLStreamException
+	{
+		ConnectorWoodstox connector = new ConnectorWoodstox();
+		BioCCollection collection = new BioCCollection();
+		collection = connector.startRead(new InputStreamReader(new FileInputStream(input), "UTF-8"));
+		
+		/*
+		 * Per document
+		 */
+		while (connector.hasNext()) 
+		{
+			BioCDocument document = connector.next();
+			PMIDs.add(document.getID());
+			
+			ArrayList<String> PassageName= new ArrayList<String>(); // array of Passage name
+			ArrayList<Integer> PassageOffset= new ArrayList<Integer>(); // array of Passage offset
+			ArrayList<String> PassageContext= new ArrayList<String>(); // array of Passage context
+			ArrayList<ArrayList<String>> AnnotationInPMID= new ArrayList(); // array of Annotations in the PassageName
+			
+			/*
+			 * Per Passage
+			 */
+			for (BioCPassage passage : document.getPassages()) 
+			{
+				PassageName.add(passage.getInfon("type")); //Paragraph
+				String txt = passage.getText();
+				if(txt.matches("[\t ]+"))
+				{
+					txt = txt.replaceAll(".","@");
+				}
+				else
+				{
+					//if(passage.getInfon("type").toLowerCase().equals("table"))
+					//{
+					//	txt=txt.replaceAll(" ", "|");
+					//}
+					txt = txt.replaceAll("ω","w");
+					txt = txt.replaceAll("μ","u");
+					txt = txt.replaceAll("κ","k");
+					txt = txt.replaceAll("α","a");
+					txt = txt.replaceAll("γ","g");
+					txt = txt.replaceAll("ɣ","g");
+					txt = txt.replaceAll("β","b");
+					txt = txt.replaceAll("×","x");
+					txt = txt.replaceAll("‑","-");
+					txt = txt.replaceAll("¹","1");
+					txt = txt.replaceAll("²","2");
+					txt = txt.replaceAll("°","o");
+					txt = txt.replaceAll("ö","o");
+					txt = txt.replaceAll("é","e");
+					txt = txt.replaceAll("à","a");
+					txt = txt.replaceAll("Á","A");
+					txt = txt.replaceAll("ε","e");
+					txt = txt.replaceAll("θ","O");
+					txt = txt.replaceAll("•",".");
+					txt = txt.replaceAll("µ","u");
+					txt = txt.replaceAll("λ","r");
+					txt = txt.replaceAll("⁺","+");
+					txt = txt.replaceAll("ν","v");
+					txt = txt.replaceAll("ï","i");
+					txt = txt.replaceAll("ã","a");
+					txt = txt.replaceAll("≡","=");
+					txt = txt.replaceAll("ó","o");
+					txt = txt.replaceAll("³","3");
+					txt = txt.replaceAll("〖","[");
+					txt = txt.replaceAll("〗","]");
+					txt = txt.replaceAll("Å","A");
+					txt = txt.replaceAll("ρ","p");
+					txt = txt.replaceAll("ü","u");
+					txt = txt.replaceAll("ɛ","e");
+					txt = txt.replaceAll("č","c");
+					txt = txt.replaceAll("š","s");
+					txt = txt.replaceAll("ß","b");
+					txt = txt.replaceAll("═","=");
+					txt = txt.replaceAll("£","L");
+					txt = txt.replaceAll("Ł","L");
+					txt = txt.replaceAll("ƒ","f");
+					txt = txt.replaceAll("ä","a");
+					txt = txt.replaceAll("–","-");
+					txt = txt.replaceAll("⁻","-");
+					txt = txt.replaceAll("〈","<");
+					txt = txt.replaceAll("〉",">");
+					txt = txt.replaceAll("χ","X");
+					txt = txt.replaceAll("Đ","D");
+					txt = txt.replaceAll("‰","%");
+					txt = txt.replaceAll("·",".");
+					txt = txt.replaceAll("→",">");
+					txt = txt.replaceAll("←","<");
+					txt = txt.replaceAll("ζ","z");
+					txt = txt.replaceAll("π","p");
+					txt = txt.replaceAll("τ","t");
+					txt = txt.replaceAll("ξ","X");
+					txt = txt.replaceAll("η","h");
+					txt = txt.replaceAll("ø","0");
+					txt = txt.replaceAll("Δ","D");
+					txt = txt.replaceAll("∆","D");
+					txt = txt.replaceAll("∑","S");
+					txt = txt.replaceAll("Ω","O");
+					txt = txt.replaceAll("δ","d");
+					txt = txt.replaceAll("σ","s");
+					txt = txt.replaceAll("Φ","F");
+					//txt = txt.replaceAll("[^\\~\\!\\@\\#\\$\\%\\^\\&\\*\\(\\)\\_\\+\\{\\}\\|\\:\"\\<\\>\\?\\`\\-\\=\\[\\]\\;\\'\\,\\.\\/\\r\\n0-9a-zA-Z ]"," ");
+				}
+				if(passage.getText().equals("") || passage.getText().matches("[ ]+"))
+				{
+					PassageContext.add("-notext-"); //Context
+				}
+				else
+				{
+					PassageContext.add(txt); //Context
+				}
+				PassageOffset.add(passage.getOffset()); //Offset
+				ArrayList<String> AnnotationInPassage= new ArrayList<String>(); // array of Annotations in the PassageName
+				AnnotationInPMID.add(AnnotationInPassage);
+			}
+			PassageNames.add(PassageName);
+			PassageContexts.add(PassageContext);
+			PassageOffsets.add(PassageOffset);
+			Annotations.add(AnnotationInPMID);
+		}	
+	}
+	public void BioCReaderWithAnnotation(String input) throws IOException, XMLStreamException
+	{
+		ConnectorWoodstox connector = new ConnectorWoodstox();
+		BioCCollection collection = new BioCCollection();
+		collection = connector.startRead(new InputStreamReader(new FileInputStream(input), "UTF-8"));
+		
+		/*
+		 * Per document
+		 */
+		while (connector.hasNext()) 
+		{
+			BioCDocument document = connector.next();
+			PMIDs.add(document.getID());
+			
+			ArrayList<String> PassageName= new ArrayList<String>(); // array of Passage name
+			ArrayList<Integer> PassageOffset= new ArrayList<Integer>(); // array of Passage offset
+			ArrayList<String> PassageContext= new ArrayList<String>(); // array of Passage context
+			ArrayList<ArrayList<String>> AnnotationInPMID= new ArrayList(); // array of Annotations in the PassageName
+			
+			/*
+			 * Per Passage
+			 */
+			for (BioCPassage passage : document.getPassages()) 
+			{
+				PassageName.add(passage.getInfon("type")); //Paragraph
+				
+				String txt = passage.getText();
+				if(txt.matches("[\t ]+"))
+				{
+					txt = txt.replaceAll(".","@");
+				}
+				else
+				{
+					//if(passage.getInfon("type").toLowerCase().equals("table"))
+					//{
+					//	txt=txt.replaceAll(" ", "|");
+					//}
+					txt = txt.replaceAll("ω","w");
+					txt = txt.replaceAll("μ","u");
+					txt = txt.replaceAll("κ","k");
+					txt = txt.replaceAll("α","a");
+					txt = txt.replaceAll("γ","g");
+					txt = txt.replaceAll("ɣ","g");
+					txt = txt.replaceAll("β","b");
+					txt = txt.replaceAll("×","x");
+					txt = txt.replaceAll("‑","-");
+					txt = txt.replaceAll("¹","1");
+					txt = txt.replaceAll("²","2");
+					txt = txt.replaceAll("°","o");
+					txt = txt.replaceAll("ö","o");
+					txt = txt.replaceAll("é","e");
+					txt = txt.replaceAll("à","a");
+					txt = txt.replaceAll("Á","A");
+					txt = txt.replaceAll("ε","e");
+					txt = txt.replaceAll("θ","O");
+					txt = txt.replaceAll("•",".");
+					txt = txt.replaceAll("µ","u");
+					txt = txt.replaceAll("λ","r");
+					txt = txt.replaceAll("⁺","+");
+					txt = txt.replaceAll("ν","v");
+					txt = txt.replaceAll("ï","i");
+					txt = txt.replaceAll("ã","a");
+					txt = txt.replaceAll("≡","=");
+					txt = txt.replaceAll("ó","o");
+					txt = txt.replaceAll("³","3");
+					txt = txt.replaceAll("〖","[");
+					txt = txt.replaceAll("〗","]");
+					txt = txt.replaceAll("Å","A");
+					txt = txt.replaceAll("ρ","p");
+					txt = txt.replaceAll("ü","u");
+					txt = txt.replaceAll("ɛ","e");
+					txt = txt.replaceAll("č","c");
+					txt = txt.replaceAll("š","s");
+					txt = txt.replaceAll("ß","b");
+					txt = txt.replaceAll("═","=");
+					txt = txt.replaceAll("£","L");
+					txt = txt.replaceAll("Ł","L");
+					txt = txt.replaceAll("ƒ","f");
+					txt = txt.replaceAll("ä","a");
+					txt = txt.replaceAll("–","-");
+					txt = txt.replaceAll("⁻","-");
+					txt = txt.replaceAll("〈","<");
+					txt = txt.replaceAll("〉",">");
+					txt = txt.replaceAll("χ","X");
+					txt = txt.replaceAll("Đ","D");
+					txt = txt.replaceAll("‰","%");
+					txt = txt.replaceAll("·",".");
+					txt = txt.replaceAll("→",">");
+					txt = txt.replaceAll("←","<");
+					txt = txt.replaceAll("ζ","z");
+					txt = txt.replaceAll("π","p");
+					txt = txt.replaceAll("τ","t");
+					txt = txt.replaceAll("ξ","X");
+					txt = txt.replaceAll("η","h");
+					txt = txt.replaceAll("ø","0");
+					txt = txt.replaceAll("Δ","D");
+					txt = txt.replaceAll("∆","D");
+					txt = txt.replaceAll("∑","S");
+					txt = txt.replaceAll("Ω","O");
+					txt = txt.replaceAll("δ","d");
+					txt = txt.replaceAll("σ","s");
+					txt = txt.replaceAll("Φ","F");
+					//txt = txt.replaceAll("[^\\~\\!\\@\\#\\$\\%\\^\\&\\*\\(\\)\\_\\+\\{\\}\\|\\:\"\\<\\>\\?\\`\\-\\=\\[\\]\\;\\'\\,\\.\\/\\r\\n0-9a-zA-Z ]"," ");
+				}
+				if(passage.getText().equals("") || passage.getText().matches("[ ]+"))
+				{
+					PassageContext.add("-notext-"); //Context
+				}
+				else
+				{
+					PassageContext.add(txt); //Context
+				}
+				PassageOffset.add(passage.getOffset()); //Offset
+				ArrayList<String> AnnotationInPassage= new ArrayList<String>(); // array of Annotations in the PassageName
+				
+				/*
+				 * Per Annotation :
+				 * start
+				 * last
+				 * mention
+				 * type
+				 * id
+				 */
+				for (BioCAnnotation Anno : passage.getAnnotations()) 
+				{
+					int start = Anno.getLocations().get(0).getOffset()-passage.getOffset(); // start
+					int last = start + Anno.getLocations().get(0).getLength(); // last
+					String AnnoMention=Anno.getText(); // mention
+					String Annotype = Anno.getInfon("type"); // type
+					String Annoid = Anno.getInfon("Identifier"); // identifier | MESH
+					if(Annoid == null)
+					{
+						Annoid = Anno.getInfon("Identifier"); // identifier | MESH
+					}
+					if(Annoid == null || Annoid.equals("null"))
+					{
+						AnnotationInPassage.add(start+"\t"+last+"\t"+AnnoMention+"\t"+Annotype); //paragraph
+					}
+					else
+					{
+						AnnotationInPassage.add(start+"\t"+last+"\t"+AnnoMention+"\t"+Annotype+"\t"+Annoid); //paragraph
+					}
+				}
+				AnnotationInPMID.add(AnnotationInPassage);
+			}
+			PassageNames.add(PassageName);
+			PassageContexts.add(PassageContext);
+			PassageOffsets.add(PassageOffset);
+			Annotations.add(AnnotationInPMID);
+		}	
+	}
+	public void BioCOutput(String input,String output, ArrayList<ArrayList<ArrayList<String>>> Annotations,boolean Final,boolean RemovePreviousAnno) throws IOException, XMLStreamException
+	{
+		boolean ShowUnNormalizedMention = false;
+		if(GNormPlus.setup_hash.containsKey("ShowUnNormalizedMention") && GNormPlus.setup_hash.get("ShowUnNormalizedMention").equals("True"))
+		{
+			ShowUnNormalizedMention = true;
+		}
+		
+		BioCDocumentWriter BioCOutputFormat = BioCFactory.newFactory(BioCFactory.WOODSTOX).createBioCDocumentWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8"));
+		BioCCollection biocCollection_input = new BioCCollection();
+		BioCCollection biocCollection_output = new BioCCollection();
+		
+		//input: BioC
+		ConnectorWoodstox connector = new ConnectorWoodstox();
+		biocCollection_input = connector.startRead(new InputStreamReader(new FileInputStream(input), "UTF-8"));
+		BioCOutputFormat.writeCollectionInfo(biocCollection_input);
+		int i=0; //count for pmid
+		while (connector.hasNext()) 
+		{
+			BioCDocument document_output = new BioCDocument();
+			BioCDocument document_input = connector.next();
+			String PMID=document_input.getID();
+			document_output.setID(PMID);
+			int annotation_count=0;
+			int j=0; //count for paragraph
+			for (BioCPassage passage_input : document_input.getPassages()) 
+			{
+				BioCPassage passage_output = passage_input;
+				
+				if(RemovePreviousAnno == true) //clean the previous annotation, if the NER result is provided
+				{
+					passage_output.clearAnnotations();
+				}
+				else
+				{
+					for (BioCAnnotation annotation : passage_output.getAnnotations()) 
+					{
+						annotation.setID(""+annotation_count);
+						annotation_count++;
+					}
+				}
+				
+				int passage_Offset = passage_input.getOffset();
+				String passage_Text = passage_input.getText();
+				ArrayList<String> AnnotationInPassage = new ArrayList<String>();
+				//ArrayList<String> AnnotationInPassage = Annotations.get(i).get(j);
+				if(Annotations.size()>i && Annotations.get(i).size()>j)
+				{
+					for(int a=0;a<Annotations.get(i).get(j).size();a++)
+					{
+						String Anno[]=Annotations.get(i).get(j).get(a).split("\\t");
+						int start = Integer.parseInt(Anno[0]);
+						int last = Integer.parseInt(Anno[1]);
+						boolean found = false;
+						if(passage_Text.length()>last)
+						{
+							String mention = Anno[2];
+							if(Final == true && passage_Text.length()>=last)
+							{
+								mention = passage_Text.substring(start, last);
+							}
+							if(mention.matches(".*\t.*"))
+							{
+								Anno[3]=Anno[4];
+								if(Anno.length>=6)
+								{
+									Anno[4]=Anno[5];
+								}
+							}
+							String type = Anno[3];
+							String id = ""; // optional
+							if(Anno.length>=5){id = Anno[4];}
+							if(Final == true)
+							{
+								for(int b=0;b<AnnotationInPassage.size();b++)
+								{
+									String Annob[]=AnnotationInPassage.get(b).split("\\t");
+									int startb = Integer.parseInt(Annob[0]);
+									int lastb = Integer.parseInt(Annob[1]);
+									String mentionb = Annob[2];
+									if(Final == true && passage_Text.length()>=lastb)
+									{
+										mentionb = passage_Text.substring(startb, lastb);
+									}
+									if(mentionb.matches(".*\t.*"))
+									{
+										Annob[3]=Annob[4];
+										if(Annob.length>=6)
+										{
+											Annob[4]=Annob[5];
+										}
+									}
+									String typeb = Annob[3];
+									String idb = ""; // optional
+									if(Annob.length>=5){idb = Annob[4];}
+									
+									if(start == startb && last == lastb && type.equals(typeb))
+									{
+										found = true;
+										if(id.matches("(Focus|Right|Left|Prefix|GeneID|Tax):[0-9]+") && (!idb.equals("")))
+										{
+										}
+										else if(idb.matches("(Focus|Right|Left|Prefix|GeneID|Tax):[0-9]+") && (!id.matches("(Focus|Right|Left|Prefix|GeneID|Tax):[0-9]+")) && (!id.equals("")))
+										{
+											AnnotationInPassage.set(b, start+"\t"+last+"\t"+mention+"\t"+type+"\t"+id);
+										}
+										else
+										{
+											if(id.equals(""))
+											{
+											}
+											else
+											{
+												AnnotationInPassage.set(b, start+"\t"+last+"\t"+mention+"\t"+type+"\t"+idb+";"+id);
+											}
+											
+										}
+										break;
+									}
+								}
+							}
+						}
+						if(found == false)
+						{
+							AnnotationInPassage.add(Annotations.get(i).get(j).get(a));
+						}
+					}
+				}
+				for(int a=0;a<AnnotationInPassage.size();a++)
+				{
+					String Anno[]=AnnotationInPassage.get(a).split("\\t");
+					HashMap <String,String> id_hash = new HashMap <String,String>();
+					if(Anno.length>=5)
+					{
+						int start = Integer.parseInt(Anno[0]);
+						int last = Integer.parseInt(Anno[1]);
+						String mention = Anno[2];
+						if(Final == true && passage_Text.length()>=last)
+						{
+							mention = passage_Text.substring(start, last);
+						}
+						if(mention.matches(".*\t.*"))
+						{
+							Anno[3]=Anno[4];
+							if(Anno.length>=6)
+							{
+								Anno[4]=Anno[5];
+							}
+						}
+						String ids = Anno[4];
+						String idlist[]=ids.split(",");
+						for(int b=0;b<idlist.length;b++)
+						{
+							id_hash.put(idlist[b], "");
+						}
+						ids = "";
+						for(String id :id_hash.keySet())
+						{
+							if(ids.equals(""))
+							{
+								ids = id;
+							}
+							else
+							{
+								ids = ids + ";" + id;
+							}
+						}
+						AnnotationInPassage.set(a, Anno[0]+"\t"+Anno[1]+"\t"+Anno[2]+"\t"+Anno[3]+"\t"+ids);
+					}					
+				}
+				
+				for(int a=0;a<AnnotationInPassage.size();a++)
+				{
+					String Anno[]=AnnotationInPassage.get(a).split("\\t");
+					int start = Integer.parseInt(Anno[0]);
+					int last = Integer.parseInt(Anno[1]);
+					if(passage_Text.length()>last)
+					{
+						String mention = Anno[2];
+						if(Final == true && passage_Text.length()>=last)
+						{
+							mention = passage_Text.substring(start, last);
+						}
+						if(mention.matches(".*\t.*"))
+						{
+							Anno[3]=Anno[4];
+							if(Anno.length>=6)
+							{
+								Anno[4]=Anno[5];
+							}
+						}
+						String type = Anno[3];
+						if(type.equals("GeneID")){type="Gene";}
+						BioCAnnotation biocAnnotation = new BioCAnnotation();
+						Map<String, String> AnnoInfons = new HashMap<String, String>();
+						AnnoInfons.put("type", type);
+						if(Anno.length>=5)
+						{
+							String identifier = Anno[4];
+							if(Final == true && ShowUnNormalizedMention==false)
+							{
+								if(type.matches("(FamilyName|Domain|Gene)"))
+								{
+									Pattern ptmp0 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)\\|([0-9\\;]+)$");
+									Matcher mtmp0 = ptmp0.matcher(identifier);
+									Pattern ptmp1 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)\\|([0-9]+)\\-([0-9]+)$");
+									Matcher mtmp1 = ptmp1.matcher(identifier);
+									Pattern ptmp2 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)$");
+									Matcher mtmp2 = ptmp2.matcher(identifier);
+									Pattern ptmp3 = Pattern.compile("^Homo\\:([0-9]+)$");
+									Matcher mtmp3 = ptmp3.matcher(identifier);
+									if(mtmp0.find())
+									{
+										String Method_SA = mtmp0.group(1);
+										String TaxonomyID = mtmp0.group(2);
+										String NCBIGeneID = mtmp0.group(3);
+										if(GNormPlus.Normalization2Protein_hash.containsKey(NCBIGeneID))
+										{
+											AnnoInfons.put("UniProt", GNormPlus.Normalization2Protein_hash.get(NCBIGeneID));
+										}
+										if(GNormPlus.HomologeneID_hash.containsKey(NCBIGeneID))
+										{
+											AnnoInfons.put("NCBI Homologene", GNormPlus.HomologeneID_hash.get(NCBIGeneID));
+										}
+										AnnoInfons.put("NCBI Gene", NCBIGeneID);
+									}
+									else if(mtmp1.find())
+									{
+										String Method_SA = mtmp1.group(1);
+										String TaxonomyID = mtmp1.group(2);
+										String NCBIGeneID = mtmp1.group(3);
+										String HomoID = mtmp1.group(4);
+										if(GNormPlus.Normalization2Protein_hash.containsKey(NCBIGeneID))
+										{
+											AnnoInfons.put("UniProt", GNormPlus.Normalization2Protein_hash.get(NCBIGeneID));
+										}
+										if(GNormPlus.HomologeneID_hash.containsKey(NCBIGeneID))
+										{
+											AnnoInfons.put("NCBI Homologene", GNormPlus.HomologeneID_hash.get(NCBIGeneID));
+										}
+										AnnoInfons.put("NCBI Gene", NCBIGeneID);
+									}
+									else if(mtmp2.find())
+									{
+										String Method_SA = mtmp2.group(1);
+										String TaxonomyID = mtmp2.group(2);
+										AnnoInfons.put("FocusSpecies", "NCBITaxonomyID:"+TaxonomyID);
+									}
+									else if(mtmp3.find())
+									{
+										String Method_SA = mtmp3.group(1);
+										String HomoID = mtmp3.group(2);
+										AnnoInfons.put("NCBI Homologene", HomoID);
+									}
+									else
+									{
+										String identifiers[] =  identifier.split(";");
+										if(identifiers.length>1)
+										{
+											ArrayList<String> identifierSTR = new ArrayList<String>();
+											ArrayList<String> ProteinidSTR = new ArrayList<String>();
+											ArrayList<String> HomoidSTR = new ArrayList<String>();
+											for(int idi=0;idi<identifiers.length;idi++)
+											{
+												Pattern ptmp4 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)\\|([0-9]+)\\-([0-9]+)$");
+												Matcher mtmp4 = ptmp4.matcher(identifiers[idi]);
+												Pattern ptmp5 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)\\|([0-9\\;]+)$");
+												Matcher mtmp5 = ptmp5.matcher(identifiers[idi]);
+												if(mtmp4.find())
+												{
+													String Method_SA = mtmp4.group(1);
+													String TaxonomyID = mtmp4.group(2);
+													String NCBIGeneID = mtmp4.group(3);
+													String HomoID = mtmp4.group(4);
+													if(!identifierSTR.contains(NCBIGeneID))
+													{
+														identifierSTR.add(NCBIGeneID);
+													}
+													if(GNormPlus.Normalization2Protein_hash.containsKey(NCBIGeneID))
+													{
+														if(!ProteinidSTR.contains(GNormPlus.Normalization2Protein_hash.containsKey(NCBIGeneID)))
+														{
+															ProteinidSTR.add(GNormPlus.Normalization2Protein_hash.get(NCBIGeneID));
+														}
+													}
+													if(GNormPlus.HomologeneID_hash.containsKey(NCBIGeneID))
+													{
+														if(!HomoidSTR.contains(GNormPlus.HomologeneID_hash.containsKey(NCBIGeneID)))
+														{
+															HomoidSTR.add(GNormPlus.HomologeneID_hash.get(NCBIGeneID));
+														}
+													}
+													
+												}
+												else if(mtmp5.find())
+												{
+													String Method_SA = mtmp5.group(1);
+													String TaxonomyID = mtmp5.group(2);
+													String NCBIGeneID = mtmp5.group(3);
+													if(!identifierSTR.contains(NCBIGeneID))
+													{
+														identifierSTR.add(NCBIGeneID);
+													}
+												}
+											}
+											String idSTR="";
+											for(int x=0;x<identifierSTR.size();x++)
+											{
+												if(idSTR.equals(""))
+												{
+													idSTR = identifierSTR.get(x);
+												}
+												else
+												{
+													idSTR = idSTR+";"+identifierSTR.get(x);
+												}
+											}
+											AnnoInfons.put("NCBI Gene", idSTR);
+											
+											String pidSTR="";
+											for(int x=0;x<ProteinidSTR.size();x++)
+											{
+												if(pidSTR.equals(""))
+												{
+													pidSTR = ProteinidSTR.get(x);
+												}
+												else
+												{
+													pidSTR = pidSTR+";"+ProteinidSTR.get(x);
+												}
+											}
+											if(!pidSTR.equals(""))
+											{
+												AnnoInfons.put("UniProt", pidSTR);
+											}
+											
+											String hidSTR="";
+											for(int x=0;x<HomoidSTR.size();x++)
+											{
+												if(hidSTR.equals(""))
+												{
+													hidSTR = HomoidSTR.get(x);
+												}
+												else
+												{
+													hidSTR = hidSTR+";"+HomoidSTR.get(x);
+												}
+											}
+											if(!hidSTR.equals(""))
+											{
+												AnnoInfons.put("NCBI Homologene", hidSTR);
+											}
+										}
+										//else
+										//{
+										//	AnnoInfons.put("Identifier", identifier);
+										//}
+									}
+								}
+								else if (type.matches("(Species|Genus|Strain)"))
+								{
+									AnnoInfons.put("type", type);
+									AnnoInfons.put("NCBI Taxonomy", identifier);
+								}
+								else if (type.matches("Cell"))
+								{
+									AnnoInfons.put("type", "CellLine");
+									AnnoInfons.put("NCBI Taxonomy", identifier);
+								}
+								else
+								{
+									AnnoInfons.put("Identifier", identifier);
+								}
+							}
+							else
+							{
+								AnnoInfons.put("Identifier", identifier);
+							}
+						}
+						biocAnnotation.setInfons(AnnoInfons);
+						BioCLocation location = new BioCLocation();
+						location.setOffset(start+passage_Offset);
+						location.setLength(last-start);
+						biocAnnotation.setLocation(location);
+						biocAnnotation.setText(mention);
+						biocAnnotation.setID(""+annotation_count);
+						annotation_count++;
+						if(Final == true)
+						{
+							if(AnnoInfons.containsKey("Identifier") || AnnoInfons.containsKey("NCBI Homologene") || AnnoInfons.containsKey("NCBI Gene") || AnnoInfons.containsKey("NCBI Taxonomy"))
+							{
+								passage_output.addAnnotation(biocAnnotation);
+							}
+						}
+						else
+						{
+							passage_output.addAnnotation(biocAnnotation);
+						}
+					}
+				}
+				document_output.addPassage(passage_output);
+				j++;
+			}
+			biocCollection_output.addDocument(document_output);
+			BioCOutputFormat.writeDocument(document_output);
+			i++;
+		}
+		BioCOutputFormat.close();
+	}	
 }

src_Java/GNormPluslib/GN.java

@@ -1,1084 +1,1084 @@
-/**
- * Project: GNormPlus
- * Function: Gene Normalization
- */
-
-package GNormPluslib;
-
-import bioc.BioCAnnotation;
-import bioc.BioCCollection;
-import bioc.BioCDocument;
-import bioc.BioCLocation;
-import bioc.BioCPassage;
-
-import bioc.io.BioCDocumentWriter;
-import bioc.io.BioCFactory;
-import bioc.io.woodstox.ConnectorWoodstox;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.text.BreakIterator;
-import java.time.LocalDate;
-import java.time.ZoneId;
-import java.text.DecimalFormat;
-import java.math.RoundingMode;
-
-import javax.xml.stream.XMLStreamException;
-
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-
-public class GN 
-{
-	public static HashMap<String, String> MatchedTokens_hash = new HashMap<String, String>();
-	private double ScoringFunction(String geneid,HashMap<String,String> Mention_hash,String LF)
-	{
-		/*
-		 * define gene/homo id
-		 */
-		
-		//LF
-		LF = LF.toLowerCase();
-		LF = LF.replaceAll("([0-9])([a-z])", "$1 $2");
-		LF = LF.replaceAll("([a-z])([0-9])", "$1 $2");
-		LF = LF.replaceAll("([\\W\\-\\_])", " ");
-		LF = LF.replaceAll("[ ]+", " ");
-		String LF_tkn[]=LF.split(" ");
-		int LF_ParticalMatch = 0;
-		
-		Pattern ptmp = Pattern.compile("[0-9]+\\-([0-9]+)");
-		Matcher mtmp = ptmp.matcher(geneid);
-		Pattern ptmp2 = Pattern.compile("([0-9]+)");
-		Matcher mtmp2 = ptmp.matcher(geneid);
-		if(mtmp.find())
-		{
-			geneid = "Homo:"+mtmp.group(1);
-		}
-		else
-		{
-			geneid = "Gene:"+geneid;
-		}
-		
-		if(GNormPlus.GeneScoring_hash.containsKey(geneid))
-		{
-			HashMap<String,Double> TF = new HashMap<String,Double>(); // token i in gene j
-			HashMap<String,Double> TermFrequency = new HashMap<String,Double>();
-			
-			/*
-			 * Tokens in Query (Gene id lexicon)
-			 */
-			String l[]=GNormPlus.GeneScoring_hash.get(geneid).split("\t"); // Gene:2664293	cmk-1,cytidylate-1,kinase-1,mssa-1	0.4096	4	0.0625	1	2.0
-			String tkns_Gene[] = l[0].split(",");
-			for(int i=0;i<tkns_Gene.length;i++)
-			{
-				String Tkn_Freq[] = tkns_Gene[i].split("-");
-				TermFrequency.put(Tkn_Freq[0], Double.parseDouble(Tkn_Freq[1]));
-			}
-			Double Cj =  Double.parseDouble(l[1]);
-			Double AllTknNum = Double.parseDouble(l[2]);
-			//Double Cj_max =  Double.parseDouble(l[3]);
-			//Double MaxTknNum = Double.parseDouble(l[4]);
-			Double Norm = Double.parseDouble(l[5]);
-			if(Norm == 0.0){Norm=1.0;}
-			
-			/*
-			 * Tokens in Document (recognized mentions)
-			 */
-			for(String Mention : Mention_hash.keySet())
-			{
-				Mention = Mention.toLowerCase();
-				Mention = Mention.replaceAll("([0-9])([a-z])", "$1 $2");
-				Mention = Mention.replaceAll("([a-z])([0-9])", "$1 $2");
-				Mention = Mention.replaceAll("([\\W\\-\\_])", " ");
-				Mention = Mention.replaceAll("[ ]+", " ");
-				String tkns_Mention[]=Mention.split(" ");
-				for(int i=0;i<tkns_Mention.length;i++)
-				{
-					if(TermFrequency.containsKey(tkns_Mention[i]))
-					{
-						TF.put(tkns_Mention[i], TermFrequency.get(tkns_Mention[i]));
-					}
-				}
-			}
-			
-			Double score=0.0;
-			for(String Tkn : TF.keySet())
-			{
-				//LF
-				for(int t=0;t<LF_tkn.length;t++)
-				{
-					if(LF_tkn[t].equals(Tkn))
-					{
-						LF_ParticalMatch++;
-					}
-				}
-				
-				double TFij = TF.get(Tkn)/AllTknNum;
-				double IDFi=GNormPlus.GeneScoringDF_hash.get(Tkn);
-				score=score+TFij*IDFi*(1/(1-TFij));
-			}
-			//score = Cj * (1/Norm) *score;
-			if(LF_ParticalMatch>0){score = score + LF_ParticalMatch;/*System.out.println(geneid+"\t"+LF+"\t"+score);*/}
-			return score;
-		}
-		else
-		{
-			//System.out.println("Error: cannot find geneid: "+geneid+" in GeneScoring_hash");
-			return 0.0;
-		}
-	}
-	
-	public void PreProcessing4GN(String Filename,String FilenameBioC) throws IOException, XMLStreamException
-	{
-		for (int i = 0; i < GNormPlus.BioCDocobj.Annotations.size(); i++) 
-		{
-			for (int j = 0; j < GNormPlus.BioCDocobj.Annotations.get(i).size(); j++) 
-			{
-				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++)
-				{
-					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
-    				String start=anno[0];
-					String last=anno[1];
-					String mentions=anno[2];
-					String type=anno[3];
-					String id="";
-					if(anno.length>=5)
-					{
-						id=anno[4];	
-					}
-					
-					if(type.equals("Gene"))
-					{
-						String mentionArr[] = mentions.split("\\|");
-						boolean update=false;
-						for(int m=0;m<mentionArr.length;m++)
-						{
-							Pattern ptmp = Pattern.compile("^(.*[0-9A-Z])[ ]*p$");
-							Matcher mtmp = ptmp.matcher(mentionArr[m]);
-							Pattern ptmp2 = Pattern.compile("^(.+)nu$");
-							Matcher mtmp2 = ptmp2.matcher(mentionArr[m]);
-							Pattern ptmp3 = Pattern.compile("^(.*)alpha(.*)$");
-							Matcher mtmp3 = ptmp3.matcher(mentionArr[m]);
-							Pattern ptmp4 = Pattern.compile("^(.*)beta(.*)$");
-							Matcher mtmp4 = ptmp4.matcher(mentionArr[m]);
-							Pattern ptmp5 = Pattern.compile("^(.+[0-9])a$");
-							Matcher mtmp5 = ptmp5.matcher(mentionArr[m]);
-							Pattern ptmp6 = Pattern.compile("^(.+[0-9])b$");
-							Matcher mtmp6 = ptmp6.matcher(mentionArr[m]);
-							Pattern ptmp7 = Pattern.compile("^(.+)II([a-z])$");
-							Matcher mtmp7 = ptmp7.matcher(mentionArr[m]);
-							Pattern ptmp8 = Pattern.compile("^(.+)III([a-z])$");
-							Matcher mtmp8 = ptmp8.matcher(mentionArr[m]);
-							if(mtmp.find())
-							{
-								mentions=mentions+"|"+mtmp.group(1);
-								update=true;
-							}
-							if(mtmp2.find())
-							{
-								mentions=mentions+"|"+mtmp2.group(1);
-								update=true;
-							}
-							if(mtmp3.find())
-							{
-								mentions=mentions+"|"+mtmp3.group(1)+"a"+mtmp3.group(2);
-								update=true;
-							}
-							if(mtmp4.find())
-							{
-								mentions=mentions+"|"+mtmp4.group(1)+"b"+mtmp4.group(2);
-								update=true;
-							}
-							if(mtmp5.find())
-							{
-								mentions=mentions+"|"+mtmp5.group(1)+"alpha";
-								update=true;
-							}
-							if(mtmp6.find())
-							{
-								mentions=mentions+"|"+mtmp6.group(1)+"beta";
-								update=true;
-							}
-							if(mtmp7.find())
-							{
-								mentions=mentions+"|"+mtmp7.group(1)+"2"+mtmp7.group(2);
-								update=true;
-							}
-							if(mtmp8.find())
-							{
-								mentions=mentions+"|"+mtmp8.group(1)+"3"+mtmp8.group(2);
-								update=true;
-							}
-						}
-						if(update == true)
-						{
-							GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, start + "\t" + last + "\t" + mentions + "\t" + type + "\t" + id );
-						}
-					}
-				}
-			}
-		}
-		//GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,false,true);
-	}
-	
-	public void ChromosomeRecognition(String Filename,String FilenameBioC) throws IOException, XMLStreamException
-	{
-		for (int i = 0; i < GNormPlus.BioCDocobj.PMIDs.size(); i++) /** PMIDs : i */
-		{
-			String Pmid = GNormPlus.BioCDocobj.PMIDs.get(i);
-			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
-			{
-				String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j); // Passage context
-				
-				/** Chromosome recognition */
-				ArrayList<String> locations = GNormPlus.PT_GeneChromosome.SearchMentionLocation(PassageContext,"ChromosomeLocation");
-				for (int k = 0 ; k < locations.size() ; k++)
-				{
-					String anno[]=locations.get(k).split("\t");
-					//int start= Integer.parseInt(anno[0]);
-	        		//int last= Integer.parseInt(anno[1]);
-	        		//String mention = anno[2];
-	        		String ids = anno[3];
-	        		//GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tChromosomeLocation\t"+ids); //paragraph
-	        		String IDs[] = ids.split("[\\|,]");
-	        		for(int idcount=0;idcount<IDs.length;idcount++)
-	        		{
-	        			//IDs[idcount] = IDs[idcount].replaceAll("\\-[0-9]+", "");
-	        			GNormPlus.Pmid2ChromosomeGene_hash.put(Pmid+"\t"+IDs[idcount],"");
-	        		}
-				}
-			}
-		}
-		//GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,false,true);
-	}
-	
-	public void GeneNormalization(String Filename,String FilenameBioC,boolean GeneIDMatch) throws IOException, XMLStreamException
-	{
-		final DecimalFormat df = new DecimalFormat("0.####");
-        df.setRoundingMode(RoundingMode.HALF_UP);
-		
-		//Tokenization
-		for (int i = 0; i < GNormPlus.BioCDocobj.Annotations.size(); i++) /** PMIDs : i */
-		{
-			String Pmid = GNormPlus.BioCDocobj.PMIDs.get(i);
-			
-			/** Species */
-			HashMap<String,String> Species_hash = new HashMap<String,String>();
-			for (int j = 0; j < GNormPlus.BioCDocobj.Annotations.get(i).size(); j++) /** Paragraphs : j */
-			{
-				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) /** Annotation : k */
-				{
-					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
-    				String mentions=anno[2];
-					String type=anno[3];
-					if(type.matches("(Species|Genus|Strain|CellLine|Cell)"))
-					{
-						Species_hash.put(mentions,"");
-					}
-				}
-			}
-			
-			
-			/*
-			 * Collect Gene mentions :
-			 * 
-			 *  GeneMention-taxid	->	"ID" : geneid
-			 *  					->	"type" : "Gene"
-			 *  					->	start1-last1 : ""
-			 *  					->	start2-last2 : ""
-			 *  					->	start3-last3 : ""
-			 */
-
-			String tiabs="";
-			for (int j = 0; j < GNormPlus.BioCDocobj.PassageContexts.get(i).size(); j++) /** Paragraphs : j */
-			{
-				tiabs=tiabs+GNormPlus.BioCDocobj.PassageContexts.get(i).get(j).toLowerCase();
-			}
-			HashMap<String,HashMap<String,String>> GeneMention_hash = new HashMap<String,HashMap<String,String>>();
-			HashMap<String,String> Mention_hash = new HashMap<String,String>();
-			for (int j = 0; j < GNormPlus.BioCDocobj.Annotations.get(i).size(); j++) /** Paragraphs : j */
-			{
-				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) /** Annotation : k */
-				{
-					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
-    				String start=anno[0];
-					String last=anno[1];
-					String mentions=anno[2];
-					String type=anno[3];
-					String taxids="Tax:9606";
-					
-					if(anno.length>=5)
-					{
-						taxids=anno[4];
-					}
-					String mentions_tmp=mentions.toLowerCase();
-					mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
-					mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
-					taxids=taxids.replaceAll("(Focus|Right|Left|Prefix|Tax):","");
-					if(taxids.equals(""))
-					{
-						taxids="9606";
-					}
-					/** Filtering */
-					boolean found_filter = false;
-					if(GNormPlus.Filtering_hash.containsKey(mentions_tmp)) // filtering
-					{
-						found_filter=true;
-					}
-					
-					if(found_filter==false) //abbreviation
-					{
-						for(String f : GNormPlus.Filtering_WithLongForm_hash.keySet())
-						{
-							if( GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).matches(".*[\\t\\|]"+f+"\tGene.*") ||
-								GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).matches(".*\\t"+f+"\\|[^\t]+\tGene.*")
-									)
-							{
-								String lf=GNormPlus.Filtering_WithLongForm_hash.get(f);
-								if(tiabs.matches(".*"+lf+".*"))
-								{
-									found_filter=true;
-									break;
-								}
-							}
-						}
-					}
-					
-					if(found_filter==false)
-					{
-						if( GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).matches(".*[\\t\\|][a-z]\tGene.*") ||
-								GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).matches(".*\\t[a-z]\\|[^\t]+\tGene.*") //32171191	Wuhan's
-									)
-						{
-							found_filter=true;
-			 
-						}
-					}
-					
-					if(found_filter == false)
-					{
-						if(type.matches("Gene"))
-	    				{
-							if(GeneMention_hash.containsKey(mentions+"\t"+taxids))
-	    					{
-	    						GeneMention_hash.get(mentions+"\t"+taxids).put(start+"\t"+last,"");
-	    					}
-	    					else 
-	    					{
-	    						HashMap<String,String> offset_hash = new HashMap<String,String>();
-	    						offset_hash.put(start+"\t"+last,"");
-	    						GeneMention_hash.put(mentions+"\t"+taxids, offset_hash);
-	    						GeneMention_hash.get(mentions+"\t"+taxids).put("type", type);
-	    						Mention_hash.put(mentions,"Gene");
-	    					}
-	    				}
-	    				else if(type.matches("(FamilyName|DomainMotif)"))
-	    				{
-	    					String GMs[]=mentions.split("\\|");
-	    					for(int g=0;g<GMs.length;g++)
-							{
-								String mention = GMs[g];
-								Mention_hash.put(mention,"FamilyDomain");
-							}
-	    				}
-					}
-					
-				}
-			}
-			
-			/*
-			 * Gene id refinement:
-			 *  1. Official name
-			 *  2. only one gene
-			 */
-			HashMap<String,String> GuaranteedGene2ID = new HashMap<String,String>();
-			HashMap<String,String> MultiGene2ID = new HashMap<String,String>();
-			for(String GeneMentionTax : GeneMention_hash.keySet())
-			{
-				String GT[]=GeneMentionTax.split("\\t");
-				String mentions=GT[0];
-				String taxids=GT[1];
-				String GMs[]=mentions.split("\\|");
-				
-				HashMap<String,String> taxids_hash = new HashMap<String,String>();
-				String taxids_arr[]=taxids.split(",");
-				for(int t=0;t<taxids_arr.length;t++)
-				{
-					taxids_hash.put(taxids_arr[t], "");
-				}
-				
-				for(int ms=0;ms<GMs.length;ms++)
-				{
-					String mention = GMs[ms];
-					String IDstr = GNormPlus.PT_Gene.MentionMatch(mention); /** searched by PT_Gene */
-					String IDs[]=IDstr.split("\\|");
-					
-					/*
-					 * printing the ambiguous gene mentions and candidates
-					 */
-					//String IDs_s[]=IDstr.split(",");
-					//if(IDs_s.length>1)
-					//{
-					//	System.out.println(Pmid+"\t"+mention+"\t"+mentions+"\t"+IDstr);
-					//}
-					
-					for(int c=0;c<IDs.length;c++)
-					{
-						String tax2ID[]=IDs[c].split(":"); // tax2ID[0] = taxid ; tax2ID[1] = geneids
-						if(taxids_hash.containsKey(tax2ID[0]))
-						{
-							String geneid=tax2ID[1];
-							String TargetTax=tax2ID[0];
-							GeneMention_hash.get(GeneMentionTax).put("ID", geneid);
-							GeneMention_hash.get(GeneMentionTax).put("TargetTax", TargetTax);
-							break;
-						}
-					}
-					
-					//geneid refinement
-					if(GeneMention_hash.get(GeneMentionTax).containsKey("ID"))
-					{
-						Pattern ptmp = Pattern.compile("\\*([0-9]+(\\-[0-9]+|))");
-						Matcher mtmp = ptmp.matcher(GeneMention_hash.get(GeneMentionTax).get("ID"));
-						
-						if(mtmp.find()) // 1. Official Name
-						{
-							GeneMention_hash.get(GeneMentionTax).put("ID",mtmp.group(1));
-							GuaranteedGene2ID.put(GeneMentionTax,mtmp.group(1));
-						}
-						else if(GeneMention_hash.get(GeneMentionTax).get("ID").matches("[0-9]+(\\-[0-9]+|)")) // 2. only one gene
-						{
-							GuaranteedGene2ID.put(GeneMentionTax,GeneMention_hash.get(GeneMentionTax).get("ID"));
-						}
-						else
-						{
-							String ID[] = GeneMention_hash.get(GeneMentionTax).get("ID").split(",");
-							boolean FoundByChroLoca=false;
-							for(int idcount=0;idcount<ID.length;idcount++)
-							{
-								if(GNormPlus.Pmid2ChromosomeGene_hash.containsKey(Pmid+"\t"+ID[idcount])) // 3. Chromosome location
-								{
-									GuaranteedGene2ID.put(GeneMentionTax,ID[idcount]);
-									FoundByChroLoca=true;
-									break;
-								}
-							}
-							if(FoundByChroLoca == false)
-							{
-								MultiGene2ID.put(GeneMentionTax, GeneMention_hash.get(GeneMentionTax).get("ID"));
-							}
-						}
-					}
-					if(GNormPlus.suffixprefix_orig2modified.containsKey(mention) && (!IDstr.equals("-1")) && (!IDstr.equals("-2")) && (!IDstr.equals("-3")))
-					{
-						break;
-					}
-				}
-			}
-			
-			/*
-			 * Gene id refinement:
-			 *  3. multiple genes but can be inferred by 1. and 2.
-			 */
-			for(String GeneMentionTax_M : MultiGene2ID.keySet())
-			{
-				for(String GeneMentionTax_G : GuaranteedGene2ID.keySet())
-				{
-					String MG[] = MultiGene2ID.get(GeneMentionTax_M).split(",");
-					for(int m=0;m<MG.length;m++)
-					{
-						if(MG[m].equals(GuaranteedGene2ID.get(GeneMentionTax_G)))
-						{
-							GeneMention_hash.get(GeneMentionTax_M).put("ID",MG[m]);
-						}
-					}
-				}
-			}
-			
-			/*
-			 * Gene id refinement:
-			 *  4. FullName -> Abbreviation
-			 */
-			for(String GeneMentionTax : GeneMention_hash.keySet())
-			{
-				String MT[] = GeneMentionTax.split("\\t");
-				if(GNormPlus.PmidLF2Abb_hash.containsKey(Pmid+"\t"+MT[0]))
-				{
-					String GeneMentionTax_Abb = GNormPlus.PmidLF2Abb_hash.get(Pmid+"\t"+MT[0]) + "\t" + MT[1];
-					if(GeneMention_hash.containsKey(GeneMentionTax_Abb) && GeneMention_hash.get(GeneMentionTax).containsKey("ID"))
-					{
-						GeneMention_hash.get(GeneMentionTax_Abb).put("ID", GeneMention_hash.get(GeneMentionTax).get("ID"));
-					}
-				}
-			}
-			
-			/*
-			 * Gene id refinement:
-			 *  5. Ranking by scoring function (inference network)
-			 */
-			for(String GeneMentionTax : GeneMention_hash.keySet())
-			{
-				if(GeneMention_hash.get(GeneMentionTax).containsKey("ID") && GeneMention_hash.get(GeneMentionTax).get("ID").matches(".+,.+"))
-				{
-					String geneids=GeneMention_hash.get(GeneMentionTax).get("ID");
-					String geneid[] = geneids.split(",");
-					
-					String OutputStyle="Top1";
-					if(OutputStyle.equals("Top1"))
-					{
-						//only return the best one
-						double max_score=0.0;
-						String target_geneid="";
-						for(int g=0;g<geneid.length;g++)
-						{
-							String MT[] = GeneMentionTax.split("\\t");
-							String LF="";
-							if(GNormPlus.PmidAbb2LF_hash.containsKey(Pmid+"\t"+MT[0]))
-							{
-								LF = GNormPlus.PmidAbb2LF_hash.get(Pmid+"\t"+MT[0]);
-							}
-							double score = ScoringFunction(geneid[g],Mention_hash,LF);
-							if(score>max_score)
-							{
-								max_score=score;
-								target_geneid=geneid[g];
-							}
-							else if(score == 0.0)
-							{
-								//System.out.println(GeneMentionTax);
-							}
-						}
-						GeneMention_hash.get(GeneMentionTax).put("ID", target_geneid);
-					}
-					else // "All"
-					{
-						//return all geneids
-						String geneSTR="";
-						for(int g=0;g<geneid.length;g++)
-						{
-							String MT[] = GeneMentionTax.split("\\t");
-							String LF="";
-							if(GNormPlus.PmidAbb2LF_hash.containsKey(Pmid+"\t"+MT[0]))
-							{
-								LF = GNormPlus.PmidAbb2LF_hash.get(Pmid+"\t"+MT[0]);
-							}
-							double score = ScoringFunction(geneid[g],Mention_hash,LF);
-							String hoge = df.format(score);
-							score=Double.parseDouble(hoge);
-							
-							if(geneSTR.equals(""))
-							{
-								geneSTR=geneid[g]+"-"+score;
-							}
-							else
-							{
-								geneSTR=geneSTR+","+geneid[g]+"-"+score;
-							}
-						}
-						GeneMention_hash.get(GeneMentionTax).put("ID", geneSTR);
-					}
-				}
-			}
-			
-			/*
-			 * Gene id refinement: - removed (Reason: cause too much False Positive)
-			 *  6. Abbreviation -> FullName
-			 *  
-			 */
-			for(String GeneMentionTax : GeneMention_hash.keySet())
-			{
-				String MT[] = GeneMentionTax.split("\\t");
-				if(GNormPlus.PmidAbb2LF_hash.containsKey(Pmid+"\t"+MT[0]))
-				{
-					String GeneMentionTax_LF = GNormPlus.PmidAbb2LF_hash.get(Pmid+"\t"+MT[0]) + "\t" + MT[1];
-					if(GeneMention_hash.containsKey(GeneMentionTax_LF) && GeneMention_hash.get(GeneMentionTax).containsKey("ID"))
-					{
-						GeneMention_hash.get(GeneMentionTax_LF).put("ID", GeneMention_hash.get(GeneMentionTax).get("ID"));
-					}
-				}
-			}
-			
-			/*
-			 * Gene id refinement:
-			 *  7. The inference network tokens of Abbreviation.ID should contain at least LF tokens
-			 *  8. The short mention should be filtered if not long form support
-			 */
-			ArrayList<String> removeGMT = new ArrayList<String>();
-			for(String GeneMentionTax : GeneMention_hash.keySet())
-			{
-				String GT[]=GeneMentionTax.split("\\t");
-				String mentions=GT[0];
-				String tax=GT[1];
-				if(GeneMention_hash.get(GeneMentionTax).containsKey("type") && GeneMention_hash.get(GeneMentionTax).get("type").equals("Gene") && GeneMention_hash.get(GeneMentionTax).containsKey("ID"))
-				{
-					String type = GeneMention_hash.get(GeneMentionTax).get("type");
-					String id = GeneMention_hash.get(GeneMentionTax).get("ID");
-					String geneid="";
-					Pattern ptmp1 = Pattern.compile("^([0-9]+)\\-([0-9]+)$");
-					Pattern ptmp2 = Pattern.compile("^([0-9]+)$");
-					Matcher mtmp1 = ptmp1.matcher(id);
-					Matcher mtmp2 = ptmp2.matcher(id);
-					//System.out.println(id);
-					if(mtmp1.find())
-					{
-						geneid = "Homo:"+mtmp1.group(2);
-					}
-					else if(mtmp2.find())
-					{
-						geneid = "Gene:"+mtmp2.group(1);
-					}
-					
-					boolean LongFormTknMatch= false;
-					boolean LongFormExist= true;
-					if(GNormPlus.GeneScoring_hash.containsKey(geneid))
-					{
-						if(GNormPlus.PmidAbb2LF_lc_hash.containsKey(Pmid+"\t"+mentions.toLowerCase()))
-						{
-							/*
-							 * token in lexicon : tkn_lexicon
-							 * token in mention : tkn_mention
-							 */
-							String l[]=GNormPlus.GeneScoring_hash.get(geneid).split("\t"); // Gene:2664293	cmk-1,cytidylate-1,kinase-1,mssa-1	0.4096	4	0.0625	1	2.0
-							String tkns_Gene[] = l[0].split(",");
-							ArrayList<String> tkn_lexicon = new ArrayList<String>();
-							for(int ti=0;ti<tkns_Gene.length;ti++)
-							{
-								String Tkn_Freq[] = tkns_Gene[ti].split("-");
-								tkn_lexicon.add(Tkn_Freq[0]);
-							}
-							
-							String LF_lc=GNormPlus.PmidAbb2LF_lc_hash.get(Pmid+"\t"+mentions.toLowerCase());
-							LF_lc = LF_lc.replaceAll("([0-9])([A-Za-z])", "$1 $2");
-							LF_lc = LF_lc.replaceAll("([A-Za-z])([0-9])", "$1 $2");
-							String tkn_mention[] = LF_lc.split("[\\W\\-\\_]");
-							for(int tl=0;tl<tkn_lexicon.size();tl++)
-							{
-								for(int tm=0;tm<tkn_mention.length;tm++)
-								{
-									if(tkn_lexicon.get(tl).equals(tkn_mention[tm]) && (!tkn_mention[tm].matches("[0-9]+")))
-									{
-										LongFormTknMatch = true;
-									}
-								}	
-							}
-						}
-						else{LongFormExist = false;}
-					}
-					else{LongFormTknMatch = true;} // exception
-					
-					if(LongFormTknMatch == false && LongFormExist == true) // 7.
-					{
-						removeGMT.add(GeneMentionTax); //remove short form
-						removeGMT.add(GNormPlus.PmidAbb2LF_hash.get(Pmid+"\t"+mentions)+"\t"+tax); //remove long form
-					}
-					else if(mentions.length()<=2 && LongFormExist == false) // 8.
-					{
-						removeGMT.add(GeneMentionTax);
-					}
-				}
-			}
-			
-			for(int gmti=0;gmti<removeGMT.size();gmti++) // remove
-			{
-				GeneMention_hash.remove(removeGMT.get(gmti));
-			}
-						
-			// Append gene ids
-			for (int j = 0; j < GNormPlus.BioCDocobj.Annotations.get(i).size(); j++) // Paragraphs : j
-			{
-				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) // Annotation : k
-				{
-					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
-    				String start=anno[0];
-					String last=anno[1];
-					String mentions=anno[2];
-					String type=anno[3];
-					String taxid_org="Tax:9606";
-					if(anno.length>=5)
-					{
-						taxid_org=anno[4];
-					}
-					String taxids=taxid_org.replaceAll("(Focus|Right|Left|Prefix|Tax):","");
-					String GMs[]=mentions.split("\\|");
-					
-					if(GeneMention_hash.containsKey(mentions+"\t"+taxids) && GeneMention_hash.get(mentions+"\t"+taxids).containsKey("TargetTax"))
-					{
-						String taxtype=taxid_org.replaceAll(":([0-9,]+)","");
-						String taxid=GeneMention_hash.get(mentions+"\t"+taxids).get("TargetTax");
-						GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, start+"\t"+last+"\t"+mentions+"\t"+type+"\t"+taxtype+":"+taxid);
-					}
-					
-					if(type.equals("Gene"))
-					{
-						GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k) + "|");
-						
-					
-						if(GeneMention_hash.containsKey(mentions+"\t"+taxids) && GeneMention_hash.get(mentions+"\t"+taxids).containsKey("ID"))
-						{
-							GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k) + GeneMention_hash.get(mentions+"\t"+taxids).get("ID") + "," );
-						}
-						else // cannot find appropriate species
-						{
-							//System.out.println(mention+"\t"+taxid);
-						}
-						GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).substring(0, GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).length()-1)); // remove ",$"
-					}
-				}
-			}
-			
-			//Extend to all gene mentions
-			HashMap<String,String> GeneMentions = new HashMap<String,String>(); // Extending Gene mentions
-			HashMap<String,String> GeneMentionLocation = new HashMap<String,String>(); // Extending Gene mentions
-			for(int j=0;j<GNormPlus.BioCDocobj.Annotations.get(i).size();j++) // Paragraph
-			{
-				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) // Annotation : k
-				{
-					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
-					int start = Integer.parseInt(anno[0]);
-					int last = Integer.parseInt(anno[1]);
-					String mentions=anno[2];
-					String type=anno[3];
-					String id="Tax:9606";
-					if(anno.length>=5)
-					{
-						id=anno[4];
-					}
-					if(type.equals("Gene") && id.matches("(Focus|Right|Left|Prefix|Tax)\\:([0-9]+)\\|([0-9]+)\\-([0-9]+)"))
-					{
-						GeneMentions.put(mentions.toLowerCase(), id);
-						for (int s=start ;s<=last;s++)
-						{
-							GeneMentionLocation.put(j+"\t"+s,"");
-						}
-					}
-					else if(type.equals("Gene") && id.matches("(Focus|Right|Left|Prefix|Tax)\\:([0-9]+)\\|([0-9]+)"))
-					{
-						GeneMentions.put(mentions.toLowerCase(), id);
-						for (int s=start ;s<=last;s++)
-						{
-							GeneMentionLocation.put(j+"\t"+s,"");
-						}
-					}
-				}
-			}
-			for(int j=0;j<GNormPlus.BioCDocobj.Annotations.get(i).size();j++) // Paragraph
-			{
-				if(GNormPlus.BioCDocobj.PassageContexts.size()>i && GNormPlus.BioCDocobj.PassageContexts.get(i).size()>j)
-				{
-					String PassageContexts = " " + GNormPlus.BioCDocobj.PassageContexts.get(i).get(j) + " ";
-					String PassageContexts_tmp = PassageContexts.toLowerCase();
-					for(String gm : GeneMentions.keySet())
-					{
-						String id = GeneMentions.get(gm);
-						if(gm.length()>=3)
-						{
-							gm = gm.replaceAll("[ ]*[\\|]*$", "");
-							gm = gm.replaceAll("^[\\|]*[ ]*", "");
-							gm = gm.replaceAll("[\\|][\\|]+", "\\|");
-							if(!gm.matches("[\\W\\-\\_]*"))
-							{
-								gm = gm.replaceAll("([^A-Za-z0-9\\| ])", "\\\\$1");
-								Pattern ptmp = Pattern.compile("^(.*[\\W\\-\\_])("+gm+")([\\W\\-\\_].*)$");
-								Matcher mtmp = ptmp.matcher(PassageContexts_tmp);
-								while(mtmp.find())
-								{
-									String pre = mtmp.group(1);
-									String gmtmp = mtmp.group(2);
-									String post = mtmp.group(3);
-			
-									int start = pre.length()-1;
-									int last = start+gmtmp.length();
-									if(PassageContexts.length()>=last+1)
-									{
-										String mention = PassageContexts.substring(start+1,last+1);
-										if(!GeneMentionLocation.containsKey(j+"\t"+start) && !GeneMentionLocation.containsKey(j+"\t"+last))
-										{
-											GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tGene\t"+id);
-										}
-									}
-									gmtmp = gmtmp.replaceAll(".", "\\@");
-									PassageContexts_tmp=pre+""+gmtmp+""+post;
-									mtmp = ptmp.matcher(PassageContexts_tmp);
-								}
-							}
-						}
-					}
-				}
-			}
-			
-			//Apply to FamilyNames
-			HashMap<String,String> geneids = new HashMap<String,String>(); // Extending Gene mentions
-			for(int j=0;j<GNormPlus.BioCDocobj.Annotations.get(i).size();j++) // Paragraph
-			{
-				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) // Annotation : k
-				{
-					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
-					String type=anno[3];
-					if(type.equals("Gene"))
-					{
-						String id="Tax:9606";
-						if(anno.length>=5)
-						{
-							id=anno[4];
-						}
-						Pattern ptmp0 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)\\|([0-9]+)$");
-						Matcher mtmp0 = ptmp0.matcher(id);
-						Pattern ptmp1 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)\\|([0-9]+)\\-([0-9]+)$");
-						Matcher mtmp1 = ptmp1.matcher(id);
-						if(mtmp0.find())
-						{
-							geneids.put(mtmp0.group(3), "");
-						}
-						if(mtmp1.find())
-						{
-							geneids.put(mtmp1.group(3), "");
-						}
-					}
-				}
-			}
-			for(int j=0;j<GNormPlus.BioCDocobj.Annotations.get(i).size();j++) // Paragraph
-			{
-				for (int k = GNormPlus.BioCDocobj.Annotations.get(i).get(j).size()-1; k >=0 ; k--) // Annotation : k
-				{
-					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
-					String mention=anno[2];
-					String type=anno[3];
-					if(type.matches("(FamilyName|DomainMotif)"))
-					{
-						String id="Tax:9606";
-						if(anno.length>=5)
-						{
-							id=anno[4];
-						}
-						String IDstrs = GNormPlus.PT_FamilyName.MentionMatch(mention);
-						String IDstr[]=IDstrs.split("\\|");
-						String ids="";
-						for(int id_i=0;id_i<IDstr.length;id_i++)
-						{
-							if(geneids.containsKey(IDstr[id_i]))
-							{
-								 if(ids.equals(""))
-								 {
-									 ids=IDstr[id_i];
-								 }
-								 else
-								 {
-									 ids=ids+";"+IDstr[id_i];
-								 }
-							}
-						}
-						if(!ids.equals(""))
-						{
-							if(type.equals("FamilyName")){type="Gene";}
-							String Annotation_k=anno[0]+"\t"+anno[1]+"\t"+anno[2]+"\t"+type+"\tTax:9606";
-							if(anno.length>=5)
-							{
-								Annotation_k=anno[0]+"\t"+anno[1]+"\t"+anno[2]+"\t"+type+"\t"+anno[4];
-							}
-							GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k,Annotation_k+"|"+ids);
-						}
-						else
-						{
-							GNormPlus.BioCDocobj.Annotations.get(i).get(j).remove(k);
-						}
-					}
-				}
-			}
-			//Species "*" and "(anti)" removed.
-			for(int j=0;j<GNormPlus.BioCDocobj.Annotations.get(i).size();j++) // Paragraph
-			{
-				for (int k = GNormPlus.BioCDocobj.Annotations.get(i).get(j).size()-1; k >=0 ; k--) // Annotation : k
-				{
-					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
-					String type=anno[3];
-					if(type.equals("Species") || type.equals("Genus") || type.equals("Strain") || type.equals("CellLine") || type.equals("Cell"))
-					{
-						String id=anno[4];
-						id=id.replaceAll("\\*", "");
-						id=id.replaceAll("\\(anti\\)", "");
-						String Annotation_k=anno[0]+"\t"+anno[1]+"\t"+anno[2]+"\t"+type+"\t"+id;
-						GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k,Annotation_k);
-					}
-				}
-			}
-			
-			for(int j=0;j<GNormPlus.BioCDocobj.Annotations.get(i).size();j++) // Paragraph
-			{
-				
-				for (int k = GNormPlus.BioCDocobj.Annotations.get(i).get(j).size()-1; k >=0 ; k--) // Annotation : k
-				{
-					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
-					int start = Integer.parseInt(anno[0]);
-					int last = Integer.parseInt(anno[1]);
-					String mention = anno[2];
-					String type = anno[3];
-					String id = anno[4];
-					if(type.equals("Gene") && Species_hash.containsKey(mention))
-					{
-						GNormPlus.BioCDocobj.Annotations.get(i).get(j).remove(k);
-					}
-					else if(type.equals("Gene") && id.equals(""))
-					{
-						GNormPlus.BioCDocobj.Annotations.get(i).get(j).remove(k);
-					}
-					else
-					{
-						for (int k1 = GNormPlus.BioCDocobj.Annotations.get(i).get(j).size()-1; k1 >=0 ; k1--) // Annotation : k
-						{
-							if(k1 != k)
-							{
-								String anno1[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k1).split("\t");
-								int start1 = Integer.parseInt(anno1[0]);
-								int last1 = Integer.parseInt(anno1[1]);
-								if((start1<start && last1>=last) || (start1<=start && last1>last))
-								{
-									GNormPlus.BioCDocobj.Annotations.get(i).get(j).remove(k);
-									break;
-								}
-							}
-						}
-					}
-				}
-			}
-		}
-		if(GeneIDMatch == true)
-		{
-			//GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,false,true);
-		}
-		else
-		{
-			GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,true,true);
-		}
-	}
-	/*
-	 * Search Potential GeneID in the Prefix Tree
-	 */
-	public ArrayList<String> SearchGeneIDLocation(String Doc)
-	{
-		ArrayList<String> location = new ArrayList<String>();
-		
-		String Doc_tmp=" "+Doc+" ";
-		Pattern ptmp = Pattern.compile("^(.*[^A-Za-z0-9]+)([0-9]+\\S*[A-Za-z]+|[A-Za-z]+\\S*[0-9]+|[0-9]+\\S*[A-Za-z]+\\S*[0-9]+|[A-Za-z]+\\S*[0-9]+\\S*[A-Za-z]+)([^A-Za-z0-9]+.*)$");
-		Matcher mtmp = ptmp.matcher(Doc_tmp);
-		while(mtmp.find())
-		{
-			String str1=mtmp.group(1);
-			String str2=mtmp.group(2);
-			String str3=mtmp.group(3);
-			for(int m=str1.length();m<=(str1.length()+str2.length());m++)
-			{
-				int start = str1.length()-1;
-				int last = start+str2.length();
-				String mention = Doc.substring(start, last);
-				if(!mention.matches(".*[\\'\\;\\[\\]\\+\\*\\\\].*"))
-				{
-					if(last-start>6 && (mention.matches(".*\\(.*\\).*") || mention.matches("[^\\(\\)]+")) )
-					{
-						Pattern ptmp1 = Pattern.compile("^(.+[^0-9])([0-9]+)\\-([0-9]+)$");
-						Matcher mtmp1 = ptmp1.matcher(mention);
-						Pattern ptmp2 = Pattern.compile("^(.+[^0-9])([0-9]+)\\-(.+[^0-9])([0-9]+)$");
-						Matcher mtmp2 = ptmp2.matcher(mention);
-						if(mtmp1.find())
-						{
-							String S1 = mtmp1.group(1);
-							if(mtmp1.group(2).length()<=6 && mtmp1.group(3).length()<=6)
-							{
-								int Num1 = Integer.parseInt(mtmp1.group(2));
-								int Num2 = Integer.parseInt(mtmp1.group(3));
-								String prefix = "";
-								Pattern ptmp3 = Pattern.compile("^([0]+)");
-								Matcher mtmp3 = ptmp3.matcher(mtmp1.group(2));
-								if(mtmp3.find())
-								{
-									prefix = mtmp3.group(1);
-								}
-								if(Num2-Num1>0 && (Num2-Num1<=20))
-								{
-									for(int n=Num1;n<=Num2;n++)
-									{
-										String StrNum=S1+prefix+n;
-										if(StrNum.length()>=5)
-										{
-											location.add(start+"\t"+last+"\t"+StrNum+"\tGeneID");
-										}
-									}
-								}
-							}
-						}
-						else if(mtmp2.find())
-						{
-							if(mtmp2.group(2).length()<=6 && mtmp2.group(4).length()<=6)
-							{
-								String S1 = mtmp2.group(1);
-								int Num1 = Integer.parseInt(mtmp2.group(2));
-								String S2 = mtmp2.group(3);
-								int Num2 = Integer.parseInt(mtmp2.group(4));
-								if(S1.equals(S2))
-								{
-									String prefix = "";
-									Pattern ptmp3 = Pattern.compile("^([0]+)");
-									Matcher mtmp3 = ptmp3.matcher(mtmp2.group(2));
-									if(mtmp3.find())
-									{
-										prefix = mtmp3.group(1);
-									}
-									if(Num2-Num1>0 && (Num2-Num1<=20))
-									{
-										for(int n=Num1;n<=Num2;n++)
-										{
-											String StrNum=S1+prefix+n;
-											if(StrNum.length()>=5)
-											{
-												location.add(start+"\t"+last+"\t"+StrNum+"\tGeneID");
-											}
-										}
-									}
-								}
-							}
-						}
-					}
-					location.add(start+"\t"+last+"\t"+mention+"\tGeneID");
-				}
-			}
-			String men="";
-			for(int m=0;m<str2.length();m++){men=men+"@";}
-			Doc_tmp=str1+men+str3;
-			mtmp = ptmp.matcher(Doc_tmp);
-		}
-		return location;
-	}
-	public void GeneIDRecognition(String Filename,String FilenameBioC) throws IOException, XMLStreamException
-	{
-		for (int i = 0; i < GNormPlus.BioCDocobj.PMIDs.size(); i++) /** PMIDs : i */
-		{
-			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
-			{
-				String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j); // Passage context
-				/** GeneID recognition by pattern match */
-				ArrayList<String> locations = SearchGeneIDLocation(PassageContext);
-				for (int k = 0 ; k < locations.size() ; k++)
-				{
-					String anno[]=locations.get(k).split("\t");
-					String mention = anno[2].toLowerCase();
-	        		mention = mention.replaceAll("[\\W\\-\\_]+", "");
-	        		if(GNormPlus.GeneIDs_hash.containsKey(mention))
-	        		{
-	        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(locations.get(k)+"\tGeneID:"+GNormPlus.GeneIDs_hash.get(mention)); //paragraph
-	        		}
-				}
-			}
-		}
-		GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,true,true);
-	}
+/**
+ * Project: GNormPlus
+ * Function: Gene Normalization
+ */
+
+package GNormPluslib;
+
+import bioc.BioCAnnotation;
+import bioc.BioCCollection;
+import bioc.BioCDocument;
+import bioc.BioCLocation;
+import bioc.BioCPassage;
+
+import bioc.io.BioCDocumentWriter;
+import bioc.io.BioCFactory;
+import bioc.io.woodstox.ConnectorWoodstox;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.text.BreakIterator;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.text.DecimalFormat;
+import java.math.RoundingMode;
+
+import javax.xml.stream.XMLStreamException;
+
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+
+public class GN 
+{
+	public static HashMap<String, String> MatchedTokens_hash = new HashMap<String, String>();
+	private double ScoringFunction(String geneid,HashMap<String,String> Mention_hash,String LF)
+	{
+		/*
+		 * define gene/homo id
+		 */
+		
+		//LF
+		LF = LF.toLowerCase();
+		LF = LF.replaceAll("([0-9])([a-z])", "$1 $2");
+		LF = LF.replaceAll("([a-z])([0-9])", "$1 $2");
+		LF = LF.replaceAll("([\\W\\-\\_])", " ");
+		LF = LF.replaceAll("[ ]+", " ");
+		String LF_tkn[]=LF.split(" ");
+		int LF_ParticalMatch = 0;
+		
+		Pattern ptmp = Pattern.compile("[0-9]+\\-([0-9]+)");
+		Matcher mtmp = ptmp.matcher(geneid);
+		Pattern ptmp2 = Pattern.compile("([0-9]+)");
+		Matcher mtmp2 = ptmp.matcher(geneid);
+		if(mtmp.find())
+		{
+			geneid = "Homo:"+mtmp.group(1);
+		}
+		else
+		{
+			geneid = "Gene:"+geneid;
+		}
+		
+		if(GNormPlus.GeneScoring_hash.containsKey(geneid))
+		{
+			HashMap<String,Double> TF = new HashMap<String,Double>(); // token i in gene j
+			HashMap<String,Double> TermFrequency = new HashMap<String,Double>();
+			
+			/*
+			 * Tokens in Query (Gene id lexicon)
+			 */
+			String l[]=GNormPlus.GeneScoring_hash.get(geneid).split("\t"); // Gene:2664293	cmk-1,cytidylate-1,kinase-1,mssa-1	0.4096	4	0.0625	1	2.0
+			String tkns_Gene[] = l[0].split(",");
+			for(int i=0;i<tkns_Gene.length;i++)
+			{
+				String Tkn_Freq[] = tkns_Gene[i].split("-");
+				TermFrequency.put(Tkn_Freq[0], Double.parseDouble(Tkn_Freq[1]));
+			}
+			Double Cj =  Double.parseDouble(l[1]);
+			Double AllTknNum = Double.parseDouble(l[2]);
+			//Double Cj_max =  Double.parseDouble(l[3]);
+			//Double MaxTknNum = Double.parseDouble(l[4]);
+			Double Norm = Double.parseDouble(l[5]);
+			if(Norm == 0.0){Norm=1.0;}
+			
+			/*
+			 * Tokens in Document (recognized mentions)
+			 */
+			for(String Mention : Mention_hash.keySet())
+			{
+				Mention = Mention.toLowerCase();
+				Mention = Mention.replaceAll("([0-9])([a-z])", "$1 $2");
+				Mention = Mention.replaceAll("([a-z])([0-9])", "$1 $2");
+				Mention = Mention.replaceAll("([\\W\\-\\_])", " ");
+				Mention = Mention.replaceAll("[ ]+", " ");
+				String tkns_Mention[]=Mention.split(" ");
+				for(int i=0;i<tkns_Mention.length;i++)
+				{
+					if(TermFrequency.containsKey(tkns_Mention[i]))
+					{
+						TF.put(tkns_Mention[i], TermFrequency.get(tkns_Mention[i]));
+					}
+				}
+			}
+			
+			Double score=0.0;
+			for(String Tkn : TF.keySet())
+			{
+				//LF
+				for(int t=0;t<LF_tkn.length;t++)
+				{
+					if(LF_tkn[t].equals(Tkn))
+					{
+						LF_ParticalMatch++;
+					}
+				}
+				
+				double TFij = TF.get(Tkn)/AllTknNum;
+				double IDFi=GNormPlus.GeneScoringDF_hash.get(Tkn);
+				score=score+TFij*IDFi*(1/(1-TFij));
+			}
+			//score = Cj * (1/Norm) *score;
+			if(LF_ParticalMatch>0){score = score + LF_ParticalMatch;/*System.out.println(geneid+"\t"+LF+"\t"+score);*/}
+			return score;
+		}
+		else
+		{
+			//System.out.println("Error: cannot find geneid: "+geneid+" in GeneScoring_hash");
+			return 0.0;
+		}
+	}
+	
+	public void PreProcessing4GN(String Filename,String FilenameBioC) throws IOException, XMLStreamException
+	{
+		for (int i = 0; i < GNormPlus.BioCDocobj.Annotations.size(); i++) 
+		{
+			for (int j = 0; j < GNormPlus.BioCDocobj.Annotations.get(i).size(); j++) 
+			{
+				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++)
+				{
+					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
+    				String start=anno[0];
+					String last=anno[1];
+					String mentions=anno[2];
+					String type=anno[3];
+					String id="";
+					if(anno.length>=5)
+					{
+						id=anno[4];	
+					}
+					
+					if(type.equals("Gene"))
+					{
+						String mentionArr[] = mentions.split("\\|");
+						boolean update=false;
+						for(int m=0;m<mentionArr.length;m++)
+						{
+							Pattern ptmp = Pattern.compile("^(.*[0-9A-Z])[ ]*p$");
+							Matcher mtmp = ptmp.matcher(mentionArr[m]);
+							Pattern ptmp2 = Pattern.compile("^(.+)nu$");
+							Matcher mtmp2 = ptmp2.matcher(mentionArr[m]);
+							Pattern ptmp3 = Pattern.compile("^(.*)alpha(.*)$");
+							Matcher mtmp3 = ptmp3.matcher(mentionArr[m]);
+							Pattern ptmp4 = Pattern.compile("^(.*)beta(.*)$");
+							Matcher mtmp4 = ptmp4.matcher(mentionArr[m]);
+							Pattern ptmp5 = Pattern.compile("^(.+[0-9])a$");
+							Matcher mtmp5 = ptmp5.matcher(mentionArr[m]);
+							Pattern ptmp6 = Pattern.compile("^(.+[0-9])b$");
+							Matcher mtmp6 = ptmp6.matcher(mentionArr[m]);
+							Pattern ptmp7 = Pattern.compile("^(.+)II([a-z])$");
+							Matcher mtmp7 = ptmp7.matcher(mentionArr[m]);
+							Pattern ptmp8 = Pattern.compile("^(.+)III([a-z])$");
+							Matcher mtmp8 = ptmp8.matcher(mentionArr[m]);
+							if(mtmp.find())
+							{
+								mentions=mentions+"|"+mtmp.group(1);
+								update=true;
+							}
+							if(mtmp2.find())
+							{
+								mentions=mentions+"|"+mtmp2.group(1);
+								update=true;
+							}
+							if(mtmp3.find())
+							{
+								mentions=mentions+"|"+mtmp3.group(1)+"a"+mtmp3.group(2);
+								update=true;
+							}
+							if(mtmp4.find())
+							{
+								mentions=mentions+"|"+mtmp4.group(1)+"b"+mtmp4.group(2);
+								update=true;
+							}
+							if(mtmp5.find())
+							{
+								mentions=mentions+"|"+mtmp5.group(1)+"alpha";
+								update=true;
+							}
+							if(mtmp6.find())
+							{
+								mentions=mentions+"|"+mtmp6.group(1)+"beta";
+								update=true;
+							}
+							if(mtmp7.find())
+							{
+								mentions=mentions+"|"+mtmp7.group(1)+"2"+mtmp7.group(2);
+								update=true;
+							}
+							if(mtmp8.find())
+							{
+								mentions=mentions+"|"+mtmp8.group(1)+"3"+mtmp8.group(2);
+								update=true;
+							}
+						}
+						if(update == true)
+						{
+							GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, start + "\t" + last + "\t" + mentions + "\t" + type + "\t" + id );
+						}
+					}
+				}
+			}
+		}
+		//GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,false,true);
+	}
+	
+	public void ChromosomeRecognition(String Filename,String FilenameBioC) throws IOException, XMLStreamException
+	{
+		for (int i = 0; i < GNormPlus.BioCDocobj.PMIDs.size(); i++) /** PMIDs : i */
+		{
+			String Pmid = GNormPlus.BioCDocobj.PMIDs.get(i);
+			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
+			{
+				String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j); // Passage context
+				
+				/** Chromosome recognition */
+				ArrayList<String> locations = GNormPlus.PT_GeneChromosome.SearchMentionLocation(PassageContext,"ChromosomeLocation");
+				for (int k = 0 ; k < locations.size() ; k++)
+				{
+					String anno[]=locations.get(k).split("\t");
+					//int start= Integer.parseInt(anno[0]);
+	        		//int last= Integer.parseInt(anno[1]);
+	        		//String mention = anno[2];
+	        		String ids = anno[3];
+	        		//GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tChromosomeLocation\t"+ids); //paragraph
+	        		String IDs[] = ids.split("[\\|,]");
+	        		for(int idcount=0;idcount<IDs.length;idcount++)
+	        		{
+	        			//IDs[idcount] = IDs[idcount].replaceAll("\\-[0-9]+", "");
+	        			GNormPlus.Pmid2ChromosomeGene_hash.put(Pmid+"\t"+IDs[idcount],"");
+	        		}
+				}
+			}
+		}
+		//GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,false,true);
+	}
+	
+	public void GeneNormalization(String Filename,String FilenameBioC,boolean GeneIDMatch) throws IOException, XMLStreamException
+	{
+		final DecimalFormat df = new DecimalFormat("0.####");
+        df.setRoundingMode(RoundingMode.HALF_UP);
+		
+		//Tokenization
+		for (int i = 0; i < GNormPlus.BioCDocobj.Annotations.size(); i++) /** PMIDs : i */
+		{
+			String Pmid = GNormPlus.BioCDocobj.PMIDs.get(i);
+			
+			/** Species */
+			HashMap<String,String> Species_hash = new HashMap<String,String>();
+			for (int j = 0; j < GNormPlus.BioCDocobj.Annotations.get(i).size(); j++) /** Paragraphs : j */
+			{
+				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) /** Annotation : k */
+				{
+					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
+    				String mentions=anno[2];
+					String type=anno[3];
+					if(type.matches("(Species|Genus|Strain|CellLine|Cell)"))
+					{
+						Species_hash.put(mentions,"");
+					}
+				}
+			}
+			
+			
+			/*
+			 * Collect Gene mentions :
+			 * 
+			 *  GeneMention-taxid	->	"ID" : geneid
+			 *  					->	"type" : "Gene"
+			 *  					->	start1-last1 : ""
+			 *  					->	start2-last2 : ""
+			 *  					->	start3-last3 : ""
+			 */
+
+			String tiabs="";
+			for (int j = 0; j < GNormPlus.BioCDocobj.PassageContexts.get(i).size(); j++) /** Paragraphs : j */
+			{
+				tiabs=tiabs+GNormPlus.BioCDocobj.PassageContexts.get(i).get(j).toLowerCase();
+			}
+			HashMap<String,HashMap<String,String>> GeneMention_hash = new HashMap<String,HashMap<String,String>>();
+			HashMap<String,String> Mention_hash = new HashMap<String,String>();
+			for (int j = 0; j < GNormPlus.BioCDocobj.Annotations.get(i).size(); j++) /** Paragraphs : j */
+			{
+				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) /** Annotation : k */
+				{
+					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
+    				String start=anno[0];
+					String last=anno[1];
+					String mentions=anno[2];
+					String type=anno[3];
+					String taxids="Tax:9606";
+					
+					if(anno.length>=5)
+					{
+						taxids=anno[4];
+					}
+					String mentions_tmp=mentions.toLowerCase();
+					mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
+					mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
+					taxids=taxids.replaceAll("(Focus|Right|Left|Prefix|Tax):","");
+					if(taxids.equals(""))
+					{
+						taxids="9606";
+					}
+					/** Filtering */
+					boolean found_filter = false;
+					if(GNormPlus.Filtering_hash.containsKey(mentions_tmp)) // filtering
+					{
+						found_filter=true;
+					}
+					
+					if(found_filter==false) //abbreviation
+					{
+						for(String f : GNormPlus.Filtering_WithLongForm_hash.keySet())
+						{
+							if( GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).matches(".*[\\t\\|]"+f+"\tGene.*") ||
+								GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).matches(".*\\t"+f+"\\|[^\t]+\tGene.*")
+									)
+							{
+								String lf=GNormPlus.Filtering_WithLongForm_hash.get(f);
+								if(tiabs.matches(".*"+lf+".*"))
+								{
+									found_filter=true;
+									break;
+								}
+							}
+						}
+					}
+					
+					if(found_filter==false)
+					{
+						if( GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).matches(".*[\\t\\|][a-z]\tGene.*") ||
+								GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).matches(".*\\t[a-z]\\|[^\t]+\tGene.*") //32171191	Wuhan's
+									)
+						{
+							found_filter=true;
+			 
+						}
+					}
+					
+					if(found_filter == false)
+					{
+						if(type.matches("Gene"))
+	    				{
+							if(GeneMention_hash.containsKey(mentions+"\t"+taxids))
+	    					{
+	    						GeneMention_hash.get(mentions+"\t"+taxids).put(start+"\t"+last,"");
+	    					}
+	    					else 
+	    					{
+	    						HashMap<String,String> offset_hash = new HashMap<String,String>();
+	    						offset_hash.put(start+"\t"+last,"");
+	    						GeneMention_hash.put(mentions+"\t"+taxids, offset_hash);
+	    						GeneMention_hash.get(mentions+"\t"+taxids).put("type", type);
+	    						Mention_hash.put(mentions,"Gene");
+	    					}
+	    				}
+	    				else if(type.matches("(FamilyName|DomainMotif)"))
+	    				{
+	    					String GMs[]=mentions.split("\\|");
+	    					for(int g=0;g<GMs.length;g++)
+							{
+								String mention = GMs[g];
+								Mention_hash.put(mention,"FamilyDomain");
+							}
+	    				}
+					}
+					
+				}
+			}
+			
+			/*
+			 * Gene id refinement:
+			 *  1. Official name
+			 *  2. only one gene
+			 */
+			HashMap<String,String> GuaranteedGene2ID = new HashMap<String,String>();
+			HashMap<String,String> MultiGene2ID = new HashMap<String,String>();
+			for(String GeneMentionTax : GeneMention_hash.keySet())
+			{
+				String GT[]=GeneMentionTax.split("\\t");
+				String mentions=GT[0];
+				String taxids=GT[1];
+				String GMs[]=mentions.split("\\|");
+				
+				HashMap<String,String> taxids_hash = new HashMap<String,String>();
+				String taxids_arr[]=taxids.split(",");
+				for(int t=0;t<taxids_arr.length;t++)
+				{
+					taxids_hash.put(taxids_arr[t], "");
+				}
+				
+				for(int ms=0;ms<GMs.length;ms++)
+				{
+					String mention = GMs[ms];
+					String IDstr = GNormPlus.PT_Gene.MentionMatch(mention); /** searched by PT_Gene */
+					String IDs[]=IDstr.split("\\|");
+					
+					/*
+					 * printing the ambiguous gene mentions and candidates
+					 */
+					//String IDs_s[]=IDstr.split(",");
+					//if(IDs_s.length>1)
+					//{
+					//	System.out.println(Pmid+"\t"+mention+"\t"+mentions+"\t"+IDstr);
+					//}
+					
+					for(int c=0;c<IDs.length;c++)
+					{
+						String tax2ID[]=IDs[c].split(":"); // tax2ID[0] = taxid ; tax2ID[1] = geneids
+						if(taxids_hash.containsKey(tax2ID[0]))
+						{
+							String geneid=tax2ID[1];
+							String TargetTax=tax2ID[0];
+							GeneMention_hash.get(GeneMentionTax).put("ID", geneid);
+							GeneMention_hash.get(GeneMentionTax).put("TargetTax", TargetTax);
+							break;
+						}
+					}
+					
+					//geneid refinement
+					if(GeneMention_hash.get(GeneMentionTax).containsKey("ID"))
+					{
+						Pattern ptmp = Pattern.compile("\\*([0-9]+(\\-[0-9]+|))");
+						Matcher mtmp = ptmp.matcher(GeneMention_hash.get(GeneMentionTax).get("ID"));
+						
+						if(mtmp.find()) // 1. Official Name
+						{
+							GeneMention_hash.get(GeneMentionTax).put("ID",mtmp.group(1));
+							GuaranteedGene2ID.put(GeneMentionTax,mtmp.group(1));
+						}
+						else if(GeneMention_hash.get(GeneMentionTax).get("ID").matches("[0-9]+(\\-[0-9]+|)")) // 2. only one gene
+						{
+							GuaranteedGene2ID.put(GeneMentionTax,GeneMention_hash.get(GeneMentionTax).get("ID"));
+						}
+						else
+						{
+							String ID[] = GeneMention_hash.get(GeneMentionTax).get("ID").split(",");
+							boolean FoundByChroLoca=false;
+							for(int idcount=0;idcount<ID.length;idcount++)
+							{
+								if(GNormPlus.Pmid2ChromosomeGene_hash.containsKey(Pmid+"\t"+ID[idcount])) // 3. Chromosome location
+								{
+									GuaranteedGene2ID.put(GeneMentionTax,ID[idcount]);
+									FoundByChroLoca=true;
+									break;
+								}
+							}
+							if(FoundByChroLoca == false)
+							{
+								MultiGene2ID.put(GeneMentionTax, GeneMention_hash.get(GeneMentionTax).get("ID"));
+							}
+						}
+					}
+					if(GNormPlus.suffixprefix_orig2modified.containsKey(mention) && (!IDstr.equals("-1")) && (!IDstr.equals("-2")) && (!IDstr.equals("-3")))
+					{
+						break;
+					}
+				}
+			}
+			
+			/*
+			 * Gene id refinement:
+			 *  3. multiple genes but can be inferred by 1. and 2.
+			 */
+			for(String GeneMentionTax_M : MultiGene2ID.keySet())
+			{
+				for(String GeneMentionTax_G : GuaranteedGene2ID.keySet())
+				{
+					String MG[] = MultiGene2ID.get(GeneMentionTax_M).split(",");
+					for(int m=0;m<MG.length;m++)
+					{
+						if(MG[m].equals(GuaranteedGene2ID.get(GeneMentionTax_G)))
+						{
+							GeneMention_hash.get(GeneMentionTax_M).put("ID",MG[m]);
+						}
+					}
+				}
+			}
+			
+			/*
+			 * Gene id refinement:
+			 *  4. FullName -> Abbreviation
+			 */
+			for(String GeneMentionTax : GeneMention_hash.keySet())
+			{
+				String MT[] = GeneMentionTax.split("\\t");
+				if(GNormPlus.PmidLF2Abb_hash.containsKey(Pmid+"\t"+MT[0]))
+				{
+					String GeneMentionTax_Abb = GNormPlus.PmidLF2Abb_hash.get(Pmid+"\t"+MT[0]) + "\t" + MT[1];
+					if(GeneMention_hash.containsKey(GeneMentionTax_Abb) && GeneMention_hash.get(GeneMentionTax).containsKey("ID"))
+					{
+						GeneMention_hash.get(GeneMentionTax_Abb).put("ID", GeneMention_hash.get(GeneMentionTax).get("ID"));
+					}
+				}
+			}
+			
+			/*
+			 * Gene id refinement:
+			 *  5. Ranking by scoring function (inference network)
+			 */
+			for(String GeneMentionTax : GeneMention_hash.keySet())
+			{
+				if(GeneMention_hash.get(GeneMentionTax).containsKey("ID") && GeneMention_hash.get(GeneMentionTax).get("ID").matches(".+,.+"))
+				{
+					String geneids=GeneMention_hash.get(GeneMentionTax).get("ID");
+					String geneid[] = geneids.split(",");
+					
+					String OutputStyle="Top1";
+					if(OutputStyle.equals("Top1"))
+					{
+						//only return the best one
+						double max_score=0.0;
+						String target_geneid="";
+						for(int g=0;g<geneid.length;g++)
+						{
+							String MT[] = GeneMentionTax.split("\\t");
+							String LF="";
+							if(GNormPlus.PmidAbb2LF_hash.containsKey(Pmid+"\t"+MT[0]))
+							{
+								LF = GNormPlus.PmidAbb2LF_hash.get(Pmid+"\t"+MT[0]);
+							}
+							double score = ScoringFunction(geneid[g],Mention_hash,LF);
+							if(score>max_score)
+							{
+								max_score=score;
+								target_geneid=geneid[g];
+							}
+							else if(score == 0.0)
+							{
+								//System.out.println(GeneMentionTax);
+							}
+						}
+						GeneMention_hash.get(GeneMentionTax).put("ID", target_geneid);
+					}
+					else // "All"
+					{
+						//return all geneids
+						String geneSTR="";
+						for(int g=0;g<geneid.length;g++)
+						{
+							String MT[] = GeneMentionTax.split("\\t");
+							String LF="";
+							if(GNormPlus.PmidAbb2LF_hash.containsKey(Pmid+"\t"+MT[0]))
+							{
+								LF = GNormPlus.PmidAbb2LF_hash.get(Pmid+"\t"+MT[0]);
+							}
+							double score = ScoringFunction(geneid[g],Mention_hash,LF);
+							String hoge = df.format(score);
+							score=Double.parseDouble(hoge);
+							
+							if(geneSTR.equals(""))
+							{
+								geneSTR=geneid[g]+"-"+score;
+							}
+							else
+							{
+								geneSTR=geneSTR+","+geneid[g]+"-"+score;
+							}
+						}
+						GeneMention_hash.get(GeneMentionTax).put("ID", geneSTR);
+					}
+				}
+			}
+			
+			/*
+			 * Gene id refinement: - removed (Reason: cause too much False Positive)
+			 *  6. Abbreviation -> FullName
+			 *  
+			 */
+			for(String GeneMentionTax : GeneMention_hash.keySet())
+			{
+				String MT[] = GeneMentionTax.split("\\t");
+				if(GNormPlus.PmidAbb2LF_hash.containsKey(Pmid+"\t"+MT[0]))
+				{
+					String GeneMentionTax_LF = GNormPlus.PmidAbb2LF_hash.get(Pmid+"\t"+MT[0]) + "\t" + MT[1];
+					if(GeneMention_hash.containsKey(GeneMentionTax_LF) && GeneMention_hash.get(GeneMentionTax).containsKey("ID"))
+					{
+						GeneMention_hash.get(GeneMentionTax_LF).put("ID", GeneMention_hash.get(GeneMentionTax).get("ID"));
+					}
+				}
+			}
+			
+			/*
+			 * Gene id refinement:
+			 *  7. The inference network tokens of Abbreviation.ID should contain at least LF tokens
+			 *  8. The short mention should be filtered if not long form support
+			 */
+			ArrayList<String> removeGMT = new ArrayList<String>();
+			for(String GeneMentionTax : GeneMention_hash.keySet())
+			{
+				String GT[]=GeneMentionTax.split("\\t");
+				String mentions=GT[0];
+				String tax=GT[1];
+				if(GeneMention_hash.get(GeneMentionTax).containsKey("type") && GeneMention_hash.get(GeneMentionTax).get("type").equals("Gene") && GeneMention_hash.get(GeneMentionTax).containsKey("ID"))
+				{
+					String type = GeneMention_hash.get(GeneMentionTax).get("type");
+					String id = GeneMention_hash.get(GeneMentionTax).get("ID");
+					String geneid="";
+					Pattern ptmp1 = Pattern.compile("^([0-9]+)\\-([0-9]+)$");
+					Pattern ptmp2 = Pattern.compile("^([0-9]+)$");
+					Matcher mtmp1 = ptmp1.matcher(id);
+					Matcher mtmp2 = ptmp2.matcher(id);
+					//System.out.println(id);
+					if(mtmp1.find())
+					{
+						geneid = "Homo:"+mtmp1.group(2);
+					}
+					else if(mtmp2.find())
+					{
+						geneid = "Gene:"+mtmp2.group(1);
+					}
+					
+					boolean LongFormTknMatch= false;
+					boolean LongFormExist= true;
+					if(GNormPlus.GeneScoring_hash.containsKey(geneid))
+					{
+						if(GNormPlus.PmidAbb2LF_lc_hash.containsKey(Pmid+"\t"+mentions.toLowerCase()))
+						{
+							/*
+							 * token in lexicon : tkn_lexicon
+							 * token in mention : tkn_mention
+							 */
+							String l[]=GNormPlus.GeneScoring_hash.get(geneid).split("\t"); // Gene:2664293	cmk-1,cytidylate-1,kinase-1,mssa-1	0.4096	4	0.0625	1	2.0
+							String tkns_Gene[] = l[0].split(",");
+							ArrayList<String> tkn_lexicon = new ArrayList<String>();
+							for(int ti=0;ti<tkns_Gene.length;ti++)
+							{
+								String Tkn_Freq[] = tkns_Gene[ti].split("-");
+								tkn_lexicon.add(Tkn_Freq[0]);
+							}
+							
+							String LF_lc=GNormPlus.PmidAbb2LF_lc_hash.get(Pmid+"\t"+mentions.toLowerCase());
+							LF_lc = LF_lc.replaceAll("([0-9])([A-Za-z])", "$1 $2");
+							LF_lc = LF_lc.replaceAll("([A-Za-z])([0-9])", "$1 $2");
+							String tkn_mention[] = LF_lc.split("[\\W\\-\\_]");
+							for(int tl=0;tl<tkn_lexicon.size();tl++)
+							{
+								for(int tm=0;tm<tkn_mention.length;tm++)
+								{
+									if(tkn_lexicon.get(tl).equals(tkn_mention[tm]) && (!tkn_mention[tm].matches("[0-9]+")))
+									{
+										LongFormTknMatch = true;
+									}
+								}	
+							}
+						}
+						else{LongFormExist = false;}
+					}
+					else{LongFormTknMatch = true;} // exception
+					
+					if(LongFormTknMatch == false && LongFormExist == true) // 7.
+					{
+						removeGMT.add(GeneMentionTax); //remove short form
+						removeGMT.add(GNormPlus.PmidAbb2LF_hash.get(Pmid+"\t"+mentions)+"\t"+tax); //remove long form
+					}
+					else if(mentions.length()<=2 && LongFormExist == false) // 8.
+					{
+						removeGMT.add(GeneMentionTax);
+					}
+				}
+			}
+			
+			for(int gmti=0;gmti<removeGMT.size();gmti++) // remove
+			{
+				GeneMention_hash.remove(removeGMT.get(gmti));
+			}
+						
+			// Append gene ids
+			for (int j = 0; j < GNormPlus.BioCDocobj.Annotations.get(i).size(); j++) // Paragraphs : j
+			{
+				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) // Annotation : k
+				{
+					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
+    				String start=anno[0];
+					String last=anno[1];
+					String mentions=anno[2];
+					String type=anno[3];
+					String taxid_org="Tax:9606";
+					if(anno.length>=5)
+					{
+						taxid_org=anno[4];
+					}
+					String taxids=taxid_org.replaceAll("(Focus|Right|Left|Prefix|Tax):","");
+					String GMs[]=mentions.split("\\|");
+					
+					if(GeneMention_hash.containsKey(mentions+"\t"+taxids) && GeneMention_hash.get(mentions+"\t"+taxids).containsKey("TargetTax"))
+					{
+						String taxtype=taxid_org.replaceAll(":([0-9,]+)","");
+						String taxid=GeneMention_hash.get(mentions+"\t"+taxids).get("TargetTax");
+						GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, start+"\t"+last+"\t"+mentions+"\t"+type+"\t"+taxtype+":"+taxid);
+					}
+					
+					if(type.equals("Gene"))
+					{
+						GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k) + "|");
+						
+					
+						if(GeneMention_hash.containsKey(mentions+"\t"+taxids) && GeneMention_hash.get(mentions+"\t"+taxids).containsKey("ID"))
+						{
+							GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k) + GeneMention_hash.get(mentions+"\t"+taxids).get("ID") + "," );
+						}
+						else // cannot find appropriate species
+						{
+							//System.out.println(mention+"\t"+taxid);
+						}
+						GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).substring(0, GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).length()-1)); // remove ",$"
+					}
+				}
+			}
+			
+			//Extend to all gene mentions
+			HashMap<String,String> GeneMentions = new HashMap<String,String>(); // Extending Gene mentions
+			HashMap<String,String> GeneMentionLocation = new HashMap<String,String>(); // Extending Gene mentions
+			for(int j=0;j<GNormPlus.BioCDocobj.Annotations.get(i).size();j++) // Paragraph
+			{
+				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) // Annotation : k
+				{
+					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
+					int start = Integer.parseInt(anno[0]);
+					int last = Integer.parseInt(anno[1]);
+					String mentions=anno[2];
+					String type=anno[3];
+					String id="Tax:9606";
+					if(anno.length>=5)
+					{
+						id=anno[4];
+					}
+					if(type.equals("Gene") && id.matches("(Focus|Right|Left|Prefix|Tax)\\:([0-9]+)\\|([0-9]+)\\-([0-9]+)"))
+					{
+						GeneMentions.put(mentions.toLowerCase(), id);
+						for (int s=start ;s<=last;s++)
+						{
+							GeneMentionLocation.put(j+"\t"+s,"");
+						}
+					}
+					else if(type.equals("Gene") && id.matches("(Focus|Right|Left|Prefix|Tax)\\:([0-9]+)\\|([0-9]+)"))
+					{
+						GeneMentions.put(mentions.toLowerCase(), id);
+						for (int s=start ;s<=last;s++)
+						{
+							GeneMentionLocation.put(j+"\t"+s,"");
+						}
+					}
+				}
+			}
+			for(int j=0;j<GNormPlus.BioCDocobj.Annotations.get(i).size();j++) // Paragraph
+			{
+				if(GNormPlus.BioCDocobj.PassageContexts.size()>i && GNormPlus.BioCDocobj.PassageContexts.get(i).size()>j)
+				{
+					String PassageContexts = " " + GNormPlus.BioCDocobj.PassageContexts.get(i).get(j) + " ";
+					String PassageContexts_tmp = PassageContexts.toLowerCase();
+					for(String gm : GeneMentions.keySet())
+					{
+						String id = GeneMentions.get(gm);
+						if(gm.length()>=3)
+						{
+							gm = gm.replaceAll("[ ]*[\\|]*$", "");
+							gm = gm.replaceAll("^[\\|]*[ ]*", "");
+							gm = gm.replaceAll("[\\|][\\|]+", "\\|");
+							if(!gm.matches("[\\W\\-\\_]*"))
+							{
+								gm = gm.replaceAll("([^A-Za-z0-9\\| ])", "\\\\$1");
+								Pattern ptmp = Pattern.compile("^(.*[\\W\\-\\_])("+gm+")([\\W\\-\\_].*)$");
+								Matcher mtmp = ptmp.matcher(PassageContexts_tmp);
+								while(mtmp.find())
+								{
+									String pre = mtmp.group(1);
+									String gmtmp = mtmp.group(2);
+									String post = mtmp.group(3);
+			
+									int start = pre.length()-1;
+									int last = start+gmtmp.length();
+									if(PassageContexts.length()>=last+1)
+									{
+										String mention = PassageContexts.substring(start+1,last+1);
+										if(!GeneMentionLocation.containsKey(j+"\t"+start) && !GeneMentionLocation.containsKey(j+"\t"+last))
+										{
+											GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tGene\t"+id);
+										}
+									}
+									gmtmp = gmtmp.replaceAll(".", "\\@");
+									PassageContexts_tmp=pre+""+gmtmp+""+post;
+									mtmp = ptmp.matcher(PassageContexts_tmp);
+								}
+							}
+						}
+					}
+				}
+			}
+			
+			//Apply to FamilyNames
+			HashMap<String,String> geneids = new HashMap<String,String>(); // Extending Gene mentions
+			for(int j=0;j<GNormPlus.BioCDocobj.Annotations.get(i).size();j++) // Paragraph
+			{
+				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) // Annotation : k
+				{
+					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
+					String type=anno[3];
+					if(type.equals("Gene"))
+					{
+						String id="Tax:9606";
+						if(anno.length>=5)
+						{
+							id=anno[4];
+						}
+						Pattern ptmp0 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)\\|([0-9]+)$");
+						Matcher mtmp0 = ptmp0.matcher(id);
+						Pattern ptmp1 = Pattern.compile("^(Focus|Right|Left|Prefix|GeneID|Tax)\\:([0-9]+)\\|([0-9]+)\\-([0-9]+)$");
+						Matcher mtmp1 = ptmp1.matcher(id);
+						if(mtmp0.find())
+						{
+							geneids.put(mtmp0.group(3), "");
+						}
+						if(mtmp1.find())
+						{
+							geneids.put(mtmp1.group(3), "");
+						}
+					}
+				}
+			}
+			for(int j=0;j<GNormPlus.BioCDocobj.Annotations.get(i).size();j++) // Paragraph
+			{
+				for (int k = GNormPlus.BioCDocobj.Annotations.get(i).get(j).size()-1; k >=0 ; k--) // Annotation : k
+				{
+					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
+					String mention=anno[2];
+					String type=anno[3];
+					if(type.matches("(FamilyName|DomainMotif)"))
+					{
+						String id="Tax:9606";
+						if(anno.length>=5)
+						{
+							id=anno[4];
+						}
+						String IDstrs = GNormPlus.PT_FamilyName.MentionMatch(mention);
+						String IDstr[]=IDstrs.split("\\|");
+						String ids="";
+						for(int id_i=0;id_i<IDstr.length;id_i++)
+						{
+							if(geneids.containsKey(IDstr[id_i]))
+							{
+								 if(ids.equals(""))
+								 {
+									 ids=IDstr[id_i];
+								 }
+								 else
+								 {
+									 ids=ids+";"+IDstr[id_i];
+								 }
+							}
+						}
+						if(!ids.equals(""))
+						{
+							if(type.equals("FamilyName")){type="Gene";}
+							String Annotation_k=anno[0]+"\t"+anno[1]+"\t"+anno[2]+"\t"+type+"\tTax:9606";
+							if(anno.length>=5)
+							{
+								Annotation_k=anno[0]+"\t"+anno[1]+"\t"+anno[2]+"\t"+type+"\t"+anno[4];
+							}
+							GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k,Annotation_k+"|"+ids);
+						}
+						else
+						{
+							GNormPlus.BioCDocobj.Annotations.get(i).get(j).remove(k);
+						}
+					}
+				}
+			}
+			//Species "*" and "(anti)" removed.
+			for(int j=0;j<GNormPlus.BioCDocobj.Annotations.get(i).size();j++) // Paragraph
+			{
+				for (int k = GNormPlus.BioCDocobj.Annotations.get(i).get(j).size()-1; k >=0 ; k--) // Annotation : k
+				{
+					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
+					String type=anno[3];
+					if(type.equals("Species") || type.equals("Genus") || type.equals("Strain") || type.equals("CellLine") || type.equals("Cell"))
+					{
+						String id=anno[4];
+						id=id.replaceAll("\\*", "");
+						id=id.replaceAll("\\(anti\\)", "");
+						String Annotation_k=anno[0]+"\t"+anno[1]+"\t"+anno[2]+"\t"+type+"\t"+id;
+						GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k,Annotation_k);
+					}
+				}
+			}
+			
+			for(int j=0;j<GNormPlus.BioCDocobj.Annotations.get(i).size();j++) // Paragraph
+			{
+				
+				for (int k = GNormPlus.BioCDocobj.Annotations.get(i).get(j).size()-1; k >=0 ; k--) // Annotation : k
+				{
+					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
+					int start = Integer.parseInt(anno[0]);
+					int last = Integer.parseInt(anno[1]);
+					String mention = anno[2];
+					String type = anno[3];
+					String id = anno[4];
+					if(type.equals("Gene") && Species_hash.containsKey(mention))
+					{
+						GNormPlus.BioCDocobj.Annotations.get(i).get(j).remove(k);
+					}
+					else if(type.equals("Gene") && id.equals(""))
+					{
+						GNormPlus.BioCDocobj.Annotations.get(i).get(j).remove(k);
+					}
+					else
+					{
+						for (int k1 = GNormPlus.BioCDocobj.Annotations.get(i).get(j).size()-1; k1 >=0 ; k1--) // Annotation : k
+						{
+							if(k1 != k)
+							{
+								String anno1[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k1).split("\t");
+								int start1 = Integer.parseInt(anno1[0]);
+								int last1 = Integer.parseInt(anno1[1]);
+								if((start1<start && last1>=last) || (start1<=start && last1>last))
+								{
+									GNormPlus.BioCDocobj.Annotations.get(i).get(j).remove(k);
+									break;
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+		if(GeneIDMatch == true)
+		{
+			//GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,false,true);
+		}
+		else
+		{
+			GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,true,true);
+		}
+	}
+	/*
+	 * Search Potential GeneID in the Prefix Tree
+	 */
+	public ArrayList<String> SearchGeneIDLocation(String Doc)
+	{
+		ArrayList<String> location = new ArrayList<String>();
+		
+		String Doc_tmp=" "+Doc+" ";
+		Pattern ptmp = Pattern.compile("^(.*[^A-Za-z0-9]+)([0-9]+\\S*[A-Za-z]+|[A-Za-z]+\\S*[0-9]+|[0-9]+\\S*[A-Za-z]+\\S*[0-9]+|[A-Za-z]+\\S*[0-9]+\\S*[A-Za-z]+)([^A-Za-z0-9]+.*)$");
+		Matcher mtmp = ptmp.matcher(Doc_tmp);
+		while(mtmp.find())
+		{
+			String str1=mtmp.group(1);
+			String str2=mtmp.group(2);
+			String str3=mtmp.group(3);
+			for(int m=str1.length();m<=(str1.length()+str2.length());m++)
+			{
+				int start = str1.length()-1;
+				int last = start+str2.length();
+				String mention = Doc.substring(start, last);
+				if(!mention.matches(".*[\\'\\;\\[\\]\\+\\*\\\\].*"))
+				{
+					if(last-start>6 && (mention.matches(".*\\(.*\\).*") || mention.matches("[^\\(\\)]+")) )
+					{
+						Pattern ptmp1 = Pattern.compile("^(.+[^0-9])([0-9]+)\\-([0-9]+)$");
+						Matcher mtmp1 = ptmp1.matcher(mention);
+						Pattern ptmp2 = Pattern.compile("^(.+[^0-9])([0-9]+)\\-(.+[^0-9])([0-9]+)$");
+						Matcher mtmp2 = ptmp2.matcher(mention);
+						if(mtmp1.find())
+						{
+							String S1 = mtmp1.group(1);
+							if(mtmp1.group(2).length()<=6 && mtmp1.group(3).length()<=6)
+							{
+								int Num1 = Integer.parseInt(mtmp1.group(2));
+								int Num2 = Integer.parseInt(mtmp1.group(3));
+								String prefix = "";
+								Pattern ptmp3 = Pattern.compile("^([0]+)");
+								Matcher mtmp3 = ptmp3.matcher(mtmp1.group(2));
+								if(mtmp3.find())
+								{
+									prefix = mtmp3.group(1);
+								}
+								if(Num2-Num1>0 && (Num2-Num1<=20))
+								{
+									for(int n=Num1;n<=Num2;n++)
+									{
+										String StrNum=S1+prefix+n;
+										if(StrNum.length()>=5)
+										{
+											location.add(start+"\t"+last+"\t"+StrNum+"\tGeneID");
+										}
+									}
+								}
+							}
+						}
+						else if(mtmp2.find())
+						{
+							if(mtmp2.group(2).length()<=6 && mtmp2.group(4).length()<=6)
+							{
+								String S1 = mtmp2.group(1);
+								int Num1 = Integer.parseInt(mtmp2.group(2));
+								String S2 = mtmp2.group(3);
+								int Num2 = Integer.parseInt(mtmp2.group(4));
+								if(S1.equals(S2))
+								{
+									String prefix = "";
+									Pattern ptmp3 = Pattern.compile("^([0]+)");
+									Matcher mtmp3 = ptmp3.matcher(mtmp2.group(2));
+									if(mtmp3.find())
+									{
+										prefix = mtmp3.group(1);
+									}
+									if(Num2-Num1>0 && (Num2-Num1<=20))
+									{
+										for(int n=Num1;n<=Num2;n++)
+										{
+											String StrNum=S1+prefix+n;
+											if(StrNum.length()>=5)
+											{
+												location.add(start+"\t"+last+"\t"+StrNum+"\tGeneID");
+											}
+										}
+									}
+								}
+							}
+						}
+					}
+					location.add(start+"\t"+last+"\t"+mention+"\tGeneID");
+				}
+			}
+			String men="";
+			for(int m=0;m<str2.length();m++){men=men+"@";}
+			Doc_tmp=str1+men+str3;
+			mtmp = ptmp.matcher(Doc_tmp);
+		}
+		return location;
+	}
+	public void GeneIDRecognition(String Filename,String FilenameBioC) throws IOException, XMLStreamException
+	{
+		for (int i = 0; i < GNormPlus.BioCDocobj.PMIDs.size(); i++) /** PMIDs : i */
+		{
+			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
+			{
+				String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j); // Passage context
+				/** GeneID recognition by pattern match */
+				ArrayList<String> locations = SearchGeneIDLocation(PassageContext);
+				for (int k = 0 ; k < locations.size() ; k++)
+				{
+					String anno[]=locations.get(k).split("\t");
+					String mention = anno[2].toLowerCase();
+	        		mention = mention.replaceAll("[\\W\\-\\_]+", "");
+	        		if(GNormPlus.GeneIDs_hash.containsKey(mention))
+	        		{
+	        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(locations.get(k)+"\tGeneID:"+GNormPlus.GeneIDs_hash.get(mention)); //paragraph
+	        		}
+				}
+			}
+		}
+		GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,true,true);
+	}
 }

src_Java/GNormPluslib/GNormPlus.java

@@ -1,696 +1,696 @@
-package GNormPluslib;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.xml.stream.XMLStreamException;
-
-import GNormPluslib.PrefixTree;
-import GNormPluslib.GNR;
-import GNormPluslib.SR;
-
-public class GNormPlus
-{
-	public static BioCDoc BioCDocobj = new BioCDoc();
-	public static PrefixTree PT_Species = new PrefixTree();
-	public static PrefixTree PT_Cell = new PrefixTree();
-	public static PrefixTree PT_CTDGene = new PrefixTree();
-	public static PrefixTree PT_Gene = new PrefixTree();
-	public static PrefixTree PT_GeneChromosome = new PrefixTree();
-	public static PrefixTree PT_FamilyName = new PrefixTree();
-	public static HashMap<String, String> ent_hash = new HashMap<String, String>();
-	public static HashMap<String, String> GenusID_hash = new HashMap<String, String>();
-	public static HashMap<String, String> PrefixID_hash = new HashMap<String, String>();
-	public static HashMap<String, Double> TaxFreq_hash = new HashMap<String, Double>();
-	public static HashMap<String, String> GeneScoring_hash = new HashMap<String, String>();
-	public static HashMap<String, Double> GeneScoringDF_hash = new HashMap<String, Double>();
-	public static HashMap<String, String> GeneIDs_hash = new HashMap<String, String>();
-	public static HashMap<String, String> Normalization2Protein_hash = new HashMap<String, String>();
-	public static HashMap<String, String> HomologeneID_hash = new HashMap<String, String>();
-	public static HashMap<String,String> SuffixTranslationMap_hash = new HashMap<String,String>();
-	public static HashMap<String,String> SuffixTranslationMap2_hash = new HashMap<String,String>();
-	public static HashMap<String, String> Pmid2Abb_hash = new HashMap<String, String>();
-	public static HashMap<String, String> PmidAbb2LF_lc_hash = new HashMap<String, String>();
-	public static HashMap<String, String> PmidLF2Abb_lc_hash = new HashMap<String, String>();
-	public static HashMap<String, String> PmidAbb2LF_hash = new HashMap<String, String>();
-	public static HashMap<String, String> PmidLF2Abb_hash = new HashMap<String, String>();
-	public static HashMap<String, String> Pmid2ChromosomeGene_hash = new HashMap<String, String>();
-	public static HashMap<String, String> SimConceptMention2Type_hash = new HashMap<String, String>();
-	public static HashMap<String, String> Filtering_hash = new HashMap<String, String>();
-	public static HashMap<String, String> Filtering_WithLongForm_hash = new HashMap<String, String>();
-	public static HashMap<String, String> SP_Virus2Human_hash = new HashMap<String, String>();
-	public static HashMap<String, String> GeneWithoutSPPrefix_hash = new HashMap<String, String>();
-	public static ArrayList <String> taxid4gene = new ArrayList <String>();
-	public static HashMap<String, String> setup_hash = new HashMap<String, String>();
-	public static HashMap<String, String> suffixprefix_orig2modified = new HashMap<String, String>();
-	public static HashMap<String, String> Abb2Longformtok_hash = new HashMap<String, String>();
-	public static HashMap<String, String> StrainID_ancestor2tax_hash = new HashMap<String, String>();
-	public static HashMap<String, String> StrainID_taxid2names_hash = new HashMap<String, String>();
-	
-	public static String SetupFile = "setup.txt";
-	public static void main(String [] args) throws IOException, InterruptedException, XMLStreamException, SQLException 
-	{
-		String InputFolder="input";
-		String OutputFolder="output";
-		String tmpFolder="tmp";
-		String FocusSpecies = "";
-		if(args.length<2)
-		{
-			System.out.println("\n$ java -Xmx30G -Xms10G -jar GNormPlus.jar [InputFolder] [OutputFolder] [SetupFile]");
-			System.out.println("[InputFolder] Default : input");
-			System.out.println("[OutputFolder] Default : output");
-			System.out.println("[SetupFile] Default : setup.txt\n\n");
-		}
-		else
-		{
-			/*
-			 * Parameters
-			 */
-			InputFolder=args[0];
-			OutputFolder=args[1];
-			if(args.length>=3)
-			{
-				SetupFile = args[2];
-			}
-			if(args.length>=4)
-			{
-				FocusSpecies=args[3];
-			}
-		}
-		
-		BufferedReader br = new BufferedReader(new FileReader(SetupFile));
-		String line="";
-		Pattern ptmp = Pattern.compile("^	([A-Za-z0-9]+) = ([^ \\t\\n\\r]+)$");
-		while ((line = br.readLine()) != null)  
-		{
-			Matcher mtmp = ptmp.matcher(line);
-			if(mtmp.find())
-			{
-				setup_hash.put(mtmp.group(1), mtmp.group(2));
-			}
-		}
-		br.close();
-		if(!setup_hash.containsKey("GeneIDMatch"))
-		{
-			setup_hash.put("GeneIDMatch","True");
-		}
-		if(!setup_hash.containsKey("HomologeneID"))
-		{
-			setup_hash.put("HomologeneID","False");
-		}
-		if(!FocusSpecies.equals(""))
-		{
-			setup_hash.put("FocusSpecies",FocusSpecies);
-		}
-		if(!setup_hash.containsKey("ShowUnNormalizedMention"))
-		{
-			setup_hash.put("ShowUnNormalizedMention","False");
-		}
-		if(setup_hash.containsKey("tmpFolder"))
-		{
-			tmpFolder=setup_hash.get("tmpFolder");
-		}
-		
-		/*
-		 * Time stamp - start : All
-		 */
-		double startTime,endTime,totTime;
-		startTime = System.currentTimeMillis();//start time
-	
-		int NumFiles=0;
-		File folder = new File(InputFolder);
-		File[] listOfFiles = folder.listFiles();
-		for (int i = 0; i < listOfFiles.length; i++)
-		{
-			if (listOfFiles[i].isFile()) 
-			{
-				String InputFile = listOfFiles[i].getName();
-				File f = new File(OutputFolder+"/"+InputFile);
-				if(f.exists() && !f.isDirectory()) 
-				{ 
-				}
-				else
-				{
-					NumFiles++;
-				}
-			}
-		}
-		
-		System.out.println("Total "+NumFiles+" file(s) wait(s) for process.");
-		
-		if(NumFiles>0)
-		{
-			/* 
-			 * Start & Load Dictionary
-			 */
-			String TrainTest = "Test";
-			if(setup_hash.containsKey("TrainTest"))
-			{
-				TrainTest = setup_hash.get("TrainTest");
-			}
-			
-			
-			/** Load Dictionary */
-			if(setup_hash.containsKey("GeneRecognition") && setup_hash.get("GeneRecognition").toLowerCase().equals("true"))
-			{
-				System.out.print("Loading Gene NER Dictionary : Processing ... \r");
-				/** CTDGene  */
-				if(setup_hash.containsKey("IgnoreNER") && setup_hash.get("IgnoreNER").toLowerCase().equals("true")){} // not NER (entities are pre-annotated)
-				else if(setup_hash.containsKey("SpeciesAssignmentOnly") && setup_hash.get("SpeciesAssignmentOnly").toLowerCase().equals("true")) {} // species assignment 
-				else
-				{
-					PT_CTDGene.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_CTDGene.txt");
-				}
-				/** ent */
-				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/ent.rev.txt"));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					String l[]=line.split("\t"); //&#x00391;	Alpha
-					ent_hash.put(l[0], l[1]);
-				}
-				br.close();	
-	
-				/** FamilyName */
-				if((!setup_hash.containsKey("IgnoreNER")) || setup_hash.get("IgnoreNER").toLowerCase() != "true")
-				{
-					PT_FamilyName.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_FamilyName.txt");
-				}
-			
-				/** GeneChromosome */
-				//PT_GeneChromosome.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_GeneChromosome.txt");
-				System.out.println("Loading Gene NER Dictionary : Processing ... done.");
-			}
-			
-			if(setup_hash.containsKey("SpeciesRecognition") && setup_hash.get("SpeciesRecognition").toLowerCase().equals("true"))
-			{
-				System.out.print("Loading Species NER Dictionary : Processing ... \r");
-				/** Species */
-				PT_Species.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_Species.txt");
-
-				/** Cell */
-				PT_Cell.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_Cell.txt");
-				
-				/** Genus */
-				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/SPGenus.txt"));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					String l[]=line.split("\t");
-					GenusID_hash.put(l[0], l[1]); // tax id -> Genus
-				}
-				br.close();	
-				
-				/** taxid4gene */
-				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/tax4gene.txt"));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					taxid4gene.add(line); // tax id -> Genus
-				}
-				br.close();	
-				System.out.println("Loading Species NER Dictionary : Processing ... done.");
-				
-			}
-			
-			if(setup_hash.containsKey("SpeciesAssignment") && setup_hash.get("SpeciesAssignment").toLowerCase().equals("true"))
-			{
-				System.out.print("Loading Species Assignment Dictionary : Processing ... \r");
-				/** GeneWithoutSPPrefix */
-				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/GeneWithoutSPPrefix.txt"));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					GeneWithoutSPPrefix_hash.put(line, "");
-				}
-				br.close();	
-				
-				/** Prefix */
-				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/SPPrefix.txt"));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					String l[]=line.split("\t");
-					PrefixID_hash.put(l[0], l[1]); //tax id -> prefix
-				}
-				br.close();
-				PrefixID_hash.put("9606", "h");
-				PrefixID_hash.put("10090", "m");
-				PrefixID_hash.put("10116", "r");
-				PrefixID_hash.put("4932", "y");
-				PrefixID_hash.put("7227", "d");
-				PrefixID_hash.put("7955", "z|dr|Dr|Zf|zf");
-				PrefixID_hash.put("3702", "at|At");
-				
-				/** Frequency */
-				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/taxonomy_freq.txt"));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					String l[]=line.split("\t");
-					TaxFreq_hash.put(l[0], Double.parseDouble(l[1])/200000000); //tax id -> prefix
-				}
-				br.close();	
-				
-				/** SP_Virus2Human_hash */
-				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/SP_Virus2HumanList.txt"));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					SP_Virus2Human_hash.put(line,"9606");
-				}
-				br.close();	
-				
-				/** SPStrain */
-				/*
-				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/SPStrain.txt"));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					String l[]=line.split("\t");
-					String ancestor_id = l[0];
-					String tax_id = l[1];
-					String tax_names = l[2];
-					StrainID_ancestor2tax_hash.put(ancestor_id, tax_id); // ancestor -> tax_id
-					StrainID_taxid2names_hash.put(tax_id, tax_names); // tax id -> strain
-				}
-				br.close();
-				*/
-				System.out.println("Loading Species Assignment Dictionary : Processing ... done.");
-				
-			}
-			
-			if(setup_hash.containsKey("GeneNormalization") && setup_hash.get("GeneNormalization").toLowerCase().equals("true"))
-			{
-				System.out.print("Loading Gene normalization Dictionary : Processing ... \r");
-				/** gene_prefix & gene_suffix */
-				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/PrefixSuffix.txt"));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					String l[]=line.split("\t");
-					String org=l[0];
-					String mod=l[1];
-					suffixprefix_orig2modified.put(org,mod);
-				}
-				br.close();	
-				
-				/** gene_prefix & gene_suffix */
-				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/NonGeneAbbr.txt"));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					String l[]=line.split("\t");
-					String shortform=l[0];
-					String longform_toks=l[1];
-					Abb2Longformtok_hash.put(shortform,longform_toks);
-				}
-				br.close();
-				
-				/** SimConcept.MentionType */
-				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/SimConcept.MentionType.txt"));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					String l[]=line.split("\t");
-					SimConceptMention2Type_hash.put(l[0], l[1]);
-				}
-				br.close();	
-				
-				/** Filtering */
-				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/Filtering.txt"));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					Filtering_hash.put(line, "");
-				}
-				br.close();	
-				
-				/** Filtering_WithLongForm.txt */
-				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/Filtering_WithLongForm.txt"));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					String l[]=line.split("\t");
-					Filtering_WithLongForm_hash.put(l[0], l[1]);
-				}
-				br.close();	
-				
-				/** Gene Dictionary */
-				if(setup_hash.containsKey("FocusSpecies") && !setup_hash.get("FocusSpecies").equals("All"))
-				{
-					PT_Gene.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_Gene."+setup_hash.get("FocusSpecies")+".txt");
-				}
-				else if((!FocusSpecies.equals("")) && (!FocusSpecies.equals("All")))
-				{
-					PT_Gene.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_Gene."+FocusSpecies+".txt");
-				}
-				else
-				{
-					PT_Gene.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_Gene.txt");
-				}
-			
-				/** GeneScoring */
-				String FileName=setup_hash.get("DictionaryFolder")+"/GeneScoring.txt";
-				
-				if(setup_hash.containsKey("FocusSpecies") && !setup_hash.get("FocusSpecies").equals("All"))
-				{
-					FileName = setup_hash.get("DictionaryFolder")+"/GeneScoring."+setup_hash.get("FocusSpecies")+".txt";
-				}
-				else if((!FocusSpecies.equals("")) && (!FocusSpecies.equals("All")))
-				{
-					FileName = setup_hash.get("DictionaryFolder")+"/GeneScoring."+FocusSpecies+".txt";
-				}
-				br = new BufferedReader(new FileReader(FileName));
-				line="";
-				while ((line = br.readLine()) != null)  
-				{
-					String l[]=line.split("\t");
-					GeneScoring_hash.put(l[0], l[1]+"\t"+l[2]+"\t"+l[3]+"\t"+l[4]+"\t"+l[5]+"\t"+l[6]);
-				}
-				br.close();	
-				
-				/** GeneScoring.DF */
-				FileName=setup_hash.get("DictionaryFolder")+"/GeneScoring.DF.txt";
-				if(setup_hash.containsKey("FocusSpecies") && !setup_hash.get("FocusSpecies").equals("All"))
-				{
-					FileName = setup_hash.get("DictionaryFolder")+"/GeneScoring.DF."+setup_hash.get("FocusSpecies")+".txt";
-				}
-				else if((!FocusSpecies.equals("")) && (!FocusSpecies.equals("All")))
-				{
-					FileName = setup_hash.get("DictionaryFolder")+"/GeneScoring.DF."+FocusSpecies+".txt";
-				}
-				br = new BufferedReader(new FileReader(FileName));
-				double Sum = Double.parseDouble(br.readLine());
-				while ((line = br.readLine()) != null)  
-				{
-					String l[]=line.split("\t");
-					// token -> idf
-					GeneScoringDF_hash.put(l[0], Math.log10(Sum/Double.parseDouble(l[1])));
-				}
-				br.close();
-				
-				/** Suffix Translation */
-				SuffixTranslationMap_hash.put("alpha","a");
-				SuffixTranslationMap_hash.put("a","alpha");
-				SuffixTranslationMap_hash.put("beta","b");
-				SuffixTranslationMap_hash.put("b","beta");
-				SuffixTranslationMap_hash.put("delta","d");
-				SuffixTranslationMap_hash.put("d","delta");
-				SuffixTranslationMap_hash.put("z","zeta");
-				SuffixTranslationMap_hash.put("zeta","z");
-				SuffixTranslationMap_hash.put("gamma","g");
-				SuffixTranslationMap_hash.put("g","gamma");
-				SuffixTranslationMap_hash.put("r","gamma");
-				SuffixTranslationMap_hash.put("y","gamma");
-				
-				SuffixTranslationMap2_hash.put("2","ii");
-				SuffixTranslationMap2_hash.put("ii","2");
-				SuffixTranslationMap2_hash.put("II","2");
-				SuffixTranslationMap2_hash.put("1","i");
-				SuffixTranslationMap2_hash.put("i","1");
-				SuffixTranslationMap2_hash.put("I","1");
-				
-				/** GeneID */
-				if(setup_hash.containsKey("GeneIDMatch") && setup_hash.get("GeneIDMatch").toLowerCase().equals("true"))
-				{
-					br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/GeneIDs.txt"));
-					line="";
-					while ((line = br.readLine()) != null)  
-					{
-						String l[]=line.split("\t");
-						GeneIDs_hash.put(l[0],l[1]);
-					}
-					br.close();
-				}
-				
-				/** Normalization2Protein */
-				if(setup_hash.containsKey("Normalization2Protein") && setup_hash.get("Normalization2Protein").toLowerCase().equals("true"))
-				{
-					br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/Gene2Protein.txt"));
-					line="";
-					while ((line = br.readLine()) != null)  
-					{
-						String l[]=line.split("\t");
-						Normalization2Protein_hash.put(l[0],l[1]);
-					}
-					br.close();
-				}
-				
-				/** HomologeneID */
-				if(setup_hash.containsKey("HomologeneID") && setup_hash.get("HomologeneID").toLowerCase().equals("true"))
-				{
-					br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/Gene2Homoid.txt"));
-					line="";
-					while ((line = br.readLine()) != null)  
-					{
-						String l[]=line.split("\t");
-						HomologeneID_hash.put(l[0],l[1]);
-					}
-					br.close();
-				}
-				System.out.println("Loading Gene normalization Dictionary : Processing ... done.");
-			}
-			
-			endTime = System.currentTimeMillis();
-			totTime = endTime - startTime;
-			System.out.println("Loading Dictionary : Processing Time:"+totTime/1000+"sec");
-			
-			folder = new File(InputFolder);
-			listOfFiles = folder.listFiles();
-			for (int i = 0; i < listOfFiles.length; i++)
-			{
-				if (listOfFiles[i].isFile()) 
-				{
-					String InputFile = listOfFiles[i].getName();
-					File f = new File(OutputFolder+"/"+InputFile);
-					if(f.exists() && !f.isDirectory()) 
-					{ 
-						System.out.println(InputFolder+"/"+InputFile+" - Done. (The output file exists in output folder)");
-					}
-					else
-					{
-						String path=tmpFolder; 
-				        File file = new File(path);
-				        File[] files = file.listFiles(); 
-				        for (File ftmp:files) 
-				        {
-				        	if (ftmp.isFile() && ftmp.exists()) 
-				            {
-				        		if(ftmp.toString().matches(tmpFolder+"/"+InputFile+".*"))
-					        	{
-				        			ftmp.delete();
-					        	}
-				        	}
-				        }
-						
-						BioCDocobj = new BioCDoc();
-						
-						/*
-						 * Format Check 
-						 */
-						String Format = "";
-						String checkR = BioCDocobj.BioCFormatCheck(InputFolder+"/"+InputFile);
-						if(checkR.equals("BioC"))
-						{
-							Format = "BioC";
-						}
-						else if(checkR.equals("PubTator"))
-						{
-							Format = "PubTator";
-						}
-						else
-						{
-							System.out.println(checkR);
-							System.exit(0);
-						}
-						
-						System.out.print(InputFolder+"/"+InputFile+" - ("+Format+" format) : Processing ... \r");
-						
-						/** PubTator2BioC*/
-						if(Format.equals("PubTator"))
-						{
-							BioCDocobj.PubTator2BioC(InputFolder+"/"+InputFile,tmpFolder+"/"+InputFile);
-						}
-						else
-						{
-							br = new BufferedReader(new FileReader(InputFolder+"/"+InputFile));
-							BufferedWriter fr = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(tmpFolder+"/"+InputFile), "UTF-8"));
-							line="";
-							while ((line = br.readLine()) != null)  
-							{
-								fr.write(line);
-							}
-							br.close();
-							fr.close();
-						}
-						
-						/** load file */
-						GNR GNRobj = new GNR();
-						GNRobj.LoadInputFile(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".Abb",TrainTest);
-						SR SRobj = new SR();
-						SimConcept SCobj = new SimConcept();
-						GN GNobj = new GN();
-						String FinalStep="";
-						
-						/** SpeciesRecognition */
-						if(setup_hash.containsKey("SpeciesRecognition")  && setup_hash.get("SpeciesRecognition").toLowerCase().equals("true") ) // pre-annotated name entities
-						{
-							SRobj.SpeciesRecognition(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".SR.xml",setup_hash.get("DictionaryFolder")+"/SPStrain.txt",setup_hash.get("FilterAntibody"));
-							FinalStep="SpeciesRecognition";
-						}
-						
-						/** GeneRecognition */
-						if( setup_hash.containsKey("GeneRecognition") && setup_hash.get("GeneRecognition").toLowerCase().equals("true") )
-						{
-							GNRobj.FeatureExtraction(tmpFolder+"/"+InputFile+".data",tmpFolder+"/"+InputFile+".loca",TrainTest);
-							GNRobj.CRF_test(setup_hash.get("GNRModel"),tmpFolder+"/"+InputFile+".data",tmpFolder+"/"+InputFile+".output","top3"); //top3
-							GNRobj.ReadCRFresult(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".loca",tmpFolder+"/"+InputFile+".output",tmpFolder+"/"+InputFile+".GNR.xml",0.005,0.05); //0.005,0.05
-							f = new File(tmpFolder+"/"+InputFile+".SR.xml");
-							if(f.exists()) 
-							{
-								GNRobj.PostProcessing(tmpFolder+"/"+InputFile+".SR.xml",tmpFolder+"/"+InputFile+".GNR.xml");
-							}
-							else
-							{
-								GNRobj.PostProcessing(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".GNR.xml");
-							}
-							FinalStep="GeneRecognition";
-						}
-						
-						/** SpeciesAssignment */
-						if(setup_hash.containsKey("SpeciesAssignment")  && setup_hash.get("SpeciesAssignment").toLowerCase().equals("true") ) // pre-annotated name entities
-						{
-							if(setup_hash.containsKey("FocusSpecies") && !setup_hash.get("FocusSpecies").equals("All")) // FocusSpecies
-							{
-								f = new File(tmpFolder+"/"+InputFile+".GNR.xml");
-								if(f.exists()) 
-								{
-									SRobj.SpeciesAssignment(tmpFolder+"/"+InputFile+".GNR.xml",tmpFolder+"/"+InputFile+".SA.xml",setup_hash.get("FocusSpecies"));
-								}
-								else
-								{
-									SRobj.SpeciesAssignment(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".SA.xml",setup_hash.get("FocusSpecies"));
-								}
-							}
-							else// All Species
-							{
-								f = new File(tmpFolder+"/"+InputFile+".GNR.xml");
-								if(f.exists()) 
-								{
-									SRobj.SpeciesAssignment(tmpFolder+"/"+InputFile+".GNR.xml",tmpFolder+"/"+InputFile+".SA.xml");
-								}
-								else
-								{
-									SRobj.SpeciesAssignment(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".SA.xml");
-								}
-							}
-							FinalStep="SpeciesAssignment";
-						}
-						
-						/** GeneNormalization */
-						if((setup_hash.containsKey("GeneNormalization")) && setup_hash.get("GeneNormalization").toLowerCase().equals("true") )
-						{
-							/** SimConcept */
-							{
-								SCobj.FeatureExtraction_Test(tmpFolder+"/"+InputFile+".SC.data");
-								SCobj.CRF_test(setup_hash.get("SCModel"),tmpFolder+"/"+InputFile+".SC.data",tmpFolder+"/"+InputFile+".SC.output");
-								SCobj.ReadCRFresult(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".SC.output",tmpFolder+"/"+InputFile+".SC.xml");
-							}
-							
-							/** GeneNormalization */
-							{
-								GNobj.PreProcessing4GN(InputFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".PreProcessing4GN.xml");
-								GNobj.ChromosomeRecognition(InputFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".GN.xml");
-								if(setup_hash.containsKey("GeneIDMatch") && setup_hash.get("GeneIDMatch").equals("True"))
-								{
-									
-									GNobj.GeneNormalization(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".GN.xml",true);
-									GNobj.GeneIDRecognition(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".GN.xml");
-								}
-								else
-								{
-									GNobj.GeneNormalization(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".GN.xml",false);
-								}
-							}
-							FinalStep="GeneNormalization";
-						}
-						
-						/** BioC2PubTator*/
-						String final_output="";
-						if(FinalStep.equals("GeneNormalization"))
-						{
-							final_output=tmpFolder+"/"+InputFile+".GN.xml";
-						}
-						else if(FinalStep.equals("SpeciesAssignment"))
-						{
-							final_output=tmpFolder+"/"+InputFile+".SA.xml";
-						}
-						else if(FinalStep.equals("SpeciesRecognition"))
-						{
-							final_output=tmpFolder+"/"+InputFile+".SR.xml";
-						}
-						else if(FinalStep.equals("GeneRecognition"))
-						{
-							final_output=tmpFolder+"/"+InputFile+".GNR.xml";
-						}
-						
-						if(Format.equals("PubTator"))
-						{
-							BioCDocobj.BioC2PubTator(final_output,OutputFolder+"/"+InputFile);
-						}
-						else
-						{
-							br = new BufferedReader(new FileReader(final_output));
-							BufferedWriter fr = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(OutputFolder+"/"+InputFile), "UTF-8"));
-							line="";
-							while ((line = br.readLine()) != null)  
-							{
-								fr.write(line);
-							}
-							br.close();
-							fr.close();
-						}
-						
-						/*
-						 * remove tmp files
-						 */
-						if((!setup_hash.containsKey("DeleteTmp")) || setup_hash.get("DeleteTmp").toLowerCase().equals("true"))
-						{
-							path="tmp"; 
-					        file = new File(path);
-					        files = file.listFiles(); 
-					        for (File ftmp:files) 
-					        {
-					        	if (ftmp.isFile() && ftmp.exists()) 
-					            {
-					        		if(ftmp.toString().matches(tmpFolder+"/"+InputFile+".*"))
-						        	{
-					        			ftmp.delete();
-						        	}
-					        	}
-					        }
-						}
-						
-						/*
-						 * Time stamp - last
-						 */
-						endTime = System.currentTimeMillis();
-						totTime = endTime - startTime;
-						System.out.println(InputFolder+"/"+InputFile+" - ("+Format+" format) : Processing Time:"+totTime/1000+"sec");
-					}
-				}
-			}
-		}
-	}
-}
+package GNormPluslib;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.xml.stream.XMLStreamException;
+
+import GNormPluslib.PrefixTree;
+import GNormPluslib.GNR;
+import GNormPluslib.SR;
+
+public class GNormPlus
+{
+	public static BioCDoc BioCDocobj = new BioCDoc();
+	public static PrefixTree PT_Species = new PrefixTree();
+	public static PrefixTree PT_Cell = new PrefixTree();
+	public static PrefixTree PT_CTDGene = new PrefixTree();
+	public static PrefixTree PT_Gene = new PrefixTree();
+	public static PrefixTree PT_GeneChromosome = new PrefixTree();
+	public static PrefixTree PT_FamilyName = new PrefixTree();
+	public static HashMap<String, String> ent_hash = new HashMap<String, String>();
+	public static HashMap<String, String> GenusID_hash = new HashMap<String, String>();
+	public static HashMap<String, String> PrefixID_hash = new HashMap<String, String>();
+	public static HashMap<String, Double> TaxFreq_hash = new HashMap<String, Double>();
+	public static HashMap<String, String> GeneScoring_hash = new HashMap<String, String>();
+	public static HashMap<String, Double> GeneScoringDF_hash = new HashMap<String, Double>();
+	public static HashMap<String, String> GeneIDs_hash = new HashMap<String, String>();
+	public static HashMap<String, String> Normalization2Protein_hash = new HashMap<String, String>();
+	public static HashMap<String, String> HomologeneID_hash = new HashMap<String, String>();
+	public static HashMap<String,String> SuffixTranslationMap_hash = new HashMap<String,String>();
+	public static HashMap<String,String> SuffixTranslationMap2_hash = new HashMap<String,String>();
+	public static HashMap<String, String> Pmid2Abb_hash = new HashMap<String, String>();
+	public static HashMap<String, String> PmidAbb2LF_lc_hash = new HashMap<String, String>();
+	public static HashMap<String, String> PmidLF2Abb_lc_hash = new HashMap<String, String>();
+	public static HashMap<String, String> PmidAbb2LF_hash = new HashMap<String, String>();
+	public static HashMap<String, String> PmidLF2Abb_hash = new HashMap<String, String>();
+	public static HashMap<String, String> Pmid2ChromosomeGene_hash = new HashMap<String, String>();
+	public static HashMap<String, String> SimConceptMention2Type_hash = new HashMap<String, String>();
+	public static HashMap<String, String> Filtering_hash = new HashMap<String, String>();
+	public static HashMap<String, String> Filtering_WithLongForm_hash = new HashMap<String, String>();
+	public static HashMap<String, String> SP_Virus2Human_hash = new HashMap<String, String>();
+	public static HashMap<String, String> GeneWithoutSPPrefix_hash = new HashMap<String, String>();
+	public static ArrayList <String> taxid4gene = new ArrayList <String>();
+	public static HashMap<String, String> setup_hash = new HashMap<String, String>();
+	public static HashMap<String, String> suffixprefix_orig2modified = new HashMap<String, String>();
+	public static HashMap<String, String> Abb2Longformtok_hash = new HashMap<String, String>();
+	public static HashMap<String, String> StrainID_ancestor2tax_hash = new HashMap<String, String>();
+	public static HashMap<String, String> StrainID_taxid2names_hash = new HashMap<String, String>();
+	
+	public static String SetupFile = "setup.txt";
+	public static void main(String [] args) throws IOException, InterruptedException, XMLStreamException, SQLException 
+	{
+		String InputFolder="input";
+		String OutputFolder="output";
+		String tmpFolder="tmp";
+		String FocusSpecies = "";
+		if(args.length<2)
+		{
+			System.out.println("\n$ java -Xmx30G -Xms10G -jar GNormPlus.jar [InputFolder] [OutputFolder] [SetupFile]");
+			System.out.println("[InputFolder] Default : input");
+			System.out.println("[OutputFolder] Default : output");
+			System.out.println("[SetupFile] Default : setup.txt\n\n");
+		}
+		else
+		{
+			/*
+			 * Parameters
+			 */
+			InputFolder=args[0];
+			OutputFolder=args[1];
+			if(args.length>=3)
+			{
+				SetupFile = args[2];
+			}
+			if(args.length>=4)
+			{
+				FocusSpecies=args[3];
+			}
+		}
+		
+		BufferedReader br = new BufferedReader(new FileReader(SetupFile));
+		String line="";
+		Pattern ptmp = Pattern.compile("^	([A-Za-z0-9]+) = ([^ \\t\\n\\r]+)$");
+		while ((line = br.readLine()) != null)  
+		{
+			Matcher mtmp = ptmp.matcher(line);
+			if(mtmp.find())
+			{
+				setup_hash.put(mtmp.group(1), mtmp.group(2));
+			}
+		}
+		br.close();
+		if(!setup_hash.containsKey("GeneIDMatch"))
+		{
+			setup_hash.put("GeneIDMatch","True");
+		}
+		if(!setup_hash.containsKey("HomologeneID"))
+		{
+			setup_hash.put("HomologeneID","False");
+		}
+		if(!FocusSpecies.equals(""))
+		{
+			setup_hash.put("FocusSpecies",FocusSpecies);
+		}
+		if(!setup_hash.containsKey("ShowUnNormalizedMention"))
+		{
+			setup_hash.put("ShowUnNormalizedMention","False");
+		}
+		if(setup_hash.containsKey("tmpFolder"))
+		{
+			tmpFolder=setup_hash.get("tmpFolder");
+		}
+		
+		/*
+		 * Time stamp - start : All
+		 */
+		double startTime,endTime,totTime;
+		startTime = System.currentTimeMillis();//start time
+	
+		int NumFiles=0;
+		File folder = new File(InputFolder);
+		File[] listOfFiles = folder.listFiles();
+		for (int i = 0; i < listOfFiles.length; i++)
+		{
+			if (listOfFiles[i].isFile()) 
+			{
+				String InputFile = listOfFiles[i].getName();
+				File f = new File(OutputFolder+"/"+InputFile);
+				if(f.exists() && !f.isDirectory()) 
+				{ 
+				}
+				else
+				{
+					NumFiles++;
+				}
+			}
+		}
+		
+		System.out.println("Total "+NumFiles+" file(s) wait(s) for process.");
+		
+		if(NumFiles>0)
+		{
+			/* 
+			 * Start & Load Dictionary
+			 */
+			String TrainTest = "Test";
+			if(setup_hash.containsKey("TrainTest"))
+			{
+				TrainTest = setup_hash.get("TrainTest");
+			}
+			
+			
+			/** Load Dictionary */
+			if(setup_hash.containsKey("GeneRecognition") && setup_hash.get("GeneRecognition").toLowerCase().equals("true"))
+			{
+				System.out.print("Loading Gene NER Dictionary : Processing ... \r");
+				/** CTDGene  */
+				if(setup_hash.containsKey("IgnoreNER") && setup_hash.get("IgnoreNER").toLowerCase().equals("true")){} // not NER (entities are pre-annotated)
+				else if(setup_hash.containsKey("SpeciesAssignmentOnly") && setup_hash.get("SpeciesAssignmentOnly").toLowerCase().equals("true")) {} // species assignment 
+				else
+				{
+					PT_CTDGene.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_CTDGene.txt");
+				}
+				/** ent */
+				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/ent.rev.txt"));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					String l[]=line.split("\t"); //&#x00391;	Alpha
+					ent_hash.put(l[0], l[1]);
+				}
+				br.close();	
+	
+				/** FamilyName */
+				if((!setup_hash.containsKey("IgnoreNER")) || setup_hash.get("IgnoreNER").toLowerCase() != "true")
+				{
+					PT_FamilyName.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_FamilyName.txt");
+				}
+			
+				/** GeneChromosome */
+				//PT_GeneChromosome.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_GeneChromosome.txt");
+				System.out.println("Loading Gene NER Dictionary : Processing ... done.");
+			}
+			
+			if(setup_hash.containsKey("SpeciesRecognition") && setup_hash.get("SpeciesRecognition").toLowerCase().equals("true"))
+			{
+				System.out.print("Loading Species NER Dictionary : Processing ... \r");
+				/** Species */
+				PT_Species.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_Species.txt");
+
+				/** Cell */
+				PT_Cell.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_Cell.txt");
+				
+				/** Genus */
+				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/SPGenus.txt"));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					String l[]=line.split("\t");
+					GenusID_hash.put(l[0], l[1]); // tax id -> Genus
+				}
+				br.close();	
+				
+				/** taxid4gene */
+				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/tax4gene.txt"));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					taxid4gene.add(line); // tax id -> Genus
+				}
+				br.close();	
+				System.out.println("Loading Species NER Dictionary : Processing ... done.");
+				
+			}
+			
+			if(setup_hash.containsKey("SpeciesAssignment") && setup_hash.get("SpeciesAssignment").toLowerCase().equals("true"))
+			{
+				System.out.print("Loading Species Assignment Dictionary : Processing ... \r");
+				/** GeneWithoutSPPrefix */
+				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/GeneWithoutSPPrefix.txt"));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					GeneWithoutSPPrefix_hash.put(line, "");
+				}
+				br.close();	
+				
+				/** Prefix */
+				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/SPPrefix.txt"));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					String l[]=line.split("\t");
+					PrefixID_hash.put(l[0], l[1]); //tax id -> prefix
+				}
+				br.close();
+				PrefixID_hash.put("9606", "h");
+				PrefixID_hash.put("10090", "m");
+				PrefixID_hash.put("10116", "r");
+				PrefixID_hash.put("4932", "y");
+				PrefixID_hash.put("7227", "d");
+				PrefixID_hash.put("7955", "z|dr|Dr|Zf|zf");
+				PrefixID_hash.put("3702", "at|At");
+				
+				/** Frequency */
+				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/taxonomy_freq.txt"));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					String l[]=line.split("\t");
+					TaxFreq_hash.put(l[0], Double.parseDouble(l[1])/200000000); //tax id -> prefix
+				}
+				br.close();	
+				
+				/** SP_Virus2Human_hash */
+				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/SP_Virus2HumanList.txt"));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					SP_Virus2Human_hash.put(line,"9606");
+				}
+				br.close();	
+				
+				/** SPStrain */
+				/*
+				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/SPStrain.txt"));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					String l[]=line.split("\t");
+					String ancestor_id = l[0];
+					String tax_id = l[1];
+					String tax_names = l[2];
+					StrainID_ancestor2tax_hash.put(ancestor_id, tax_id); // ancestor -> tax_id
+					StrainID_taxid2names_hash.put(tax_id, tax_names); // tax id -> strain
+				}
+				br.close();
+				*/
+				System.out.println("Loading Species Assignment Dictionary : Processing ... done.");
+				
+			}
+			
+			if(setup_hash.containsKey("GeneNormalization") && setup_hash.get("GeneNormalization").toLowerCase().equals("true"))
+			{
+				System.out.print("Loading Gene normalization Dictionary : Processing ... \r");
+				/** gene_prefix & gene_suffix */
+				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/PrefixSuffix.txt"));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					String l[]=line.split("\t");
+					String org=l[0];
+					String mod=l[1];
+					suffixprefix_orig2modified.put(org,mod);
+				}
+				br.close();	
+				
+				/** gene_prefix & gene_suffix */
+				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/NonGeneAbbr.txt"));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					String l[]=line.split("\t");
+					String shortform=l[0];
+					String longform_toks=l[1];
+					Abb2Longformtok_hash.put(shortform,longform_toks);
+				}
+				br.close();
+				
+				/** SimConcept.MentionType */
+				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/SimConcept.MentionType.txt"));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					String l[]=line.split("\t");
+					SimConceptMention2Type_hash.put(l[0], l[1]);
+				}
+				br.close();	
+				
+				/** Filtering */
+				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/Filtering.txt"));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					Filtering_hash.put(line, "");
+				}
+				br.close();	
+				
+				/** Filtering_WithLongForm.txt */
+				br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/Filtering_WithLongForm.txt"));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					String l[]=line.split("\t");
+					Filtering_WithLongForm_hash.put(l[0], l[1]);
+				}
+				br.close();	
+				
+				/** Gene Dictionary */
+				if(setup_hash.containsKey("FocusSpecies") && !setup_hash.get("FocusSpecies").equals("All"))
+				{
+					PT_Gene.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_Gene."+setup_hash.get("FocusSpecies")+".txt");
+				}
+				else if((!FocusSpecies.equals("")) && (!FocusSpecies.equals("All")))
+				{
+					PT_Gene.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_Gene."+FocusSpecies+".txt");
+				}
+				else
+				{
+					PT_Gene.TreeFile2Tree(setup_hash.get("DictionaryFolder")+"/PT_Gene.txt");
+				}
+			
+				/** GeneScoring */
+				String FileName=setup_hash.get("DictionaryFolder")+"/GeneScoring.txt";
+				
+				if(setup_hash.containsKey("FocusSpecies") && !setup_hash.get("FocusSpecies").equals("All"))
+				{
+					FileName = setup_hash.get("DictionaryFolder")+"/GeneScoring."+setup_hash.get("FocusSpecies")+".txt";
+				}
+				else if((!FocusSpecies.equals("")) && (!FocusSpecies.equals("All")))
+				{
+					FileName = setup_hash.get("DictionaryFolder")+"/GeneScoring."+FocusSpecies+".txt";
+				}
+				br = new BufferedReader(new FileReader(FileName));
+				line="";
+				while ((line = br.readLine()) != null)  
+				{
+					String l[]=line.split("\t");
+					GeneScoring_hash.put(l[0], l[1]+"\t"+l[2]+"\t"+l[3]+"\t"+l[4]+"\t"+l[5]+"\t"+l[6]);
+				}
+				br.close();	
+				
+				/** GeneScoring.DF */
+				FileName=setup_hash.get("DictionaryFolder")+"/GeneScoring.DF.txt";
+				if(setup_hash.containsKey("FocusSpecies") && !setup_hash.get("FocusSpecies").equals("All"))
+				{
+					FileName = setup_hash.get("DictionaryFolder")+"/GeneScoring.DF."+setup_hash.get("FocusSpecies")+".txt";
+				}
+				else if((!FocusSpecies.equals("")) && (!FocusSpecies.equals("All")))
+				{
+					FileName = setup_hash.get("DictionaryFolder")+"/GeneScoring.DF."+FocusSpecies+".txt";
+				}
+				br = new BufferedReader(new FileReader(FileName));
+				double Sum = Double.parseDouble(br.readLine());
+				while ((line = br.readLine()) != null)  
+				{
+					String l[]=line.split("\t");
+					// token -> idf
+					GeneScoringDF_hash.put(l[0], Math.log10(Sum/Double.parseDouble(l[1])));
+				}
+				br.close();
+				
+				/** Suffix Translation */
+				SuffixTranslationMap_hash.put("alpha","a");
+				SuffixTranslationMap_hash.put("a","alpha");
+				SuffixTranslationMap_hash.put("beta","b");
+				SuffixTranslationMap_hash.put("b","beta");
+				SuffixTranslationMap_hash.put("delta","d");
+				SuffixTranslationMap_hash.put("d","delta");
+				SuffixTranslationMap_hash.put("z","zeta");
+				SuffixTranslationMap_hash.put("zeta","z");
+				SuffixTranslationMap_hash.put("gamma","g");
+				SuffixTranslationMap_hash.put("g","gamma");
+				SuffixTranslationMap_hash.put("r","gamma");
+				SuffixTranslationMap_hash.put("y","gamma");
+				
+				SuffixTranslationMap2_hash.put("2","ii");
+				SuffixTranslationMap2_hash.put("ii","2");
+				SuffixTranslationMap2_hash.put("II","2");
+				SuffixTranslationMap2_hash.put("1","i");
+				SuffixTranslationMap2_hash.put("i","1");
+				SuffixTranslationMap2_hash.put("I","1");
+				
+				/** GeneID */
+				if(setup_hash.containsKey("GeneIDMatch") && setup_hash.get("GeneIDMatch").toLowerCase().equals("true"))
+				{
+					br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/GeneIDs.txt"));
+					line="";
+					while ((line = br.readLine()) != null)  
+					{
+						String l[]=line.split("\t");
+						GeneIDs_hash.put(l[0],l[1]);
+					}
+					br.close();
+				}
+				
+				/** Normalization2Protein */
+				if(setup_hash.containsKey("Normalization2Protein") && setup_hash.get("Normalization2Protein").toLowerCase().equals("true"))
+				{
+					br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/Gene2Protein.txt"));
+					line="";
+					while ((line = br.readLine()) != null)  
+					{
+						String l[]=line.split("\t");
+						Normalization2Protein_hash.put(l[0],l[1]);
+					}
+					br.close();
+				}
+				
+				/** HomologeneID */
+				if(setup_hash.containsKey("HomologeneID") && setup_hash.get("HomologeneID").toLowerCase().equals("true"))
+				{
+					br = new BufferedReader(new FileReader(setup_hash.get("DictionaryFolder")+"/Gene2Homoid.txt"));
+					line="";
+					while ((line = br.readLine()) != null)  
+					{
+						String l[]=line.split("\t");
+						HomologeneID_hash.put(l[0],l[1]);
+					}
+					br.close();
+				}
+				System.out.println("Loading Gene normalization Dictionary : Processing ... done.");
+			}
+			
+			endTime = System.currentTimeMillis();
+			totTime = endTime - startTime;
+			System.out.println("Loading Dictionary : Processing Time:"+totTime/1000+"sec");
+			
+			folder = new File(InputFolder);
+			listOfFiles = folder.listFiles();
+			for (int i = 0; i < listOfFiles.length; i++)
+			{
+				if (listOfFiles[i].isFile()) 
+				{
+					String InputFile = listOfFiles[i].getName();
+					File f = new File(OutputFolder+"/"+InputFile);
+					if(f.exists() && !f.isDirectory()) 
+					{ 
+						System.out.println(InputFolder+"/"+InputFile+" - Done. (The output file exists in output folder)");
+					}
+					else
+					{
+						String path=tmpFolder; 
+				        File file = new File(path);
+				        File[] files = file.listFiles(); 
+				        for (File ftmp:files) 
+				        {
+				        	if (ftmp.isFile() && ftmp.exists()) 
+				            {
+				        		if(ftmp.toString().matches(tmpFolder+"/"+InputFile+".*"))
+					        	{
+				        			ftmp.delete();
+					        	}
+				        	}
+				        }
+						
+						BioCDocobj = new BioCDoc();
+						
+						/*
+						 * Format Check 
+						 */
+						String Format = "";
+						String checkR = BioCDocobj.BioCFormatCheck(InputFolder+"/"+InputFile);
+						if(checkR.equals("BioC"))
+						{
+							Format = "BioC";
+						}
+						else if(checkR.equals("PubTator"))
+						{
+							Format = "PubTator";
+						}
+						else
+						{
+							System.out.println(checkR);
+							System.exit(0);
+						}
+						
+						System.out.print(InputFolder+"/"+InputFile+" - ("+Format+" format) : Processing ... \r");
+						
+						/** PubTator2BioC*/
+						if(Format.equals("PubTator"))
+						{
+							BioCDocobj.PubTator2BioC(InputFolder+"/"+InputFile,tmpFolder+"/"+InputFile);
+						}
+						else
+						{
+							br = new BufferedReader(new FileReader(InputFolder+"/"+InputFile));
+							BufferedWriter fr = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(tmpFolder+"/"+InputFile), "UTF-8"));
+							line="";
+							while ((line = br.readLine()) != null)  
+							{
+								fr.write(line);
+							}
+							br.close();
+							fr.close();
+						}
+						
+						/** load file */
+						GNR GNRobj = new GNR();
+						GNRobj.LoadInputFile(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".Abb",TrainTest);
+						SR SRobj = new SR();
+						SimConcept SCobj = new SimConcept();
+						GN GNobj = new GN();
+						String FinalStep="";
+						
+						/** SpeciesRecognition */
+						if(setup_hash.containsKey("SpeciesRecognition")  && setup_hash.get("SpeciesRecognition").toLowerCase().equals("true") ) // pre-annotated name entities
+						{
+							SRobj.SpeciesRecognition(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".SR.xml",setup_hash.get("DictionaryFolder")+"/SPStrain.txt",setup_hash.get("FilterAntibody"));
+							FinalStep="SpeciesRecognition";
+						}
+						
+						/** GeneRecognition */
+						if( setup_hash.containsKey("GeneRecognition") && setup_hash.get("GeneRecognition").toLowerCase().equals("true") )
+						{
+							GNRobj.FeatureExtraction(tmpFolder+"/"+InputFile+".data",tmpFolder+"/"+InputFile+".loca",TrainTest);
+							GNRobj.CRF_test(setup_hash.get("GNRModel"),tmpFolder+"/"+InputFile+".data",tmpFolder+"/"+InputFile+".output","top3"); //top3
+							GNRobj.ReadCRFresult(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".loca",tmpFolder+"/"+InputFile+".output",tmpFolder+"/"+InputFile+".GNR.xml",0.005,0.05); //0.005,0.05
+							f = new File(tmpFolder+"/"+InputFile+".SR.xml");
+							if(f.exists()) 
+							{
+								GNRobj.PostProcessing(tmpFolder+"/"+InputFile+".SR.xml",tmpFolder+"/"+InputFile+".GNR.xml");
+							}
+							else
+							{
+								GNRobj.PostProcessing(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".GNR.xml");
+							}
+							FinalStep="GeneRecognition";
+						}
+						
+						/** SpeciesAssignment */
+						if(setup_hash.containsKey("SpeciesAssignment")  && setup_hash.get("SpeciesAssignment").toLowerCase().equals("true") ) // pre-annotated name entities
+						{
+							if(setup_hash.containsKey("FocusSpecies") && !setup_hash.get("FocusSpecies").equals("All")) // FocusSpecies
+							{
+								f = new File(tmpFolder+"/"+InputFile+".GNR.xml");
+								if(f.exists()) 
+								{
+									SRobj.SpeciesAssignment(tmpFolder+"/"+InputFile+".GNR.xml",tmpFolder+"/"+InputFile+".SA.xml",setup_hash.get("FocusSpecies"));
+								}
+								else
+								{
+									SRobj.SpeciesAssignment(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".SA.xml",setup_hash.get("FocusSpecies"));
+								}
+							}
+							else// All Species
+							{
+								f = new File(tmpFolder+"/"+InputFile+".GNR.xml");
+								if(f.exists()) 
+								{
+									SRobj.SpeciesAssignment(tmpFolder+"/"+InputFile+".GNR.xml",tmpFolder+"/"+InputFile+".SA.xml");
+								}
+								else
+								{
+									SRobj.SpeciesAssignment(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".SA.xml");
+								}
+							}
+							FinalStep="SpeciesAssignment";
+						}
+						
+						/** GeneNormalization */
+						if((setup_hash.containsKey("GeneNormalization")) && setup_hash.get("GeneNormalization").toLowerCase().equals("true") )
+						{
+							/** SimConcept */
+							{
+								SCobj.FeatureExtraction_Test(tmpFolder+"/"+InputFile+".SC.data");
+								SCobj.CRF_test(setup_hash.get("SCModel"),tmpFolder+"/"+InputFile+".SC.data",tmpFolder+"/"+InputFile+".SC.output");
+								SCobj.ReadCRFresult(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".SC.output",tmpFolder+"/"+InputFile+".SC.xml");
+							}
+							
+							/** GeneNormalization */
+							{
+								GNobj.PreProcessing4GN(InputFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".PreProcessing4GN.xml");
+								GNobj.ChromosomeRecognition(InputFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".GN.xml");
+								if(setup_hash.containsKey("GeneIDMatch") && setup_hash.get("GeneIDMatch").equals("True"))
+								{
+									
+									GNobj.GeneNormalization(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".GN.xml",true);
+									GNobj.GeneIDRecognition(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".GN.xml");
+								}
+								else
+								{
+									GNobj.GeneNormalization(tmpFolder+"/"+InputFile,tmpFolder+"/"+InputFile+".GN.xml",false);
+								}
+							}
+							FinalStep="GeneNormalization";
+						}
+						
+						/** BioC2PubTator*/
+						String final_output="";
+						if(FinalStep.equals("GeneNormalization"))
+						{
+							final_output=tmpFolder+"/"+InputFile+".GN.xml";
+						}
+						else if(FinalStep.equals("SpeciesAssignment"))
+						{
+							final_output=tmpFolder+"/"+InputFile+".SA.xml";
+						}
+						else if(FinalStep.equals("SpeciesRecognition"))
+						{
+							final_output=tmpFolder+"/"+InputFile+".SR.xml";
+						}
+						else if(FinalStep.equals("GeneRecognition"))
+						{
+							final_output=tmpFolder+"/"+InputFile+".GNR.xml";
+						}
+						
+						if(Format.equals("PubTator"))
+						{
+							BioCDocobj.BioC2PubTator(final_output,OutputFolder+"/"+InputFile);
+						}
+						else
+						{
+							br = new BufferedReader(new FileReader(final_output));
+							BufferedWriter fr = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(OutputFolder+"/"+InputFile), "UTF-8"));
+							line="";
+							while ((line = br.readLine()) != null)  
+							{
+								fr.write(line);
+							}
+							br.close();
+							fr.close();
+						}
+						
+						/*
+						 * remove tmp files
+						 */
+						if((!setup_hash.containsKey("DeleteTmp")) || setup_hash.get("DeleteTmp").toLowerCase().equals("true"))
+						{
+							path="tmp"; 
+					        file = new File(path);
+					        files = file.listFiles(); 
+					        for (File ftmp:files) 
+					        {
+					        	if (ftmp.isFile() && ftmp.exists()) 
+					            {
+					        		if(ftmp.toString().matches(tmpFolder+"/"+InputFile+".*"))
+						        	{
+					        			ftmp.delete();
+						        	}
+					        	}
+					        }
+						}
+						
+						/*
+						 * Time stamp - last
+						 */
+						endTime = System.currentTimeMillis();
+						totTime = endTime - startTime;
+						System.out.println(InputFolder+"/"+InputFile+" - ("+Format+" format) : Processing Time:"+totTime/1000+"sec");
+					}
+				}
+			}
+		}
+	}
+}

src_Java/GNormPluslib/PrefixTree.java

@@ -1,893 +1,893 @@
-/**
- * Project: GNormPlus
- * Function: Dictionary lookup by Prefix Tree
- */
-
-package GNormPluslib;
-
-import java.io.*;
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class PrefixTree
-{
-	private Tree Tr=new Tree();
-	
-	/*
-	 * Read Dictionary and insert Mention into the Prefix Tree
-	 */
-	public static HashMap<String, String> StopWord_hash = new HashMap<String, String>();
-	
-	public void Hash2Tree(HashMap<String, String> ID2Names)
-	{
-		for(String ID : ID2Names.keySet())  
-		{
-			String NameColumn[]=ID2Names.get(ID).split("\\|");
-			for(int i=0;i<NameColumn.length;i++)
-			{
-				Tr.insertMention(NameColumn[i],ID);
-			}
-		}
-	}
-	public void Dictionary2Tree_Combine(String Filename,String StopWords,String MentionType)	
-	{
-		try 
-		{
-			//System.out.println("Dictionary2Tree_Combine : " + Filename);
-			
-			/** Stop Word */
-			BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(StopWords), "UTF-8"));
-			String line="";
-			while ((line = br.readLine()) != null)  
-			{
-				StopWord_hash.put(line, "StopWord");
-			}
-			br.close();	
-			
-			BufferedReader inputfile = new BufferedReader(new InputStreamReader(new FileInputStream(Filename), "UTF-8"));
-			line="";
-			//int count=0;
-			while ((line = inputfile.readLine()) != null)  
-			{
-				//count++;
-				//if(count%10000==0){	System.out.println(count);	}
-				String Column[]=line.split("\t");
-				if(Column.length>1)
-				{
-					Column[0]=Column[0].replace("species:ncbi:","");
-					Column[1]=Column[1].replaceAll(" strain=", " ");
-					Column[1]=Column[1].replaceAll("[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_]", " ");
-					Column[1]=Column[1].replaceAll("[\\(\\)]", " ");
-					String SpNameColumn[]=Column[1].split("\\|");
-					for(int i=0;i<SpNameColumn.length;i++)
-					{
-						String tmp = SpNameColumn[i];
-						tmp=tmp.replaceAll("[\\W\\-\\_]", "");
-						
-						/*
-						 * Criteria for Species
-						 */
-						if(	MentionType.equals("Species") &&
-							(!SpNameColumn[i].substring(0, 1).matches("[\\W\\-\\_]")) &&
-							(!SpNameColumn[i].matches("a[\\W\\-\\_].*")) &&
-							tmp.length()>=3
-							)
-						{
-							boolean stopword_boolean=false;
-							for(String stopword_RegEx : StopWord_hash.keySet())
-							{
-								Pattern ptmp = Pattern.compile("^"+stopword_RegEx+"$");
-								Matcher mtmp = ptmp.matcher(SpNameColumn[i].toLowerCase());
-								if(mtmp.find())
-								{
-									stopword_boolean=true;
-								}
-							}
-							if(stopword_boolean == false)
-							{
-								Tr.insertMention(SpNameColumn[i],Column[0]);
-							}
-						}
-						/*
-						 * Criteria for Gene
-						 */
-						else if (MentionType.equals("Gene") &&  
-								(!SpNameColumn[i].substring(0, 1).matches("[\\W\\-\\_]")) &&
-								tmp.length()>=3
-								)
-						{
-							if(!StopWord_hash.containsKey(SpNameColumn[i].toLowerCase()))
-							{
-								Tr.insertMention(SpNameColumn[i],Column[0]);
-							}
-						}
-						/*
-						 * Criteria for Cell
-						 */
-						else if (MentionType.equals("Cell") && 
-								(!SpNameColumn[i].substring(0, 1).matches("[\\W\\-\\_]")) &&
-								tmp.length()>=3
-								)
-						{
-							if(!StopWord_hash.containsKey(SpNameColumn[i].toLowerCase()))
-							{
-								Tr.insertMention(SpNameColumn[i],Column[0]);
-							}
-						}
-						/*
-						 * others
-						 */
-						else if ((!SpNameColumn[i].substring(0, 1).matches("[\\W\\-\\_]")) &&
-								tmp.length()>=3
-								)
-						{
-							if(!StopWord_hash.containsKey(SpNameColumn[i].toLowerCase()))
-							{
-								Tr.insertMention(SpNameColumn[i],Column[0]);
-							}
-						}
-					}
-				}
-			}
-			inputfile.close();	
-		}
-		catch(IOException e1){ System.out.println("[Dictionary2Tree_Combine]: Input file is not exist.");}
-	}
-	public void Dictionary2Tree_UniqueGene(String Filename,String StopWords,String Preifx)	
-	{
-		try 
-		{
-			//System.out.println("Dictionary2Tree_UniqueGene : " + Filename);
-			
-			/** Stop Word */
-			BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(StopWords), "UTF-8"));
-			String line="";
-			while ((line = br.readLine()) != null)  
-			{
-				StopWord_hash.put(line, "StopWord");
-			}
-			br.close();	
-			
-			BufferedReader inputfile = new BufferedReader(new InputStreamReader(new FileInputStream(Filename), "UTF-8"));
-			line="";
-			//int count=0;
-			while ((line = inputfile.readLine()) != null)  
-			{
-				//count++;
-				//if(count%10000==0){	System.out.println(count);	}
-				String Column[]=line.split("\t");
-				if(Column.length>1)
-				{
-					if(!StopWord_hash.containsKey(Column[0].toLowerCase()))
-					{
-						if(Preifx.equals(""))
-						{
-							Tr.insertMention(Column[0],Column[1]);
-						}
-						else if(Preifx.equals("Num") && Column[0].matches("[0-9].*"))
-						{
-							Tr.insertMention(Column[0],Column[1]);
-						}
-						else if(Preifx.equals("AZNum") && Column[0].matches("[a-z][0-9].*"))
-						{
-							Tr.insertMention(Column[0],Column[1]);
-						}
-						else if(Preifx.equals("lo") && Column[0].length()>2 && Column[0].substring(0,2).equals(Preifx))
-						{
-							if( ! Column[0].matches("loc[0-9]+"))
-							{
-								Tr.insertMention(Column[0],Column[1]);
-							}
-						}
-						else if(Preifx.equals("un") && Column[0].length()>2 && Column[0].substring(0,2).equals(Preifx))
-						{
-							if(Column[0].length()>=6 && Column[0].substring(0,6).equals("unchar"))
-							{
-								// remove uncharacterized
-							}
-							else
-							{
-								Tr.insertMention(Column[0],Column[1]);
-							}
-						}
-						else if(Column[0].length()>2 && Column[0].substring(0,2).equals(Preifx))
-						{
-							Tr.insertMention(Column[0],Column[1]);
-						}
-					}
-				}
-			}
-			inputfile.close();	
-		}
-		catch(IOException e1){ System.out.println("[Dictionary2Tree_UniqueGene]: Input file is not exist.");}
-	}
-	public void Dictionary2Tree_UniqueSpecies(String Filename,String StopWords,String Preifx)	
-	{
-		try 
-		{
-			//System.out.println("Dictionary2Tree_UniqueGene : " + Filename);
-			
-			/** Stop Word */
-			BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(StopWords), "UTF-8"));
-			String line="";
-			while ((line = br.readLine()) != null)  
-			{
-				StopWord_hash.put(line, "StopWord");
-			}
-			br.close();	
-			
-			BufferedReader inputfile = new BufferedReader(new InputStreamReader(new FileInputStream(Filename), "UTF-8"));
-			line="";
-			while ((line = inputfile.readLine()) != null)  
-			{
-				//count++;
-				//if(count%10000==0){	System.out.println(count);	}
-				String Column[]=line.split("\t");
-				if(Column.length>1)
-				{
-					if(!StopWord_hash.containsKey(Column[0].toLowerCase()))
-					{
-						if(Preifx.equals("")) //all
-						{
-							if(Column[0].matches(".*[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_].*"))
-							{
-								String mention_rev=Column[0].replaceAll("[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_]", " ");
-								String mention_tmp=mention_rev.replaceAll("[\\W\\-\\_]","");
-								if(mention_tmp.length()>=10)
-								{
-									Tr.insertMention(mention_rev,Column[1]);
-								}
-							}
-							else
-							{
-								Tr.insertMention(Column[0],Column[1]); // mention, id
-							}
-							
-						}
-						else if(Column[0].matches("[0-9][0-9].*"))
-						{
-							if(Preifx.equals("Num"))
-							{
-								if(Column[0].matches(".*[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_].*"))
-								{
-									String mention_rev=Column[0].replaceAll("[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_]", " ");
-									String mention_tmp=mention_rev.replaceAll("[\\W\\-\\_]","");
-									if(mention_tmp.length()>=10)
-									{
-										Tr.insertMention(mention_rev,Column[1]);
-									}
-								}
-								else
-								{
-									Tr.insertMention(Column[0],Column[1]); // mention, id
-								}
-							}
-						}
-						/*
-						else if(Column[0].matches("[a-z][0-9].*"))
-						{
-							if(Preifx.equals("AZNum"))
-							{
-								if(Column[0].matches(".*[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_].*"))
-								{
-									String mention_rev=Column[0].replaceAll("[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_]", " ");
-									String mention_tmp=mention_rev.replaceAll("[\\W\\-\\_]","");
-									if(mention_tmp.length()>=10)
-									{
-										Tr.insertMention(mention_rev,Column[1]);
-									}
-								}
-								else
-								{
-									Tr.insertMention(Column[0],Column[1]); // mention, id
-								}
-							}
-						}
-						*/
-						else if(Column[0].matches("[a-z][a-z].*"))
-						{
-							if(Column[0].length()>2 && Column[0].substring(0,2).equals(Preifx))
-							{
-								if(Column[0].matches(".*[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_].*"))
-								{
-									String mention_rev=Column[0].replaceAll("[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_]", " ");
-									String mention_tmp=mention_rev.replaceAll("[\\W\\-\\_]","");
-									if(mention_tmp.length()>=10)
-									{
-										Tr.insertMention(mention_rev,Column[1]);
-									}
-								}
-								else
-								{
-									Tr.insertMention(Column[0],Column[1]); // mention, id
-								}
-							}
-						}
-						else if(Preifx.equals("Others"))
-						{
-							if(Column[0].matches(".*[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_].*"))
-							{
-								String mention_rev=Column[0].replaceAll("[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_]", " ");
-								String mention_tmp=mention_rev.replaceAll("[\\W\\-\\_]","");
-								if(mention_tmp.length()>=10)
-								{
-									Tr.insertMention(mention_rev,Column[1]);
-								}
-							}
-							else
-							{
-								Tr.insertMention(Column[0],Column[1]); // mention, id
-							}
-						}
-					}
-				}
-			}
-			inputfile.close();	
-		}
-		catch(IOException e1){ System.out.println("[Dictionary2Tree_UniqueGene]: Input file is not exist.");}
-	}
-	public void TreeFile2Tree(String Filename)	
-	{
-		try 
-		{
-			//System.out.println("TreeFile2Tree : " + Filename);
-			
-			BufferedReader inputfile = new BufferedReader(new InputStreamReader(new FileInputStream(Filename), "UTF-8"));
-			String line="";
-			int count=0;
-			while ((line = inputfile.readLine()) != null)  
-			{
-				String Anno[]=line.split("\t");
-				if(Anno.length<2){System.out.println(count+"\t"+line);} //check error
-				String LocationInTree = Anno[0];
-				String token = Anno[1];
-				String identifier="";
-				if(Anno.length==3)
-				{
-					identifier = Anno[2];
-				}
-				String LocationsInTree[]=LocationInTree.split("-");
-				TreeNode tmp = Tr.root;
-				for(int i=0;i<LocationsInTree.length-1;i++)
-				{
-					tmp=tmp.links.get(Integer.parseInt(LocationsInTree[i])-1);
-				}
-				tmp.InsertToken(token,identifier);
-				//if(count%10000==0){System.out.println(count);}
-				count++;
-			}
-			inputfile.close();	
-		}
-		catch(IOException e1){ System.out.println("[TreeFile2Tee]: Input file: "+ Filename +" is not exist.");}
-	}
-	
-	/*
-	 * Search target mention in the Prefix Tree
-	 */
-	public String MentionMatch(String Mentions)
-	{
-		ArrayList<String> location = new ArrayList<String>();
-		String Menlist[]=Mentions.split("\\|");
-		for(int m=0;m<Menlist.length;m++)
-		{
-			String Mention=Menlist[m];
-			String Mention_lc=Mention.toLowerCase();
-			Mention_lc = Mention_lc.replaceAll("[\\W\\-\\_]+", "");
-			Mention_lc = Mention_lc.replaceAll("([0-9])([a-z])", "$1 $2");
-			Mention_lc = Mention_lc.replaceAll("([a-z])([0-9])", "$1 $2");
-			String Tkns[]=Mention_lc.split(" ");
-			
-			int PrefixTranslation=0;
-			int i=0;
-			boolean find=false;
-			TreeNode tmp = Tr.root;
-			
-			while( i<Tkns.length && tmp.CheckChild(Tkns[i],PrefixTranslation)>=0) //Find Tokens in the links
-			{
-				if(i == Tkns.length-1){PrefixTranslation = 1;}
-				tmp=tmp.links.get(tmp.CheckChild(Tkns[i],PrefixTranslation)); //move point to the link
-				find=true;
-				i++;
-			}
-			if(find == true)
-			{
-				if(i==Tkns.length)
-				{
-					if(!tmp.Concept.equals(""))
-					{
-						return tmp.Concept;
-					}
-					else
-					{
-						return "-1";
-						//gene id is not found.
-					}
-				}
-				else
-				{
-					return "-2";
-					//the gene mention matched a substring in PrefixTree.
-				}
-			}
-			else
-			{
-				return "-3";
-				//mention is not found
-			}
-		}
-		return "-3"; //mention is not found
-	}
-	
-	/*
-	 * Search target mention in the Prefix Tree
-	 */
-	public String MentionMatch_species(String Mentions)
-	{
-		ArrayList<String> location = new ArrayList<String>();
-		String Menlist[]=Mentions.split("\\|");
-		for(int m=0;m<Menlist.length;m++)
-		{
-			String Mention=Menlist[m];
-			String Mention_lc=Mention.toLowerCase();
-			Mention_lc = Mention_lc.replaceAll("[\\W\\-\\_]+", " ");
-			Mention_lc = Mention_lc.replaceAll("([0-9])([a-z])", "$1 $2");
-			Mention_lc = Mention_lc.replaceAll("([a-z])([0-9])", "$1 $2");
-			Mention_lc = Mention_lc.replaceAll("^[ ]+", "");
-			Mention_lc = Mention_lc.replaceAll("[ ]+$", "");
-			String Tkns[]=Mention_lc.split(" ");
-			
-			int PrefixTranslation=0;
-			int i=0;
-			boolean find=false;
-			TreeNode tmp = Tr.root;
-			
-			while( i<Tkns.length && tmp.CheckChild(Tkns[i],PrefixTranslation)>=0) //Find Tokens in the links
-			{
-				if(i == Tkns.length-1){PrefixTranslation = 1;}
-				tmp=tmp.links.get(tmp.CheckChild(Tkns[i],PrefixTranslation)); //move point to the link
-				find=true;
-				i++;
-			}
-			if(find == true)
-			{
-				if(i==Tkns.length)
-				{
-					if(!tmp.Concept.equals(""))
-					{
-						return tmp.Concept;
-					}
-					else
-					{
-						return "-1";
-						//gene id is not found.
-					}
-				}
-				else
-				{
-					return "-2";
-					//the gene mention matched a substring in PrefixTree.
-				}
-			}
-			else
-			{
-				return "-3";
-				//mention is not found
-			}
-		}
-		return "-3"; //mention is not found
-	}
-	
-	/*
-	 * Search target mention in the Prefix Tree
-	 * ConceptType: Species|Genus|Cell|CTDGene
-	 */
-	public ArrayList<String> SearchMentionLocation(String Doc,String ConceptType)
-	{
-		ArrayList<String> location = new ArrayList<String>();
-		Doc=Doc+" XXXX XXXX";
-		String Doc_org=Doc;
-		Doc=Doc.toLowerCase();
-		String Doc_lc=Doc;
-		Doc = Doc.replaceAll("([0-9])([A-Za-z])", "$1 $2");
-		Doc = Doc.replaceAll("([A-Za-z])([0-9])", "$1 $2");
-		Doc = Doc.replaceAll("[\\W^;:,]+", " ");
-		
-		/* = keep special characters =
-		 * 
-		String regex="\\s+|(?=\\p{Punct})|(?<=\\p{Punct})";
-		String DocTkns[]=Doc.split(regex);
-		 */
-		
-		String DocTkns[]=Doc.split(" ");
-		int Offset=0;
-		int Start=0;
-		int Last=0;
-		int FirstTime=0;
-		
-		while(Doc_lc.length()>0 && Doc_lc.substring(0,1).matches("[\\W]")) //clean the forward whitespace
-		{
-			Doc_lc=Doc_lc.substring(1);
-			Offset++;
-		}
-		
-		for(int i=0;i<DocTkns.length;i++)
-		{
-			//System.out.println(i+"\t"+Start+"\t"+Last+"\t"+Offset+"\t"+Doc_lc);
-			
-			int pre_i=i;
-			int pre_Start=Start;
-			int pre_Last=Last;
-			String pre_Doc_lc=Doc_lc;
-			int pre_Offset=Offset;
-			
-			TreeNode tmp = Tr.root;
-			boolean find=false;
-			int PrefixTranslation=2;
-			if(ConceptType.equals("Species"))
-			{
-				PrefixTranslation=3;
-			}
-			int ConceptFound=i; //Keep found concept
-			String ConceptFound_STR="";//Keep found concept
-			int FirstTime_while = -1;
-			
-			while( tmp.CheckChild(DocTkns[i],PrefixTranslation)>=0 ) //Find Tokens in the links
-			{
-				FirstTime_while++;
-				tmp=tmp.links.get(tmp.CheckChild(DocTkns[i],PrefixTranslation)); //move point to the link
-				if(Start==0 && FirstTime>0){Start = Offset;} //Start <- Offset 
-				if(Doc_lc.length()>=DocTkns[i].length() && Doc_lc.substring(0,DocTkns[i].length()).equals(DocTkns[i]))
-				{
-					if(DocTkns[i].length()>0)
-					{
-						Doc_lc=Doc_lc.substring(DocTkns[i].length());
-						Offset=Offset+DocTkns[i].length();
-					}
-				}
-				Last = Offset;
-				while(Doc_lc.length()>0 && Doc_lc.substring(0,1).matches("[\\W]")) //clean the forward whitespace
-				{
-					Doc_lc=Doc_lc.substring(1);
-					Offset++;
-				}
-				i++;
-				
-				if(ConceptType.equals("Species"))
-				{
-					if(i<DocTkns.length-3 && DocTkns[i].matches("(str|strain|substr|substrain|subspecies|subsp|var|variant|pathovars|pv|biovar|bv)"))
-					{
-						Doc_lc=Doc_lc.substring(DocTkns[i].length());
-						Offset=Offset+DocTkns[i].length();
-						Last = Offset;
-						while(Doc_lc.length()>0 && Doc_lc.substring(0,1).matches("[\\W]")) //clean the forward whitespace
-						{
-							Doc_lc=Doc_lc.substring(1);
-							Offset++;
-						}
-						i++;
-					}
-				}
-				
-				if(!tmp.Concept.equals("") && (Last-Start>0)) //Keep found concept
-				{
-					if(Last<Doc_org.length())
-					{
-						ConceptFound=i;
-						ConceptFound_STR=Start+"\t"+Last+"\t"+Doc_org.substring(Start, Last)+"\t"+tmp.Concept;
-						//System.out.println(ConceptFound_STR);
-					}
-				}
-				
-				find=true;
-				if(i>=DocTkns.length){break;}
-				else if(i==DocTkns.length-1){PrefixTranslation=2;}
-				
-				//System.out.println(i+"\t"+Start+"\t"+Last+"\t("+FirstTime_while+")\t"+Offset+"\t"+Doc_lc);
-				
-				if(FirstTime_while==0) // first matched token
-				{
-					pre_i=i;
-					pre_Start=Start;
-					pre_Last=Last;
-					pre_Doc_lc=Doc_lc;
-					pre_Offset=Offset;
-				}
-			}
-			
-			if(find == true)
-			{
-				//System.out.println(find+"\t"+FirstTime_while+"\t"+Start+"\t"+Last+"\t"+Doc_org.substring(Start, Last)+"\t"+tmp.Concept);
-				if(!tmp.Concept.equals("")) //the last matched token has concept id 
-				{
-					if(Last<Doc_org.length() && Last>Start)
-					{
-						location.add(Start+"\t"+Last+"\t"+Doc_org.substring(Start, Last)+"\t"+tmp.Concept);
-					}
-				}
-				else
-				{
-					if(!ConceptFound_STR.equals("")) //Keep found concept
-					{
-						location.add(ConceptFound_STR);
-						i = ConceptFound + 1;
-					}
-					
-					if(FirstTime_while>=1)
-					{
-						i=pre_i;
-						Start=pre_Start;
-						Last=pre_Last;
-						Doc_lc=pre_Doc_lc;
-						Offset=pre_Offset;
-					}
-				}
-				Start=0;
-				Last=0;
-				if(i>0){i--;}
-				ConceptFound=i; //Keep found concept
-				ConceptFound_STR="";//Keep found concept
-			}
-			else //if(find == false)
-			{
-				//System.out.println(find+"\t"+FirstTime_while+"\t"+Start+"\t"+Last+"\t"+Doc_org.substring(Start, Last)+"\t"+tmp.Concept);
-				
-				if(FirstTime_while>=1 && tmp.Concept.equals(""))
-				{
-					i=pre_i;
-					Start=pre_Start;
-					Last=pre_Last;
-					Doc_lc=pre_Doc_lc;
-					Offset=pre_Offset;
-				}
-				
-				if(Doc_lc.length()>=DocTkns[i].length() && Doc_lc.substring(0,DocTkns[i].length()).equals(DocTkns[i]))
-				{
-					if(DocTkns[i].length()>0)
-					{
-						Doc_lc=Doc_lc.substring(DocTkns[i].length());
-						Offset=Offset+DocTkns[i].length();
-					}
-				}
-			}
-			
-			while(Doc_lc.length()>0 && Doc_lc.substring(0,1).matches("[\\W]")) //clean the forward whitespace
-			{
-				Doc_lc=Doc_lc.substring(1);
-				Offset++;
-			}
-			FirstTime++;
-			
-			//System.out.println();
-		}
-		return location;
-	}
-	
-	/*
-	 * Print out the Prefix Tree
-	 */
-	public String PrintTree()
-	{
-		return Tr.PrintTree_preorder(Tr.root,"");
-	}
-	
-	public void SaveTree(String outputfile) throws IOException
-	{
-		BufferedWriter fr = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputfile), "UTF-8"));
-		Tr.SaveTree_preorder(Tr.root,"",fr);
-		fr.close();
-	}
-	
-	
-	public void insertMention(String Mention, String Identifier)
-	{
-		Tr.insertMention(Mention,Identifier);
-	}
-}
-
-class Tree 
-{
-	/*
-	 * Prefix Tree - root node
-	 */
-	public TreeNode root;
-	
-	public Tree() 
-	{ 
-		root = new TreeNode("-ROOT-"); 
-	}
-	
-	/*
-	 * Insert mention into the tree
-	 */
-	public void insertMention(String Mention, String Identifier)
-	{
-		Mention=Mention.toLowerCase();
-		
-		Mention = Mention.replaceAll("([0-9])([A-Za-z])", "$1 $2");
-		Mention = Mention.replaceAll("([A-Za-z])([0-9])", "$1 $2");
-		Mention = Mention.replaceAll("[\\W\\-\\_]+", " ");
-		/* = keep special characters =
-		 * 
-		String regex="\\s+|(?=\\p{Punct})|(?<=\\p{Punct})";
-		String Tokens[]=Mention.split(regex);
-		 */
-		String Tokens[]=Mention.split(" ");
-		TreeNode tmp = root;
-		for(int i=0;i<Tokens.length;i++)
-		{
-			if(tmp.CheckChild(Tokens[i],0)>=0)
-			{
-				tmp=tmp.links.get( tmp.CheckChild(Tokens[i],0) ); //go through next generation (exist node)
-				if(i == Tokens.length-1)
-				{
-					tmp.Concept=Identifier;
-				}
-			}
-			else //not exist
-			{
-				if(i == Tokens.length-1)
-				{
-					tmp.InsertToken(Tokens[i],Identifier);
-				}
-				else
-				{
-					tmp.InsertToken(Tokens[i]);
-				}
-				tmp=tmp.links.get(tmp.NumOflinks-1); //go to the next generation (new node)
-			}
-		}
-	}
-	
-	/*
-	 * Print the tree by pre-order
-	 */
-	public String PrintTree_preorder(TreeNode node, String LocationInTree)
-	{
-		String opt="";
-		if(!node.token.equals("-ROOT-"))//Ignore root
-		{
-			if(node.Concept.equals(""))
-			{
-				opt=opt+LocationInTree+"\t"+node.token+"\n";
-			}
-			else
-			{
-				opt=opt+LocationInTree+"\t"+node.token+"\t"+node.Concept+"\n";
-			}
-		} 
-		if(!LocationInTree.equals("")){LocationInTree=LocationInTree+"-";}
-		for(int i=0;i<node.NumOflinks;i++)
-		{
-			opt=opt+PrintTree_preorder(node.links.get(i),LocationInTree+(i+1));
-		}
-		return opt;
-	}
-	
-	/*
-	 * Print the tree by pre-order
-	 */
-	public void SaveTree_preorder(TreeNode node, String LocationInTree, BufferedWriter fr) throws IOException
-	{
-		if(!node.token.equals("-ROOT-"))//Ignore root
-		{
-			if(node.Concept.equals(""))
-			{
-				fr.write(LocationInTree+"\t"+node.token+"\n");
-			}
-			else
-			{
-				fr.write(LocationInTree+"\t"+node.token+"\t"+node.Concept+"\n");
-			}
-		} 
-		if(!LocationInTree.equals("")){LocationInTree=LocationInTree+"-";}
-		for(int i=0;i<node.NumOflinks;i++)
-		{
-			SaveTree_preorder(node.links.get(i),LocationInTree+(i+1),fr);
-		}
-	}
-}
-
-class TreeNode 
-{
-	String token; //token of the node
-	int NumOflinks; //Number of links
-	public String Concept;
-	HashMap<String,Integer> Hashs;
-	ArrayList<TreeNode> links;
-	
-	public TreeNode(String Tok,String ID)
-	{
-		token = Tok;
-		NumOflinks = 0;
-		Concept = ID;
-		links = new ArrayList<TreeNode>();/*link*/
-		Hashs = new HashMap<String,Integer>();/*hash*/
-	}
-	public TreeNode(String Tok)
-	{
-		token = Tok;
-		NumOflinks = 0;
-		Concept = "";
-		links = new ArrayList<TreeNode>();/*link*/
-		Hashs = new HashMap<String,Integer>();/*hash*/
-	}
-	public TreeNode()
-	{
-		token = "";
-		NumOflinks = 0;
-		Concept = "";
-		links = new ArrayList<TreeNode>();/*link*/
-		Hashs = new HashMap<String,Integer>();/*hash*/
-	}
-	
-	public String toString()
-	{
-		return (token+"\t"+Concept);
-	}
-	
-	/*
-	 * Insert an new node under the target node
-	 */
-	public void InsertToken(String Tok)
-	{
-		TreeNode NewNode = new TreeNode(Tok);
-		
-		/*link*/
-		links.add(NewNode);
-		
-		/*hash*/
-		Hashs.put(Tok, NumOflinks);
-		
-		NumOflinks++;
-	}
-	public void InsertToken(String Tok,String ID)
-	{
-		TreeNode NewNode = new TreeNode(Tok,ID);
-		/*link*/
-		links.add(NewNode);
-		
-		/*hash*/
-		Hashs.put(Tok, NumOflinks);
-		
-		NumOflinks++;
-	}
-	
-	/*
-	 * Check the tokens of children
-	 */
-	public int CheckChild(String Tok, Integer PrefixTranslation)
-	{
-		if(Hashs.containsKey(Tok))
-		{
-			return(Hashs.get(Tok));
-		}
-		
-		if(PrefixTranslation == 1 && Tok.matches("(alpha|beta|gamam|[abg]|[12])")) // SuffixTranslationMap
-		{
-			if(Hashs.containsKey(GNormPlus.SuffixTranslationMap_hash.get(Tok)))
-			{
-				return(Hashs.get(GNormPlus.SuffixTranslationMap_hash.get(Tok)));
-			}
-			
-		}
-		else if(PrefixTranslation == 2 && Tok.matches("[1-5]")) // for CTDGene feature
-		{
-			for(int i=0;i<links.size();i++)
-	        {
-				if(links.get(i).token.matches("[1-5]"))
-				{
-					return(i);
-				}
-	        }
-			
-			for(int i=1;i<=5;i++)
-			{
-				if(Hashs.containsKey(i)){return(Hashs.get(i));}
-			}
-		}
-		
-		return(-1);
-	}
-}
+/**
+ * Project: GNormPlus
+ * Function: Dictionary lookup by Prefix Tree
+ */
+
+package GNormPluslib;
+
+import java.io.*;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class PrefixTree
+{
+	private Tree Tr=new Tree();
+	
+	/*
+	 * Read Dictionary and insert Mention into the Prefix Tree
+	 */
+	public static HashMap<String, String> StopWord_hash = new HashMap<String, String>();
+	
+	public void Hash2Tree(HashMap<String, String> ID2Names)
+	{
+		for(String ID : ID2Names.keySet())  
+		{
+			String NameColumn[]=ID2Names.get(ID).split("\\|");
+			for(int i=0;i<NameColumn.length;i++)
+			{
+				Tr.insertMention(NameColumn[i],ID);
+			}
+		}
+	}
+	public void Dictionary2Tree_Combine(String Filename,String StopWords,String MentionType)	
+	{
+		try 
+		{
+			//System.out.println("Dictionary2Tree_Combine : " + Filename);
+			
+			/** Stop Word */
+			BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(StopWords), "UTF-8"));
+			String line="";
+			while ((line = br.readLine()) != null)  
+			{
+				StopWord_hash.put(line, "StopWord");
+			}
+			br.close();	
+			
+			BufferedReader inputfile = new BufferedReader(new InputStreamReader(new FileInputStream(Filename), "UTF-8"));
+			line="";
+			//int count=0;
+			while ((line = inputfile.readLine()) != null)  
+			{
+				//count++;
+				//if(count%10000==0){	System.out.println(count);	}
+				String Column[]=line.split("\t");
+				if(Column.length>1)
+				{
+					Column[0]=Column[0].replace("species:ncbi:","");
+					Column[1]=Column[1].replaceAll(" strain=", " ");
+					Column[1]=Column[1].replaceAll("[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_]", " ");
+					Column[1]=Column[1].replaceAll("[\\(\\)]", " ");
+					String SpNameColumn[]=Column[1].split("\\|");
+					for(int i=0;i<SpNameColumn.length;i++)
+					{
+						String tmp = SpNameColumn[i];
+						tmp=tmp.replaceAll("[\\W\\-\\_]", "");
+						
+						/*
+						 * Criteria for Species
+						 */
+						if(	MentionType.equals("Species") &&
+							(!SpNameColumn[i].substring(0, 1).matches("[\\W\\-\\_]")) &&
+							(!SpNameColumn[i].matches("a[\\W\\-\\_].*")) &&
+							tmp.length()>=3
+							)
+						{
+							boolean stopword_boolean=false;
+							for(String stopword_RegEx : StopWord_hash.keySet())
+							{
+								Pattern ptmp = Pattern.compile("^"+stopword_RegEx+"$");
+								Matcher mtmp = ptmp.matcher(SpNameColumn[i].toLowerCase());
+								if(mtmp.find())
+								{
+									stopword_boolean=true;
+								}
+							}
+							if(stopword_boolean == false)
+							{
+								Tr.insertMention(SpNameColumn[i],Column[0]);
+							}
+						}
+						/*
+						 * Criteria for Gene
+						 */
+						else if (MentionType.equals("Gene") &&  
+								(!SpNameColumn[i].substring(0, 1).matches("[\\W\\-\\_]")) &&
+								tmp.length()>=3
+								)
+						{
+							if(!StopWord_hash.containsKey(SpNameColumn[i].toLowerCase()))
+							{
+								Tr.insertMention(SpNameColumn[i],Column[0]);
+							}
+						}
+						/*
+						 * Criteria for Cell
+						 */
+						else if (MentionType.equals("Cell") && 
+								(!SpNameColumn[i].substring(0, 1).matches("[\\W\\-\\_]")) &&
+								tmp.length()>=3
+								)
+						{
+							if(!StopWord_hash.containsKey(SpNameColumn[i].toLowerCase()))
+							{
+								Tr.insertMention(SpNameColumn[i],Column[0]);
+							}
+						}
+						/*
+						 * others
+						 */
+						else if ((!SpNameColumn[i].substring(0, 1).matches("[\\W\\-\\_]")) &&
+								tmp.length()>=3
+								)
+						{
+							if(!StopWord_hash.containsKey(SpNameColumn[i].toLowerCase()))
+							{
+								Tr.insertMention(SpNameColumn[i],Column[0]);
+							}
+						}
+					}
+				}
+			}
+			inputfile.close();	
+		}
+		catch(IOException e1){ System.out.println("[Dictionary2Tree_Combine]: Input file is not exist.");}
+	}
+	public void Dictionary2Tree_UniqueGene(String Filename,String StopWords,String Preifx)	
+	{
+		try 
+		{
+			//System.out.println("Dictionary2Tree_UniqueGene : " + Filename);
+			
+			/** Stop Word */
+			BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(StopWords), "UTF-8"));
+			String line="";
+			while ((line = br.readLine()) != null)  
+			{
+				StopWord_hash.put(line, "StopWord");
+			}
+			br.close();	
+			
+			BufferedReader inputfile = new BufferedReader(new InputStreamReader(new FileInputStream(Filename), "UTF-8"));
+			line="";
+			//int count=0;
+			while ((line = inputfile.readLine()) != null)  
+			{
+				//count++;
+				//if(count%10000==0){	System.out.println(count);	}
+				String Column[]=line.split("\t");
+				if(Column.length>1)
+				{
+					if(!StopWord_hash.containsKey(Column[0].toLowerCase()))
+					{
+						if(Preifx.equals(""))
+						{
+							Tr.insertMention(Column[0],Column[1]);
+						}
+						else if(Preifx.equals("Num") && Column[0].matches("[0-9].*"))
+						{
+							Tr.insertMention(Column[0],Column[1]);
+						}
+						else if(Preifx.equals("AZNum") && Column[0].matches("[a-z][0-9].*"))
+						{
+							Tr.insertMention(Column[0],Column[1]);
+						}
+						else if(Preifx.equals("lo") && Column[0].length()>2 && Column[0].substring(0,2).equals(Preifx))
+						{
+							if( ! Column[0].matches("loc[0-9]+"))
+							{
+								Tr.insertMention(Column[0],Column[1]);
+							}
+						}
+						else if(Preifx.equals("un") && Column[0].length()>2 && Column[0].substring(0,2).equals(Preifx))
+						{
+							if(Column[0].length()>=6 && Column[0].substring(0,6).equals("unchar"))
+							{
+								// remove uncharacterized
+							}
+							else
+							{
+								Tr.insertMention(Column[0],Column[1]);
+							}
+						}
+						else if(Column[0].length()>2 && Column[0].substring(0,2).equals(Preifx))
+						{
+							Tr.insertMention(Column[0],Column[1]);
+						}
+					}
+				}
+			}
+			inputfile.close();	
+		}
+		catch(IOException e1){ System.out.println("[Dictionary2Tree_UniqueGene]: Input file is not exist.");}
+	}
+	public void Dictionary2Tree_UniqueSpecies(String Filename,String StopWords,String Preifx)	
+	{
+		try 
+		{
+			//System.out.println("Dictionary2Tree_UniqueGene : " + Filename);
+			
+			/** Stop Word */
+			BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(StopWords), "UTF-8"));
+			String line="";
+			while ((line = br.readLine()) != null)  
+			{
+				StopWord_hash.put(line, "StopWord");
+			}
+			br.close();	
+			
+			BufferedReader inputfile = new BufferedReader(new InputStreamReader(new FileInputStream(Filename), "UTF-8"));
+			line="";
+			while ((line = inputfile.readLine()) != null)  
+			{
+				//count++;
+				//if(count%10000==0){	System.out.println(count);	}
+				String Column[]=line.split("\t");
+				if(Column.length>1)
+				{
+					if(!StopWord_hash.containsKey(Column[0].toLowerCase()))
+					{
+						if(Preifx.equals("")) //all
+						{
+							if(Column[0].matches(".*[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_].*"))
+							{
+								String mention_rev=Column[0].replaceAll("[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_]", " ");
+								String mention_tmp=mention_rev.replaceAll("[\\W\\-\\_]","");
+								if(mention_tmp.length()>=10)
+								{
+									Tr.insertMention(mention_rev,Column[1]);
+								}
+							}
+							else
+							{
+								Tr.insertMention(Column[0],Column[1]); // mention, id
+							}
+							
+						}
+						else if(Column[0].matches("[0-9][0-9].*"))
+						{
+							if(Preifx.equals("Num"))
+							{
+								if(Column[0].matches(".*[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_].*"))
+								{
+									String mention_rev=Column[0].replaceAll("[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_]", " ");
+									String mention_tmp=mention_rev.replaceAll("[\\W\\-\\_]","");
+									if(mention_tmp.length()>=10)
+									{
+										Tr.insertMention(mention_rev,Column[1]);
+									}
+								}
+								else
+								{
+									Tr.insertMention(Column[0],Column[1]); // mention, id
+								}
+							}
+						}
+						/*
+						else if(Column[0].matches("[a-z][0-9].*"))
+						{
+							if(Preifx.equals("AZNum"))
+							{
+								if(Column[0].matches(".*[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_].*"))
+								{
+									String mention_rev=Column[0].replaceAll("[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_]", " ");
+									String mention_tmp=mention_rev.replaceAll("[\\W\\-\\_]","");
+									if(mention_tmp.length()>=10)
+									{
+										Tr.insertMention(mention_rev,Column[1]);
+									}
+								}
+								else
+								{
+									Tr.insertMention(Column[0],Column[1]); // mention, id
+								}
+							}
+						}
+						*/
+						else if(Column[0].matches("[a-z][a-z].*"))
+						{
+							if(Column[0].length()>2 && Column[0].substring(0,2).equals(Preifx))
+							{
+								if(Column[0].matches(".*[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_].*"))
+								{
+									String mention_rev=Column[0].replaceAll("[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_]", " ");
+									String mention_tmp=mention_rev.replaceAll("[\\W\\-\\_]","");
+									if(mention_tmp.length()>=10)
+									{
+										Tr.insertMention(mention_rev,Column[1]);
+									}
+								}
+								else
+								{
+									Tr.insertMention(Column[0],Column[1]); // mention, id
+								}
+							}
+						}
+						else if(Preifx.equals("Others"))
+						{
+							if(Column[0].matches(".*[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_].*"))
+							{
+								String mention_rev=Column[0].replaceAll("[\\W\\-\\_](str\\.|strain|substr\\.|substrain|var\\.|variety|variant|subsp\\.|subspecies|pv\\.|pathovars|pathovar|br\\.|biovar)[\\W\\-\\_]", " ");
+								String mention_tmp=mention_rev.replaceAll("[\\W\\-\\_]","");
+								if(mention_tmp.length()>=10)
+								{
+									Tr.insertMention(mention_rev,Column[1]);
+								}
+							}
+							else
+							{
+								Tr.insertMention(Column[0],Column[1]); // mention, id
+							}
+						}
+					}
+				}
+			}
+			inputfile.close();	
+		}
+		catch(IOException e1){ System.out.println("[Dictionary2Tree_UniqueGene]: Input file is not exist.");}
+	}
+	public void TreeFile2Tree(String Filename)	
+	{
+		try 
+		{
+			//System.out.println("TreeFile2Tree : " + Filename);
+			
+			BufferedReader inputfile = new BufferedReader(new InputStreamReader(new FileInputStream(Filename), "UTF-8"));
+			String line="";
+			int count=0;
+			while ((line = inputfile.readLine()) != null)  
+			{
+				String Anno[]=line.split("\t");
+				if(Anno.length<2){System.out.println(count+"\t"+line);} //check error
+				String LocationInTree = Anno[0];
+				String token = Anno[1];
+				String identifier="";
+				if(Anno.length==3)
+				{
+					identifier = Anno[2];
+				}
+				String LocationsInTree[]=LocationInTree.split("-");
+				TreeNode tmp = Tr.root;
+				for(int i=0;i<LocationsInTree.length-1;i++)
+				{
+					tmp=tmp.links.get(Integer.parseInt(LocationsInTree[i])-1);
+				}
+				tmp.InsertToken(token,identifier);
+				//if(count%10000==0){System.out.println(count);}
+				count++;
+			}
+			inputfile.close();	
+		}
+		catch(IOException e1){ System.out.println("[TreeFile2Tee]: Input file: "+ Filename +" is not exist.");}
+	}
+	
+	/*
+	 * Search target mention in the Prefix Tree
+	 */
+	public String MentionMatch(String Mentions)
+	{
+		ArrayList<String> location = new ArrayList<String>();
+		String Menlist[]=Mentions.split("\\|");
+		for(int m=0;m<Menlist.length;m++)
+		{
+			String Mention=Menlist[m];
+			String Mention_lc=Mention.toLowerCase();
+			Mention_lc = Mention_lc.replaceAll("[\\W\\-\\_]+", "");
+			Mention_lc = Mention_lc.replaceAll("([0-9])([a-z])", "$1 $2");
+			Mention_lc = Mention_lc.replaceAll("([a-z])([0-9])", "$1 $2");
+			String Tkns[]=Mention_lc.split(" ");
+			
+			int PrefixTranslation=0;
+			int i=0;
+			boolean find=false;
+			TreeNode tmp = Tr.root;
+			
+			while( i<Tkns.length && tmp.CheckChild(Tkns[i],PrefixTranslation)>=0) //Find Tokens in the links
+			{
+				if(i == Tkns.length-1){PrefixTranslation = 1;}
+				tmp=tmp.links.get(tmp.CheckChild(Tkns[i],PrefixTranslation)); //move point to the link
+				find=true;
+				i++;
+			}
+			if(find == true)
+			{
+				if(i==Tkns.length)
+				{
+					if(!tmp.Concept.equals(""))
+					{
+						return tmp.Concept;
+					}
+					else
+					{
+						return "-1";
+						//gene id is not found.
+					}
+				}
+				else
+				{
+					return "-2";
+					//the gene mention matched a substring in PrefixTree.
+				}
+			}
+			else
+			{
+				return "-3";
+				//mention is not found
+			}
+		}
+		return "-3"; //mention is not found
+	}
+	
+	/*
+	 * Search target mention in the Prefix Tree
+	 */
+	public String MentionMatch_species(String Mentions)
+	{
+		ArrayList<String> location = new ArrayList<String>();
+		String Menlist[]=Mentions.split("\\|");
+		for(int m=0;m<Menlist.length;m++)
+		{
+			String Mention=Menlist[m];
+			String Mention_lc=Mention.toLowerCase();
+			Mention_lc = Mention_lc.replaceAll("[\\W\\-\\_]+", " ");
+			Mention_lc = Mention_lc.replaceAll("([0-9])([a-z])", "$1 $2");
+			Mention_lc = Mention_lc.replaceAll("([a-z])([0-9])", "$1 $2");
+			Mention_lc = Mention_lc.replaceAll("^[ ]+", "");
+			Mention_lc = Mention_lc.replaceAll("[ ]+$", "");
+			String Tkns[]=Mention_lc.split(" ");
+			
+			int PrefixTranslation=0;
+			int i=0;
+			boolean find=false;
+			TreeNode tmp = Tr.root;
+			
+			while( i<Tkns.length && tmp.CheckChild(Tkns[i],PrefixTranslation)>=0) //Find Tokens in the links
+			{
+				if(i == Tkns.length-1){PrefixTranslation = 1;}
+				tmp=tmp.links.get(tmp.CheckChild(Tkns[i],PrefixTranslation)); //move point to the link
+				find=true;
+				i++;
+			}
+			if(find == true)
+			{
+				if(i==Tkns.length)
+				{
+					if(!tmp.Concept.equals(""))
+					{
+						return tmp.Concept;
+					}
+					else
+					{
+						return "-1";
+						//gene id is not found.
+					}
+				}
+				else
+				{
+					return "-2";
+					//the gene mention matched a substring in PrefixTree.
+				}
+			}
+			else
+			{
+				return "-3";
+				//mention is not found
+			}
+		}
+		return "-3"; //mention is not found
+	}
+	
+	/*
+	 * Search target mention in the Prefix Tree
+	 * ConceptType: Species|Genus|Cell|CTDGene
+	 */
+	public ArrayList<String> SearchMentionLocation(String Doc,String ConceptType)
+	{
+		ArrayList<String> location = new ArrayList<String>();
+		Doc=Doc+" XXXX XXXX";
+		String Doc_org=Doc;
+		Doc=Doc.toLowerCase();
+		String Doc_lc=Doc;
+		Doc = Doc.replaceAll("([0-9])([A-Za-z])", "$1 $2");
+		Doc = Doc.replaceAll("([A-Za-z])([0-9])", "$1 $2");
+		Doc = Doc.replaceAll("[\\W^;:,]+", " ");
+		
+		/* = keep special characters =
+		 * 
+		String regex="\\s+|(?=\\p{Punct})|(?<=\\p{Punct})";
+		String DocTkns[]=Doc.split(regex);
+		 */
+		
+		String DocTkns[]=Doc.split(" ");
+		int Offset=0;
+		int Start=0;
+		int Last=0;
+		int FirstTime=0;
+		
+		while(Doc_lc.length()>0 && Doc_lc.substring(0,1).matches("[\\W]")) //clean the forward whitespace
+		{
+			Doc_lc=Doc_lc.substring(1);
+			Offset++;
+		}
+		
+		for(int i=0;i<DocTkns.length;i++)
+		{
+			//System.out.println(i+"\t"+Start+"\t"+Last+"\t"+Offset+"\t"+Doc_lc);
+			
+			int pre_i=i;
+			int pre_Start=Start;
+			int pre_Last=Last;
+			String pre_Doc_lc=Doc_lc;
+			int pre_Offset=Offset;
+			
+			TreeNode tmp = Tr.root;
+			boolean find=false;
+			int PrefixTranslation=2;
+			if(ConceptType.equals("Species"))
+			{
+				PrefixTranslation=3;
+			}
+			int ConceptFound=i; //Keep found concept
+			String ConceptFound_STR="";//Keep found concept
+			int FirstTime_while = -1;
+			
+			while( tmp.CheckChild(DocTkns[i],PrefixTranslation)>=0 ) //Find Tokens in the links
+			{
+				FirstTime_while++;
+				tmp=tmp.links.get(tmp.CheckChild(DocTkns[i],PrefixTranslation)); //move point to the link
+				if(Start==0 && FirstTime>0){Start = Offset;} //Start <- Offset 
+				if(Doc_lc.length()>=DocTkns[i].length() && Doc_lc.substring(0,DocTkns[i].length()).equals(DocTkns[i]))
+				{
+					if(DocTkns[i].length()>0)
+					{
+						Doc_lc=Doc_lc.substring(DocTkns[i].length());
+						Offset=Offset+DocTkns[i].length();
+					}
+				}
+				Last = Offset;
+				while(Doc_lc.length()>0 && Doc_lc.substring(0,1).matches("[\\W]")) //clean the forward whitespace
+				{
+					Doc_lc=Doc_lc.substring(1);
+					Offset++;
+				}
+				i++;
+				
+				if(ConceptType.equals("Species"))
+				{
+					if(i<DocTkns.length-3 && DocTkns[i].matches("(str|strain|substr|substrain|subspecies|subsp|var|variant|pathovars|pv|biovar|bv)"))
+					{
+						Doc_lc=Doc_lc.substring(DocTkns[i].length());
+						Offset=Offset+DocTkns[i].length();
+						Last = Offset;
+						while(Doc_lc.length()>0 && Doc_lc.substring(0,1).matches("[\\W]")) //clean the forward whitespace
+						{
+							Doc_lc=Doc_lc.substring(1);
+							Offset++;
+						}
+						i++;
+					}
+				}
+				
+				if(!tmp.Concept.equals("") && (Last-Start>0)) //Keep found concept
+				{
+					if(Last<Doc_org.length())
+					{
+						ConceptFound=i;
+						ConceptFound_STR=Start+"\t"+Last+"\t"+Doc_org.substring(Start, Last)+"\t"+tmp.Concept;
+						//System.out.println(ConceptFound_STR);
+					}
+				}
+				
+				find=true;
+				if(i>=DocTkns.length){break;}
+				else if(i==DocTkns.length-1){PrefixTranslation=2;}
+				
+				//System.out.println(i+"\t"+Start+"\t"+Last+"\t("+FirstTime_while+")\t"+Offset+"\t"+Doc_lc);
+				
+				if(FirstTime_while==0) // first matched token
+				{
+					pre_i=i;
+					pre_Start=Start;
+					pre_Last=Last;
+					pre_Doc_lc=Doc_lc;
+					pre_Offset=Offset;
+				}
+			}
+			
+			if(find == true)
+			{
+				//System.out.println(find+"\t"+FirstTime_while+"\t"+Start+"\t"+Last+"\t"+Doc_org.substring(Start, Last)+"\t"+tmp.Concept);
+				if(!tmp.Concept.equals("")) //the last matched token has concept id 
+				{
+					if(Last<Doc_org.length() && Last>Start)
+					{
+						location.add(Start+"\t"+Last+"\t"+Doc_org.substring(Start, Last)+"\t"+tmp.Concept);
+					}
+				}
+				else
+				{
+					if(!ConceptFound_STR.equals("")) //Keep found concept
+					{
+						location.add(ConceptFound_STR);
+						i = ConceptFound + 1;
+					}
+					
+					if(FirstTime_while>=1)
+					{
+						i=pre_i;
+						Start=pre_Start;
+						Last=pre_Last;
+						Doc_lc=pre_Doc_lc;
+						Offset=pre_Offset;
+					}
+				}
+				Start=0;
+				Last=0;
+				if(i>0){i--;}
+				ConceptFound=i; //Keep found concept
+				ConceptFound_STR="";//Keep found concept
+			}
+			else //if(find == false)
+			{
+				//System.out.println(find+"\t"+FirstTime_while+"\t"+Start+"\t"+Last+"\t"+Doc_org.substring(Start, Last)+"\t"+tmp.Concept);
+				
+				if(FirstTime_while>=1 && tmp.Concept.equals(""))
+				{
+					i=pre_i;
+					Start=pre_Start;
+					Last=pre_Last;
+					Doc_lc=pre_Doc_lc;
+					Offset=pre_Offset;
+				}
+				
+				if(Doc_lc.length()>=DocTkns[i].length() && Doc_lc.substring(0,DocTkns[i].length()).equals(DocTkns[i]))
+				{
+					if(DocTkns[i].length()>0)
+					{
+						Doc_lc=Doc_lc.substring(DocTkns[i].length());
+						Offset=Offset+DocTkns[i].length();
+					}
+				}
+			}
+			
+			while(Doc_lc.length()>0 && Doc_lc.substring(0,1).matches("[\\W]")) //clean the forward whitespace
+			{
+				Doc_lc=Doc_lc.substring(1);
+				Offset++;
+			}
+			FirstTime++;
+			
+			//System.out.println();
+		}
+		return location;
+	}
+	
+	/*
+	 * Print out the Prefix Tree
+	 */
+	public String PrintTree()
+	{
+		return Tr.PrintTree_preorder(Tr.root,"");
+	}
+	
+	public void SaveTree(String outputfile) throws IOException
+	{
+		BufferedWriter fr = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputfile), "UTF-8"));
+		Tr.SaveTree_preorder(Tr.root,"",fr);
+		fr.close();
+	}
+	
+	
+	public void insertMention(String Mention, String Identifier)
+	{
+		Tr.insertMention(Mention,Identifier);
+	}
+}
+
+class Tree 
+{
+	/*
+	 * Prefix Tree - root node
+	 */
+	public TreeNode root;
+	
+	public Tree() 
+	{ 
+		root = new TreeNode("-ROOT-"); 
+	}
+	
+	/*
+	 * Insert mention into the tree
+	 */
+	public void insertMention(String Mention, String Identifier)
+	{
+		Mention=Mention.toLowerCase();
+		
+		Mention = Mention.replaceAll("([0-9])([A-Za-z])", "$1 $2");
+		Mention = Mention.replaceAll("([A-Za-z])([0-9])", "$1 $2");
+		Mention = Mention.replaceAll("[\\W\\-\\_]+", " ");
+		/* = keep special characters =
+		 * 
+		String regex="\\s+|(?=\\p{Punct})|(?<=\\p{Punct})";
+		String Tokens[]=Mention.split(regex);
+		 */
+		String Tokens[]=Mention.split(" ");
+		TreeNode tmp = root;
+		for(int i=0;i<Tokens.length;i++)
+		{
+			if(tmp.CheckChild(Tokens[i],0)>=0)
+			{
+				tmp=tmp.links.get( tmp.CheckChild(Tokens[i],0) ); //go through next generation (exist node)
+				if(i == Tokens.length-1)
+				{
+					tmp.Concept=Identifier;
+				}
+			}
+			else //not exist
+			{
+				if(i == Tokens.length-1)
+				{
+					tmp.InsertToken(Tokens[i],Identifier);
+				}
+				else
+				{
+					tmp.InsertToken(Tokens[i]);
+				}
+				tmp=tmp.links.get(tmp.NumOflinks-1); //go to the next generation (new node)
+			}
+		}
+	}
+	
+	/*
+	 * Print the tree by pre-order
+	 */
+	public String PrintTree_preorder(TreeNode node, String LocationInTree)
+	{
+		String opt="";
+		if(!node.token.equals("-ROOT-"))//Ignore root
+		{
+			if(node.Concept.equals(""))
+			{
+				opt=opt+LocationInTree+"\t"+node.token+"\n";
+			}
+			else
+			{
+				opt=opt+LocationInTree+"\t"+node.token+"\t"+node.Concept+"\n";
+			}
+		} 
+		if(!LocationInTree.equals("")){LocationInTree=LocationInTree+"-";}
+		for(int i=0;i<node.NumOflinks;i++)
+		{
+			opt=opt+PrintTree_preorder(node.links.get(i),LocationInTree+(i+1));
+		}
+		return opt;
+	}
+	
+	/*
+	 * Print the tree by pre-order
+	 */
+	public void SaveTree_preorder(TreeNode node, String LocationInTree, BufferedWriter fr) throws IOException
+	{
+		if(!node.token.equals("-ROOT-"))//Ignore root
+		{
+			if(node.Concept.equals(""))
+			{
+				fr.write(LocationInTree+"\t"+node.token+"\n");
+			}
+			else
+			{
+				fr.write(LocationInTree+"\t"+node.token+"\t"+node.Concept+"\n");
+			}
+		} 
+		if(!LocationInTree.equals("")){LocationInTree=LocationInTree+"-";}
+		for(int i=0;i<node.NumOflinks;i++)
+		{
+			SaveTree_preorder(node.links.get(i),LocationInTree+(i+1),fr);
+		}
+	}
+}
+
+class TreeNode 
+{
+	String token; //token of the node
+	int NumOflinks; //Number of links
+	public String Concept;
+	HashMap<String,Integer> Hashs;
+	ArrayList<TreeNode> links;
+	
+	public TreeNode(String Tok,String ID)
+	{
+		token = Tok;
+		NumOflinks = 0;
+		Concept = ID;
+		links = new ArrayList<TreeNode>();/*link*/
+		Hashs = new HashMap<String,Integer>();/*hash*/
+	}
+	public TreeNode(String Tok)
+	{
+		token = Tok;
+		NumOflinks = 0;
+		Concept = "";
+		links = new ArrayList<TreeNode>();/*link*/
+		Hashs = new HashMap<String,Integer>();/*hash*/
+	}
+	public TreeNode()
+	{
+		token = "";
+		NumOflinks = 0;
+		Concept = "";
+		links = new ArrayList<TreeNode>();/*link*/
+		Hashs = new HashMap<String,Integer>();/*hash*/
+	}
+	
+	public String toString()
+	{
+		return (token+"\t"+Concept);
+	}
+	
+	/*
+	 * Insert an new node under the target node
+	 */
+	public void InsertToken(String Tok)
+	{
+		TreeNode NewNode = new TreeNode(Tok);
+		
+		/*link*/
+		links.add(NewNode);
+		
+		/*hash*/
+		Hashs.put(Tok, NumOflinks);
+		
+		NumOflinks++;
+	}
+	public void InsertToken(String Tok,String ID)
+	{
+		TreeNode NewNode = new TreeNode(Tok,ID);
+		/*link*/
+		links.add(NewNode);
+		
+		/*hash*/
+		Hashs.put(Tok, NumOflinks);
+		
+		NumOflinks++;
+	}
+	
+	/*
+	 * Check the tokens of children
+	 */
+	public int CheckChild(String Tok, Integer PrefixTranslation)
+	{
+		if(Hashs.containsKey(Tok))
+		{
+			return(Hashs.get(Tok));
+		}
+		
+		if(PrefixTranslation == 1 && Tok.matches("(alpha|beta|gamam|[abg]|[12])")) // SuffixTranslationMap
+		{
+			if(Hashs.containsKey(GNormPlus.SuffixTranslationMap_hash.get(Tok)))
+			{
+				return(Hashs.get(GNormPlus.SuffixTranslationMap_hash.get(Tok)));
+			}
+			
+		}
+		else if(PrefixTranslation == 2 && Tok.matches("[1-5]")) // for CTDGene feature
+		{
+			for(int i=0;i<links.size();i++)
+	        {
+				if(links.get(i).token.matches("[1-5]"))
+				{
+					return(i);
+				}
+	        }
+			
+			for(int i=1;i<=5;i++)
+			{
+				if(Hashs.containsKey(i)){return(Hashs.get(i));}
+			}
+		}
+		
+		return(-1);
+	}
+}
 	

src_Java/GNormPluslib/SR.java

@@ -1,1044 +1,1044 @@
-/**
- * Project: GNormPlus
- * Function: Species recognition and Species assignment
- */
-
-package GNormPluslib;
-
-import bioc.BioCAnnotation;
-import bioc.BioCCollection;
-import bioc.BioCDocument;
-import bioc.BioCLocation;
-import bioc.BioCPassage;
-
-import bioc.io.BioCDocumentWriter;
-import bioc.io.BioCFactory;
-import bioc.io.woodstox.ConnectorWoodstox;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.text.BreakIterator;
-import java.time.LocalDate;
-import java.time.ZoneId;
-
-import javax.xml.stream.XMLStreamException;
-
-import org.tartarus.snowball.SnowballStemmer;
-import org.tartarus.snowball.ext.englishStemmer;
-
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Collections;
-
-public class SR 
-{
-	@SuppressWarnings("null")
-	public void SpeciesRecognition(String Filename,String FilenameBioC,String StrainFilename,String FilterAntibody) throws IOException, XMLStreamException
-	{
-		/** Recognizing Species Names: SP */
-		for (int i = 0; i < GNormPlus.BioCDocobj.PMIDs.size(); i++) /** PMIDs : i */
-		{
-			String Pmid = GNormPlus.BioCDocobj.PMIDs.get(i);
-			PrefixTree PT_Genus = new PrefixTree();
-			HashMap<String, String> SPID_hash = new HashMap<String, String>();
-			ArrayList<String> TargetedLocation = new ArrayList<String>();
-			HashMap<String, String> GenusNames = new HashMap<String, String>();
-			HashMap<String, String> Mention2ID_lc = new HashMap<String, String>();
-			ArrayList<String> IDset = new ArrayList<String>();
-			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
-			{
-				String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j); // Passage context
-				
-				/** Species recognition */
-				ArrayList<String> locations = GNormPlus.PT_Species.SearchMentionLocation(PassageContext,"Species"); /** PT_Species */
-				for (int k = 0 ; k < locations.size() ; k++)
-				{
-					String anno[]=locations.get(k).split("\t");
-					int start= Integer.parseInt(anno[0]);
-	        		int last= Integer.parseInt(anno[1]);
-	        		
-	        		// For anti-serum filtering
-	        		String ForwardSTR="";
-	        		String BackwardSTR="";
-	        		if(start>21)
-	        		{
-	        			ForwardSTR = (PassageContext+"ZZZZZZZZZZZZZZZZZZZZZZZZZZZ").substring(start-21,last);
-	        		}
-	        		else
-	        		{
-	        			ForwardSTR = (PassageContext+"ZZZZZZZZZZZZZZZZZZZZZZZZZZZ").substring(0,last);
-	        		}
-	        		if(PassageContext.length()>last+21)
-	        		{
-	        			BackwardSTR = PassageContext.substring(start,last+21);
-	        		}
-	        		else
-	        		{
-	        			BackwardSTR = PassageContext.substring(start,PassageContext.length());
-	        		}
-	        			        		
-	        		String mention = anno[2];
-	        		String id = anno[3];
-	        		String mention_tmp=mention.toLowerCase();
-	        		mention_tmp = mention_tmp.replaceAll("([^A-Za-z0-9@ ])", "\\\\$1");
-	        		String antibody="";
-	        		if(ForwardSTR.toLowerCase().matches(".*(anti|antibody|antibodies|serum|polyclonal|monoclonal|igg)[\\W\\-\\_]+"+mention_tmp)) {antibody="(anti)";}//filtering : antibody
-	        		else if(BackwardSTR.toLowerCase().matches(mention_tmp+"[\\W\\-\\_]+(anti|antibody|antibodies|serum|polyclonal|monoclonal|igg).*")){antibody="(anti)";} //filtering : antibody
-	        		else if(BackwardSTR.toLowerCase().matches(mention_tmp+"[\\W\\-\\_]+[A-Za-z0-9]+[\\W\\-\\_]+(anti|antibody|antibodies|serum|polyclonal|monoclonal|igg).*")){antibody="(anti)";} //filtering : antibody
-	        		
-					if(mention.matches(".*[\\(\\[\\{].*") && BackwardSTR.toLowerCase().matches(mention_tmp+"\\).*") )
-    				{
-	        			last=last+1;
-	        			mention=mention+")";
-    				}
-	        		
-	        		if(BackwardSTR.toLowerCase().matches(mention_tmp+"[0-9].*")){} // filtered: Bee1p
-	        		else if((mention.matches(".*[;:,].*")) && mention.length()<=10){} // filtered : x, XXX
-	        		else if(mention.matches("to[\\W\\-\\_]+[0-9]+")){} // to 7
-	        		else if(mention.matches("[a-z][\\)\\]\\}].*") && (!mention.matches(".*[\\(\\[\\{].*")) && mention.length()<=10){} // s). Major
-	        		else if(mention.matches(".*[\\(\\[\\{].*") && (!mention.matches(".*[\\)\\]\\}].*")) && mention.length()<=10){} // s). Major
-	        		else if(!id.equals("NA"))
-	        		{
-	        			if(GNormPlus.BioCDocobj.Annotations.size()>i && GNormPlus.BioCDocobj.Annotations.get(i).size()>j)
-						{
-	        				if((!mention.matches("^[A-Za-z] [A-Za-z0-9]+$")) && (mention.length()>=3)) // invalid species: "a group/a GAL4/a strain"
-	        				{
-	        					if(FilterAntibody.equals("False") || (!antibody.equals("(anti)")))
-	        					{
-	        						String patt="^(.+?) [sS]train";
-									Pattern ptmp = Pattern.compile(patt);
-									Matcher mtmp = ptmp.matcher(mention);
-									if(mtmp.find())
-									{
-										mention=mtmp.group(1);
-										last=last-7;
-									}
-			        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tSpecies\t"+id); //+antibody
-			        				String mentions_tmp=mention.toLowerCase();
-		    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
-		    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
-		    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
-			        				Mention2ID_lc.put(mention.toLowerCase(), id); //+antibody
-			        				
-			         				String mention_genus = "";
-			        				patt="^([A-Za-z]+) ";
-									ptmp = Pattern.compile(patt);
-									mtmp = ptmp.matcher(mention);
-									if(mtmp.find())
-									{
-										mention_genus=mtmp.group(1); // get genus
-									}
-									
-			        				IDset.add(id);
-			        				for(int s=start;s<last;s++)
-									{
-										TargetedLocation.add(j+"\t"+s);
-									}
-			        				String ids[]=id.split(";");
-			        				for(int x=0;x<ids.length;x++)
-			        				{
-				        				patt="^\\**([0-9]+)";
-										ptmp = Pattern.compile(patt);
-										mtmp = ptmp.matcher(ids[x]);
-										if(mtmp.find())
-										{
-											SPID_hash.put(mtmp.group(1), mention_genus);
-										}
-			        				}
-	        					}
-							}
-						}
-	        		}
-				}
-				
-				/** Cell Line recognition */
-				locations = GNormPlus.PT_Cell.SearchMentionLocation(PassageContext,"Cell"); /** PT_Cell */
-				for (int k = 0 ; k < locations.size() ; k++)
-				{
-					String anno[]=locations.get(k).split("\t");
-					int start= Integer.parseInt(anno[0]);
-	        		int last= Integer.parseInt(anno[1]);
-	        		String mention = anno[2];
-	        		String id = anno[3];
-	        		if(GNormPlus.BioCDocobj.Annotations.size()>i && GNormPlus.BioCDocobj.Annotations.get(i).size()>j)
-					{
-	        			if(!TargetedLocation.contains(j+"\t"+start)) //already exists
-	        			{
-	        				int last40=0;
-		        			if(PassageContext.length()>=last+40)
-		        			{
-		        				last40=last+40;
-		        			}
-		        			else
-		        			{
-		        				last40=PassageContext.length();
-		        			}
-		        			
-		        			// For anti-serum filtering
-			        		String ForwardSTR="";
-			        		String BackwardSTR="";
-			        		if(start>21)
-			        		{
-			        			ForwardSTR = PassageContext.substring(start-21,last);
-			        		}
-			        		else
-			        		{
-			        			ForwardSTR = PassageContext.substring(0,last);
-			        		}
-			        		if(PassageContext.length()>last+21)
-			        		{
-			        			BackwardSTR = PassageContext.substring(start,last+21);
-			        		}
-			        		else
-			        		{
-			        			BackwardSTR = PassageContext.substring(start,PassageContext.length());
-			        		}
-			        		String mention_tmp=mention.toLowerCase();
-			        		mention_tmp = mention_tmp.replaceAll("([^A-Za-z0-9@ ])", "\\\\$1");
-			        		if(mention_tmp.matches(".*[\\[\\]\\(\\)\\{\\}].*")){}
-			        		else if(BackwardSTR.toLowerCase().matches(mention_tmp+"[0-9\\-\\_].*")){} // filtered: Bee1p
-			        		else if(ForwardSTR.toLowerCase().matches(".*[0-9\\-\\_]"+mention_tmp)){} // filtered: IL-22RA1
-			        		else
-			        		{
-			        			String patt="[\\W\\-]cell([\\- ]*line|)[s]*[\\W\\-]";
-			    				Pattern ptmp = Pattern.compile(patt);
-			    				Matcher mtmp = ptmp.matcher(PassageContext.substring(last, last40).toLowerCase());
-			    				if(mtmp.find())
-			    				{
-			    					if(GNormPlus.taxid4gene.contains(id)) // for gene
-				        			{
-				        				id="*"+id;
-					        		}
-			    					GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tCell\t"+id);
-									String mentions_tmp=mention.toLowerCase();
-		    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
-		    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
-		    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
-			    					IDset.add(id);
-			    					for(int s=start;s<last;s++)
-									{
-										TargetedLocation.add(j+"\t"+s);
-									}
-			    				}
-			        		}
-		        		}
-					}
-				}
-				
-				/** Genus names*/
-				for(String ID: SPID_hash.keySet())
-				{
-					if(GNormPlus.GenusID_hash.containsKey(ID))
-					{
-						GenusNames.put(ID,GNormPlus.GenusID_hash.get(ID));
-					}
-					if(SPID_hash.get(ID).length()>=7)
-					{
-						GenusNames.put(ID,SPID_hash.get(ID));
-					}
-				}
-			}
-			
-			GenusNames.put("3702", "arabidopsis");
-			GenusNames.put("4932", "saccharomyces");
-			GenusNames.put("562", "escherichia");
-			GenusNames.put("7227", "drosophila");
-			GenusNames.put("8355", "xenopus");
-			
-			PT_Genus.Hash2Tree(GenusNames);
-			
-			/** Genus recognition */
-			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
-			{
-				if(GNormPlus.BioCDocobj.PassageContexts.size()>i && 
-					GNormPlus.BioCDocobj.PassageContexts.get(i).size()>j &&  
-					GNormPlus.BioCDocobj.Annotations.size()>i &&  
-					GNormPlus.BioCDocobj.Annotations.get(i).size()>j 
-					)
-				{
-					String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j);
-					ArrayList<String> locations_Genus = PT_Genus.SearchMentionLocation(PassageContext,"Genus"); /** PT_Genus*/
-					for (int k = 0 ; k < locations_Genus.size() ; k++)
-					{
-						String anno[]=locations_Genus.get(k).split("\t");
-						String start= anno[0];
-		        		String last= anno[1];
-		        		String mention = anno[2];
-		        		String id = anno[3];
-		        		if(!TargetedLocation.contains(j+"\t"+start)) //already exists
-	        			{
-		        			String patt="^\\**([0-9]+)$";
-							Pattern ptmp = Pattern.compile(patt);
-							Matcher mtmp = ptmp.matcher(id);
-							if(mtmp.find())
-							{
-								id = mtmp.group(1);
-							}
-							
-							if(GNormPlus.taxid4gene.contains(id)) // for gene
-		        			{
-		        				id="*"+id;
-			        		}
-							GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tGenus\t"+id);
-							String mentions_tmp=mention.toLowerCase();
-    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
-    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
-    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
-							IDset.add(id);
-							for(int s=Integer.parseInt(start);s<Integer.parseInt(last);s++)
-							{
-								TargetedLocation.add(j+"\t"+s);
-							}
-		        		}
-		        	}
-				}
-			}
-			
-			/** Strain Tree */
-			PrefixTree PT_Strain = new PrefixTree();
-			HashMap<String, String> StrainID_hash = new HashMap<String, String>();
-			BufferedReader br = new BufferedReader(new FileReader(StrainFilename));
-			String line="";
-			while ((line = br.readLine()) != null)  
-			{
-				String l[]=line.split("\t");
-				String ancestor = l[0];
-				String tax_id = l[1];
-				String tax_names = l[2];
-				if(SPID_hash.containsKey(ancestor))
-				{
-					StrainID_hash.put(tax_id, tax_names); // tax id -> strain
-				}
-				else if(SPID_hash.containsKey(tax_id))
-				{
-					StrainID_hash.put(tax_id, tax_names); // tax id -> strain
-				}
-			}
-			br.close();
-			HashMap<String, String> StrainNames = new HashMap<String, String>();
-			for(String ID: StrainID_hash.keySet())
-			{
-				StrainNames.put(ID,StrainID_hash.get(ID));
-			}
-			
-			PT_Strain.Hash2Tree(StrainNames);
-			
-			/** Strain recognition */
-			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
-			{
-				if(GNormPlus.BioCDocobj.PassageContexts.size()>i && 
-					GNormPlus.BioCDocobj.PassageContexts.get(i).size()>j &&  
-					GNormPlus.BioCDocobj.Annotations.size()>i &&  
-					GNormPlus.BioCDocobj.Annotations.get(i).size()>j 
-					)
-				{
-					String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j); // Passage context
-					ArrayList<String> locations_Strain = PT_Strain.SearchMentionLocation(PassageContext,"Strain"); /** PT_Strain*/
-					for (int k = 0 ; k < locations_Strain.size() ; k++)
-					{
-						String anno[]=locations_Strain.get(k).split("\t");
-						String start= anno[0];
-		        		String last= anno[1];
-		        		String mention = anno[2];
-		        		String id = anno[3];
-		        		if(!TargetedLocation.contains(j+"\t"+start)) //already exists
-	        			{
-		        			if((!mention.matches(".*[;,\\{\\}\\(\\)\\[\\]].*")) && !mention.matches("[a-z]{1,4} [0-9]{1,3}"))
-		        			{
-			        			if(GNormPlus.taxid4gene.contains(id)) // for gene
-			        			{
-			        				id="*"+id;
-				        		}
-			        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tStrain\t"+id);
-								String mentions_tmp=mention.toLowerCase();
-	    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
-	    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
-	    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
-			        			IDset.add(id);
-			        			for(int s=Integer.parseInt(start);s<Integer.parseInt(last);s++)
-								{
-									TargetedLocation.add(j+"\t"+s);
-								}
-		        			}
-		        		}
-					}
-				}
-			}
-			
-			HashMap<String, String> OtherNames = new HashMap<String, String>();
-			for(String men : Mention2ID_lc.keySet())
-			{
-				String men_id= Mention2ID_lc.get(men);
-				if(GNormPlus.PmidLF2Abb_lc_hash.containsKey(Pmid+"\t"+men))
-				{
-					String Abb = GNormPlus.PmidLF2Abb_lc_hash.get(Pmid+"\t"+men);
-					// Abbreviation
-					if(OtherNames.containsKey(men_id))
-					{
-						OtherNames.put(men_id, OtherNames.get(men_id)+"|"+Abb);
-					}
-					else
-					{
-						OtherNames.put(men_id,Abb);
-					}
-				}
-				String men_nospace=men.replaceAll(" ", "");
-				// no space
-				if(OtherNames.containsKey(men_id))
-				{
-					OtherNames.put(men_id, OtherNames.get(men_id)+"|"+men_nospace);
-				}
-				else
-				{
-					OtherNames.put(men_id,men_nospace);
-				}
-			}
-			PrefixTree PT_Others = new PrefixTree();
-			PT_Others.Hash2Tree(OtherNames);
-			
-			/** 
-			 *
-			 * Others: 
-			 * 1) Abbreviation 
-			 * 2) no space
-			 * 
-			 * */
-			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
-			{
-				if(GNormPlus.BioCDocobj.PassageContexts.size()>i && 
-					GNormPlus.BioCDocobj.PassageContexts.get(i).size()>j &&  
-					GNormPlus.BioCDocobj.Annotations.size()>i &&  
-					GNormPlus.BioCDocobj.Annotations.get(i).size()>j 
-					)
-				{
-					String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j); // Passage context
-					ArrayList<String> locations_Abb = PT_Others.SearchMentionLocation(PassageContext,"Species"); /** PT_Abb*/
-					for (int k = 0 ; k < locations_Abb.size() ; k++)
-					{
-						String anno[]=locations_Abb.get(k).split("\t");
-						String start= anno[0];
-		        		String last= anno[1];
-		        		String mention = anno[2];
-		        		String id = anno[3];
-		        		if(!TargetedLocation.contains(j+"\t"+start)) //already exists
-	        			{
-		        			if(GNormPlus.taxid4gene.contains(id)) // for gene
-		        			{
-		        				id="*"+id;
-			        		}
-		        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tSpecies\t"+id);
-							String mentions_tmp=mention.toLowerCase();
-    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
-    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
-    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
-		        			Mention2ID_lc.put(mention.toLowerCase(), id);
-		        			IDset.add(id);
-		        			for(int s=Integer.parseInt(start);s<Integer.parseInt(last);s++)
-							{
-								TargetedLocation.add(j+"\t"+s);
-							}
-		        		}
-					}
-				}
-			}
-			
-			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
-			{
-				if(GNormPlus.BioCDocobj.PassageContexts.size()>i && GNormPlus.BioCDocobj.PassageContexts.get(i).size()>j && GNormPlus.BioCDocobj.Annotations.size()>i && GNormPlus.BioCDocobj.Annotations.get(i).size()>j)
-				{
-					ArrayList <Integer> remove_anno = new ArrayList <Integer>();
-					for (int a = 0; a < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); a++) /** Annotations : a */
-					{
-						String SpAnno[]=GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(a).split("\t");
-						String start= SpAnno[0];
-		        		String last= SpAnno[1];
-		        		String mention = SpAnno[2];
-		        		String type = SpAnno[3];
-		        		
-		        		if(type.matches("Gene|FamilyName"))
-		        		{
-		        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(a,start+"\t"+last+"\t"+mention+"\t"+type);
-			        	}
-		        		else if(type.matches("Species|Genus|Strain|Cell") && SpAnno.length==5)
-		        		{
-		        			//System.out.println(GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(a));
-			        		/** Abbreviation solution */
-			    			if(GNormPlus.PmidAbb2LF_lc_hash.containsKey(Pmid+"\t"+mention.toLowerCase()) && Mention2ID_lc.containsKey(GNormPlus.PmidAbb2LF_lc_hash.containsKey(Pmid+"\t"+mention.toLowerCase())))
-							{
-								String LF_lc=GNormPlus.PmidAbb2LF_lc_hash.get(Pmid+"\t"+mention.toLowerCase());
-								if(Mention2ID_lc.containsKey(LF_lc))
-								{
-									String LF_ID=Mention2ID_lc.get(LF_lc);
-									GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(a, start+"\t"+last+"\t"+mention+"\t"+type+"\t"+LF_ID);
-									String mentions_tmp=mention.toLowerCase();
-		    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
-		    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
-		    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
-								}
-							}
-			    			else if (SpAnno.length>4) 
-			    			{
-			    				String id = SpAnno[4];
-			    				String id_split[]=id.split(";");
-		    					if(id_split.length>=2)
-		    					{
-		    						/** Smallest set of tax ids */
-				    				boolean found=false;
-		    						for(int x=0;x<IDset.size();x++)
-				    				{
-				    					String id_tmp= IDset.get(x);
-				    					for(int y=0;y<id_split.length;y++) // if any other id is a component of the target id
-				    					{
-				    						if(id_split[y].equals(id_tmp))
-				    						{
-				    							found=true;
-				    						}
-				    					}
-				    					if(found == true)
-				    					{
-				    						GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(a, start+"\t"+last+"\t"+mention+"\t"+type+"\t"+id_tmp);
-											String mentions_tmp=mention.toLowerCase();
-				    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
-				    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
-				    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
-				    						x=1000000;
-				    					}
-			    					}
-		    						
-		    						/** smallest tax id number */
-		    						if(found == false)
-			    					{
-		    							int min=10000000;
-				    					String min_id="";
-			    						for(int y=0;y<id_split.length;y++) // if any other id is a component of the target id
-				    					{
-			    							String id_tmp = id_split[y];
-			    							String patt="^\\**([0-9]+)";
-			    							Pattern ptmp = Pattern.compile(patt);
-			    							Matcher mtmp = ptmp.matcher(id_tmp);
-			    							if(mtmp.find())
-			    							{
-			    								id_tmp = mtmp.group(1);
-			    							}
-			    								
-			    							if(y==0)
-			    							{
-			    								min_id=id_split[y];
-			    								min=Integer.parseInt(id_tmp);
-			    							}
-			    							else if(Integer.parseInt(id_tmp)<min)
-			    							{
-			    								min=Integer.parseInt(id_tmp);
-			    								min_id=id_tmp;
-			    							}
-				    					}
-			    						if(GNormPlus.taxid4gene.contains(min_id)) // for gene
-			    						{
-			    							min_id="*"+min_id;
-			    						}
-					        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(a,start+"\t"+last+"\t"+mention+"\tSpecies\t"+min_id);
-										String mentions_tmp=mention.toLowerCase();
-			    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
-			    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
-			    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
-					        		}
-			    				}
-			    			}
-		        		}
-		        		else //disease, and other concepts
-		        		{
-		        			remove_anno.add(a);
-		        		}
-					}
-					
-					Collections.sort(remove_anno);
-					for (int counter = remove_anno.size()-1; counter >= 0 ; counter--) 
-					{ 		      
-						int ai=remove_anno.get(counter);
-						//System.out.println("\n"+ai+"\t"+GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(ai));
-						GNormPlus.BioCDocobj.Annotations.get(i).get(j).remove(ai);
-					}
-				}
-			}	
-		}
-		GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,false,true); //save in BioC file
-	}
-	public void SpeciesAssignment(String Filename,String FilenameBioC) throws IOException, XMLStreamException
-	{
-		GNormPlus.BioCDocobj.Annotations = new ArrayList();
-		GNormPlus.BioCDocobj.BioCReaderWithAnnotation(Filename);
-		
-		BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.US);	
-		for (int i = 0; i < GNormPlus.BioCDocobj.Annotations.size(); i++) /** PMIDs : i */
-		{
-			HashMap<String, String> PrefixIDTarget_hash = new HashMap<String, String>();
-			PrefixIDTarget_hash.put("9606", "h");
-			PrefixIDTarget_hash.put("10090", "m");
-			PrefixIDTarget_hash.put("10116", "r");
-			PrefixIDTarget_hash.put("4932", "y");
-			PrefixIDTarget_hash.put("7227", "d");
-			PrefixIDTarget_hash.put("7955", "z|zf|Zf|dr|Dr");
-			PrefixIDTarget_hash.put("3702", "at|At");
-			
-			HashMap<String, Double> SP2Num_hash = new HashMap<String, Double>();
-			for (int j = 0; j < GNormPlus.BioCDocobj.Annotations.get(i).size(); j++) /** Paragraphs : j */
-			{
-				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) // Annotation : k
-				{
-					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
-					if(anno.length==5) //Species
-	        		{
-						String patt="^\\**([0-9]+)$";
-						Pattern ptmp = Pattern.compile(patt);
-						Matcher mtmp = ptmp.matcher(anno[4]);
-						if(mtmp.find())
-						{
-							String id = mtmp.group(1);
-							
-							if(!PrefixIDTarget_hash.containsKey(id))
-							{
-								PrefixIDTarget_hash.put(id,GNormPlus.PrefixID_hash.get(id)); // taxid -> prefix
-							}
-							if(j == 0)//title
-			        		{
-			        			if(SP2Num_hash.containsKey(id))
-			        			{
-			        				SP2Num_hash.put(id, SP2Num_hash.get(id)+2);
-			        			}
-			        			else
-			        			{
-			        				if(GNormPlus.TaxFreq_hash.containsKey(id))
-			        				{
-			        					SP2Num_hash.put(id, GNormPlus.TaxFreq_hash.get(id)+2);
-			        				}
-			        				else
-			        				{
-			        					SP2Num_hash.put(id, 2.0);
-			        				}
-			        			}
-			        			// Virus -> Human (not to double weight human to virus)
-			        			/*if(GNormPlus.SP_Virus2Human_hash.containsKey(id))
-		        				{
-			        				if(SP2Num_hash.containsKey("9606"))
-				        			{
-				        				SP2Num_hash.put("9606", SP2Num_hash.get("9606")+2);
-				        			}
-				        			else
-				        			{
-				        				SP2Num_hash.put("9606", 2 + GNormPlus.TaxFreq_hash.get("9606")+1);
-				        			}
-		        				}*/
-			        		}
-			        		else
-			        		{
-			        			if(SP2Num_hash.containsKey(id))
-			        			{
-			        				SP2Num_hash.put(id, SP2Num_hash.get(id)+1);
-			        			}
-			        			else
-			        			{
-			        				if(GNormPlus.TaxFreq_hash.containsKey(id))
-			        				{
-			        					SP2Num_hash.put(id, 1 + GNormPlus.TaxFreq_hash.get(id));
-			        				}
-			        				else
-			        				{
-			        					SP2Num_hash.put(id, 1.0);
-			        				}
-			        			}
-			        			// Virus -> Human
-			        			/*if(GNormPlus.SP_Virus2Human_hash.containsKey(id))
-		        				{
-			        				if(SP2Num_hash.containsKey("9606"))
-				        			{
-				        				SP2Num_hash.put("9606", SP2Num_hash.get("9606")+1);
-				        			}
-				        			else
-				        			{
-				        				SP2Num_hash.put("9606", GNormPlus.TaxFreq_hash.get("9606")+1);
-				        			}
-		        				}*/
-			        		}
-						}
-	        		}
-				}
-			}
-			String MajorSP="9606";
-			double MaxSP=0;
-			for(String tid : SP2Num_hash.keySet())
-			{
-				if(SP2Num_hash.get(tid)>MaxSP)
-				{
-					MajorSP=tid;
-					MaxSP=SP2Num_hash.get(tid);
-				}
-			}
-			
-			for (int j = 0; j < GNormPlus.BioCDocobj.PassageContexts.get(i).size(); j++) /** Paragraphs : j */
-			{
-				String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j); // Passage context
-				//int PassageOffset = GNormPlus.BioCDocobj.PassageOffsets.get(i).get(j); // Passage offset
-				iterator.setText(PassageContext);
-				ArrayList<Integer> Sentence_offsets = new ArrayList<Integer>();
-				int Sent_start = iterator.first();
-				for (int Sent_last = iterator.next(); Sent_last != BreakIterator.DONE; Sent_start = Sent_last, Sent_last = iterator.next()) 
-				{
-					Sentence_offsets.add(Sent_start);
-				}
-				
-				HashMap<Integer,String> Annotations_Gene_hash = new HashMap<Integer,String>();
-				ArrayList<String> Annotations_Species = new ArrayList<String>();
-				if(GNormPlus.BioCDocobj.Annotations.get(i).size()>j)
-				{
-					for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) // Annotation : k
-					{
-						String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
-						if(anno.length==5) //Species
-		        		{
-							Annotations_Species.add(GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k));
-		        		}
-		        		else //Gene : if(anno.length==3)
-		        		{
-		        			//String mention = PassageContext.substring(Integer.parseInt(anno[0]), Integer.parseInt(anno[1]));
-		        			Annotations_Gene_hash.put(k,GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k)); // k -> Gene Annotation
-		        		}
-					}
-	
-					//Gene --> Species Inference (PMID:28777492)
-					HashMap<String,HashMap<Integer,String>> mention2Location2Species_hash = new HashMap<String,HashMap<Integer,String>>(); 
-					HashMap<Integer,String> Location2Species_hash = new HashMap<Integer,String>(); 
-					for (int k : Annotations_Gene_hash.keySet()) // k is the index of GNormPlus.BioCDocobj.Annotations.get(i).get(j) 
-	    			{
-						boolean SPfound = false;
-						String anno[] = Annotations_Gene_hash.get(k).split("\t");
-	    				int G_Start= Integer.parseInt(anno[0]);
-		        		int G_Last= Integer.parseInt(anno[1]);
-		        		String G_mentions = anno[2];
-		        		/**
-		        		 *  2. Co-occurring word
-		        		 *  boundary : 
-		        		 *  Sentence Start: Sentence_offsets.get(Target_Sentence)
-		        		 *  Sentence Last: Sentence_offsets.get(Target_Sentence+1)
-		        		 */
-		        		//Find the target sentence
-		        		int Target_Sentence=0;
-		        		if(SPfound == false) // 1. left : Closed to start of the gene mention 
-		        		{
-		        			for(int s=0;s<Sentence_offsets.size();s++)
-		        		
-			        		{
-			        			int Sentence_last=1000000;
-			        			if(s<Sentence_offsets.size()-1)
-			        			{
-			        				Sentence_last=Sentence_offsets.get(s+1);
-			        			}
-			        			if(G_Start<Sentence_last)
-			        			{
-			        				Target_Sentence=s;
-			        				break;
-			        			}
-			        		}
-		        		}
-		        		int Sentence_Start = Sentence_offsets.get(Target_Sentence);
-		        		int Sentence_Last = 1000000;
-		        		if(Sentence_offsets.size() > Target_Sentence+1){ Sentence_Last = Sentence_offsets.get(Target_Sentence+1); }
-		        		if(SPfound == false) // 1. left : Closed to start of the gene mention 
-		        		{
-		        			int closet_Sp_Start=0;
-		        			for(int sp=0;sp<Annotations_Species.size();sp++) // Find the closet species
-			        		{
-			        			String AnnoSp[]=Annotations_Species.get(sp).split("\t");
-			        			int Sp_Start = Integer.parseInt(AnnoSp[0]);
-				        		String patt="^\\**([0-9]+)$";
-								Pattern ptmp = Pattern.compile(patt);
-								Matcher mtmp = ptmp.matcher(AnnoSp[4]);
-								if(mtmp.find())
-								{
-									String taxid = mtmp.group(1);
-									Location2Species_hash.put(Sp_Start,taxid);
-				        			if(Sp_Start <= G_Start && Sp_Start >= Sentence_Start && Sp_Start >closet_Sp_Start)
-					        		{
-					        			closet_Sp_Start=Sp_Start;
-					        			Location2Species_hash.put(Integer.parseInt(anno[0]), taxid);
-
-					        			if(mention2Location2Species_hash.containsKey(G_mentions.toLowerCase()))
-					        			{
-					        				mention2Location2Species_hash.get(G_mentions.toLowerCase()).put(Integer.parseInt(anno[0]), taxid);
-					        			}
-					        			else
-					        			{
-					        				mention2Location2Species_hash.put(G_mentions.toLowerCase(),Location2Species_hash);
-					        			}
-					        			
-					        			SPfound=true;
-					        		}
-								}
-				        	}
-			        	}
-		        		if(SPfound == false) // 2. right : Closed to last of the gene mention
-		        		{
-		        			int closet_Sp_Last=1000000;
-		        			for(int sp=0;sp<Annotations_Species.size();sp++) // Find the closet species
-			        		{
-			        			String AnnoSp[]=Annotations_Species.get(sp).split("\t");
-			        			int Sp_Last = Integer.parseInt(AnnoSp[1]);
-				        		String patt="^\\**([0-9]+)$";
-								Pattern ptmp = Pattern.compile(patt);
-								Matcher mtmp = ptmp.matcher(AnnoSp[4]);
-								if(mtmp.find())
-								{
-									String taxid = mtmp.group(1);
-				        			if(Sp_Last >= G_Last && Sp_Last <= Sentence_Last && Sp_Last < closet_Sp_Last)
-					        		{
-					        			closet_Sp_Last=Sp_Last;
-					        			Location2Species_hash.put(Integer.parseInt(anno[0]), taxid);
-					        			
-					        			if(mention2Location2Species_hash.containsKey(G_mentions.toLowerCase()))
-					        			{
-					        				mention2Location2Species_hash.get(G_mentions.toLowerCase()).put(Integer.parseInt(anno[0]), taxid);
-					        			}
-					        			else
-					        			{
-					        				mention2Location2Species_hash.put(G_mentions.toLowerCase(),Location2Species_hash);
-					        			}
-					        			
-					        			SPfound=true;
-					        		}
-								}
-				        	}
-		        		}
-	    			}
-					
-					for (int k : Annotations_Gene_hash.keySet()) // k is the index of GNormPlus.BioCDocobj.Annotations.get(i).get(j) 
-	    			{
-						String anno[] = Annotations_Gene_hash.get(k).split("\t");
-	    				int G_Start= Integer.parseInt(anno[0]);
-		        		int G_Last= Integer.parseInt(anno[1]);
-		        		String G_mentions = anno[2];
-		        		String G_type = anno[3];
-		        		String G_mention_list[]=G_mentions.split("\\|");
-		        		String G_mention=G_mention_list[0]; // only use the first term to detect species ; should be updated after SimConcept
-		        		
-		        		/** 1. prefix */
-		        		boolean SPfound = false;
-		        		for(String taxid: PrefixIDTarget_hash.keySet())
-		        		{
-							if(GNormPlus.GeneWithoutSPPrefix_hash.containsKey(G_mention.toLowerCase()))
-							{
-								//special case, and no need for prefix - SA
-							}
-							else
-							{
-								Pattern ptmp = Pattern.compile("^("+PrefixIDTarget_hash.get(taxid)+")([A-Z].*)$");
-								Matcher mtmp = ptmp.matcher(G_mention);
-								if(mtmp.find())
-								{
-									String MentionWoPrefix=mtmp.group(2);
-									GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, anno[0]+"\t"+anno[1]+"\t"+anno[2]+"|"+MentionWoPrefix+"\t"+anno[3]+"\tPrefix:"+taxid);
-									SPfound=true;
-									break;
-								}
-							}
-		        		}
-		        		
-		        		/**
-		        		 *  2. Co-occurring word
-		        		 *  boundary : 
-		        		 *  Sentence Start: Sentence_offsets.get(Target_Sentence)
-		        		 *  Sentence Last: Sentence_offsets.get(Target_Sentence+1)
-		        		 */
-		        		//Find the target sentence
-		        		int Target_Sentence=0;
-		        		if(SPfound == false) // 1. left : Closed to start of the gene mention 
-		        		{
-		        			for(int s=0;s<Sentence_offsets.size();s++)
-		        		
-			        		{
-			        			int Sentence_last=1000000;
-			        			if(s<Sentence_offsets.size()-1)
-			        			{
-			        				Sentence_last=Sentence_offsets.get(s+1);
-			        			}
-			        			if(G_Start<Sentence_last)
-			        			{
-			        				Target_Sentence=s;
-			        				break;
-			        			}
-			        		}
-		        		}
-		        		int Sentence_Start = Sentence_offsets.get(Target_Sentence);
-		        		int Sentence_Last = 1000000;
-		        		if(Sentence_offsets.size() > Target_Sentence+1){ Sentence_Last = Sentence_offsets.get(Target_Sentence+1); }
-		        		if(SPfound == false) // 1. left : Closed to start of the gene mention 
-		        		{
-		        			int closet_Sp_Start=0;
-		        			for(int sp=0;sp<Annotations_Species.size();sp++) // Find the closet species
-			        		{
-			        			String AnnoSp[]=Annotations_Species.get(sp).split("\t");
-			        			int Sp_Start = Integer.parseInt(AnnoSp[0]);
-				        		String patt="^\\**([0-9]+)$";
-								Pattern ptmp = Pattern.compile(patt);
-								Matcher mtmp = ptmp.matcher(AnnoSp[4]);
-								if(mtmp.find())
-								{
-									String taxid = mtmp.group(1);
-				        			if(Sp_Start <= G_Start && Sp_Start >= Sentence_Start && Sp_Start >closet_Sp_Start)
-					        		{
-					        			closet_Sp_Start=Sp_Start;
-										if(GNormPlus.SP_Virus2Human_hash.containsKey(taxid))
-				        				{
-					        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tLeft:"+taxid+"&9606");
-				        				}
-					        			else
-					        			{
-					        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tLeft:"+taxid);
-					        			}
-					        			SPfound=true;
-					        		}
-								}
-				        	}
-			        	}
-		        		if(SPfound == false) // 2. right : Closed to last of the gene mention
-		        		{
-		        			int closet_Sp_Last=1000000;
-		        			for(int sp=0;sp<Annotations_Species.size();sp++) // Find the closet species
-			        		{
-			        			String AnnoSp[]=Annotations_Species.get(sp).split("\t");
-			        			int Sp_Last = Integer.parseInt(AnnoSp[1]);
-				        		String patt="^\\**([0-9]+)$";
-								Pattern ptmp = Pattern.compile(patt);
-								Matcher mtmp = ptmp.matcher(AnnoSp[4]);
-								if(mtmp.find())
-								{
-									String taxid = mtmp.group(1);
-				        			if(Sp_Last >= G_Last && Sp_Last <= Sentence_Last && Sp_Last < closet_Sp_Last)
-					        		{
-					        			closet_Sp_Last=Sp_Last;
-										if(GNormPlus.SP_Virus2Human_hash.containsKey(taxid))
-				        				{
-					        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tRight:"+taxid+"&9606");
-				        				}
-					        			else
-					        			{
-					        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tRight:"+taxid);
-					        			}
-					        			SPfound=true;
-					        		}
-								}
-				        	}
-		        		}
-		        		
-		    			/** 3. Focus species */
-		        		if(SPfound == false) // 2. right : Closed to last of the gene mention
-		        		{
-		        			// 1. only the mentions appeared earlier are inferred
-		        			//
-		        			if(mention2Location2Species_hash.containsKey(G_mentions.toLowerCase()))
-		        			{
-		        				int closed_loca=0;
-		        				for (int loca_start : mention2Location2Species_hash.get(G_mentions.toLowerCase()).keySet())
-			        			{
-		        					if(loca_start<G_Start)
-									{
-										if(loca_start>closed_loca)
-										{
-											closed_loca=loca_start;
-										}
-									}
-			        			}
-		        				if(closed_loca>0)
-								{
-		        					if(GNormPlus.SP_Virus2Human_hash.containsKey(Location2Species_hash.get(closed_loca)))
-			        				{
-				        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tFocus:"+Location2Species_hash.get(closed_loca)+"&9606");
-			        				}
-				        			else
-				        			{
-					        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tFocus:"+Location2Species_hash.get(closed_loca));
-				        			}
-								}
-								else
-								{
-									if(GNormPlus.SP_Virus2Human_hash.containsKey(MajorSP))
-			        				{
-				        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tFocus:"+MajorSP+"&9606");
-			        				}
-				        			else
-				        			{
-					        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tFocus:"+MajorSP);
-				        			}
-								}
-		        			}
-		        			else
-							{
-		        				if(GNormPlus.SP_Virus2Human_hash.containsKey(MajorSP))
-		        				{
-			        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tFocus:"+MajorSP+"&9606");
-		        				}
-			        			else
-			        			{
-				        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tFocus:"+MajorSP);
-			        			}
-							}
-		        		}
-	    			}
-				}
-			}
-		}
-		GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,false,true);
-	}
-	public void SpeciesAssignment(String Filename,String FilenameBioC,String FocusSpecies) throws IOException, XMLStreamException
-	{
-		for (int i = 0; i < GNormPlus.BioCDocobj.Annotations.size(); i++) /** PMIDs : i */
-		{
-			for (int j = 0; j < GNormPlus.BioCDocobj.Annotations.get(i).size(); j++) /** Paragraphs : j */
-			{
-				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) // Annotation : k
-				{
-					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
-					if(anno.length==5) //Species
-	        		{
-						String id=anno[4].replaceAll("\\*", "");
-						GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, anno[0]+"\t"+anno[1]+"\t"+anno[2]+"\t"+anno[3]+"\t"+id);
-	        		}
-	        		else //Gene : if(anno.length==3)
-	        		{
-	        			/** 1. prefix */
-		        		boolean SPfound = false;
-		        		if(GNormPlus.GeneWithoutSPPrefix_hash.containsKey(anno[2].toLowerCase()))
-						{
-							//special case, and no need for prefix - SA
-						}
-						else
-						{
-							Pattern ptmp = Pattern.compile("^("+GNormPlus.PrefixID_hash.get(FocusSpecies)+")([A-Z].*)$");
-							Matcher mtmp = ptmp.matcher(anno[2]);
-							if(mtmp.find())
-							{
-								String MentionWoPrefix=mtmp.group(2);
-								GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, anno[0]+"\t"+anno[1]+"\t"+anno[2]+"|"+MentionWoPrefix+"\t"+anno[3]+"\tPrefix:"+FocusSpecies);
-								SPfound=true;
-							}
-						}
-		        		if(SPfound == false)
-		        		{
-		        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k,  GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k)+"\tFocus:"+FocusSpecies);
-		        		}
-	        		}
-				}
-			}
-		}
-		GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,false,true);
-	}
+/**
+ * Project: GNormPlus
+ * Function: Species recognition and Species assignment
+ */
+
+package GNormPluslib;
+
+import bioc.BioCAnnotation;
+import bioc.BioCCollection;
+import bioc.BioCDocument;
+import bioc.BioCLocation;
+import bioc.BioCPassage;
+
+import bioc.io.BioCDocumentWriter;
+import bioc.io.BioCFactory;
+import bioc.io.woodstox.ConnectorWoodstox;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.text.BreakIterator;
+import java.time.LocalDate;
+import java.time.ZoneId;
+
+import javax.xml.stream.XMLStreamException;
+
+import org.tartarus.snowball.SnowballStemmer;
+import org.tartarus.snowball.ext.englishStemmer;
+
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Collections;
+
+public class SR 
+{
+	@SuppressWarnings("null")
+	public void SpeciesRecognition(String Filename,String FilenameBioC,String StrainFilename,String FilterAntibody) throws IOException, XMLStreamException
+	{
+		/** Recognizing Species Names: SP */
+		for (int i = 0; i < GNormPlus.BioCDocobj.PMIDs.size(); i++) /** PMIDs : i */
+		{
+			String Pmid = GNormPlus.BioCDocobj.PMIDs.get(i);
+			PrefixTree PT_Genus = new PrefixTree();
+			HashMap<String, String> SPID_hash = new HashMap<String, String>();
+			ArrayList<String> TargetedLocation = new ArrayList<String>();
+			HashMap<String, String> GenusNames = new HashMap<String, String>();
+			HashMap<String, String> Mention2ID_lc = new HashMap<String, String>();
+			ArrayList<String> IDset = new ArrayList<String>();
+			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
+			{
+				String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j); // Passage context
+				
+				/** Species recognition */
+				ArrayList<String> locations = GNormPlus.PT_Species.SearchMentionLocation(PassageContext,"Species"); /** PT_Species */
+				for (int k = 0 ; k < locations.size() ; k++)
+				{
+					String anno[]=locations.get(k).split("\t");
+					int start= Integer.parseInt(anno[0]);
+	        		int last= Integer.parseInt(anno[1]);
+	        		
+	        		// For anti-serum filtering
+	        		String ForwardSTR="";
+	        		String BackwardSTR="";
+	        		if(start>21)
+	        		{
+	        			ForwardSTR = (PassageContext+"ZZZZZZZZZZZZZZZZZZZZZZZZZZZ").substring(start-21,last);
+	        		}
+	        		else
+	        		{
+	        			ForwardSTR = (PassageContext+"ZZZZZZZZZZZZZZZZZZZZZZZZZZZ").substring(0,last);
+	        		}
+	        		if(PassageContext.length()>last+21)
+	        		{
+	        			BackwardSTR = PassageContext.substring(start,last+21);
+	        		}
+	        		else
+	        		{
+	        			BackwardSTR = PassageContext.substring(start,PassageContext.length());
+	        		}
+	        			        		
+	        		String mention = anno[2];
+	        		String id = anno[3];
+	        		String mention_tmp=mention.toLowerCase();
+	        		mention_tmp = mention_tmp.replaceAll("([^A-Za-z0-9@ ])", "\\\\$1");
+	        		String antibody="";
+	        		if(ForwardSTR.toLowerCase().matches(".*(anti|antibody|antibodies|serum|polyclonal|monoclonal|igg)[\\W\\-\\_]+"+mention_tmp)) {antibody="(anti)";}//filtering : antibody
+	        		else if(BackwardSTR.toLowerCase().matches(mention_tmp+"[\\W\\-\\_]+(anti|antibody|antibodies|serum|polyclonal|monoclonal|igg).*")){antibody="(anti)";} //filtering : antibody
+	        		else if(BackwardSTR.toLowerCase().matches(mention_tmp+"[\\W\\-\\_]+[A-Za-z0-9]+[\\W\\-\\_]+(anti|antibody|antibodies|serum|polyclonal|monoclonal|igg).*")){antibody="(anti)";} //filtering : antibody
+	        		
+					if(mention.matches(".*[\\(\\[\\{].*") && BackwardSTR.toLowerCase().matches(mention_tmp+"\\).*") )
+    				{
+	        			last=last+1;
+	        			mention=mention+")";
+    				}
+	        		
+	        		if(BackwardSTR.toLowerCase().matches(mention_tmp+"[0-9].*")){} // filtered: Bee1p
+	        		else if((mention.matches(".*[;:,].*")) && mention.length()<=10){} // filtered : x, XXX
+	        		else if(mention.matches("to[\\W\\-\\_]+[0-9]+")){} // to 7
+	        		else if(mention.matches("[a-z][\\)\\]\\}].*") && (!mention.matches(".*[\\(\\[\\{].*")) && mention.length()<=10){} // s). Major
+	        		else if(mention.matches(".*[\\(\\[\\{].*") && (!mention.matches(".*[\\)\\]\\}].*")) && mention.length()<=10){} // s). Major
+	        		else if(!id.equals("NA"))
+	        		{
+	        			if(GNormPlus.BioCDocobj.Annotations.size()>i && GNormPlus.BioCDocobj.Annotations.get(i).size()>j)
+						{
+	        				if((!mention.matches("^[A-Za-z] [A-Za-z0-9]+$")) && (mention.length()>=3)) // invalid species: "a group/a GAL4/a strain"
+	        				{
+	        					if(FilterAntibody.equals("False") || (!antibody.equals("(anti)")))
+	        					{
+	        						String patt="^(.+?) [sS]train";
+									Pattern ptmp = Pattern.compile(patt);
+									Matcher mtmp = ptmp.matcher(mention);
+									if(mtmp.find())
+									{
+										mention=mtmp.group(1);
+										last=last-7;
+									}
+			        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tSpecies\t"+id); //+antibody
+			        				String mentions_tmp=mention.toLowerCase();
+		    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
+		    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
+		    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
+			        				Mention2ID_lc.put(mention.toLowerCase(), id); //+antibody
+			        				
+			         				String mention_genus = "";
+			        				patt="^([A-Za-z]+) ";
+									ptmp = Pattern.compile(patt);
+									mtmp = ptmp.matcher(mention);
+									if(mtmp.find())
+									{
+										mention_genus=mtmp.group(1); // get genus
+									}
+									
+			        				IDset.add(id);
+			        				for(int s=start;s<last;s++)
+									{
+										TargetedLocation.add(j+"\t"+s);
+									}
+			        				String ids[]=id.split(";");
+			        				for(int x=0;x<ids.length;x++)
+			        				{
+				        				patt="^\\**([0-9]+)";
+										ptmp = Pattern.compile(patt);
+										mtmp = ptmp.matcher(ids[x]);
+										if(mtmp.find())
+										{
+											SPID_hash.put(mtmp.group(1), mention_genus);
+										}
+			        				}
+	        					}
+							}
+						}
+	        		}
+				}
+				
+				/** Cell Line recognition */
+				locations = GNormPlus.PT_Cell.SearchMentionLocation(PassageContext,"Cell"); /** PT_Cell */
+				for (int k = 0 ; k < locations.size() ; k++)
+				{
+					String anno[]=locations.get(k).split("\t");
+					int start= Integer.parseInt(anno[0]);
+	        		int last= Integer.parseInt(anno[1]);
+	        		String mention = anno[2];
+	        		String id = anno[3];
+	        		if(GNormPlus.BioCDocobj.Annotations.size()>i && GNormPlus.BioCDocobj.Annotations.get(i).size()>j)
+					{
+	        			if(!TargetedLocation.contains(j+"\t"+start)) //already exists
+	        			{
+	        				int last40=0;
+		        			if(PassageContext.length()>=last+40)
+		        			{
+		        				last40=last+40;
+		        			}
+		        			else
+		        			{
+		        				last40=PassageContext.length();
+		        			}
+		        			
+		        			// For anti-serum filtering
+			        		String ForwardSTR="";
+			        		String BackwardSTR="";
+			        		if(start>21)
+			        		{
+			        			ForwardSTR = PassageContext.substring(start-21,last);
+			        		}
+			        		else
+			        		{
+			        			ForwardSTR = PassageContext.substring(0,last);
+			        		}
+			        		if(PassageContext.length()>last+21)
+			        		{
+			        			BackwardSTR = PassageContext.substring(start,last+21);
+			        		}
+			        		else
+			        		{
+			        			BackwardSTR = PassageContext.substring(start,PassageContext.length());
+			        		}
+			        		String mention_tmp=mention.toLowerCase();
+			        		mention_tmp = mention_tmp.replaceAll("([^A-Za-z0-9@ ])", "\\\\$1");
+			        		if(mention_tmp.matches(".*[\\[\\]\\(\\)\\{\\}].*")){}
+			        		else if(BackwardSTR.toLowerCase().matches(mention_tmp+"[0-9\\-\\_].*")){} // filtered: Bee1p
+			        		else if(ForwardSTR.toLowerCase().matches(".*[0-9\\-\\_]"+mention_tmp)){} // filtered: IL-22RA1
+			        		else
+			        		{
+			        			String patt="[\\W\\-]cell([\\- ]*line|)[s]*[\\W\\-]";
+			    				Pattern ptmp = Pattern.compile(patt);
+			    				Matcher mtmp = ptmp.matcher(PassageContext.substring(last, last40).toLowerCase());
+			    				if(mtmp.find())
+			    				{
+			    					if(GNormPlus.taxid4gene.contains(id)) // for gene
+				        			{
+				        				id="*"+id;
+					        		}
+			    					GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tCell\t"+id);
+									String mentions_tmp=mention.toLowerCase();
+		    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
+		    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
+		    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
+			    					IDset.add(id);
+			    					for(int s=start;s<last;s++)
+									{
+										TargetedLocation.add(j+"\t"+s);
+									}
+			    				}
+			        		}
+		        		}
+					}
+				}
+				
+				/** Genus names*/
+				for(String ID: SPID_hash.keySet())
+				{
+					if(GNormPlus.GenusID_hash.containsKey(ID))
+					{
+						GenusNames.put(ID,GNormPlus.GenusID_hash.get(ID));
+					}
+					if(SPID_hash.get(ID).length()>=7)
+					{
+						GenusNames.put(ID,SPID_hash.get(ID));
+					}
+				}
+			}
+			
+			GenusNames.put("3702", "arabidopsis");
+			GenusNames.put("4932", "saccharomyces");
+			GenusNames.put("562", "escherichia");
+			GenusNames.put("7227", "drosophila");
+			GenusNames.put("8355", "xenopus");
+			
+			PT_Genus.Hash2Tree(GenusNames);
+			
+			/** Genus recognition */
+			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
+			{
+				if(GNormPlus.BioCDocobj.PassageContexts.size()>i && 
+					GNormPlus.BioCDocobj.PassageContexts.get(i).size()>j &&  
+					GNormPlus.BioCDocobj.Annotations.size()>i &&  
+					GNormPlus.BioCDocobj.Annotations.get(i).size()>j 
+					)
+				{
+					String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j);
+					ArrayList<String> locations_Genus = PT_Genus.SearchMentionLocation(PassageContext,"Genus"); /** PT_Genus*/
+					for (int k = 0 ; k < locations_Genus.size() ; k++)
+					{
+						String anno[]=locations_Genus.get(k).split("\t");
+						String start= anno[0];
+		        		String last= anno[1];
+		        		String mention = anno[2];
+		        		String id = anno[3];
+		        		if(!TargetedLocation.contains(j+"\t"+start)) //already exists
+	        			{
+		        			String patt="^\\**([0-9]+)$";
+							Pattern ptmp = Pattern.compile(patt);
+							Matcher mtmp = ptmp.matcher(id);
+							if(mtmp.find())
+							{
+								id = mtmp.group(1);
+							}
+							
+							if(GNormPlus.taxid4gene.contains(id)) // for gene
+		        			{
+		        				id="*"+id;
+			        		}
+							GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tGenus\t"+id);
+							String mentions_tmp=mention.toLowerCase();
+    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
+    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
+    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
+							IDset.add(id);
+							for(int s=Integer.parseInt(start);s<Integer.parseInt(last);s++)
+							{
+								TargetedLocation.add(j+"\t"+s);
+							}
+		        		}
+		        	}
+				}
+			}
+			
+			/** Strain Tree */
+			PrefixTree PT_Strain = new PrefixTree();
+			HashMap<String, String> StrainID_hash = new HashMap<String, String>();
+			BufferedReader br = new BufferedReader(new FileReader(StrainFilename));
+			String line="";
+			while ((line = br.readLine()) != null)  
+			{
+				String l[]=line.split("\t");
+				String ancestor = l[0];
+				String tax_id = l[1];
+				String tax_names = l[2];
+				if(SPID_hash.containsKey(ancestor))
+				{
+					StrainID_hash.put(tax_id, tax_names); // tax id -> strain
+				}
+				else if(SPID_hash.containsKey(tax_id))
+				{
+					StrainID_hash.put(tax_id, tax_names); // tax id -> strain
+				}
+			}
+			br.close();
+			HashMap<String, String> StrainNames = new HashMap<String, String>();
+			for(String ID: StrainID_hash.keySet())
+			{
+				StrainNames.put(ID,StrainID_hash.get(ID));
+			}
+			
+			PT_Strain.Hash2Tree(StrainNames);
+			
+			/** Strain recognition */
+			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
+			{
+				if(GNormPlus.BioCDocobj.PassageContexts.size()>i && 
+					GNormPlus.BioCDocobj.PassageContexts.get(i).size()>j &&  
+					GNormPlus.BioCDocobj.Annotations.size()>i &&  
+					GNormPlus.BioCDocobj.Annotations.get(i).size()>j 
+					)
+				{
+					String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j); // Passage context
+					ArrayList<String> locations_Strain = PT_Strain.SearchMentionLocation(PassageContext,"Strain"); /** PT_Strain*/
+					for (int k = 0 ; k < locations_Strain.size() ; k++)
+					{
+						String anno[]=locations_Strain.get(k).split("\t");
+						String start= anno[0];
+		        		String last= anno[1];
+		        		String mention = anno[2];
+		        		String id = anno[3];
+		        		if(!TargetedLocation.contains(j+"\t"+start)) //already exists
+	        			{
+		        			if((!mention.matches(".*[;,\\{\\}\\(\\)\\[\\]].*")) && !mention.matches("[a-z]{1,4} [0-9]{1,3}"))
+		        			{
+			        			if(GNormPlus.taxid4gene.contains(id)) // for gene
+			        			{
+			        				id="*"+id;
+				        		}
+			        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tStrain\t"+id);
+								String mentions_tmp=mention.toLowerCase();
+	    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
+	    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
+	    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
+			        			IDset.add(id);
+			        			for(int s=Integer.parseInt(start);s<Integer.parseInt(last);s++)
+								{
+									TargetedLocation.add(j+"\t"+s);
+								}
+		        			}
+		        		}
+					}
+				}
+			}
+			
+			HashMap<String, String> OtherNames = new HashMap<String, String>();
+			for(String men : Mention2ID_lc.keySet())
+			{
+				String men_id= Mention2ID_lc.get(men);
+				if(GNormPlus.PmidLF2Abb_lc_hash.containsKey(Pmid+"\t"+men))
+				{
+					String Abb = GNormPlus.PmidLF2Abb_lc_hash.get(Pmid+"\t"+men);
+					// Abbreviation
+					if(OtherNames.containsKey(men_id))
+					{
+						OtherNames.put(men_id, OtherNames.get(men_id)+"|"+Abb);
+					}
+					else
+					{
+						OtherNames.put(men_id,Abb);
+					}
+				}
+				String men_nospace=men.replaceAll(" ", "");
+				// no space
+				if(OtherNames.containsKey(men_id))
+				{
+					OtherNames.put(men_id, OtherNames.get(men_id)+"|"+men_nospace);
+				}
+				else
+				{
+					OtherNames.put(men_id,men_nospace);
+				}
+			}
+			PrefixTree PT_Others = new PrefixTree();
+			PT_Others.Hash2Tree(OtherNames);
+			
+			/** 
+			 *
+			 * Others: 
+			 * 1) Abbreviation 
+			 * 2) no space
+			 * 
+			 * */
+			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
+			{
+				if(GNormPlus.BioCDocobj.PassageContexts.size()>i && 
+					GNormPlus.BioCDocobj.PassageContexts.get(i).size()>j &&  
+					GNormPlus.BioCDocobj.Annotations.size()>i &&  
+					GNormPlus.BioCDocobj.Annotations.get(i).size()>j 
+					)
+				{
+					String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j); // Passage context
+					ArrayList<String> locations_Abb = PT_Others.SearchMentionLocation(PassageContext,"Species"); /** PT_Abb*/
+					for (int k = 0 ; k < locations_Abb.size() ; k++)
+					{
+						String anno[]=locations_Abb.get(k).split("\t");
+						String start= anno[0];
+		        		String last= anno[1];
+		        		String mention = anno[2];
+		        		String id = anno[3];
+		        		if(!TargetedLocation.contains(j+"\t"+start)) //already exists
+	        			{
+		        			if(GNormPlus.taxid4gene.contains(id)) // for gene
+		        			{
+		        				id="*"+id;
+			        		}
+		        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).add(start+"\t"+last+"\t"+mention+"\tSpecies\t"+id);
+							String mentions_tmp=mention.toLowerCase();
+    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
+    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
+    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
+		        			Mention2ID_lc.put(mention.toLowerCase(), id);
+		        			IDset.add(id);
+		        			for(int s=Integer.parseInt(start);s<Integer.parseInt(last);s++)
+							{
+								TargetedLocation.add(j+"\t"+s);
+							}
+		        		}
+					}
+				}
+			}
+			
+			for (int j = 0; j < GNormPlus.BioCDocobj.PassageNames.get(i).size(); j++) /** Paragraphs : j */
+			{
+				if(GNormPlus.BioCDocobj.PassageContexts.size()>i && GNormPlus.BioCDocobj.PassageContexts.get(i).size()>j && GNormPlus.BioCDocobj.Annotations.size()>i && GNormPlus.BioCDocobj.Annotations.get(i).size()>j)
+				{
+					ArrayList <Integer> remove_anno = new ArrayList <Integer>();
+					for (int a = 0; a < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); a++) /** Annotations : a */
+					{
+						String SpAnno[]=GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(a).split("\t");
+						String start= SpAnno[0];
+		        		String last= SpAnno[1];
+		        		String mention = SpAnno[2];
+		        		String type = SpAnno[3];
+		        		
+		        		if(type.matches("Gene|FamilyName"))
+		        		{
+		        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(a,start+"\t"+last+"\t"+mention+"\t"+type);
+			        	}
+		        		else if(type.matches("Species|Genus|Strain|Cell") && SpAnno.length==5)
+		        		{
+		        			//System.out.println(GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(a));
+			        		/** Abbreviation solution */
+			    			if(GNormPlus.PmidAbb2LF_lc_hash.containsKey(Pmid+"\t"+mention.toLowerCase()) && Mention2ID_lc.containsKey(GNormPlus.PmidAbb2LF_lc_hash.containsKey(Pmid+"\t"+mention.toLowerCase())))
+							{
+								String LF_lc=GNormPlus.PmidAbb2LF_lc_hash.get(Pmid+"\t"+mention.toLowerCase());
+								if(Mention2ID_lc.containsKey(LF_lc))
+								{
+									String LF_ID=Mention2ID_lc.get(LF_lc);
+									GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(a, start+"\t"+last+"\t"+mention+"\t"+type+"\t"+LF_ID);
+									String mentions_tmp=mention.toLowerCase();
+		    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
+		    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
+		    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
+								}
+							}
+			    			else if (SpAnno.length>4) 
+			    			{
+			    				String id = SpAnno[4];
+			    				String id_split[]=id.split(";");
+		    					if(id_split.length>=2)
+		    					{
+		    						/** Smallest set of tax ids */
+				    				boolean found=false;
+		    						for(int x=0;x<IDset.size();x++)
+				    				{
+				    					String id_tmp= IDset.get(x);
+				    					for(int y=0;y<id_split.length;y++) // if any other id is a component of the target id
+				    					{
+				    						if(id_split[y].equals(id_tmp))
+				    						{
+				    							found=true;
+				    						}
+				    					}
+				    					if(found == true)
+				    					{
+				    						GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(a, start+"\t"+last+"\t"+mention+"\t"+type+"\t"+id_tmp);
+											String mentions_tmp=mention.toLowerCase();
+				    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
+				    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
+				    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
+				    						x=1000000;
+				    					}
+			    					}
+		    						
+		    						/** smallest tax id number */
+		    						if(found == false)
+			    					{
+		    							int min=10000000;
+				    					String min_id="";
+			    						for(int y=0;y<id_split.length;y++) // if any other id is a component of the target id
+				    					{
+			    							String id_tmp = id_split[y];
+			    							String patt="^\\**([0-9]+)";
+			    							Pattern ptmp = Pattern.compile(patt);
+			    							Matcher mtmp = ptmp.matcher(id_tmp);
+			    							if(mtmp.find())
+			    							{
+			    								id_tmp = mtmp.group(1);
+			    							}
+			    								
+			    							if(y==0)
+			    							{
+			    								min_id=id_split[y];
+			    								min=Integer.parseInt(id_tmp);
+			    							}
+			    							else if(Integer.parseInt(id_tmp)<min)
+			    							{
+			    								min=Integer.parseInt(id_tmp);
+			    								min_id=id_tmp;
+			    							}
+				    					}
+			    						if(GNormPlus.taxid4gene.contains(min_id)) // for gene
+			    						{
+			    							min_id="*"+min_id;
+			    						}
+					        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(a,start+"\t"+last+"\t"+mention+"\tSpecies\t"+min_id);
+										String mentions_tmp=mention.toLowerCase();
+			    						mentions_tmp=mentions_tmp.replaceAll("[\\W\\-\\_]","");
+			    						mentions_tmp=mentions_tmp.replaceAll("[0-9]","0");
+			    						GNormPlus.Filtering_hash.put(mentions_tmp,"");
+					        		}
+			    				}
+			    			}
+		        		}
+		        		else //disease, and other concepts
+		        		{
+		        			remove_anno.add(a);
+		        		}
+					}
+					
+					Collections.sort(remove_anno);
+					for (int counter = remove_anno.size()-1; counter >= 0 ; counter--) 
+					{ 		      
+						int ai=remove_anno.get(counter);
+						//System.out.println("\n"+ai+"\t"+GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(ai));
+						GNormPlus.BioCDocobj.Annotations.get(i).get(j).remove(ai);
+					}
+				}
+			}	
+		}
+		GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,false,true); //save in BioC file
+	}
+	public void SpeciesAssignment(String Filename,String FilenameBioC) throws IOException, XMLStreamException
+	{
+		GNormPlus.BioCDocobj.Annotations = new ArrayList();
+		GNormPlus.BioCDocobj.BioCReaderWithAnnotation(Filename);
+		
+		BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.US);	
+		for (int i = 0; i < GNormPlus.BioCDocobj.Annotations.size(); i++) /** PMIDs : i */
+		{
+			HashMap<String, String> PrefixIDTarget_hash = new HashMap<String, String>();
+			PrefixIDTarget_hash.put("9606", "h");
+			PrefixIDTarget_hash.put("10090", "m");
+			PrefixIDTarget_hash.put("10116", "r");
+			PrefixIDTarget_hash.put("4932", "y");
+			PrefixIDTarget_hash.put("7227", "d");
+			PrefixIDTarget_hash.put("7955", "z|zf|Zf|dr|Dr");
+			PrefixIDTarget_hash.put("3702", "at|At");
+			
+			HashMap<String, Double> SP2Num_hash = new HashMap<String, Double>();
+			for (int j = 0; j < GNormPlus.BioCDocobj.Annotations.get(i).size(); j++) /** Paragraphs : j */
+			{
+				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) // Annotation : k
+				{
+					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
+					if(anno.length==5) //Species
+	        		{
+						String patt="^\\**([0-9]+)$";
+						Pattern ptmp = Pattern.compile(patt);
+						Matcher mtmp = ptmp.matcher(anno[4]);
+						if(mtmp.find())
+						{
+							String id = mtmp.group(1);
+							
+							if(!PrefixIDTarget_hash.containsKey(id))
+							{
+								PrefixIDTarget_hash.put(id,GNormPlus.PrefixID_hash.get(id)); // taxid -> prefix
+							}
+							if(j == 0)//title
+			        		{
+			        			if(SP2Num_hash.containsKey(id))
+			        			{
+			        				SP2Num_hash.put(id, SP2Num_hash.get(id)+2);
+			        			}
+			        			else
+			        			{
+			        				if(GNormPlus.TaxFreq_hash.containsKey(id))
+			        				{
+			        					SP2Num_hash.put(id, GNormPlus.TaxFreq_hash.get(id)+2);
+			        				}
+			        				else
+			        				{
+			        					SP2Num_hash.put(id, 2.0);
+			        				}
+			        			}
+			        			// Virus -> Human (not to double weight human to virus)
+			        			/*if(GNormPlus.SP_Virus2Human_hash.containsKey(id))
+		        				{
+			        				if(SP2Num_hash.containsKey("9606"))
+				        			{
+				        				SP2Num_hash.put("9606", SP2Num_hash.get("9606")+2);
+				        			}
+				        			else
+				        			{
+				        				SP2Num_hash.put("9606", 2 + GNormPlus.TaxFreq_hash.get("9606")+1);
+				        			}
+		        				}*/
+			        		}
+			        		else
+			        		{
+			        			if(SP2Num_hash.containsKey(id))
+			        			{
+			        				SP2Num_hash.put(id, SP2Num_hash.get(id)+1);
+			        			}
+			        			else
+			        			{
+			        				if(GNormPlus.TaxFreq_hash.containsKey(id))
+			        				{
+			        					SP2Num_hash.put(id, 1 + GNormPlus.TaxFreq_hash.get(id));
+			        				}
+			        				else
+			        				{
+			        					SP2Num_hash.put(id, 1.0);
+			        				}
+			        			}
+			        			// Virus -> Human
+			        			/*if(GNormPlus.SP_Virus2Human_hash.containsKey(id))
+		        				{
+			        				if(SP2Num_hash.containsKey("9606"))
+				        			{
+				        				SP2Num_hash.put("9606", SP2Num_hash.get("9606")+1);
+				        			}
+				        			else
+				        			{
+				        				SP2Num_hash.put("9606", GNormPlus.TaxFreq_hash.get("9606")+1);
+				        			}
+		        				}*/
+			        		}
+						}
+	        		}
+				}
+			}
+			String MajorSP="9606";
+			double MaxSP=0;
+			for(String tid : SP2Num_hash.keySet())
+			{
+				if(SP2Num_hash.get(tid)>MaxSP)
+				{
+					MajorSP=tid;
+					MaxSP=SP2Num_hash.get(tid);
+				}
+			}
+			
+			for (int j = 0; j < GNormPlus.BioCDocobj.PassageContexts.get(i).size(); j++) /** Paragraphs : j */
+			{
+				String PassageContext = GNormPlus.BioCDocobj.PassageContexts.get(i).get(j); // Passage context
+				//int PassageOffset = GNormPlus.BioCDocobj.PassageOffsets.get(i).get(j); // Passage offset
+				iterator.setText(PassageContext);
+				ArrayList<Integer> Sentence_offsets = new ArrayList<Integer>();
+				int Sent_start = iterator.first();
+				for (int Sent_last = iterator.next(); Sent_last != BreakIterator.DONE; Sent_start = Sent_last, Sent_last = iterator.next()) 
+				{
+					Sentence_offsets.add(Sent_start);
+				}
+				
+				HashMap<Integer,String> Annotations_Gene_hash = new HashMap<Integer,String>();
+				ArrayList<String> Annotations_Species = new ArrayList<String>();
+				if(GNormPlus.BioCDocobj.Annotations.get(i).size()>j)
+				{
+					for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) // Annotation : k
+					{
+						String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
+						if(anno.length==5) //Species
+		        		{
+							Annotations_Species.add(GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k));
+		        		}
+		        		else //Gene : if(anno.length==3)
+		        		{
+		        			//String mention = PassageContext.substring(Integer.parseInt(anno[0]), Integer.parseInt(anno[1]));
+		        			Annotations_Gene_hash.put(k,GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k)); // k -> Gene Annotation
+		        		}
+					}
+	
+					//Gene --> Species Inference (PMID:28777492)
+					HashMap<String,HashMap<Integer,String>> mention2Location2Species_hash = new HashMap<String,HashMap<Integer,String>>(); 
+					HashMap<Integer,String> Location2Species_hash = new HashMap<Integer,String>(); 
+					for (int k : Annotations_Gene_hash.keySet()) // k is the index of GNormPlus.BioCDocobj.Annotations.get(i).get(j) 
+	    			{
+						boolean SPfound = false;
+						String anno[] = Annotations_Gene_hash.get(k).split("\t");
+	    				int G_Start= Integer.parseInt(anno[0]);
+		        		int G_Last= Integer.parseInt(anno[1]);
+		        		String G_mentions = anno[2];
+		        		/**
+		        		 *  2. Co-occurring word
+		        		 *  boundary : 
+		        		 *  Sentence Start: Sentence_offsets.get(Target_Sentence)
+		        		 *  Sentence Last: Sentence_offsets.get(Target_Sentence+1)
+		        		 */
+		        		//Find the target sentence
+		        		int Target_Sentence=0;
+		        		if(SPfound == false) // 1. left : Closed to start of the gene mention 
+		        		{
+		        			for(int s=0;s<Sentence_offsets.size();s++)
+		        		
+			        		{
+			        			int Sentence_last=1000000;
+			        			if(s<Sentence_offsets.size()-1)
+			        			{
+			        				Sentence_last=Sentence_offsets.get(s+1);
+			        			}
+			        			if(G_Start<Sentence_last)
+			        			{
+			        				Target_Sentence=s;
+			        				break;
+			        			}
+			        		}
+		        		}
+		        		int Sentence_Start = Sentence_offsets.get(Target_Sentence);
+		        		int Sentence_Last = 1000000;
+		        		if(Sentence_offsets.size() > Target_Sentence+1){ Sentence_Last = Sentence_offsets.get(Target_Sentence+1); }
+		        		if(SPfound == false) // 1. left : Closed to start of the gene mention 
+		        		{
+		        			int closet_Sp_Start=0;
+		        			for(int sp=0;sp<Annotations_Species.size();sp++) // Find the closet species
+			        		{
+			        			String AnnoSp[]=Annotations_Species.get(sp).split("\t");
+			        			int Sp_Start = Integer.parseInt(AnnoSp[0]);
+				        		String patt="^\\**([0-9]+)$";
+								Pattern ptmp = Pattern.compile(patt);
+								Matcher mtmp = ptmp.matcher(AnnoSp[4]);
+								if(mtmp.find())
+								{
+									String taxid = mtmp.group(1);
+									Location2Species_hash.put(Sp_Start,taxid);
+				        			if(Sp_Start <= G_Start && Sp_Start >= Sentence_Start && Sp_Start >closet_Sp_Start)
+					        		{
+					        			closet_Sp_Start=Sp_Start;
+					        			Location2Species_hash.put(Integer.parseInt(anno[0]), taxid);
+
+					        			if(mention2Location2Species_hash.containsKey(G_mentions.toLowerCase()))
+					        			{
+					        				mention2Location2Species_hash.get(G_mentions.toLowerCase()).put(Integer.parseInt(anno[0]), taxid);
+					        			}
+					        			else
+					        			{
+					        				mention2Location2Species_hash.put(G_mentions.toLowerCase(),Location2Species_hash);
+					        			}
+					        			
+					        			SPfound=true;
+					        		}
+								}
+				        	}
+			        	}
+		        		if(SPfound == false) // 2. right : Closed to last of the gene mention
+		        		{
+		        			int closet_Sp_Last=1000000;
+		        			for(int sp=0;sp<Annotations_Species.size();sp++) // Find the closet species
+			        		{
+			        			String AnnoSp[]=Annotations_Species.get(sp).split("\t");
+			        			int Sp_Last = Integer.parseInt(AnnoSp[1]);
+				        		String patt="^\\**([0-9]+)$";
+								Pattern ptmp = Pattern.compile(patt);
+								Matcher mtmp = ptmp.matcher(AnnoSp[4]);
+								if(mtmp.find())
+								{
+									String taxid = mtmp.group(1);
+				        			if(Sp_Last >= G_Last && Sp_Last <= Sentence_Last && Sp_Last < closet_Sp_Last)
+					        		{
+					        			closet_Sp_Last=Sp_Last;
+					        			Location2Species_hash.put(Integer.parseInt(anno[0]), taxid);
+					        			
+					        			if(mention2Location2Species_hash.containsKey(G_mentions.toLowerCase()))
+					        			{
+					        				mention2Location2Species_hash.get(G_mentions.toLowerCase()).put(Integer.parseInt(anno[0]), taxid);
+					        			}
+					        			else
+					        			{
+					        				mention2Location2Species_hash.put(G_mentions.toLowerCase(),Location2Species_hash);
+					        			}
+					        			
+					        			SPfound=true;
+					        		}
+								}
+				        	}
+		        		}
+	    			}
+					
+					for (int k : Annotations_Gene_hash.keySet()) // k is the index of GNormPlus.BioCDocobj.Annotations.get(i).get(j) 
+	    			{
+						String anno[] = Annotations_Gene_hash.get(k).split("\t");
+	    				int G_Start= Integer.parseInt(anno[0]);
+		        		int G_Last= Integer.parseInt(anno[1]);
+		        		String G_mentions = anno[2];
+		        		String G_type = anno[3];
+		        		String G_mention_list[]=G_mentions.split("\\|");
+		        		String G_mention=G_mention_list[0]; // only use the first term to detect species ; should be updated after SimConcept
+		        		
+		        		/** 1. prefix */
+		        		boolean SPfound = false;
+		        		for(String taxid: PrefixIDTarget_hash.keySet())
+		        		{
+							if(GNormPlus.GeneWithoutSPPrefix_hash.containsKey(G_mention.toLowerCase()))
+							{
+								//special case, and no need for prefix - SA
+							}
+							else
+							{
+								Pattern ptmp = Pattern.compile("^("+PrefixIDTarget_hash.get(taxid)+")([A-Z].*)$");
+								Matcher mtmp = ptmp.matcher(G_mention);
+								if(mtmp.find())
+								{
+									String MentionWoPrefix=mtmp.group(2);
+									GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, anno[0]+"\t"+anno[1]+"\t"+anno[2]+"|"+MentionWoPrefix+"\t"+anno[3]+"\tPrefix:"+taxid);
+									SPfound=true;
+									break;
+								}
+							}
+		        		}
+		        		
+		        		/**
+		        		 *  2. Co-occurring word
+		        		 *  boundary : 
+		        		 *  Sentence Start: Sentence_offsets.get(Target_Sentence)
+		        		 *  Sentence Last: Sentence_offsets.get(Target_Sentence+1)
+		        		 */
+		        		//Find the target sentence
+		        		int Target_Sentence=0;
+		        		if(SPfound == false) // 1. left : Closed to start of the gene mention 
+		        		{
+		        			for(int s=0;s<Sentence_offsets.size();s++)
+		        		
+			        		{
+			        			int Sentence_last=1000000;
+			        			if(s<Sentence_offsets.size()-1)
+			        			{
+			        				Sentence_last=Sentence_offsets.get(s+1);
+			        			}
+			        			if(G_Start<Sentence_last)
+			        			{
+			        				Target_Sentence=s;
+			        				break;
+			        			}
+			        		}
+		        		}
+		        		int Sentence_Start = Sentence_offsets.get(Target_Sentence);
+		        		int Sentence_Last = 1000000;
+		        		if(Sentence_offsets.size() > Target_Sentence+1){ Sentence_Last = Sentence_offsets.get(Target_Sentence+1); }
+		        		if(SPfound == false) // 1. left : Closed to start of the gene mention 
+		        		{
+		        			int closet_Sp_Start=0;
+		        			for(int sp=0;sp<Annotations_Species.size();sp++) // Find the closet species
+			        		{
+			        			String AnnoSp[]=Annotations_Species.get(sp).split("\t");
+			        			int Sp_Start = Integer.parseInt(AnnoSp[0]);
+				        		String patt="^\\**([0-9]+)$";
+								Pattern ptmp = Pattern.compile(patt);
+								Matcher mtmp = ptmp.matcher(AnnoSp[4]);
+								if(mtmp.find())
+								{
+									String taxid = mtmp.group(1);
+				        			if(Sp_Start <= G_Start && Sp_Start >= Sentence_Start && Sp_Start >closet_Sp_Start)
+					        		{
+					        			closet_Sp_Start=Sp_Start;
+										if(GNormPlus.SP_Virus2Human_hash.containsKey(taxid))
+				        				{
+					        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tLeft:"+taxid+"&9606");
+				        				}
+					        			else
+					        			{
+					        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tLeft:"+taxid);
+					        			}
+					        			SPfound=true;
+					        		}
+								}
+				        	}
+			        	}
+		        		if(SPfound == false) // 2. right : Closed to last of the gene mention
+		        		{
+		        			int closet_Sp_Last=1000000;
+		        			for(int sp=0;sp<Annotations_Species.size();sp++) // Find the closet species
+			        		{
+			        			String AnnoSp[]=Annotations_Species.get(sp).split("\t");
+			        			int Sp_Last = Integer.parseInt(AnnoSp[1]);
+				        		String patt="^\\**([0-9]+)$";
+								Pattern ptmp = Pattern.compile(patt);
+								Matcher mtmp = ptmp.matcher(AnnoSp[4]);
+								if(mtmp.find())
+								{
+									String taxid = mtmp.group(1);
+				        			if(Sp_Last >= G_Last && Sp_Last <= Sentence_Last && Sp_Last < closet_Sp_Last)
+					        		{
+					        			closet_Sp_Last=Sp_Last;
+										if(GNormPlus.SP_Virus2Human_hash.containsKey(taxid))
+				        				{
+					        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tRight:"+taxid+"&9606");
+				        				}
+					        			else
+					        			{
+					        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tRight:"+taxid);
+					        			}
+					        			SPfound=true;
+					        		}
+								}
+				        	}
+		        		}
+		        		
+		    			/** 3. Focus species */
+		        		if(SPfound == false) // 2. right : Closed to last of the gene mention
+		        		{
+		        			// 1. only the mentions appeared earlier are inferred
+		        			//
+		        			if(mention2Location2Species_hash.containsKey(G_mentions.toLowerCase()))
+		        			{
+		        				int closed_loca=0;
+		        				for (int loca_start : mention2Location2Species_hash.get(G_mentions.toLowerCase()).keySet())
+			        			{
+		        					if(loca_start<G_Start)
+									{
+										if(loca_start>closed_loca)
+										{
+											closed_loca=loca_start;
+										}
+									}
+			        			}
+		        				if(closed_loca>0)
+								{
+		        					if(GNormPlus.SP_Virus2Human_hash.containsKey(Location2Species_hash.get(closed_loca)))
+			        				{
+				        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tFocus:"+Location2Species_hash.get(closed_loca)+"&9606");
+			        				}
+				        			else
+				        			{
+					        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tFocus:"+Location2Species_hash.get(closed_loca));
+				        			}
+								}
+								else
+								{
+									if(GNormPlus.SP_Virus2Human_hash.containsKey(MajorSP))
+			        				{
+				        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tFocus:"+MajorSP+"&9606");
+			        				}
+				        			else
+				        			{
+					        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tFocus:"+MajorSP);
+				        			}
+								}
+		        			}
+		        			else
+							{
+		        				if(GNormPlus.SP_Virus2Human_hash.containsKey(MajorSP))
+		        				{
+			        				GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tFocus:"+MajorSP+"&9606");
+		        				}
+			        			else
+			        			{
+				        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, Annotations_Gene_hash.get(k)+"\tFocus:"+MajorSP);
+			        			}
+							}
+		        		}
+	    			}
+				}
+			}
+		}
+		GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,false,true);
+	}
+	public void SpeciesAssignment(String Filename,String FilenameBioC,String FocusSpecies) throws IOException, XMLStreamException
+	{
+		for (int i = 0; i < GNormPlus.BioCDocobj.Annotations.size(); i++) /** PMIDs : i */
+		{
+			for (int j = 0; j < GNormPlus.BioCDocobj.Annotations.get(i).size(); j++) /** Paragraphs : j */
+			{
+				for (int k = 0; k < GNormPlus.BioCDocobj.Annotations.get(i).get(j).size(); k++) // Annotation : k
+				{
+					String anno[] = GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k).split("\t");
+					if(anno.length==5) //Species
+	        		{
+						String id=anno[4].replaceAll("\\*", "");
+						GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, anno[0]+"\t"+anno[1]+"\t"+anno[2]+"\t"+anno[3]+"\t"+id);
+	        		}
+	        		else //Gene : if(anno.length==3)
+	        		{
+	        			/** 1. prefix */
+		        		boolean SPfound = false;
+		        		if(GNormPlus.GeneWithoutSPPrefix_hash.containsKey(anno[2].toLowerCase()))
+						{
+							//special case, and no need for prefix - SA
+						}
+						else
+						{
+							Pattern ptmp = Pattern.compile("^("+GNormPlus.PrefixID_hash.get(FocusSpecies)+")([A-Z].*)$");
+							Matcher mtmp = ptmp.matcher(anno[2]);
+							if(mtmp.find())
+							{
+								String MentionWoPrefix=mtmp.group(2);
+								GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k, anno[0]+"\t"+anno[1]+"\t"+anno[2]+"|"+MentionWoPrefix+"\t"+anno[3]+"\tPrefix:"+FocusSpecies);
+								SPfound=true;
+							}
+						}
+		        		if(SPfound == false)
+		        		{
+		        			GNormPlus.BioCDocobj.Annotations.get(i).get(j).set(k,  GNormPlus.BioCDocobj.Annotations.get(i).get(j).get(k)+"\tFocus:"+FocusSpecies);
+		        		}
+	        		}
+				}
+			}
+		}
+		GNormPlus.BioCDocobj.BioCOutput(Filename,FilenameBioC,GNormPlus.BioCDocobj.Annotations,false,true);
+	}
 }

src_python/GeneNER/BIO_format.py

@@ -1,257 +1,257 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Wed Sep  7 08:58:22 2022
-
-@author: luol2
-"""
-
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Jun 24 11:27:57 2022
-
-@author: luol2
-"""
-
-
-import stanza
-import sys
-import os
-import io
-import json
-import re
-#sort entity by position in text
-def pubtator_entitysort(infile):
-    
-    fin=open(infile,'r',encoding='utf-8')    
-    # fout=open(path+'LitCoin/sort/Train_sort.PubTator','w',encoding='utf-8')
-    fout=io.StringIO()
-    all_in=fin.read().strip().split('\n\n')
-    fin.close()
-    error_dict={} #use to debug error
-    for doc in all_in:
-        entity_dict={}
-        lines=doc.split('\n')
-        fout.write(lines[0]+'\n'+lines[1]+'\n')
-        for i in range(2,len(lines)):
-            segs=lines[i].split('\t')
-            if len(segs)>=5:
-                if lines[i] not in entity_dict.keys():
-                    entity_dict[lines[i]]=int(segs[1])
-                else:
-                    print('entity have in',lines[i])
-                    if segs[0] not in error_dict.keys():
-                        error_dict[segs[0]]=[lines[i]]
-                    else:
-                        if lines[i] not in error_dict[segs[0]]:
-                            error_dict[segs[0]].append(lines[i])
-
-        entity_sort=sorted(entity_dict.items(), key=lambda kv:(kv[1]), reverse=False)
-        for ele in entity_sort:
-            fout.write(ele[0]+'\n')
-        fout.write('\n')
-    return fout
-
-def filter_overlap(infile): #nonest
-
-    fin=io.StringIO(infile.getvalue())
-    fout=io.StringIO()
-    
-    documents=fin.read().strip().split('\n\n')
-    fin.close()
-    total_entity=0
-    over_entity=0
-    nest_entity=0
-    for doc in documents:
-        lines=doc.split('\n')
-        entity_list=[]
-        if len(lines)>2:
-            first_entity=lines[2].split('\t')
-            nest_list=[first_entity]
-            max_eid=int(first_entity[2])
-            total_entity+=len(lines)-2
-            for i in range(3,len(lines)):
-                segs=lines[i].split('\t')
-                if int(segs[1])> max_eid:
-                    if len(nest_list)==1:
-                        entity_list.append(nest_list[0])
-                        nest_list=[]
-                        nest_list.append(segs)
-                        if int(segs[2])>max_eid:
-                            max_eid=int(segs[2])
-                    else:
-                        # print(nest_list)
-                        nest_entity+=len(nest_list)-1
-                        tem=find_max_entity(nest_list)#find max entity
-                        # if len(tem)>1:
-                        #     print('max nest >1:',tem)
-                        entity_list.extend(tem)
-                        nest_list=[]
-                        nest_list.append(segs)
-                        if int(segs[2])>max_eid:
-                            max_eid=int(segs[2])
-                        
-                else:
-                    nest_list.append(segs)
-                    if int(segs[2])>max_eid:
-                        max_eid=int(segs[2])
-            if nest_list!=[]:
-                if len(nest_list)==1:
-                    entity_list.append(nest_list[0])
-
-                else:
-                    tem=find_max_entity(nest_list)#find max entity
-                    # if len(tem)>1:
-                    #     print('max nest >1:',tem)
-                    entity_list.extend(tem)
-        fout.write(lines[0]+'\n'+lines[1]+'\n')
-        for ele in entity_list:
-            fout.write('\t'.join(ele)+'\n')
-        fout.write('\n')
-    # print(total_entity,over_entity, nest_entity)
-    return fout
-def find_max_entity(nest_list): #longest entity
-    max_len=0
-    final_tem=[]
-    max_index=0
-    for i in range(0, len(nest_list)):
-        cur_len=int(nest_list[i][2])-int(nest_list[i][1])
-        if cur_len>max_len:
-            max_len=cur_len
-            max_index=i
-
-    final_tem.append(nest_list[max_index])
-    return final_tem    
-                
-# change ori pubtator format to labeled text , entity begin with " ssss", end with 'eeee '
-def pubtator_to_labeltext(infile):
-    
-    fin=io.StringIO(infile.getvalue())
-    all_context=fin.read().strip().split('\n\n')
-    fin.close()
-    fout=io.StringIO()
-    label_dic={}
-    
-    for doc in all_context:
-        lines=doc.split('\n')
-        ori_text=lines[0].split('|t|')[1]+' '+lines[1].split('|a|')[1]
-        pmid=lines[0].split('|t|')[0]
-        s_index=0
-        e_index=0
-        new_text=''
-        for i in range(2,len(lines)):
-            segs=lines[i].split('\t')
-            label_dic[segs[4].lower()]=segs[4]
-            if len(segs)==6:
-                e_index=int(segs[1])
-                new_text+=ori_text[s_index:e_index]+' ssss'+segs[4].lower()+' '+ori_text[int(segs[1]):int(segs[2])]+' eeee'+segs[4].lower()+' '  
-                s_index=int(segs[2])
-                # if ori_text[int(segs[1]):int(segs[2])]!=segs[3]:
-                #     print('error(ori,label):',ori_text[int(segs[1]):int(segs[2])],segs[3])
-
-        new_text+=ori_text[s_index:] 
-        fout.write(pmid+'\t'+' '.join(new_text.strip().split())+'\n')
-    return fout, label_dic
-  
-
-def pre_token(sentence):
-    sentence=re.sub("([\=\/\(\)\<\>\+\-\_])"," \\1 ",sentence)
-    sentence=re.sub("[ ]+"," ",sentence);
-    return sentence
-
-# labeltext to conll format (BIO), a token (including features) per line. sentences are split by '\n', or docs are split by '\n'
-def labeltext_to_conll_fasttoken(infile,label_dic):
-    
-    fin=io.StringIO(infile.getvalue())
-    all_context=fin.read().strip().split('\n')
-    fin.close()
-    fout=io.StringIO()
-    
-    # nlp = stanza.Pipeline(lang='en', processors='tokenize',package='craft') #package='craft'
-    nlp = stanza.Pipeline(lang='en', processors={'tokenize': 'spacy'},package='None') #package='craft'
-
-    doc_i=0
-    for doc in all_context:
-        doc_text=doc.split('\t')[1]
-        doc_text=pre_token(doc_text)
-        doc_stanza = nlp(doc_text)
-        doc_i+=1
-        #print(doc_i)
-        inentity_flag=0
-        last_label='O'
-        for sent in doc_stanza.sentences:
-            temp_sent=[]
-            word_num=0
-            for word in sent.words:
-                word_num+=1
-                # print(word.text)
-                if word.text.strip()=='':
-                    continue
-                temp_sent.append(word.text)
-                if word.text.startswith('ssss')==True:
-                    last_label=word.text
-                    inentity_flag=1
-                elif word.text.startswith('eeee')==True:
-                    last_label=word.text
-                    inentity_flag=0                    
-                else:
-                    if last_label=='O':
-                        now_label='O'
-                    elif last_label.startswith('ssss')==True:
-                        now_label='B-'+label_dic[last_label[4:]]
-                        
-                    elif last_label.startswith('B-')==True:
-                        now_label='I-'+last_label[2:]
-                    elif last_label.startswith('I-')==True:
-                        now_label='I-'+last_label[2:]    
-                    elif last_label.startswith('eeee')==True:
-                        now_label='O'
-                        
-                    fout.write(word.text+'\t'+now_label+'\n')
-                    last_label=now_label
-            if inentity_flag==1: # if entity is split by sentence, will connate the sentence
-                # print('sentence error!!!')
-                # print(word.text,word_num)
-                # print(temp_sent)
-                pass
-            else:
-                fout.write('\n')
-    return fout
-        
-def pubtator_to_conll(infile):
-    
-    #1.entity sort 
-    input_sort=pubtator_entitysort(infile)
-    #print(input_sort.getvalue())
-    
-    #2. no overlap, if overlap get longest entity
-    input_nonest=filter_overlap(input_sort)
-    # print('......sort.....\n',input_sort.getvalue())
-    
-    #3. pubtator to label text
-    input_labtext,label_dic=pubtator_to_labeltext(input_nonest)
-    # print('......label.....\n',input_labtext.getvalue())
-    #print(label_dic)
-    
-    #4. label text to conll
-    output = labeltext_to_conll_fasttoken(input_labtext,label_dic)
-    # print('......output.....\n',output.getvalue())
-    # fout=open(outfile,'w',encoding='utf-8')
-    # fout.write(input_nonest.getvalue())
-    # fout.close()
-    return output
-
-if __name__=='__main__':
-    
-
-    infile='../../TrainingSet/No100/NER.Train.txt'  
-    output=pubtator_to_conll(infile)
-    fout=open('../../TrainingSet/No100/NER.Train.conll','w',encoding='utf-8')
-    fout.write(output.getvalue())
-    fout.close()
-    output.close()
-    
-
-
-    
-   
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Sep  7 08:58:22 2022
+
+@author: luol2
+"""
+
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jun 24 11:27:57 2022
+
+@author: luol2
+"""
+
+
+import stanza
+import sys
+import os
+import io
+import json
+import re
+#sort entity by position in text
+def pubtator_entitysort(infile):
+    
+    fin=open(infile,'r',encoding='utf-8')    
+    # fout=open(path+'LitCoin/sort/Train_sort.PubTator','w',encoding='utf-8')
+    fout=io.StringIO()
+    all_in=fin.read().strip().split('\n\n')
+    fin.close()
+    error_dict={} #use to debug error
+    for doc in all_in:
+        entity_dict={}
+        lines=doc.split('\n')
+        fout.write(lines[0]+'\n'+lines[1]+'\n')
+        for i in range(2,len(lines)):
+            segs=lines[i].split('\t')
+            if len(segs)>=5:
+                if lines[i] not in entity_dict.keys():
+                    entity_dict[lines[i]]=int(segs[1])
+                else:
+                    print('entity have in',lines[i])
+                    if segs[0] not in error_dict.keys():
+                        error_dict[segs[0]]=[lines[i]]
+                    else:
+                        if lines[i] not in error_dict[segs[0]]:
+                            error_dict[segs[0]].append(lines[i])
+
+        entity_sort=sorted(entity_dict.items(), key=lambda kv:(kv[1]), reverse=False)
+        for ele in entity_sort:
+            fout.write(ele[0]+'\n')
+        fout.write('\n')
+    return fout
+
+def filter_overlap(infile): #nonest
+
+    fin=io.StringIO(infile.getvalue())
+    fout=io.StringIO()
+    
+    documents=fin.read().strip().split('\n\n')
+    fin.close()
+    total_entity=0
+    over_entity=0
+    nest_entity=0
+    for doc in documents:
+        lines=doc.split('\n')
+        entity_list=[]
+        if len(lines)>2:
+            first_entity=lines[2].split('\t')
+            nest_list=[first_entity]
+            max_eid=int(first_entity[2])
+            total_entity+=len(lines)-2
+            for i in range(3,len(lines)):
+                segs=lines[i].split('\t')
+                if int(segs[1])> max_eid:
+                    if len(nest_list)==1:
+                        entity_list.append(nest_list[0])
+                        nest_list=[]
+                        nest_list.append(segs)
+                        if int(segs[2])>max_eid:
+                            max_eid=int(segs[2])
+                    else:
+                        # print(nest_list)
+                        nest_entity+=len(nest_list)-1
+                        tem=find_max_entity(nest_list)#find max entity
+                        # if len(tem)>1:
+                        #     print('max nest >1:',tem)
+                        entity_list.extend(tem)
+                        nest_list=[]
+                        nest_list.append(segs)
+                        if int(segs[2])>max_eid:
+                            max_eid=int(segs[2])
+                        
+                else:
+                    nest_list.append(segs)
+                    if int(segs[2])>max_eid:
+                        max_eid=int(segs[2])
+            if nest_list!=[]:
+                if len(nest_list)==1:
+                    entity_list.append(nest_list[0])
+
+                else:
+                    tem=find_max_entity(nest_list)#find max entity
+                    # if len(tem)>1:
+                    #     print('max nest >1:',tem)
+                    entity_list.extend(tem)
+        fout.write(lines[0]+'\n'+lines[1]+'\n')
+        for ele in entity_list:
+            fout.write('\t'.join(ele)+'\n')
+        fout.write('\n')
+    # print(total_entity,over_entity, nest_entity)
+    return fout
+def find_max_entity(nest_list): #longest entity
+    max_len=0
+    final_tem=[]
+    max_index=0
+    for i in range(0, len(nest_list)):
+        cur_len=int(nest_list[i][2])-int(nest_list[i][1])
+        if cur_len>max_len:
+            max_len=cur_len
+            max_index=i
+
+    final_tem.append(nest_list[max_index])
+    return final_tem    
+                
+# change ori pubtator format to labeled text , entity begin with " ssss", end with 'eeee '
+def pubtator_to_labeltext(infile):
+    
+    fin=io.StringIO(infile.getvalue())
+    all_context=fin.read().strip().split('\n\n')
+    fin.close()
+    fout=io.StringIO()
+    label_dic={}
+    
+    for doc in all_context:
+        lines=doc.split('\n')
+        ori_text=lines[0].split('|t|')[1]+' '+lines[1].split('|a|')[1]
+        pmid=lines[0].split('|t|')[0]
+        s_index=0
+        e_index=0
+        new_text=''
+        for i in range(2,len(lines)):
+            segs=lines[i].split('\t')
+            label_dic[segs[4].lower()]=segs[4]
+            if len(segs)==6:
+                e_index=int(segs[1])
+                new_text+=ori_text[s_index:e_index]+' ssss'+segs[4].lower()+' '+ori_text[int(segs[1]):int(segs[2])]+' eeee'+segs[4].lower()+' '  
+                s_index=int(segs[2])
+                # if ori_text[int(segs[1]):int(segs[2])]!=segs[3]:
+                #     print('error(ori,label):',ori_text[int(segs[1]):int(segs[2])],segs[3])
+
+        new_text+=ori_text[s_index:] 
+        fout.write(pmid+'\t'+' '.join(new_text.strip().split())+'\n')
+    return fout, label_dic
+  
+
+def pre_token(sentence):
+    sentence=re.sub("([\=\/\(\)\<\>\+\-\_])"," \\1 ",sentence)
+    sentence=re.sub("[ ]+"," ",sentence);
+    return sentence
+
+# labeltext to conll format (BIO), a token (including features) per line. sentences are split by '\n', or docs are split by '\n'
+def labeltext_to_conll_fasttoken(infile,label_dic):
+    
+    fin=io.StringIO(infile.getvalue())
+    all_context=fin.read().strip().split('\n')
+    fin.close()
+    fout=io.StringIO()
+    
+    # nlp = stanza.Pipeline(lang='en', processors='tokenize',package='craft') #package='craft'
+    nlp = stanza.Pipeline(lang='en', processors={'tokenize': 'spacy'},package='None') #package='craft'
+
+    doc_i=0
+    for doc in all_context:
+        doc_text=doc.split('\t')[1]
+        doc_text=pre_token(doc_text)
+        doc_stanza = nlp(doc_text)
+        doc_i+=1
+        #print(doc_i)
+        inentity_flag=0
+        last_label='O'
+        for sent in doc_stanza.sentences:
+            temp_sent=[]
+            word_num=0
+            for word in sent.words:
+                word_num+=1
+                # print(word.text)
+                if word.text.strip()=='':
+                    continue
+                temp_sent.append(word.text)
+                if word.text.startswith('ssss')==True:
+                    last_label=word.text
+                    inentity_flag=1
+                elif word.text.startswith('eeee')==True:
+                    last_label=word.text
+                    inentity_flag=0                    
+                else:
+                    if last_label=='O':
+                        now_label='O'
+                    elif last_label.startswith('ssss')==True:
+                        now_label='B-'+label_dic[last_label[4:]]
+                        
+                    elif last_label.startswith('B-')==True:
+                        now_label='I-'+last_label[2:]
+                    elif last_label.startswith('I-')==True:
+                        now_label='I-'+last_label[2:]    
+                    elif last_label.startswith('eeee')==True:
+                        now_label='O'
+                        
+                    fout.write(word.text+'\t'+now_label+'\n')
+                    last_label=now_label
+            if inentity_flag==1: # if entity is split by sentence, will connate the sentence
+                # print('sentence error!!!')
+                # print(word.text,word_num)
+                # print(temp_sent)
+                pass
+            else:
+                fout.write('\n')
+    return fout
+        
+def pubtator_to_conll(infile):
+    
+    #1.entity sort 
+    input_sort=pubtator_entitysort(infile)
+    #print(input_sort.getvalue())
+    
+    #2. no overlap, if overlap get longest entity
+    input_nonest=filter_overlap(input_sort)
+    # print('......sort.....\n',input_sort.getvalue())
+    
+    #3. pubtator to label text
+    input_labtext,label_dic=pubtator_to_labeltext(input_nonest)
+    # print('......label.....\n',input_labtext.getvalue())
+    #print(label_dic)
+    
+    #4. label text to conll
+    output = labeltext_to_conll_fasttoken(input_labtext,label_dic)
+    # print('......output.....\n',output.getvalue())
+    # fout=open(outfile,'w',encoding='utf-8')
+    # fout.write(input_nonest.getvalue())
+    # fout.close()
+    return output
+
+if __name__=='__main__':
+    
+
+    infile='../../TrainingSet/No100/NER.Train.txt'  
+    output=pubtator_to_conll(infile)
+    fout=open('../../TrainingSet/No100/NER.Train.conll','w',encoding='utf-8')
+    fout.write(output.getvalue())
+    fout.close()
+    output.close()
+    
+
+
+    
+   
     

src_python/GeneNER/Evaluation_ner.py

@@ -1,243 +1,243 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Mon Mar  1 15:33:54 2021
-
-@author: luol2
-"""
-# from BIO format to entity
-def BIO_tag(tokens):
-    gold_entity={}
-    pre_entity={}
-    gold_start,gold_end=0,0
-    pre_start,pre_end=0,0
-    for i in range(0,len(tokens)):
-        segs=tokens[i].split('\t')
-        
-        # generate gold entity
-        if segs[1].startswith('B-')>0:
-            gold_start=i
-            gold_type=segs[1][2:]
-            if i+1>=len(tokens): # the last word
-                gold_end=i
-                if gold_type in gold_entity.keys():
-                    gold_entity[gold_type].append([gold_start,gold_end])
-                else:
-                    gold_entity[gold_type]=[[gold_start,gold_end]]
-            else: # non last word
-                next_seg=tokens[i+1].split('\t')
-                if next_seg[1].startswith('B-')>0 or next_seg[1]=='O':
-                    gold_end=i
-                    if gold_type in gold_entity.keys():
-                        gold_entity[gold_type].append([gold_start,gold_end])
-                    else:
-                        gold_entity[gold_type]=[[gold_start,gold_end]]
-                elif next_seg[1].startswith('I-')>0:
-                    pass
-        elif segs[1].startswith('I-')>0:
-            if i+1>=len(tokens): # the last word
-                gold_end=i
-                if gold_type in gold_entity.keys():
-                    gold_entity[gold_type].append([gold_start,gold_end])
-                else:
-                    gold_entity[gold_type]=[[gold_start,gold_end]]
-            else: # non last word
-                next_seg=tokens[i+1].split('\t')
-                if next_seg[1].startswith('B-')>0 or next_seg[1]=='O':
-                    gold_end=i
-                    if gold_type in gold_entity.keys():
-                        gold_entity[gold_type].append([gold_start,gold_end])
-                    else:
-                        gold_entity[gold_type]=[[gold_start,gold_end]]
-                elif next_seg[1].startswith('I-')>0:
-                    pass
-        elif segs[1]=='O':
-            pass
-        
-        # generate prediction entity            
-        if segs[2].startswith('B-')>0:
-            pre_start=i
-            pre_type=segs[2][2:]
-            if i+1>=len(tokens): # the last word
-                pre_end=i
-                if pre_type in pre_entity.keys():
-                    pre_entity[pre_type].append([pre_start,pre_end])
-                else:
-                    pre_entity[pre_type]=[[pre_start,pre_end]]
-            else: # non last word
-                next_seg=tokens[i+1].split('\t')
-                if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                    pre_end=i
-                    if pre_type in pre_entity.keys():
-                        pre_entity[pre_type].append([pre_start,pre_end])
-                    else:
-                        pre_entity[pre_type]=[[pre_start,pre_end]]
-                elif next_seg[2].startswith('I-')>0:
-                    pass
-        elif segs[2].startswith('I-')>0:
-            if i==0 and i+1<len(tokens): # the first word and not only a word
-                pre_start=i
-                pre_type=segs[2][2:]
-                next_seg=tokens[i+1].split('\t')
-                if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                    pre_end=i
-                    if pre_type in pre_entity.keys():
-                        pre_entity[pre_type].append([pre_start,pre_end])
-                    else:
-                        pre_entity[pre_type]=[[pre_start,pre_end]]
-                elif next_seg[2].startswith('I-')>0:
-                    pass
-            elif i==0 and i+1==len(tokens):# only one word:
-                pre_start=i
-                pre_type=segs[2][2:]
-                pre_end=i
-                if pre_type in pre_entity.keys():
-                    pre_entity[pre_type].append([pre_start,pre_end])
-                else:
-                    pre_entity[pre_type]=[[pre_start,pre_end]]
-            elif i+1>=len(tokens): # the last word
-                last_seg=tokens[i-1].split('\t')
-                if last_seg[2]=='O':
-                    pre_start=i
-                    pre_type=segs[2][2:]                
-                pre_end=i
-                if pre_type in pre_entity.keys():
-                    pre_entity[pre_type].append([pre_start,pre_end])
-                else:
-                    pre_entity[pre_type]=[[pre_start,pre_end]]
-            elif i+1< len(tokens): # non last word
-                next_seg=tokens[i+1].split('\t')
-                last_seg=tokens[i-1].split('\t')
-                if last_seg[2]=='O':
-                    pre_start=i
-                    pre_type=segs[2][2:]  
-                if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                    pre_end=i
-                    if pre_type in pre_entity.keys():
-                        pre_entity[pre_type].append([pre_start,pre_end])
-                    else:
-                        pre_entity[pre_type]=[[pre_start,pre_end]]
-                elif next_seg[2].startswith('I-')>0:
-                    pass
-        elif segs[2]=='O':
-            pass        
-    # print(tokens)
-    # print(gold_entity)
-    # print(pre_entity)
-    return gold_entity,pre_entity
-        
-# input: token \t Gold \t Prediction\n, sentence is split "\n"
-def NER_Evaluation():
-    path='//panfs/pan1/bionlp/lulab/luoling/OpenBioIE_project/models/Kfold/BiLSTM-CRF/'
-    fin=open(path+'dev_pre.conll_all','r',encoding='utf-8')
-    all_sentence=fin.read().strip().split('\n\n')
-    fin.close()
-    Metrics={} #{'entity_type':[TP,gold_num,pre_num]}
-
-    for sentence in all_sentence:
-        tokens=sentence.split('\n')
-        gold_entity,pre_entity=BIO_tag(tokens)
-        # print(tokens)
-        for entity_type in gold_entity.keys():
-            if entity_type not in Metrics.keys():
-                Metrics[entity_type]=[0,len(gold_entity[entity_type]),0]
-            else:
-                Metrics[entity_type][1]+=len(gold_entity[entity_type])
-        for entity_type in pre_entity.keys():
-            if entity_type not in Metrics.keys():
-                Metrics[entity_type]=[0,0,len(pre_entity[entity_type])]
-            else:
-                Metrics[entity_type][2]+=len(pre_entity[entity_type])
-                for mention in pre_entity[entity_type]:
-                    if entity_type in gold_entity.keys():
-                        if mention in gold_entity[entity_type]:
-                            Metrics[entity_type][0]+=1
-    print(Metrics)
-    TP,Gold_num,Pre_num=0,0,0
-    for ele in Metrics.keys():
-        if Metrics[ele][2]==0:
-            p=0
-        else:
-            p=Metrics[ele][0]/Metrics[ele][2]
-        if Metrics[ele][1]==0:
-            r=0
-        else:
-            r=Metrics[ele][0]/Metrics[ele][1]
-        if p+r==0:
-            f1=0
-        else:
-            f1=2*p*r/(p+r)
-        TP+=Metrics[ele][0]
-        Gold_num+=Metrics[ele][1]
-        Pre_num+=Metrics[ele][2]
-        print(ele+': P=%.5f, R=%.5f, F1=%.5f' % (p,r,f1))
-        # break
-    if Pre_num==0:
-        P=0
-    else:
-        P=TP/Pre_num
-    R=TP/Gold_num
-    F1=2*P*R/(P+R)
-    print("Overall: P=%.5f, R=%.5f, F1=%.5f"% (P,R,F1))
-    
-def NER_Evaluation_fn(file):
-    
-    fin=open(file,'r',encoding='utf-8')
-    all_sentence=fin.read().strip().split('\n\n')
-    fin.close()
-    Metrics={} #{'entity_type':[TP,gold_num,pre_num]}
-    breai=0
-    for sentence in all_sentence:
-        breai+=1
-        if breai>5000:
-            break
-        tokens=sentence.split('\n')
-        gold_entity,pre_entity=BIO_tag(tokens)
-        # print(tokens)
-        for entity_type in gold_entity.keys():
-            if entity_type not in Metrics.keys():
-                Metrics[entity_type]=[0,len(gold_entity[entity_type]),0]
-            else:
-                Metrics[entity_type][1]+=len(gold_entity[entity_type])
-        for entity_type in pre_entity.keys():
-            if entity_type not in Metrics.keys():
-                Metrics[entity_type]=[0,0,len(pre_entity[entity_type])]
-            else:
-                Metrics[entity_type][2]+=len(pre_entity[entity_type])
-                for mention in pre_entity[entity_type]:
-                    if entity_type in gold_entity.keys():
-                        if mention in gold_entity[entity_type]:
-                            Metrics[entity_type][0]+=1
-    print(Metrics)
-    TP,Gold_num,Pre_num=0,0,0
-    for ele in Metrics.keys():
-        if Metrics[ele][2]==0:
-            p=0
-        else:
-            p=Metrics[ele][0]/Metrics[ele][2]
-        if Metrics[ele][1]==0:
-            r=0
-        else:
-            r=Metrics[ele][0]/Metrics[ele][1]
-        if p+r==0:
-            f1=0
-        else:
-            f1=2*p*r/(p+r)
-        TP+=Metrics[ele][0]
-        Gold_num+=Metrics[ele][1]
-        Pre_num+=Metrics[ele][2]
-        print(ele+': P=%.5f, R=%.5f, F1=%.5f' % (p,r,f1))
-        # break
-    if Pre_num==0:
-        P=0
-    else:
-        P=TP/Pre_num
-    R=TP/Gold_num
-    if P+R==0:
-        F1=0
-    else:
-        F1=2*P*R/(P+R)
-    print("Overall: P=%.5f, R=%.5f, F1=%.5f"% (P,R,F1))
-    return F1
-            
-if __name__=='__main__':
-    NER_Evaluation()
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Mar  1 15:33:54 2021
+
+@author: luol2
+"""
+# from BIO format to entity
+def BIO_tag(tokens):
+    gold_entity={}
+    pre_entity={}
+    gold_start,gold_end=0,0
+    pre_start,pre_end=0,0
+    for i in range(0,len(tokens)):
+        segs=tokens[i].split('\t')
+        
+        # generate gold entity
+        if segs[1].startswith('B-')>0:
+            gold_start=i
+            gold_type=segs[1][2:]
+            if i+1>=len(tokens): # the last word
+                gold_end=i
+                if gold_type in gold_entity.keys():
+                    gold_entity[gold_type].append([gold_start,gold_end])
+                else:
+                    gold_entity[gold_type]=[[gold_start,gold_end]]
+            else: # non last word
+                next_seg=tokens[i+1].split('\t')
+                if next_seg[1].startswith('B-')>0 or next_seg[1]=='O':
+                    gold_end=i
+                    if gold_type in gold_entity.keys():
+                        gold_entity[gold_type].append([gold_start,gold_end])
+                    else:
+                        gold_entity[gold_type]=[[gold_start,gold_end]]
+                elif next_seg[1].startswith('I-')>0:
+                    pass
+        elif segs[1].startswith('I-')>0:
+            if i+1>=len(tokens): # the last word
+                gold_end=i
+                if gold_type in gold_entity.keys():
+                    gold_entity[gold_type].append([gold_start,gold_end])
+                else:
+                    gold_entity[gold_type]=[[gold_start,gold_end]]
+            else: # non last word
+                next_seg=tokens[i+1].split('\t')
+                if next_seg[1].startswith('B-')>0 or next_seg[1]=='O':
+                    gold_end=i
+                    if gold_type in gold_entity.keys():
+                        gold_entity[gold_type].append([gold_start,gold_end])
+                    else:
+                        gold_entity[gold_type]=[[gold_start,gold_end]]
+                elif next_seg[1].startswith('I-')>0:
+                    pass
+        elif segs[1]=='O':
+            pass
+        
+        # generate prediction entity            
+        if segs[2].startswith('B-')>0:
+            pre_start=i
+            pre_type=segs[2][2:]
+            if i+1>=len(tokens): # the last word
+                pre_end=i
+                if pre_type in pre_entity.keys():
+                    pre_entity[pre_type].append([pre_start,pre_end])
+                else:
+                    pre_entity[pre_type]=[[pre_start,pre_end]]
+            else: # non last word
+                next_seg=tokens[i+1].split('\t')
+                if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                    pre_end=i
+                    if pre_type in pre_entity.keys():
+                        pre_entity[pre_type].append([pre_start,pre_end])
+                    else:
+                        pre_entity[pre_type]=[[pre_start,pre_end]]
+                elif next_seg[2].startswith('I-')>0:
+                    pass
+        elif segs[2].startswith('I-')>0:
+            if i==0 and i+1<len(tokens): # the first word and not only a word
+                pre_start=i
+                pre_type=segs[2][2:]
+                next_seg=tokens[i+1].split('\t')
+                if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                    pre_end=i
+                    if pre_type in pre_entity.keys():
+                        pre_entity[pre_type].append([pre_start,pre_end])
+                    else:
+                        pre_entity[pre_type]=[[pre_start,pre_end]]
+                elif next_seg[2].startswith('I-')>0:
+                    pass
+            elif i==0 and i+1==len(tokens):# only one word:
+                pre_start=i
+                pre_type=segs[2][2:]
+                pre_end=i
+                if pre_type in pre_entity.keys():
+                    pre_entity[pre_type].append([pre_start,pre_end])
+                else:
+                    pre_entity[pre_type]=[[pre_start,pre_end]]
+            elif i+1>=len(tokens): # the last word
+                last_seg=tokens[i-1].split('\t')
+                if last_seg[2]=='O':
+                    pre_start=i
+                    pre_type=segs[2][2:]                
+                pre_end=i
+                if pre_type in pre_entity.keys():
+                    pre_entity[pre_type].append([pre_start,pre_end])
+                else:
+                    pre_entity[pre_type]=[[pre_start,pre_end]]
+            elif i+1< len(tokens): # non last word
+                next_seg=tokens[i+1].split('\t')
+                last_seg=tokens[i-1].split('\t')
+                if last_seg[2]=='O':
+                    pre_start=i
+                    pre_type=segs[2][2:]  
+                if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                    pre_end=i
+                    if pre_type in pre_entity.keys():
+                        pre_entity[pre_type].append([pre_start,pre_end])
+                    else:
+                        pre_entity[pre_type]=[[pre_start,pre_end]]
+                elif next_seg[2].startswith('I-')>0:
+                    pass
+        elif segs[2]=='O':
+            pass        
+    # print(tokens)
+    # print(gold_entity)
+    # print(pre_entity)
+    return gold_entity,pre_entity
+        
+# input: token \t Gold \t Prediction\n, sentence is split "\n"
+def NER_Evaluation():
+    path='//panfs/pan1/bionlp/lulab/luoling/OpenBioIE_project/models/Kfold/BiLSTM-CRF/'
+    fin=open(path+'dev_pre.conll_all','r',encoding='utf-8')
+    all_sentence=fin.read().strip().split('\n\n')
+    fin.close()
+    Metrics={} #{'entity_type':[TP,gold_num,pre_num]}
+
+    for sentence in all_sentence:
+        tokens=sentence.split('\n')
+        gold_entity,pre_entity=BIO_tag(tokens)
+        # print(tokens)
+        for entity_type in gold_entity.keys():
+            if entity_type not in Metrics.keys():
+                Metrics[entity_type]=[0,len(gold_entity[entity_type]),0]
+            else:
+                Metrics[entity_type][1]+=len(gold_entity[entity_type])
+        for entity_type in pre_entity.keys():
+            if entity_type not in Metrics.keys():
+                Metrics[entity_type]=[0,0,len(pre_entity[entity_type])]
+            else:
+                Metrics[entity_type][2]+=len(pre_entity[entity_type])
+                for mention in pre_entity[entity_type]:
+                    if entity_type in gold_entity.keys():
+                        if mention in gold_entity[entity_type]:
+                            Metrics[entity_type][0]+=1
+    print(Metrics)
+    TP,Gold_num,Pre_num=0,0,0
+    for ele in Metrics.keys():
+        if Metrics[ele][2]==0:
+            p=0
+        else:
+            p=Metrics[ele][0]/Metrics[ele][2]
+        if Metrics[ele][1]==0:
+            r=0
+        else:
+            r=Metrics[ele][0]/Metrics[ele][1]
+        if p+r==0:
+            f1=0
+        else:
+            f1=2*p*r/(p+r)
+        TP+=Metrics[ele][0]
+        Gold_num+=Metrics[ele][1]
+        Pre_num+=Metrics[ele][2]
+        print(ele+': P=%.5f, R=%.5f, F1=%.5f' % (p,r,f1))
+        # break
+    if Pre_num==0:
+        P=0
+    else:
+        P=TP/Pre_num
+    R=TP/Gold_num
+    F1=2*P*R/(P+R)
+    print("Overall: P=%.5f, R=%.5f, F1=%.5f"% (P,R,F1))
+    
+def NER_Evaluation_fn(file):
+    
+    fin=open(file,'r',encoding='utf-8')
+    all_sentence=fin.read().strip().split('\n\n')
+    fin.close()
+    Metrics={} #{'entity_type':[TP,gold_num,pre_num]}
+    breai=0
+    for sentence in all_sentence:
+        breai+=1
+        if breai>5000:
+            break
+        tokens=sentence.split('\n')
+        gold_entity,pre_entity=BIO_tag(tokens)
+        # print(tokens)
+        for entity_type in gold_entity.keys():
+            if entity_type not in Metrics.keys():
+                Metrics[entity_type]=[0,len(gold_entity[entity_type]),0]
+            else:
+                Metrics[entity_type][1]+=len(gold_entity[entity_type])
+        for entity_type in pre_entity.keys():
+            if entity_type not in Metrics.keys():
+                Metrics[entity_type]=[0,0,len(pre_entity[entity_type])]
+            else:
+                Metrics[entity_type][2]+=len(pre_entity[entity_type])
+                for mention in pre_entity[entity_type]:
+                    if entity_type in gold_entity.keys():
+                        if mention in gold_entity[entity_type]:
+                            Metrics[entity_type][0]+=1
+    print(Metrics)
+    TP,Gold_num,Pre_num=0,0,0
+    for ele in Metrics.keys():
+        if Metrics[ele][2]==0:
+            p=0
+        else:
+            p=Metrics[ele][0]/Metrics[ele][2]
+        if Metrics[ele][1]==0:
+            r=0
+        else:
+            r=Metrics[ele][0]/Metrics[ele][1]
+        if p+r==0:
+            f1=0
+        else:
+            f1=2*p*r/(p+r)
+        TP+=Metrics[ele][0]
+        Gold_num+=Metrics[ele][1]
+        Pre_num+=Metrics[ele][2]
+        print(ele+': P=%.5f, R=%.5f, F1=%.5f' % (p,r,f1))
+        # break
+    if Pre_num==0:
+        P=0
+    else:
+        P=TP/Pre_num
+    R=TP/Gold_num
+    if P+R==0:
+        F1=0
+    else:
+        F1=2*P*R/(P+R)
+    print("Overall: P=%.5f, R=%.5f, F1=%.5f"% (P,R,F1))
+    return F1
+            
+if __name__=='__main__':
+    NER_Evaluation()

src_python/GeneNER/model_ner.py

@@ -1,102 +1,102 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Wed Feb 10 09:08:09 2021
-
-@author: luol2
-"""
-import tensorflow as tf
-from src_python.GeneNER.represent_ner import Hugface_RepresentationLayer
-from tensorflow.keras.layers import *
-from tensorflow.keras.models import Model
-from tensorflow.keras.optimizers import RMSprop, SGD, Adam, Adadelta, Adagrad,Nadam
-from transformers import TFBertModel, BertConfig,TFElectraModel,TFAutoModel
-import numpy as np
-import sys
-
-
-class LRSchedule_LINEAR(tf.keras.optimizers.schedules.LearningRateSchedule):
-    def __init__(
-        self,
-        init_lr=5e-5,
-        init_warmup_lr=0.0,
-        final_lr=5e-7,
-        warmup_steps=0,
-        decay_steps=0,
-    ):
-        super().__init__()
-        self.init_lr = init_lr
-        self.init_warmup_lr=init_warmup_lr
-        self.final_lr = final_lr
-        self.warmup_steps = warmup_steps
-        self.decay_steps = decay_steps
-        
-    def __call__(self, step):
-        """ linear warm up - linear decay """
-        if self.warmup_steps>0:
-            warmup_lr = (self.init_lr - self.init_warmup_lr)/self.warmup_steps * step+self.init_warmup_lr
-        else:
-            warmup_lr=1000.0
-        #print('\n.......warmup_lr:',warmup_lr)
-        decay_lr = tf.math.maximum(
-            self.final_lr,
-            self.init_lr - (step - self.warmup_steps)/self.decay_steps*(self.init_lr - self.final_lr)
-        )
-        #print('\n.....decay_lr:',decay_lr)
-        return tf.math.minimum(warmup_lr,decay_lr)
-
-        
-class HUGFACE_NER(): #huggingface transformers
-    def __init__(self, model_files):
-        self.model_type='HUGFACE'
-        self.maxlen = 256  #sent 256 doc-512,pretrain-sent 128
-        self.checkpoint_path = model_files['checkpoint_path']
-        self.label_file=model_files['labelfile']
-        self.lowercase=model_files['lowercase']
-        self.rep = Hugface_RepresentationLayer(self.checkpoint_path, self.label_file, lowercase=self.lowercase)
-        
-            
-    def build_encoder(self):
-        print('...vocab len:',self.rep.vocab_len)
-        plm_model = TFAutoModel.from_pretrained(self.checkpoint_path, from_pt=True)
-        # plm_model.resize_token_embeddings(self.rep.vocab_len) 
-        x1_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='input_ids')
-        x2_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='token_type_ids')
-        x3_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='attention_mask')
-        x = plm_model(x1_in, token_type_ids=x2_in, attention_mask=x3_in)[0]
-        #dense = TimeDistributed(Dense(512, activation='relu'), name='dense1')(x)
-        self.encoder = Model (inputs=[x1_in,x2_in,x3_in], outputs=x,name='hugface_encoder')
-        self.encoder.summary()
-        
-    def build_softmax_decoder(self):
-
-        x1_in = Input(shape=(self.maxlen,),dtype=tf.int32)
-        x2_in = Input(shape=(self.maxlen,),dtype=tf.int32)
-        x3_in = Input(shape=(self.maxlen,),dtype=tf.int32)  
-        features = self.encoder([x1_in,x2_in,x3_in])
-        #features = Dropout(0.4)(features)
-        features = TimeDistributed(Dense(128, activation='relu'), name='dense2')(features)
-        features= Dropout(0.1)(features)
-        output = TimeDistributed(Dense(self.rep.label_table_size, activation='softmax'), name='softmax')(features)
-        self.model = Model(inputs=[x1_in,x2_in,x3_in], outputs=output, name="hugface_softmax")
-
-        lr_schedule=LRSchedule_LINEAR(
-            init_lr=1e-5,
-            init_warmup_lr=1e-7,
-            final_lr=5e-6,
-            warmup_steps=0,
-            decay_steps=1000)
-
-        opt = Adam(learning_rate = lr_schedule)
-        #opt = Adam(lr=5e-6) 
-        self.model.compile(
-            optimizer=opt,
-            loss='sparse_categorical_crossentropy',
-            metrics=['accuracy'],
-        )
-        self.model.summary()
-        
-        
-    def load_model(self,model_file):
-        self.model.load_weights(model_file)
-        self.model.summary()  
-        print('load HUGFACE model done!')
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Feb 10 09:08:09 2021
+
+@author: luol2
+"""
+import tensorflow as tf
+from src_python.GeneNER.represent_ner import Hugface_RepresentationLayer
+from tensorflow.keras.layers import *
+from tensorflow.keras.models import Model
+from tensorflow.keras.optimizers import RMSprop, SGD, Adam, Adadelta, Adagrad,Nadam
+from transformers import TFBertModel, BertConfig,TFElectraModel,TFAutoModel
+import numpy as np
+import sys
+
+
+class LRSchedule_LINEAR(tf.keras.optimizers.schedules.LearningRateSchedule):
+    def __init__(
+        self,
+        init_lr=5e-5,
+        init_warmup_lr=0.0,
+        final_lr=5e-7,
+        warmup_steps=0,
+        decay_steps=0,
+    ):
+        super().__init__()
+        self.init_lr = init_lr
+        self.init_warmup_lr=init_warmup_lr
+        self.final_lr = final_lr
+        self.warmup_steps = warmup_steps
+        self.decay_steps = decay_steps
+        
+    def __call__(self, step):
+        """ linear warm up - linear decay """
+        if self.warmup_steps>0:
+            warmup_lr = (self.init_lr - self.init_warmup_lr)/self.warmup_steps * step+self.init_warmup_lr
+        else:
+            warmup_lr=1000.0
+        #print('\n.......warmup_lr:',warmup_lr)
+        decay_lr = tf.math.maximum(
+            self.final_lr,
+            self.init_lr - (step - self.warmup_steps)/self.decay_steps*(self.init_lr - self.final_lr)
+        )
+        #print('\n.....decay_lr:',decay_lr)
+        return tf.math.minimum(warmup_lr,decay_lr)
+
+        
+class HUGFACE_NER(): #huggingface transformers
+    def __init__(self, model_files):
+        self.model_type='HUGFACE'
+        self.maxlen = 256  #sent 256 doc-512,pretrain-sent 128
+        self.checkpoint_path = model_files['checkpoint_path']
+        self.label_file=model_files['labelfile']
+        self.lowercase=model_files['lowercase']
+        self.rep = Hugface_RepresentationLayer(self.checkpoint_path, self.label_file, lowercase=self.lowercase)
+        
+            
+    def build_encoder(self):
+        print('...vocab len:',self.rep.vocab_len)
+        plm_model = TFAutoModel.from_pretrained(self.checkpoint_path, from_pt=True)
+        # plm_model.resize_token_embeddings(self.rep.vocab_len) 
+        x1_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='input_ids')
+        x2_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='token_type_ids')
+        x3_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='attention_mask')
+        x = plm_model(x1_in, token_type_ids=x2_in, attention_mask=x3_in)[0]
+        #dense = TimeDistributed(Dense(512, activation='relu'), name='dense1')(x)
+        self.encoder = Model (inputs=[x1_in,x2_in,x3_in], outputs=x,name='hugface_encoder')
+        self.encoder.summary()
+        
+    def build_softmax_decoder(self):
+
+        x1_in = Input(shape=(self.maxlen,),dtype=tf.int32)
+        x2_in = Input(shape=(self.maxlen,),dtype=tf.int32)
+        x3_in = Input(shape=(self.maxlen,),dtype=tf.int32)  
+        features = self.encoder([x1_in,x2_in,x3_in])
+        #features = Dropout(0.4)(features)
+        features = TimeDistributed(Dense(128, activation='relu'), name='dense2')(features)
+        features= Dropout(0.1)(features)
+        output = TimeDistributed(Dense(self.rep.label_table_size, activation='softmax'), name='softmax')(features)
+        self.model = Model(inputs=[x1_in,x2_in,x3_in], outputs=output, name="hugface_softmax")
+
+        lr_schedule=LRSchedule_LINEAR(
+            init_lr=1e-5,
+            init_warmup_lr=1e-7,
+            final_lr=5e-6,
+            warmup_steps=0,
+            decay_steps=1000)
+
+        opt = Adam(learning_rate = lr_schedule)
+        #opt = Adam(lr=5e-6) 
+        self.model.compile(
+            optimizer=opt,
+            loss='sparse_categorical_crossentropy',
+            metrics=['accuracy'],
+        )
+        self.model.summary()
+        
+        
+    def load_model(self,model_file):
+        self.model.load_weights(model_file)
+        self.model.summary()  
+        print('load HUGFACE model done!')

src_python/GeneNER/ner_tag.py

@@ -1,85 +1,85 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Wed Jun  8 11:01:23 2022
-
-@author: luol2
-"""
-
-
-
-import io
-import re
-from src_python.GeneNER.processing_data_ner import ml_intext_fn,out_BIO_BERT_softmax_fn
-from src_python.GeneNER.restore_index_ner import NN_restore_index_fn
-import tensorflow as tf
-gpu = tf.config.list_physical_devices('GPU')
-print("Num GPUs Available: ", len(gpu))
-if len(gpu) > 0:
-    tf.config.experimental.set_memory_growth(gpu[0], True)
-
-def pre_token(sentence):
-    sentence=re.sub("([\W\-\_])"," \\1 ",sentence)
-    sentence=re.sub("[ ]+"," ",sentence);
-    return sentence
-
-def ssplit_token_pos_lemma(in_text,text_level,nlp_token, max_len=400):
-    #print('max_len:',max_len)
-    fout=io.StringIO()
-
-    in_text=in_text.strip()
-    in_text=pre_token(in_text)
-    doc_stanza = nlp_token(in_text)
-    strlen=0
-    for sent in doc_stanza.sentences:
-        for word in sent.words:
-            strlen+=1
-            if word.text.strip()=='':
-                pass
-                #print('!!!!blank token text!!!')
-            else:
-                fout.write(word.text+'\tO\n')
-            if strlen>=max_len:
-                #print('long sentence:',strlen)
-                fout.write('\n')
-                strlen=0
-        if text_level=='SENT':
-            fout.write('\n')
-            strlen=0
-    if text_level=='DOC':
-        fout.write('\n')
-           
-    return fout.getvalue()
-
-def ml_tagging(ml_input,nn_model):
-
-    test_list = ml_intext_fn(ml_input)
-    test_x,test_y, test_bert_text_label=nn_model.rep.load_data_hugface(test_list,word_max_len=nn_model.maxlen,label_type='softmax')
-    test_pre = nn_model.model.predict(test_x,batch_size=64)
-    test_decode_temp=out_BIO_BERT_softmax_fn(test_pre,test_bert_text_label,nn_model.rep.index_2_label)
-    
-    return test_decode_temp
-# only machine learning-based method
-def ML_Tag(text,ml_model,nlp_token,text_level='SENT'):
-
-#    startTime=time.time()
-    ssplit_token=ssplit_token_pos_lemma(text, text_level, nlp_token, max_len=ml_model.maxlen)
-    #print(ssplit_token) 
-#    print('ssplit token:',time.time()-startTime)
-    
-#    startTime=time.time()
-    ml_tsv=ml_tagging(ssplit_token,ml_model)
-    #print(ml_tsv)
-#    print('ml ner:',time.time()-startTime)
-   
-    final_result= NN_restore_index_fn(text,ml_tsv)
-
-    # print('final ner:',time.time()-startTime)
-    
-    return final_result
-
-
-
-
-
-    
-
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Jun  8 11:01:23 2022
+
+@author: luol2
+"""
+
+
+
+import io
+import re
+from src_python.GeneNER.processing_data_ner import ml_intext_fn,out_BIO_BERT_softmax_fn
+from src_python.GeneNER.restore_index_ner import NN_restore_index_fn
+import tensorflow as tf
+gpu = tf.config.list_physical_devices('GPU')
+print("Num GPUs Available: ", len(gpu))
+if len(gpu) > 0:
+    tf.config.experimental.set_memory_growth(gpu[0], True)
+
+def pre_token(sentence):
+    sentence=re.sub("([\W\-\_])"," \\1 ",sentence)
+    sentence=re.sub("[ ]+"," ",sentence);
+    return sentence
+
+def ssplit_token_pos_lemma(in_text,text_level,nlp_token, max_len=400):
+    #print('max_len:',max_len)
+    fout=io.StringIO()
+
+    in_text=in_text.strip()
+    in_text=pre_token(in_text)
+    doc_stanza = nlp_token(in_text)
+    strlen=0
+    for sent in doc_stanza.sentences:
+        for word in sent.words:
+            strlen+=1
+            if word.text.strip()=='':
+                pass
+                #print('!!!!blank token text!!!')
+            else:
+                fout.write(word.text+'\tO\n')
+            if strlen>=max_len:
+                #print('long sentence:',strlen)
+                fout.write('\n')
+                strlen=0
+        if text_level=='SENT':
+            fout.write('\n')
+            strlen=0
+    if text_level=='DOC':
+        fout.write('\n')
+           
+    return fout.getvalue()
+
+def ml_tagging(ml_input,nn_model):
+
+    test_list = ml_intext_fn(ml_input)
+    test_x,test_y, test_bert_text_label=nn_model.rep.load_data_hugface(test_list,word_max_len=nn_model.maxlen,label_type='softmax')
+    test_pre = nn_model.model.predict(test_x,batch_size=64)
+    test_decode_temp=out_BIO_BERT_softmax_fn(test_pre,test_bert_text_label,nn_model.rep.index_2_label)
+    
+    return test_decode_temp
+# only machine learning-based method
+def ML_Tag(text,ml_model,nlp_token,text_level='SENT'):
+
+#    startTime=time.time()
+    ssplit_token=ssplit_token_pos_lemma(text, text_level, nlp_token, max_len=ml_model.maxlen)
+    #print(ssplit_token) 
+#    print('ssplit token:',time.time()-startTime)
+    
+#    startTime=time.time()
+    ml_tsv=ml_tagging(ssplit_token,ml_model)
+    #print(ml_tsv)
+#    print('ml ner:',time.time()-startTime)
+   
+    final_result= NN_restore_index_fn(text,ml_tsv)
+
+    # print('final ner:',time.time()-startTime)
+    
+    return final_result
+
+
+
+
+
+    
+

src_python/GeneNER/processing_data_ner.py

@@ -1,210 +1,210 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Tue Mar 10 16:34:12 2020
-
-@author: luol2
-"""
-import numpy as np
-import io
-import sys
-#read ner text (word\tlabel), generate the list[[[w1,label],[w2,label]]]
-def ml_intext(file):
-    fin=open(file,'r',encoding='utf-8')
-    alltexts=fin.read().strip().split('\n\n')
-    fin.close()
-    data_list=[]
-
-    for sents in alltexts:
-        lines=sents.split('\n')
-        temp_sentece=[]
-        for i in range(0,len(lines)):
-            seg=lines[i].split('\t')
-            temp_sentece.append(seg[:])
-        
-        data_list.append(temp_sentece)
-    #print(data_list)
-    #print(label_list)
-    return data_list
-
-def ml_intext_fn(ml_input):
-    fin=io.StringIO(ml_input)
-    alltexts=fin.read().strip().split('\n\n')
-    fin.close()
-    data_list=[]
-
-    for sents in alltexts:
-        lines=sents.split('\n')
-        temp_sentece=[]
-        for i in range(0,len(lines)):
-            seg=lines[i].split('\t')
-            temp_sentece.append(seg[:])
-        
-        data_list.append(temp_sentece)
-    #print(data_list)
-    #print(label_list)
-    return data_list
-
-# model predict result to conll evalute format  [token answer predict]        
-def out_BIO_crf(file,raw_pre,raw_input,label_set):
-    fout=open(file,'w',encoding='utf-8')
-    for i in range(len(raw_input)):
-        
-        for j in range(len(raw_input[i])):
-            if j<len(raw_pre[i]):
-                label_id = raw_pre[i][j]
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    fout.close()
-def out_BIO_crf_fn(raw_pre,raw_input,label_set):
-    fout=io.StringIO()
-    for i in range(len(raw_input)):
-        
-        for j in range(len(raw_input[i])):
-            if j<len(raw_pre[i]):
-                label_id = raw_pre[i][j] 
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    return fout.getvalue()      
-def out_BIO_softmax(file,raw_pre,raw_input,label_set):
-    fout=open(file,'w',encoding='utf-8')
-    #print(raw_pre[0:2])
-    for i in range(len(raw_input)):
-        
-        for j in range(len(raw_input[i])):
-            if j<len(raw_pre[i]):
-                label_id = np.argmax(raw_pre[i][j])
-                #print(label_id)
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    fout.close()
-def out_BIO_softmax_fn(raw_pre,raw_input,label_set):
-    fout=io.StringIO()
-    #print(raw_pre[0:2])
-    for i in range(len(raw_input)):
-        
-        for j in range(len(raw_input[i])):
-            if j<len(raw_pre[i]):
-                label_id = np.argmax(raw_pre[i][j])
-                #print(label_id)
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    return fout.getvalue()   
-
-def out_BIO_BERT_softmax(file,raw_pre,raw_input,label_set):
-    fout=open(file,'w',encoding='utf-8')
-    for i in range(len(raw_input)):
-        for j in range(len(raw_input[i])):
-            if raw_input[i][j][-1]<len(raw_pre[i]):
-                # label_id = raw_pre[i][j]
-                label_id = np.argmax(raw_pre[i][raw_input[i][j][-1]])
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    fout.close() 
-def out_BIO_BERT_softmax_fn(raw_pre,raw_input,label_set):
-    fout=io.StringIO()
-    for i in range(len(raw_input)):
-        for j in range(len(raw_input[i])):
-            if raw_input[i][j][-1]<len(raw_pre[i]):
-                #label_id = raw_pre[i][j]
-                label_id = np.argmax(raw_pre[i][raw_input[i][j][-1]])
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    return fout.getvalue()
-def out_BIO_BERT_crf(file,raw_pre,raw_input,label_set):
-    fout=open(file,'w',encoding='utf-8')
-    for i in range(len(raw_input)):
-        
-        for j in range(len(raw_input[i])):
-            if raw_input[i][j][-1]<len(raw_pre[i]):
-                label_id = raw_pre[i][raw_input[i][j][-1]] 
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    fout.close() 
-def out_BIO_BERT_crf_fn(raw_pre,raw_input,label_set):
-    fout=io.StringIO()
-    for i in range(len(raw_input)):
-        
-        for j in range(len(raw_input[i])):
-            if raw_input[i][j][-1]<len(raw_pre[i]):
-                label_id = raw_pre[i][raw_input[i][j][-1]] 
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    return fout.getvalue()                  
-
-def out_BIO_BERT_softmax_score_fn(raw_pre,raw_input,label_set):
-    fout=io.StringIO()
-    for i in range(len(raw_input)):
-
-        for j in range(len(raw_input[i])):
-            if j<len(raw_pre[i]):
-                #label_id = raw_pre[i][j]
-                label_id = np.argmax(raw_pre[i][j])
-                label_score = round(raw_pre[i][j][label_id],4)
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-                label_score = 0.0
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\t'+str(label_score)+'\n')
-        fout.write('\n')
-    return fout.getvalue()
-#generate char vocab
-def char_vocab(infile,outfile_char):
-    fin=open(infile,'r',encoding='utf-8')
-    #fout=open(outfile,'w',encoding='utf-8')
-    fout_char=open(outfile_char,'w',encoding='utf-8')
-    char_vocab=['oov_char']
-    max_len=0
-    for line in fin:
-        if line.strip()!='':
-            seg=line.split('\t')
-            word_len=len(seg[0])
-            #if word_len<1000:
-            #    fout.write(line)
-            if word_len>max_len:
-                max_len=word_len
-                print(seg[0])
-            for i in range(word_len):
-                if seg[0][i] not in char_vocab:
-                    char_vocab.append(seg[0][i])
-        #else:
-        #    fout.write(line)
-    fin.close()
-    #fout.close()
-    for ele in char_vocab:
-        fout_char.write(ele+'\n')
-    fout_char.close()
-    print('max_len:',max_len)
-
-
-if __name__=='__main__':
-    # infile='//panfs/pan1/bionlp/lulab/luoling/HPO_project/AutoPhe/data/pubmed_unlabel/mutation_disease_1990.ner_BIO'
-    # #outfile='//panfs/pan1/bionlp/lulab/luoling/HPO_project/AutoPhe/data/pubmed_unlabel/mutation_disease_1990.ner_BIO_new'
-    # outfile_char='//panfs/pan1/bionlp/lulab/luoling/HPO_project/AutoPhe/src/nn_model/vocab/char_vocab'
-    # #processing_text(file)
-    # char_vocab(infile,outfile_char)
-    a=[1,2,3]
-    print(a[:-1])
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Mar 10 16:34:12 2020
+
+@author: luol2
+"""
+import numpy as np
+import io
+import sys
+#read ner text (word\tlabel), generate the list[[[w1,label],[w2,label]]]
+def ml_intext(file):
+    fin=open(file,'r',encoding='utf-8')
+    alltexts=fin.read().strip().split('\n\n')
+    fin.close()
+    data_list=[]
+
+    for sents in alltexts:
+        lines=sents.split('\n')
+        temp_sentece=[]
+        for i in range(0,len(lines)):
+            seg=lines[i].split('\t')
+            temp_sentece.append(seg[:])
+        
+        data_list.append(temp_sentece)
+    #print(data_list)
+    #print(label_list)
+    return data_list
+
+def ml_intext_fn(ml_input):
+    fin=io.StringIO(ml_input)
+    alltexts=fin.read().strip().split('\n\n')
+    fin.close()
+    data_list=[]
+
+    for sents in alltexts:
+        lines=sents.split('\n')
+        temp_sentece=[]
+        for i in range(0,len(lines)):
+            seg=lines[i].split('\t')
+            temp_sentece.append(seg[:])
+        
+        data_list.append(temp_sentece)
+    #print(data_list)
+    #print(label_list)
+    return data_list
+
+# model predict result to conll evalute format  [token answer predict]        
+def out_BIO_crf(file,raw_pre,raw_input,label_set):
+    fout=open(file,'w',encoding='utf-8')
+    for i in range(len(raw_input)):
+        
+        for j in range(len(raw_input[i])):
+            if j<len(raw_pre[i]):
+                label_id = raw_pre[i][j]
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    fout.close()
+def out_BIO_crf_fn(raw_pre,raw_input,label_set):
+    fout=io.StringIO()
+    for i in range(len(raw_input)):
+        
+        for j in range(len(raw_input[i])):
+            if j<len(raw_pre[i]):
+                label_id = raw_pre[i][j] 
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    return fout.getvalue()      
+def out_BIO_softmax(file,raw_pre,raw_input,label_set):
+    fout=open(file,'w',encoding='utf-8')
+    #print(raw_pre[0:2])
+    for i in range(len(raw_input)):
+        
+        for j in range(len(raw_input[i])):
+            if j<len(raw_pre[i]):
+                label_id = np.argmax(raw_pre[i][j])
+                #print(label_id)
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    fout.close()
+def out_BIO_softmax_fn(raw_pre,raw_input,label_set):
+    fout=io.StringIO()
+    #print(raw_pre[0:2])
+    for i in range(len(raw_input)):
+        
+        for j in range(len(raw_input[i])):
+            if j<len(raw_pre[i]):
+                label_id = np.argmax(raw_pre[i][j])
+                #print(label_id)
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    return fout.getvalue()   
+
+def out_BIO_BERT_softmax(file,raw_pre,raw_input,label_set):
+    fout=open(file,'w',encoding='utf-8')
+    for i in range(len(raw_input)):
+        for j in range(len(raw_input[i])):
+            if raw_input[i][j][-1]<len(raw_pre[i]):
+                # label_id = raw_pre[i][j]
+                label_id = np.argmax(raw_pre[i][raw_input[i][j][-1]])
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    fout.close() 
+def out_BIO_BERT_softmax_fn(raw_pre,raw_input,label_set):
+    fout=io.StringIO()
+    for i in range(len(raw_input)):
+        for j in range(len(raw_input[i])):
+            if raw_input[i][j][-1]<len(raw_pre[i]):
+                #label_id = raw_pre[i][j]
+                label_id = np.argmax(raw_pre[i][raw_input[i][j][-1]])
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    return fout.getvalue()
+def out_BIO_BERT_crf(file,raw_pre,raw_input,label_set):
+    fout=open(file,'w',encoding='utf-8')
+    for i in range(len(raw_input)):
+        
+        for j in range(len(raw_input[i])):
+            if raw_input[i][j][-1]<len(raw_pre[i]):
+                label_id = raw_pre[i][raw_input[i][j][-1]] 
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    fout.close() 
+def out_BIO_BERT_crf_fn(raw_pre,raw_input,label_set):
+    fout=io.StringIO()
+    for i in range(len(raw_input)):
+        
+        for j in range(len(raw_input[i])):
+            if raw_input[i][j][-1]<len(raw_pre[i]):
+                label_id = raw_pre[i][raw_input[i][j][-1]] 
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    return fout.getvalue()                  
+
+def out_BIO_BERT_softmax_score_fn(raw_pre,raw_input,label_set):
+    fout=io.StringIO()
+    for i in range(len(raw_input)):
+
+        for j in range(len(raw_input[i])):
+            if j<len(raw_pre[i]):
+                #label_id = raw_pre[i][j]
+                label_id = np.argmax(raw_pre[i][j])
+                label_score = round(raw_pre[i][j][label_id],4)
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+                label_score = 0.0
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\t'+str(label_score)+'\n')
+        fout.write('\n')
+    return fout.getvalue()
+#generate char vocab
+def char_vocab(infile,outfile_char):
+    fin=open(infile,'r',encoding='utf-8')
+    #fout=open(outfile,'w',encoding='utf-8')
+    fout_char=open(outfile_char,'w',encoding='utf-8')
+    char_vocab=['oov_char']
+    max_len=0
+    for line in fin:
+        if line.strip()!='':
+            seg=line.split('\t')
+            word_len=len(seg[0])
+            #if word_len<1000:
+            #    fout.write(line)
+            if word_len>max_len:
+                max_len=word_len
+                print(seg[0])
+            for i in range(word_len):
+                if seg[0][i] not in char_vocab:
+                    char_vocab.append(seg[0][i])
+        #else:
+        #    fout.write(line)
+    fin.close()
+    #fout.close()
+    for ele in char_vocab:
+        fout_char.write(ele+'\n')
+    fout_char.close()
+    print('max_len:',max_len)
+
+
+if __name__=='__main__':
+    # infile='//panfs/pan1/bionlp/lulab/luoling/HPO_project/AutoPhe/data/pubmed_unlabel/mutation_disease_1990.ner_BIO'
+    # #outfile='//panfs/pan1/bionlp/lulab/luoling/HPO_project/AutoPhe/data/pubmed_unlabel/mutation_disease_1990.ner_BIO_new'
+    # outfile_char='//panfs/pan1/bionlp/lulab/luoling/HPO_project/AutoPhe/src/nn_model/vocab/char_vocab'
+    # #processing_text(file)
+    # char_vocab(infile,outfile_char)
+    a=[1,2,3]
+    print(a[:-1])

src_python/GeneNER/represent_ner.py

@@ -1,183 +1,183 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Mon Aug 30 19:54:17 2021
-
-@author: luol2
-"""
-
-
-
-import os, sys
-import numpy as np
-from tensorflow.keras.preprocessing.sequence import pad_sequences
-from transformers import AutoTokenizer
-
-
-class Hugface_RepresentationLayer(object):
-    
-    
-    def __init__(self, tokenizer_name_or_path, label_file,lowercase=True):
-        
-
-        #load vocab
-
-        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True,do_lower_case=lowercase)
-        self.label_2_index={}
-        self.index_2_label={}
-        self.label_table_size=0
-        self.load_label_vocab(label_file,self.label_2_index,self.index_2_label)
-        self.label_table_size=len(self.label_2_index)
-        self.vocab_len=len(self.tokenizer)
-       
-    def load_label_vocab(self,fea_file,fea_index,index_2_label):
-        
-        fin=open(fea_file,'r',encoding='utf-8')
-        all_text=fin.read().strip().split('\n')
-        fin.close()
-        for i in range(0,len(all_text)):
-            fea_index[all_text[i]]=i
-            index_2_label[str(i)]=all_text[i]
-            
-
-            
-    def generate_label_list(self,ori_tokens,labels,word_index): #the lable of subtoken is the same with the label of first subtoken
-        label_list=['O']*len(word_index)
-
-        label_list_index=[]
-        old_new_token_map=[]
-        ori_i=0
-        for i in range(0,len(word_index)):
-            if word_index[i]==None:
-                label_list_index.append(self.label_2_index[label_list[i]])
-            else:
-                label_list[i]=labels[word_index[i]]
-                label_list_index.append(self.label_2_index[label_list[i]])
-                if word_index[i]==ori_i:
-                    old_new_token_map.append(i)
-                    ori_i+=1
-        
-
-        bert_text_label=[]
-        for i in range(0,len(ori_tokens)):
-            bert_text_label.append([ori_tokens[i],labels[i],old_new_token_map[i]])
-
-        return label_list_index,bert_text_label
-    
-    def generate_label_list_B(self,ori_tokens,labels,word_index): #tonly first subtoken is B, other is I
-        label_list=['O']*len(word_index)
-
-        label_list_index=[]
-        old_new_token_map=[]
-        ori_i=0
-        first_index=-1
-        i=0
-        while i <len(word_index):
-            if word_index[i]==None:
-                label_list_index.append(self.label_2_index[label_list[i]])
-                i+=1
-            else:
-                first_index=word_index[i]
-                if first_index==ori_i:
-                    old_new_token_map.append(i)
-                    ori_i+=1
-                label_list[i]=labels[word_index[i]]
-                label_list_index.append(self.label_2_index[label_list[i]])
-                i+=1
-                while word_index[i]==first_index and word_index[i]!=None:
-                    #print(first_index)
-                    if labels[first_index].startswith("B-"):
-                        label_list[i]='I-'+labels[first_index][2:]
-                        label_list_index.append(self.label_2_index[label_list[i]])
-                    else:
-                        label_list[i]=labels[word_index[i]]
-                        label_list_index.append(self.label_2_index[label_list[i]])
-                    i+=1
-                        
-
-        
-
-        bert_text_label=[]
-        #print(len(old_new_token_map))
-        for i in range(0,len(ori_tokens)):
-            if i<len(old_new_token_map):
-                bert_text_label.append([ori_tokens[i],labels[i],old_new_token_map[i]])
-            else: # after token > max len
-                break
-        return label_list_index,bert_text_label
-    
-    def load_data_hugface(self,instances,  word_max_len=100, label_type='softmax'):
-    
-        x_index=[]
-        x_seg=[]
-        x_mask=[]
-        y_list=[]
-        bert_text_labels=[]
-        max_len=0
-        over_num=0
-        maxT=word_max_len
-        ave_len=0
-
-        #print('instances:', instances)
-        #print('labels:',labels)
-        
-        
-        for sentence in instances:                           
-            sentence_text_list=[]
-            label_list=[]
-            for j in range(0,len(sentence)):
-                sentence_text_list.append(sentence[j][0])
-                label_list.append(sentence[j][-1])
-
-            token_result=self.tokenizer(
-                sentence_text_list,
-                max_length=word_max_len,
-                truncation=True,is_split_into_words=True)
-            
-            bert_tokens=self.tokenizer.convert_ids_to_tokens(token_result['input_ids'])
-            word_index=token_result.word_ids(batch_index=0)
-            ave_len+=len(bert_tokens)
-            if len(sentence_text_list)>max_len:
-                max_len=len(sentence_text_list)
-            if len(bert_tokens)==maxT:
-                over_num+=1
-
-            x_index.append(token_result['input_ids'])
-            x_seg.append(token_result['token_type_ids'])
-            x_mask.append(token_result['attention_mask'])
-            
-            #print('\nsentence_text_list:',len(sentence_text_list),sentence_text_list)
-            #print('\nlabel:',len(label_list),label_list)
-            #print('\nword_index:',len(word_index),word_index)
-            #print('\nbert_tokens:',len(bert_tokens),bert_tokens)
-            label_list,bert_text_label=self.generate_label_list_B(sentence_text_list,label_list,word_index) # the label list after bert token, ori token/lable/new index
-            #print('\nlabel list:',len(label_list),label_list)
-            #print('\nbert_text_label:',len(bert_text_label),bert_text_label)
-            #sys.exit()
-            y_list.append(label_list)
-            #print(y_list)
-            bert_text_labels.append(bert_text_label)
-
-        
-        x1_np = pad_sequences(x_index, word_max_len, value=0, padding='post',truncating='post')  # right padding
-        x2_np = pad_sequences(x_seg, word_max_len, value=0, padding='post',truncating='post')
-        x3_np = pad_sequences(x_mask, word_max_len, value=0, padding='post',truncating='post')
-        y_np = pad_sequences(y_list, word_max_len, value=0, padding='post',truncating='post')
-        #print('x1_np:',x1_np)
-        #print('\nx2_np:',x2_np)
-        #print('\ny_np:',y_np)
-        #print('\nbert_text:',bert_text_labels)
-        # print('bert max len:',max_len,',Over',maxT,':',over_num,'ave len:',ave_len/len(instances),'total:',len(instances))
-
-        if label_type=='softmax':
-            y_np = np.expand_dims(y_np, 2)
-        elif label_type=='crf':
-            pass
-        
-        return [x1_np, x2_np,x3_np], y_np,bert_text_labels  
-         
-
-if __name__ == '__main__':
-    pass
-    
- 
-            
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Aug 30 19:54:17 2021
+
+@author: luol2
+"""
+
+
+
+import os, sys
+import numpy as np
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+from transformers import AutoTokenizer
+
+
+class Hugface_RepresentationLayer(object):
+    
+    
+    def __init__(self, tokenizer_name_or_path, label_file,lowercase=True):
+        
+
+        #load vocab
+
+        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True,do_lower_case=lowercase)
+        self.label_2_index={}
+        self.index_2_label={}
+        self.label_table_size=0
+        self.load_label_vocab(label_file,self.label_2_index,self.index_2_label)
+        self.label_table_size=len(self.label_2_index)
+        self.vocab_len=len(self.tokenizer)
+       
+    def load_label_vocab(self,fea_file,fea_index,index_2_label):
+        
+        fin=open(fea_file,'r',encoding='utf-8')
+        all_text=fin.read().strip().split('\n')
+        fin.close()
+        for i in range(0,len(all_text)):
+            fea_index[all_text[i]]=i
+            index_2_label[str(i)]=all_text[i]
+            
+
+            
+    def generate_label_list(self,ori_tokens,labels,word_index): #the lable of subtoken is the same with the label of first subtoken
+        label_list=['O']*len(word_index)
+
+        label_list_index=[]
+        old_new_token_map=[]
+        ori_i=0
+        for i in range(0,len(word_index)):
+            if word_index[i]==None:
+                label_list_index.append(self.label_2_index[label_list[i]])
+            else:
+                label_list[i]=labels[word_index[i]]
+                label_list_index.append(self.label_2_index[label_list[i]])
+                if word_index[i]==ori_i:
+                    old_new_token_map.append(i)
+                    ori_i+=1
+        
+
+        bert_text_label=[]
+        for i in range(0,len(ori_tokens)):
+            bert_text_label.append([ori_tokens[i],labels[i],old_new_token_map[i]])
+
+        return label_list_index,bert_text_label
+    
+    def generate_label_list_B(self,ori_tokens,labels,word_index): #tonly first subtoken is B, other is I
+        label_list=['O']*len(word_index)
+
+        label_list_index=[]
+        old_new_token_map=[]
+        ori_i=0
+        first_index=-1
+        i=0
+        while i <len(word_index):
+            if word_index[i]==None:
+                label_list_index.append(self.label_2_index[label_list[i]])
+                i+=1
+            else:
+                first_index=word_index[i]
+                if first_index==ori_i:
+                    old_new_token_map.append(i)
+                    ori_i+=1
+                label_list[i]=labels[word_index[i]]
+                label_list_index.append(self.label_2_index[label_list[i]])
+                i+=1
+                while word_index[i]==first_index and word_index[i]!=None:
+                    #print(first_index)
+                    if labels[first_index].startswith("B-"):
+                        label_list[i]='I-'+labels[first_index][2:]
+                        label_list_index.append(self.label_2_index[label_list[i]])
+                    else:
+                        label_list[i]=labels[word_index[i]]
+                        label_list_index.append(self.label_2_index[label_list[i]])
+                    i+=1
+                        
+
+        
+
+        bert_text_label=[]
+        #print(len(old_new_token_map))
+        for i in range(0,len(ori_tokens)):
+            if i<len(old_new_token_map):
+                bert_text_label.append([ori_tokens[i],labels[i],old_new_token_map[i]])
+            else: # after token > max len
+                break
+        return label_list_index,bert_text_label
+    
+    def load_data_hugface(self,instances,  word_max_len=100, label_type='softmax'):
+    
+        x_index=[]
+        x_seg=[]
+        x_mask=[]
+        y_list=[]
+        bert_text_labels=[]
+        max_len=0
+        over_num=0
+        maxT=word_max_len
+        ave_len=0
+
+        #print('instances:', instances)
+        #print('labels:',labels)
+        
+        
+        for sentence in instances:                           
+            sentence_text_list=[]
+            label_list=[]
+            for j in range(0,len(sentence)):
+                sentence_text_list.append(sentence[j][0])
+                label_list.append(sentence[j][-1])
+
+            token_result=self.tokenizer(
+                sentence_text_list,
+                max_length=word_max_len,
+                truncation=True,is_split_into_words=True)
+            
+            bert_tokens=self.tokenizer.convert_ids_to_tokens(token_result['input_ids'])
+            word_index=token_result.word_ids(batch_index=0)
+            ave_len+=len(bert_tokens)
+            if len(sentence_text_list)>max_len:
+                max_len=len(sentence_text_list)
+            if len(bert_tokens)==maxT:
+                over_num+=1
+
+            x_index.append(token_result['input_ids'])
+            x_seg.append(token_result['token_type_ids'])
+            x_mask.append(token_result['attention_mask'])
+            
+            #print('\nsentence_text_list:',len(sentence_text_list),sentence_text_list)
+            #print('\nlabel:',len(label_list),label_list)
+            #print('\nword_index:',len(word_index),word_index)
+            #print('\nbert_tokens:',len(bert_tokens),bert_tokens)
+            label_list,bert_text_label=self.generate_label_list_B(sentence_text_list,label_list,word_index) # the label list after bert token, ori token/lable/new index
+            #print('\nlabel list:',len(label_list),label_list)
+            #print('\nbert_text_label:',len(bert_text_label),bert_text_label)
+            #sys.exit()
+            y_list.append(label_list)
+            #print(y_list)
+            bert_text_labels.append(bert_text_label)
+
+        
+        x1_np = pad_sequences(x_index, word_max_len, value=0, padding='post',truncating='post')  # right padding
+        x2_np = pad_sequences(x_seg, word_max_len, value=0, padding='post',truncating='post')
+        x3_np = pad_sequences(x_mask, word_max_len, value=0, padding='post',truncating='post')
+        y_np = pad_sequences(y_list, word_max_len, value=0, padding='post',truncating='post')
+        #print('x1_np:',x1_np)
+        #print('\nx2_np:',x2_np)
+        #print('\ny_np:',y_np)
+        #print('\nbert_text:',bert_text_labels)
+        # print('bert max len:',max_len,',Over',maxT,':',over_num,'ave len:',ave_len/len(instances),'total:',len(instances))
+
+        if label_type=='softmax':
+            y_np = np.expand_dims(y_np, 2)
+        elif label_type=='crf':
+            pass
+        
+        return [x1_np, x2_np,x3_np], y_np,bert_text_labels  
+         
+
+if __name__ == '__main__':
+    pass
+    
+ 
+            

src_python/GeneNER/restore_index_ner.py

@@ -1,447 +1,447 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Mar  5 10:40:08 2021
-
-@author: luol2
-"""
-
-# -*- coding: utf-8 -*-
-"""
-Created on Sun Jun 14 17:19:02 2020
-
-@author: luol2
-"""
-
-import io
-import sys
-
-# from BIO format to entity,list line is sentence, follwing the entity(start, end, text, entity, type)
-def NN_BIO_tag_entity(pre_BIO):
-    sentences=pre_BIO.strip().split('\n\n')
-
-    pre_result=[]
-    #print(sentences)
-    for sent in sentences:
-        tokens=sent.split('\n')
-        pre_entity=[]
-        pre_start,pre_end=0,0
-        sent_text=''
-        for i in range(0,len(tokens)):
-            segs=tokens[i].split('\t')
-            sent_text+=segs[0]+' '
-            if len(segs)<3:
-                continue
-            #print(tokens)
-            # generate prediction entity            
-            if segs[2].startswith('B-')>0:
-                pre_start=i
-                pre_type=segs[2][2:]
-                if i+1>=len(tokens): # the last word
-                    pre_end=i
-                    pre_entity.append([pre_start,pre_end,pre_type])
-                else: # non last word
-                    next_seg=tokens[i+1].split('\t')
-                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                        pre_end=i
-                        pre_entity.append([pre_start,pre_end,pre_type])
-                    elif next_seg[2].startswith('I-')>0:
-                        pass
-            elif segs[2].startswith('I-')>0:
-                if i==0 and i+1<len(tokens): # the first word and not only a word
-                    pre_start=i
-                    pre_type=segs[2][2:]
-                    next_seg=tokens[i+1].split('\t')
-                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                        pre_end=i
-                        pre_entity.append([pre_start,pre_end,pre_type])
-                    elif next_seg[2].startswith('I-')>0:
-                        pass
-                elif i==0 and i+1==len(tokens):# only one word:
-                    pre_start=i
-                    pre_type=segs[2][2:]
-                    pre_end=i
-                    pre_entity.append([pre_start,pre_end,pre_type])
-                elif i+1>=len(tokens): # the last word
-                    last_seg=tokens[i-1].split('\t')
-                    if last_seg[2]=='O':
-                        pre_start=i
-                        pre_type=segs[2][2:]                
-                    pre_end=i
-                    pre_entity.append([pre_start,pre_end,pre_type])
-                elif i+1< len(tokens): # non last word
-                    next_seg=tokens[i+1].split('\t')
-                    last_seg=tokens[i-1].split('\t')
-                    if last_seg[2]=='O':
-                        pre_start=i
-                        pre_type=segs[2][2:]  
-                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                        pre_end=i
-                        pre_entity.append([pre_start,pre_end,pre_type])
-                    elif next_seg[2].startswith('I-')>0:
-                        pass
-            elif segs[2]=='O':
-                pass        
-        pre_result.append([sent_text.rstrip(),pre_entity])
-
-
-        # print(pre_entity)
-    return pre_result
-
-def NN_restore_index_fn(ori_text,file_pre):
-
-    input_result=NN_BIO_tag_entity(file_pre)
-    #print(input_result)
-    
-    
-    new_sentence=''
-    restore_result=[]
-    
-    sentence_ori=ori_text.lower()
-
-    for sent_ele in input_result:
-
-        #print(pre_lines)
-#        print(sentence_ori)
-        if len(sent_ele[1])>0:
-            #print(pre_lines)
-            sentence_pre=sent_ele[0].lower()
-            sentence_pre=sentence_pre.split()
-            
-            pre_result=sent_ele[1]
-
-            
-            restore_sid=0
-            restore_eid=0
-            each_word_id=[]
-            
-            for i in range(0,len(sentence_pre)):
-
-                temp_id=sentence_ori.find(sentence_pre[i])
-                if temp_id<0:
-                        #print('ori:',sentence_ori)
-                        print('resotr index error:',sentence_pre[i])
-                new_sentence+=sentence_ori[0:temp_id]
-                
-                restore_sid=len(new_sentence)
-                restore_eid=len(new_sentence)+len(sentence_pre[i])
-                each_word_id.append([str(restore_sid),str(restore_eid)])
-                new_sentence+=sentence_ori[temp_id:temp_id+len(sentence_pre[i])]
-                sentence_ori=sentence_ori[temp_id+len(sentence_pre[i]):]
-#            print('each_word:',each_word_id)    
-            for pre_ele in pre_result:
-                temp_pre_result=[each_word_id[int(pre_ele[0])][0],each_word_id[int(pre_ele[1])][1],pre_ele[2]]
-                if temp_pre_result not in restore_result:
-                    restore_result.append(temp_pre_result)
-        else:
-            sentence_pre=sent_ele[0].lower()
-            sentence_pre=sentence_pre.split()
-           
-            for i in range(0,len(sentence_pre)):
-
-                temp_id=sentence_ori.find(sentence_pre[i])
-                if temp_id<0:
-                    print('resotr index error:',sentence_pre[i])
-                new_sentence+=sentence_ori[0:temp_id]
-                new_sentence+=sentence_ori[temp_id:temp_id+len(sentence_pre[i])]
-                sentence_ori=sentence_ori[temp_id+len(sentence_pre[i]):]
-    #print('resotre:',restore_result)
-    return restore_result
-
-def BERT_BIO_tag_entity(pre_BIO):
-    sentences=pre_BIO.strip().split('\n\n')
-
-    pre_result=[]
-    for sent in sentences:
-        tokens=sent.split('\n')
-        pre_entity=[]
-        pre_start,pre_end=0,0
-        sent_text=''
-        for i in range(1,len(tokens)-1):
-            segs=tokens[i].split('\t')
-            sent_text+=segs[0]+' '
-            # generate prediction entity            
-            if segs[2].startswith('B-')>0:
-                pre_start=i
-                pre_type=segs[2][2:]
-                if i+1>=len(tokens): # the last word
-                    pre_end=i
-                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                else: # non last word
-                    next_seg=tokens[i+1].split('\t')
-                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                        pre_end=i
-                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                    elif next_seg[2].startswith('I-')>0:
-                        pass
-            elif segs[2].startswith('I-')>0:
-                if i==0 and i+1<len(tokens): # the first word and not only a word
-                    pre_start=i
-                    pre_type=segs[2][2:]
-                    next_seg=tokens[i+1].split('\t')
-                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                        pre_end=i
-                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                    elif next_seg[2].startswith('I-')>0:
-                        pass
-                elif i==0 and i+1==len(tokens):# only one word:
-                    pre_start=i
-                    pre_type=segs[2][2:]
-                    pre_end=i
-                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                elif i+1>=len(tokens): # the last word
-                    last_seg=tokens[i-1].split('\t')
-                    if last_seg[2]=='O':
-                        pre_start=i
-                        pre_type=segs[2][2:]                
-                    pre_end=i
-                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                elif i+1< len(tokens): # non last word
-                    next_seg=tokens[i+1].split('\t')
-                    last_seg=tokens[i-1].split('\t')
-                    if last_seg[2]=='O':
-                        pre_start=i
-                        pre_type=segs[2][2:]  
-                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                        pre_end=i
-                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                    elif next_seg[2].startswith('I-')>0:
-                        pass
-            elif segs[2]=='O':
-                pass        
-        pre_result.append([sent_text.rstrip(),pre_entity])
-
-
-    #print(pre_result)
-    return pre_result
-
-def BERT_BIO_tag_entity_revised(pre_BIO):
-    print('revised version')
-    sentences=pre_BIO.strip().split('\n\n')
-
-    pre_result=[]
-    for sent in sentences:
-        tokens=sent.split('\n')
-        pre_entity=[]
-        pre_start,pre_end=0,0
-        sent_text=''
-        for i in range(1,len(tokens)-1):
-            segs=tokens[i].split('\t')
-            sent_text+=segs[0]+' '
-            # generate prediction entity            
-            if segs[2].startswith('B-')>0:
-                pre_start=i
-                pre_type=segs[2][2:]
-                if i+1>=len(tokens)-1: # the last word
-                    pre_end=i
-                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                else: # non last word
-                    next_seg=tokens[i+1].split('\t')
-                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                        pre_end=i
-                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                    elif next_seg[2].startswith('I-')>0:
-                        pass
-            elif segs[2].startswith('I-')>0:
-                if i==1 and i+1<len(tokens)-1: # the first word and not only a word
-                    pre_start=i
-                    pre_type=segs[2][2:]
-                    next_seg=tokens[i+1].split('\t')
-                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                        pre_end=i
-                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                    elif next_seg[2].startswith('I-')>0:
-                        pass
-                elif i==1 and i+1==len(tokens)-1:# only one word:
-                    pre_start=i
-                    pre_type=segs[2][2:]
-                    pre_end=i
-                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                elif i+1>=len(tokens)-1: # the last word
-                    last_seg=tokens[i-1].split('\t')
-                    if last_seg[2]=='O':
-                        pre_start=i
-                        pre_type=segs[2][2:]                
-                    pre_end=i
-                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                elif i+1< len(tokens)-1: # non last word
-                    next_seg=tokens[i+1].split('\t')
-                    last_seg=tokens[i-1].split('\t')
-                    if last_seg[2]=='O':
-                        pre_start=i
-                        pre_type=segs[2][2:]  
-                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                        pre_end=i
-                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                    elif next_seg[2].startswith('I-')>0:
-                        pass
-            elif segs[2]=='O':
-                pass        
-        pre_result.append([sent_text.rstrip(),pre_entity])
-
-
-    #print(pre_result)
-    return pre_result
-
-# only predict on the first token of the ori word 
-def BERT_BIO_tag_entity_word(pre_BIO):
-    sentences=pre_BIO.strip().split('\n\n')
-
-    pre_result=[]
-    for sent in sentences:
-        tokens=sent.split('\n')
-        pre_entity=[]
-        pre_start,pre_end=0,0
-        sent_text=''
-        i=1
-        while i< len(tokens)-1:
-        # for i in range(1,len(tokens)-1):
-            segs=tokens[i].split('\t')
-            sent_text+=segs[0]+' '
-            # generate prediction entity            
-            if segs[2].startswith('B-')>0:
-                pre_start=i
-                pre_type=segs[2][2:]
-                if i+1>=len(tokens)-1: # the last word
-                    pre_end=i
-                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                else: # non last word
-                    #pass a word
-                    sub_segs=tokens[i+1].split('\t')
-                    while(sub_segs[0].find('##')==0):
-                        i+=1
-                        sent_text+=sub_segs[0]+' '
-                        sub_segs=tokens[i+1].split('\t')
-                        
-                        
-                    next_seg=tokens[i+1].split('\t')
-                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                        pre_end=i
-                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                    elif next_seg[2].startswith('I-')>0:
-                        pass
-            elif segs[2].startswith('I-')>0:
-                if i==1 and i+1<len(tokens)-1: # the first word and not only a word
-                    pre_start=i
-                    pre_type=segs[2][2:]
-                    #pass a word
-                    sub_segs=tokens[i+1].split('\t')
-                    while(sub_segs[0].find('##')==0):
-                        i+=1
-                        sent_text+=sub_segs[0]+' '
-                        sub_segs=tokens[i+1].split('\t')
-                    
-                    next_seg=tokens[i+1].split('\t')
-                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                        pre_end=i
-                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                    elif next_seg[2].startswith('I-')>0:
-                        pass
-                elif i==1 and i+1==len(tokens)-1:# only one word:
-                    pre_start=i
-                    pre_type=segs[2][2:]
-                    pre_end=i
-                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                elif i+1>=len(tokens)-1: # the last word
-                    last_seg=tokens[i-1].split('\t')
-                    if last_seg[2]=='O':
-                        pre_start=i
-                        pre_type=segs[2][2:]                
-                    pre_end=i
-                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                elif i+1< len(tokens)-1: # non last word
-                    
-                    last_seg=tokens[i-1].split('\t')
-                    if last_seg[2]=='O':
-                        pre_start=i
-                        pre_type=segs[2][2:]
-                        #pass a word
-                        sub_segs=tokens[i+1].split('\t')
-                        while(sub_segs[0].find('##')==0):
-                            i+=1
-                            sent_text+=sub_segs[0]+' '
-                            sub_segs=tokens[i+1].split('\t')
-                    next_seg=tokens[i+1].split('\t')
-                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
-                        pre_end=i
-                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
-                    elif next_seg[2].startswith('I-')>0:
-                        pass
-            elif segs[2]=='O':
-                pass
-            i+=1
-        pre_result.append([sent_text.rstrip(),pre_entity])
-
-
-    #print(pre_result)
-    return pre_result
-
-
-def BERT_restore_index_fn(ori_text,file_pre):
-
-    # input_result=BERT_BIO_tag_entity_revised(file_pre)
-    input_result=BERT_BIO_tag_entity_word(file_pre)
-    #print(input_result)
-    
-    
-    new_sentence=''
-    restore_result=[]
-    
-    sentence_ori=ori_text.lower()
-
-    for sent_ele in input_result:
-
-        #print(pre_lines)
-#        print(sentence_ori)
-        if len(sent_ele[1])>0:
-            #print(pre_lines)
-            sentence_pre=sent_ele[0].lower()
-            sentence_pre=sentence_pre.split()
-            
-            pre_result=sent_ele[1]
-
-            
-            restore_sid=0
-            restore_eid=0
-            each_word_id=[]
-            
-            
-            for i in range(0,len(sentence_pre)):
-                if sentence_pre[i][0:2]=="##":
-                    sentence_pre[i]=sentence_pre[i][2:]
-                temp_id=sentence_ori.find(sentence_pre[i])
-                if temp_id<0:
-                        #print('ori:',sentence_ori)
-                        print('resotr index error:',sentence_pre[i])
-                new_sentence+=sentence_ori[0:temp_id]
-                
-                restore_sid=len(new_sentence)
-                restore_eid=len(new_sentence)+len(sentence_pre[i])
-                each_word_id.append([str(restore_sid),str(restore_eid)])
-                new_sentence+=sentence_ori[temp_id:temp_id+len(sentence_pre[i])]
-                sentence_ori=sentence_ori[temp_id+len(sentence_pre[i]):]
-#            print('each_word:',each_word_id)    
-            for pre_ele in pre_result:
-                temp_pre_result=[each_word_id[int(pre_ele[0])][0],each_word_id[int(pre_ele[1])][1],pre_ele[2]]
-                if temp_pre_result not in restore_result:
-                    restore_result.append(temp_pre_result)
-        else:
-            sentence_pre=sent_ele[0].lower()
-            sentence_pre=sentence_pre.split()
-           
-            for i in range(0,len(sentence_pre)):
-                if sentence_pre[i][0:2]=="##":
-                    sentence_pre[i]=sentence_pre[i][2:]
-                temp_id=sentence_ori.find(sentence_pre[i])
-                if temp_id<0:
-                    print('resotr index error:',sentence_pre[i])
-                new_sentence+=sentence_ori[0:temp_id]
-                new_sentence+=sentence_ori[temp_id:temp_id+len(sentence_pre[i])]
-                sentence_ori=sentence_ori[temp_id+len(sentence_pre[i]):]
-    #print('resotre:',restore_result)
-    return restore_result
-if __name__=='__main__':
-    path='//panfs/pan1/bionlp/lulab/luoling/OpenBioIE_project/models/'
-    fin=open(path+'devout_test.txt','r',encoding='utf-8')
-    file_pre=fin.read()
-    ori_text="D90A-SOD1 mediated amyotrophic lateral sclerosis: a single founder for all cases with evidence for a Cis-acting disease modifier in the recessive haplotype. More than 100 different heterozygous mutations in copper/zinc superoxide dismutase (SOD1) have been found in patients with amyotrophic lateral sclerosis (ALS), a fatal neurodegenerative disease. Uniquely, D90A-SOD1 has been identified in recessive, dominant and apparently sporadic pedigrees. The phenotype of homozygotes is stereotyped with an extended survival, whereas that of affected heterozygotes varies. The frequency of D90A-SOD1 is 50 times higher in Scandinavia (2.5%) than elsewhere, though ALS prevalence is not raised there. Our earlier study indicated separate founders for recessive and dominant/sporadic ALS and we proposed a disease-modifying factor linked to the recessive mutation. Here we have doubled our sample set and employed novel markers to characterise the mutation's origin and localise any modifying factor. Linkage disequilibrium analysis indicates that D90A homozygotes and heterozygotes share a rare haplotype and are all descended from a single ancient founder (alpha 0.974) c.895 generations ago. Homozygotes arose subsequently only c.63 generations ago (alpha 0.878). Recombination has reduced the region shared by recessive kindreds to 97-265 kb around SOD1, excluding all neighbouring genes. We propose that a cis-acting regulatory polymorphism has arisen close to D90A-SOD1 in the recessive founder, which decreases ALS susceptibility in heterozygotes and slows disease progression."
-    NN_restore_index_fn(ori_text,file_pre)
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Mar  5 10:40:08 2021
+
+@author: luol2
+"""
+
+# -*- coding: utf-8 -*-
+"""
+Created on Sun Jun 14 17:19:02 2020
+
+@author: luol2
+"""
+
+import io
+import sys
+
+# from BIO format to entity,list line is sentence, follwing the entity(start, end, text, entity, type)
+def NN_BIO_tag_entity(pre_BIO):
+    sentences=pre_BIO.strip().split('\n\n')
+
+    pre_result=[]
+    #print(sentences)
+    for sent in sentences:
+        tokens=sent.split('\n')
+        pre_entity=[]
+        pre_start,pre_end=0,0
+        sent_text=''
+        for i in range(0,len(tokens)):
+            segs=tokens[i].split('\t')
+            sent_text+=segs[0]+' '
+            if len(segs)<3:
+                continue
+            #print(tokens)
+            # generate prediction entity            
+            if segs[2].startswith('B-')>0:
+                pre_start=i
+                pre_type=segs[2][2:]
+                if i+1>=len(tokens): # the last word
+                    pre_end=i
+                    pre_entity.append([pre_start,pre_end,pre_type])
+                else: # non last word
+                    next_seg=tokens[i+1].split('\t')
+                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                        pre_end=i
+                        pre_entity.append([pre_start,pre_end,pre_type])
+                    elif next_seg[2].startswith('I-')>0:
+                        pass
+            elif segs[2].startswith('I-')>0:
+                if i==0 and i+1<len(tokens): # the first word and not only a word
+                    pre_start=i
+                    pre_type=segs[2][2:]
+                    next_seg=tokens[i+1].split('\t')
+                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                        pre_end=i
+                        pre_entity.append([pre_start,pre_end,pre_type])
+                    elif next_seg[2].startswith('I-')>0:
+                        pass
+                elif i==0 and i+1==len(tokens):# only one word:
+                    pre_start=i
+                    pre_type=segs[2][2:]
+                    pre_end=i
+                    pre_entity.append([pre_start,pre_end,pre_type])
+                elif i+1>=len(tokens): # the last word
+                    last_seg=tokens[i-1].split('\t')
+                    if last_seg[2]=='O':
+                        pre_start=i
+                        pre_type=segs[2][2:]                
+                    pre_end=i
+                    pre_entity.append([pre_start,pre_end,pre_type])
+                elif i+1< len(tokens): # non last word
+                    next_seg=tokens[i+1].split('\t')
+                    last_seg=tokens[i-1].split('\t')
+                    if last_seg[2]=='O':
+                        pre_start=i
+                        pre_type=segs[2][2:]  
+                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                        pre_end=i
+                        pre_entity.append([pre_start,pre_end,pre_type])
+                    elif next_seg[2].startswith('I-')>0:
+                        pass
+            elif segs[2]=='O':
+                pass        
+        pre_result.append([sent_text.rstrip(),pre_entity])
+
+
+        # print(pre_entity)
+    return pre_result
+
+def NN_restore_index_fn(ori_text,file_pre):
+
+    input_result=NN_BIO_tag_entity(file_pre)
+    #print(input_result)
+    
+    
+    new_sentence=''
+    restore_result=[]
+    
+    sentence_ori=ori_text.lower()
+
+    for sent_ele in input_result:
+
+        #print(pre_lines)
+#        print(sentence_ori)
+        if len(sent_ele[1])>0:
+            #print(pre_lines)
+            sentence_pre=sent_ele[0].lower()
+            sentence_pre=sentence_pre.split()
+            
+            pre_result=sent_ele[1]
+
+            
+            restore_sid=0
+            restore_eid=0
+            each_word_id=[]
+            
+            for i in range(0,len(sentence_pre)):
+
+                temp_id=sentence_ori.find(sentence_pre[i])
+                if temp_id<0:
+                        #print('ori:',sentence_ori)
+                        print('resotr index error:',sentence_pre[i])
+                new_sentence+=sentence_ori[0:temp_id]
+                
+                restore_sid=len(new_sentence)
+                restore_eid=len(new_sentence)+len(sentence_pre[i])
+                each_word_id.append([str(restore_sid),str(restore_eid)])
+                new_sentence+=sentence_ori[temp_id:temp_id+len(sentence_pre[i])]
+                sentence_ori=sentence_ori[temp_id+len(sentence_pre[i]):]
+#            print('each_word:',each_word_id)    
+            for pre_ele in pre_result:
+                temp_pre_result=[each_word_id[int(pre_ele[0])][0],each_word_id[int(pre_ele[1])][1],pre_ele[2]]
+                if temp_pre_result not in restore_result:
+                    restore_result.append(temp_pre_result)
+        else:
+            sentence_pre=sent_ele[0].lower()
+            sentence_pre=sentence_pre.split()
+           
+            for i in range(0,len(sentence_pre)):
+
+                temp_id=sentence_ori.find(sentence_pre[i])
+                if temp_id<0:
+                    print('resotr index error:',sentence_pre[i])
+                new_sentence+=sentence_ori[0:temp_id]
+                new_sentence+=sentence_ori[temp_id:temp_id+len(sentence_pre[i])]
+                sentence_ori=sentence_ori[temp_id+len(sentence_pre[i]):]
+    #print('resotre:',restore_result)
+    return restore_result
+
+def BERT_BIO_tag_entity(pre_BIO):
+    sentences=pre_BIO.strip().split('\n\n')
+
+    pre_result=[]
+    for sent in sentences:
+        tokens=sent.split('\n')
+        pre_entity=[]
+        pre_start,pre_end=0,0
+        sent_text=''
+        for i in range(1,len(tokens)-1):
+            segs=tokens[i].split('\t')
+            sent_text+=segs[0]+' '
+            # generate prediction entity            
+            if segs[2].startswith('B-')>0:
+                pre_start=i
+                pre_type=segs[2][2:]
+                if i+1>=len(tokens): # the last word
+                    pre_end=i
+                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                else: # non last word
+                    next_seg=tokens[i+1].split('\t')
+                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                        pre_end=i
+                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                    elif next_seg[2].startswith('I-')>0:
+                        pass
+            elif segs[2].startswith('I-')>0:
+                if i==0 and i+1<len(tokens): # the first word and not only a word
+                    pre_start=i
+                    pre_type=segs[2][2:]
+                    next_seg=tokens[i+1].split('\t')
+                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                        pre_end=i
+                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                    elif next_seg[2].startswith('I-')>0:
+                        pass
+                elif i==0 and i+1==len(tokens):# only one word:
+                    pre_start=i
+                    pre_type=segs[2][2:]
+                    pre_end=i
+                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                elif i+1>=len(tokens): # the last word
+                    last_seg=tokens[i-1].split('\t')
+                    if last_seg[2]=='O':
+                        pre_start=i
+                        pre_type=segs[2][2:]                
+                    pre_end=i
+                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                elif i+1< len(tokens): # non last word
+                    next_seg=tokens[i+1].split('\t')
+                    last_seg=tokens[i-1].split('\t')
+                    if last_seg[2]=='O':
+                        pre_start=i
+                        pre_type=segs[2][2:]  
+                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                        pre_end=i
+                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                    elif next_seg[2].startswith('I-')>0:
+                        pass
+            elif segs[2]=='O':
+                pass        
+        pre_result.append([sent_text.rstrip(),pre_entity])
+
+
+    #print(pre_result)
+    return pre_result
+
+def BERT_BIO_tag_entity_revised(pre_BIO):
+    print('revised version')
+    sentences=pre_BIO.strip().split('\n\n')
+
+    pre_result=[]
+    for sent in sentences:
+        tokens=sent.split('\n')
+        pre_entity=[]
+        pre_start,pre_end=0,0
+        sent_text=''
+        for i in range(1,len(tokens)-1):
+            segs=tokens[i].split('\t')
+            sent_text+=segs[0]+' '
+            # generate prediction entity            
+            if segs[2].startswith('B-')>0:
+                pre_start=i
+                pre_type=segs[2][2:]
+                if i+1>=len(tokens)-1: # the last word
+                    pre_end=i
+                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                else: # non last word
+                    next_seg=tokens[i+1].split('\t')
+                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                        pre_end=i
+                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                    elif next_seg[2].startswith('I-')>0:
+                        pass
+            elif segs[2].startswith('I-')>0:
+                if i==1 and i+1<len(tokens)-1: # the first word and not only a word
+                    pre_start=i
+                    pre_type=segs[2][2:]
+                    next_seg=tokens[i+1].split('\t')
+                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                        pre_end=i
+                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                    elif next_seg[2].startswith('I-')>0:
+                        pass
+                elif i==1 and i+1==len(tokens)-1:# only one word:
+                    pre_start=i
+                    pre_type=segs[2][2:]
+                    pre_end=i
+                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                elif i+1>=len(tokens)-1: # the last word
+                    last_seg=tokens[i-1].split('\t')
+                    if last_seg[2]=='O':
+                        pre_start=i
+                        pre_type=segs[2][2:]                
+                    pre_end=i
+                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                elif i+1< len(tokens)-1: # non last word
+                    next_seg=tokens[i+1].split('\t')
+                    last_seg=tokens[i-1].split('\t')
+                    if last_seg[2]=='O':
+                        pre_start=i
+                        pre_type=segs[2][2:]  
+                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                        pre_end=i
+                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                    elif next_seg[2].startswith('I-')>0:
+                        pass
+            elif segs[2]=='O':
+                pass        
+        pre_result.append([sent_text.rstrip(),pre_entity])
+
+
+    #print(pre_result)
+    return pre_result
+
+# only predict on the first token of the ori word 
+def BERT_BIO_tag_entity_word(pre_BIO):
+    sentences=pre_BIO.strip().split('\n\n')
+
+    pre_result=[]
+    for sent in sentences:
+        tokens=sent.split('\n')
+        pre_entity=[]
+        pre_start,pre_end=0,0
+        sent_text=''
+        i=1
+        while i< len(tokens)-1:
+        # for i in range(1,len(tokens)-1):
+            segs=tokens[i].split('\t')
+            sent_text+=segs[0]+' '
+            # generate prediction entity            
+            if segs[2].startswith('B-')>0:
+                pre_start=i
+                pre_type=segs[2][2:]
+                if i+1>=len(tokens)-1: # the last word
+                    pre_end=i
+                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                else: # non last word
+                    #pass a word
+                    sub_segs=tokens[i+1].split('\t')
+                    while(sub_segs[0].find('##')==0):
+                        i+=1
+                        sent_text+=sub_segs[0]+' '
+                        sub_segs=tokens[i+1].split('\t')
+                        
+                        
+                    next_seg=tokens[i+1].split('\t')
+                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                        pre_end=i
+                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                    elif next_seg[2].startswith('I-')>0:
+                        pass
+            elif segs[2].startswith('I-')>0:
+                if i==1 and i+1<len(tokens)-1: # the first word and not only a word
+                    pre_start=i
+                    pre_type=segs[2][2:]
+                    #pass a word
+                    sub_segs=tokens[i+1].split('\t')
+                    while(sub_segs[0].find('##')==0):
+                        i+=1
+                        sent_text+=sub_segs[0]+' '
+                        sub_segs=tokens[i+1].split('\t')
+                    
+                    next_seg=tokens[i+1].split('\t')
+                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                        pre_end=i
+                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                    elif next_seg[2].startswith('I-')>0:
+                        pass
+                elif i==1 and i+1==len(tokens)-1:# only one word:
+                    pre_start=i
+                    pre_type=segs[2][2:]
+                    pre_end=i
+                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                elif i+1>=len(tokens)-1: # the last word
+                    last_seg=tokens[i-1].split('\t')
+                    if last_seg[2]=='O':
+                        pre_start=i
+                        pre_type=segs[2][2:]                
+                    pre_end=i
+                    pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                elif i+1< len(tokens)-1: # non last word
+                    
+                    last_seg=tokens[i-1].split('\t')
+                    if last_seg[2]=='O':
+                        pre_start=i
+                        pre_type=segs[2][2:]
+                        #pass a word
+                        sub_segs=tokens[i+1].split('\t')
+                        while(sub_segs[0].find('##')==0):
+                            i+=1
+                            sent_text+=sub_segs[0]+' '
+                            sub_segs=tokens[i+1].split('\t')
+                    next_seg=tokens[i+1].split('\t')
+                    if next_seg[2].startswith('B-')>0 or next_seg[2]=='O':
+                        pre_end=i
+                        pre_entity.append([pre_start-1,pre_end-1,pre_type])
+                    elif next_seg[2].startswith('I-')>0:
+                        pass
+            elif segs[2]=='O':
+                pass
+            i+=1
+        pre_result.append([sent_text.rstrip(),pre_entity])
+
+
+    #print(pre_result)
+    return pre_result
+
+
+def BERT_restore_index_fn(ori_text,file_pre):
+
+    # input_result=BERT_BIO_tag_entity_revised(file_pre)
+    input_result=BERT_BIO_tag_entity_word(file_pre)
+    #print(input_result)
+    
+    
+    new_sentence=''
+    restore_result=[]
+    
+    sentence_ori=ori_text.lower()
+
+    for sent_ele in input_result:
+
+        #print(pre_lines)
+#        print(sentence_ori)
+        if len(sent_ele[1])>0:
+            #print(pre_lines)
+            sentence_pre=sent_ele[0].lower()
+            sentence_pre=sentence_pre.split()
+            
+            pre_result=sent_ele[1]
+
+            
+            restore_sid=0
+            restore_eid=0
+            each_word_id=[]
+            
+            
+            for i in range(0,len(sentence_pre)):
+                if sentence_pre[i][0:2]=="##":
+                    sentence_pre[i]=sentence_pre[i][2:]
+                temp_id=sentence_ori.find(sentence_pre[i])
+                if temp_id<0:
+                        #print('ori:',sentence_ori)
+                        print('resotr index error:',sentence_pre[i])
+                new_sentence+=sentence_ori[0:temp_id]
+                
+                restore_sid=len(new_sentence)
+                restore_eid=len(new_sentence)+len(sentence_pre[i])
+                each_word_id.append([str(restore_sid),str(restore_eid)])
+                new_sentence+=sentence_ori[temp_id:temp_id+len(sentence_pre[i])]
+                sentence_ori=sentence_ori[temp_id+len(sentence_pre[i]):]
+#            print('each_word:',each_word_id)    
+            for pre_ele in pre_result:
+                temp_pre_result=[each_word_id[int(pre_ele[0])][0],each_word_id[int(pre_ele[1])][1],pre_ele[2]]
+                if temp_pre_result not in restore_result:
+                    restore_result.append(temp_pre_result)
+        else:
+            sentence_pre=sent_ele[0].lower()
+            sentence_pre=sentence_pre.split()
+           
+            for i in range(0,len(sentence_pre)):
+                if sentence_pre[i][0:2]=="##":
+                    sentence_pre[i]=sentence_pre[i][2:]
+                temp_id=sentence_ori.find(sentence_pre[i])
+                if temp_id<0:
+                    print('resotr index error:',sentence_pre[i])
+                new_sentence+=sentence_ori[0:temp_id]
+                new_sentence+=sentence_ori[temp_id:temp_id+len(sentence_pre[i])]
+                sentence_ori=sentence_ori[temp_id+len(sentence_pre[i]):]
+    #print('resotre:',restore_result)
+    return restore_result
+if __name__=='__main__':
+    path='//panfs/pan1/bionlp/lulab/luoling/OpenBioIE_project/models/'
+    fin=open(path+'devout_test.txt','r',encoding='utf-8')
+    file_pre=fin.read()
+    ori_text="D90A-SOD1 mediated amyotrophic lateral sclerosis: a single founder for all cases with evidence for a Cis-acting disease modifier in the recessive haplotype. More than 100 different heterozygous mutations in copper/zinc superoxide dismutase (SOD1) have been found in patients with amyotrophic lateral sclerosis (ALS), a fatal neurodegenerative disease. Uniquely, D90A-SOD1 has been identified in recessive, dominant and apparently sporadic pedigrees. The phenotype of homozygotes is stereotyped with an extended survival, whereas that of affected heterozygotes varies. The frequency of D90A-SOD1 is 50 times higher in Scandinavia (2.5%) than elsewhere, though ALS prevalence is not raised there. Our earlier study indicated separate founders for recessive and dominant/sporadic ALS and we proposed a disease-modifying factor linked to the recessive mutation. Here we have doubled our sample set and employed novel markers to characterise the mutation's origin and localise any modifying factor. Linkage disequilibrium analysis indicates that D90A homozygotes and heterozygotes share a rare haplotype and are all descended from a single ancient founder (alpha 0.974) c.895 generations ago. Homozygotes arose subsequently only c.63 generations ago (alpha 0.878). Recombination has reduced the region shared by recessive kindreds to 97-265 kb around SOD1, excluding all neighbouring genes. We propose that a cis-acting regulatory polymorphism has arisen close to D90A-SOD1 in the recessive founder, which decreases ALS susceptibility in heterozygotes and slows disease progression."
+    NN_restore_index_fn(ori_text,file_pre)

src_python/SpeAss/Evaluation_sa.py

@@ -1,396 +1,396 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Mon Mar  1 15:33:54 2021
-
-@author: luol2
-"""
-# compute metrics using IO prefile
-#ignore arg1
-def Rel_Evaluation(prefile):
-    fin=open(prefile,'r',encoding='utf-8')
-    all_in=fin.read().strip().split('\n\n')
-    fin.close()
-    TP=0 #gold=pre=pos
-    FP=0 #gold=neg, pre=pos
-    FN=0 #gold=pos, pre=Neg
-    for sentence in all_in:
-        tokens=sentence.split('\n')
-        entity_id=0
-        token_id=0
-        temp_gold='O'
-        temp_pre='O'
-        while (token_id<len(tokens)):
-            seg=tokens[token_id].split('\t')
-            if seg[0]=='<GENE>':
-                if seg[1]=='O':
-                    temp_gold=seg[1]
-                else:
-                    temp_gold=seg[1][2:]
-                if seg[2]=='O':
-                    temp_pre=seg[2]
-                else:
-                    temp_pre=seg[2][2:]  
-                token_id+=1
-                seg=tokens[token_id].split('\t')
-                while seg[0]!='</GENE>':
-                    token_id+=1
-                    seg=tokens[token_id].split('\t')
-                    if seg[1]!='O' and temp_gold=='O':
-                        temp_gold=seg[1][2:]
-                    if seg[2]!='O' and temp_pre=='O':
-                        temp_pre=seg[2][2:]
-                if temp_pre!='O' and temp_gold!='O' and temp_pre==temp_gold:
-                    TP+=1
-                elif temp_pre!='O' and temp_gold!='O' and temp_pre!=temp_gold:
-                    FP+=1
-                    FN+=1
-                elif temp_pre!='O' and temp_gold=='O' :
-                    FP+=1
-                elif temp_pre=='O' and temp_gold!='O' :
-                    FN+=1
-                temp_pre='O'
-                temp_gold='O'
-        
-            else:
-                pass
-            token_id+=1
-    # print('TP,FP,FN:',TP,FP,FN)
-    if TP+FP==0:
-        P=0
-    else:
-        P=TP/(TP+FP)
-    if TP+FN==0:
-        R=0
-    else:
-        R=TP/(TP+FN)
-    if P+R==0:
-        F1=0
-    else:
-        F1=2*P*R/(P+R)
-    print('TP,FP,FN:',TP,FP,FN)
-    print('P,R,F1:',P,R,F1)
-
-
-def Rel_Evaluation_fn(prefile):
-    fin=open(prefile,'r',encoding='utf-8')
-    all_in=fin.read().strip().split('\n\n')
-    fin.close()
-    TP=0 #gold=pre=pos
-    FP=0 #gold=neg, pre=pos
-    FN=0 #gold=pos, pre=Neg
-    for sentence in all_in:
-        tokens=sentence.split('\n')
-        entity_id=0
-        token_id=0
-        temp_gold='O'
-        temp_pre='O'
-        while (token_id<len(tokens)):
-            seg=tokens[token_id].split('\t')
-            if seg[0]=='<GENE>':
-                if seg[1]=='O':
-                    temp_gold=seg[1]
-                else:
-                    temp_gold=seg[1][2:]
-                if seg[2]=='O':
-                    temp_pre=seg[2]
-                else:
-                    temp_pre=seg[2][2:]  
-                token_id+=1
-                seg=tokens[token_id].split('\t')
-                while seg[0]!='</GENE>':
-                    token_id+=1
-                    seg=tokens[token_id].split('\t')
-                    if seg[1]!='O' and temp_gold=='O':
-                        temp_gold=seg[1][2:]
-                    if seg[2]!='O' and temp_pre=='O':
-                        temp_pre=seg[2][2:]
-                if temp_pre!='O' and temp_gold!='O' and temp_pre==temp_gold:
-                    TP+=1
-                elif temp_pre!='O' and temp_gold!='O' and temp_pre!=temp_gold:
-                    FP+=1
-                elif temp_pre!='O' and temp_gold=='O' :
-                    FP+=1
-                elif temp_pre=='O' and temp_gold!='O' :
-                    FN+=1
-                temp_pre='O'
-                temp_gold='O'
-        
-            else:
-                pass
-            token_id+=1
-    print('TP,FP,FN:',TP,FP,FN)
-    if TP+FP==0:
-        P=0
-    else:
-        P=TP/(TP+FP)
-    if TP+FN==0:
-        R=0
-    else:
-        R=TP/(TP+FN)
-    if P+R==0:
-        F1=0
-    else:
-        F1=2*P*R/(P+R)
-    # print('TP,FP,FN:',TP,FP,FN)
-    print('P,R,F1:',P,R,F1)
-    return F1
-
-def Rel_Evaluation_Hugface_fn(prefile,ARG2_label='gene1s'):
-    fin=open(prefile,'r',encoding='utf-8')
-    all_in=fin.read().strip().split('\n\n')
-    fin.close()
-    TP=0 #gold=pre=pos
-    FP=0 #gold=neg, pre=pos
-    FN=0 #gold=pos, pre=Neg
-    result_dict={}#{'rel type':[TP,FP,FN],...,}
-    for sentence in all_in:
-        tokens=sentence.split('\n')
-        for token in tokens:
-            seg=token.split('\t')
-            if seg[0]==ARG2_label:
-                if seg[1].find('ARG2')>=0:
-                    if seg[2]==seg[1]:
-                        if seg[1] not in result_dict.keys():
-                            result_dict[seg[1]]=[1,0,0]
-                        else:
-                            result_dict[seg[1]][0]+=1
-                        TP+=1
-                    elif seg[2].find('ARG2')>=0:
-                        if seg[1] not in result_dict.keys():
-                            result_dict[seg[1]]=[0,0,1]
-                        else:
-                            result_dict[seg[1]][2]+=1
-                        if seg[2] not in result_dict.keys():
-                            result_dict[seg[2]]=[0,1,0]
-                        else:
-                            result_dict[seg[2]][1]+=1
-                        FP+=1
-                        FN+=1
-                    else:
-                        if seg[1] not in result_dict.keys():
-                            result_dict[seg[1]]=[0,0,1]
-                        else:
-                            result_dict[seg[1]][2]+=1
-                        FN+=1
-
-                else:
-                    if seg[2].find('ARG2')>=0:
-                        if seg[2] not in result_dict.keys():
-                            result_dict[seg[2]]=[0,1,0]
-                        else:
-                            result_dict[seg[2]][1]+=1
-                        FP+=1
-    # print('TP,FP,FN:',TP,FP,FN)
-    rel_metrics={}
-    for rel_type in result_dict.keys():
-        if result_dict[rel_type][0]+result_dict[rel_type][1]==0:
-            p=0
-        else:
-            p=result_dict[rel_type][0]/(result_dict[rel_type][0]+result_dict[rel_type][1])
-        if result_dict[rel_type][0]+result_dict[rel_type][2]==0:
-            r=0
-        else:
-            r=result_dict[rel_type][0]/(result_dict[rel_type][0]+result_dict[rel_type][2])
-        if p+r==0:
-            f1=0
-        else:
-            f1=2*p*r/(p+r)
-        rel_metrics[rel_type]=[round(p,4),round(r,4),round(f1,4)]
-    if TP+FP==0:
-        P=0
-    else:
-        P=TP/(TP+FP)
-    if TP+FN==0:
-        R=0
-    else:
-        R=TP/(TP+FN)
-    if P+R==0:
-        F1=0
-    else:
-        F1=2*P*R/(P+R)
-    P=round(P,4)
-    R=round(R,4)
-    F1=round(F1,4)
-    print('mertics:\n',rel_metrics)
-    print('\nTP,FP,FN:',TP,FP,FN)
-    print('Overall P,R,F1:',P,R,F1)   
-    return [P,R,F1],rel_metrics
-
-def Rel_Evaluation_AIO_fn(prefile):
-    fin=open(prefile,'r',encoding='utf-8')
-    all_in=fin.read().strip().split('\n\n')
-    fin.close()
-    TP=0 #gold=pre=pos
-    FP=0 #gold=neg, pre=pos
-    FN=0 #gold=pos, pre=Neg
-    for sentence in all_in:
-        tokens=sentence.split('\n')
-        for token in tokens:
-            seg=token.split('\t')
-            if seg[0]=='<GENE>':
-                if seg[1].find('ARG2-')>=0:
-                    if seg[2]==seg[1]:
-                        TP+=1
-                    elif seg[2].find('ARG2-')>=0:
-                        FP+=1
-                        FN+=1
-                    else:
-                        FN+=1
-
-                else:
-                    if seg[2].find('ARG2-')>=0:
-                        FP+=1
-    # print('TP,FP,FN:',TP,FP,FN)
-    if TP+FP==0:
-        P=0
-    else:
-        P=TP/(TP+FP)
-    if TP+FN==0:
-        R=0
-    else:
-        R=TP/(TP+FN)
-    if P+R==0:
-        F1=0
-    else:
-        F1=2*P*R/(P+R)
-    P=round(P,4)
-    R=round(R,4)
-    F1=round(F1,4)
-    print('TP,FP,FN:',TP,FP,FN)
-    print('P,R,F1:',P,R,F1)   
-    return [P,R,F1]
-
-def Rel_Evaluation_AIO_GC_fn(prefile):
-    fin=open(prefile,'r',encoding='utf-8')
-    all_in=fin.read().strip().split('\n\n')
-    fin.close()
-    TP=0 #gold=pre=pos
-    FP=0 #gold=neg, pre=pos
-    FN=0 #gold=pos, pre=Neg
-    for sentence in all_in:
-        tokens=sentence.split('\n')
-        for token in tokens:
-            seg=token.split('\t')
-            if seg[0]=='<CHEMICAL>':
-                if seg[1].find('ARG2-')>=0:
-                    if seg[2]==seg[1]:
-                        TP+=1
-                    elif seg[2].find('ARG2-')>=0:
-                        FP+=1
-                        FN+=1
-                    else:
-                        FN+=1
-
-                else:
-                    if seg[2].find('ARG2-')>=0:
-                        FP+=1
-    # print('TP,FP,FN:',TP,FP,FN)
-    if TP+FP==0:
-        P=0
-    else:
-        P=TP/(TP+FP)
-    if TP+FN==0:
-        R=0
-    else:
-        R=TP/(TP+FN)
-    if P+R==0:
-        F1=0
-    else:
-        F1=2*P*R/(P+R)
-    P=round(P,4)
-    R=round(R,4)
-    F1=round(F1,4)
-    print('TP,FP,FN:',TP,FP,FN)
-    print('P,R,F1:',P,R,F1)   
-    return [P,R,F1]
-    
-def office_evaluation(goldfile,prefile):
-    fin_gold=open(goldfile,'r',encoding='utf-8')
-    all_gold=fin_gold.read().strip().split('\n')
-    fin_gold.close()
-    fin_pre=open(prefile,'r',encoding='utf-8')
-    all_pre=fin_pre.read().strip().split('\n')
-    fin_pre.close()
-    
-    gold_result={}#{'relation type':set(line)}
-    pre_result={}
-    all_result={} #{'relation type':[tp,fp,fn]}
-    for line in all_gold:
-        seg=line.split('\t')
-        if seg[1] not in all_result.keys():
-            all_result[seg[1]]=[0,0,0]
-        if seg[1] not in gold_result.keys(): 
-            gold_result[seg[1]]=set()
-            gold_result[seg[1]].add(line)         
-        else:
-            gold_result[seg[1]].add(line)
-      
-    for line in all_pre:
-        seg=line.split('\t')
-        if seg[1] not in pre_result.keys():
-            pre_result[seg[1]]=set()
-            pre_result[seg[1]].add(line)
-        else:
-            pre_result[seg[1]].add(line)
-            
-    for rel_type in gold_result.keys():
-        for gold_ele in gold_result[rel_type]:
-            if rel_type not in pre_result.keys():
-                all_result[rel_type][2]+=1
-            else:
-                if gold_ele in pre_result[rel_type]:
-                    all_result[rel_type][0]+=1
-                else:
-                    all_result[rel_type][2]+=1
-        if rel_type in pre_result.keys():
-            for pre_ele in pre_result[rel_type]:
-                if pre_ele not in gold_result[rel_type]:
-                    all_result[rel_type][1]+=1
-    ave_f=0
-    TP,FP,FN=0,0,0
-    print(all_result)
-    for rel_type in all_result.keys():
-        TP+=all_result[rel_type][0]
-        FP+=all_result[rel_type][1]
-        FN+=all_result[rel_type][2]
-        tem_p,tem_r,tem_f=0,0,0
-        if all_result[rel_type][0]+all_result[rel_type][1]==0:
-            tem_p=0
-        else:
-            tem_p=all_result[rel_type][0]/(all_result[rel_type][0]+all_result[rel_type][1])
-        if all_result[rel_type][0]+all_result[rel_type][2]==0:
-            tem_r=0
-        else:
-            tem_r=all_result[rel_type][0]/(all_result[rel_type][0]+all_result[rel_type][2])
-        if tem_p+tem_r==0:
-            tem_f=0
-        else:
-            tem_f=2*tem_p*tem_r/(tem_p+tem_r)
-        ave_f+=tem_f
-        print('%s:p=%.4f,r=%.4f,f=%.4f' % (rel_type,tem_p,tem_r,tem_f))
-    
-    if TP+FP==0:
-        P=0
-    else:
-        P=TP/(TP+FP)
-    if TP+FN==0:
-        R=0
-    else:
-        R=TP/(TP+FN)
-    if P+R==0:
-        F1=0
-    else:
-        F1=2*P*R/(P+R)
-    ave_f+=tem_f
-    
-    print('Overall:')
-    print('ave_f1:',ave_f/len(all_result))
-    print('TP=%d, FP=%d, FN=%d'%(TP,FP,FN))
-    print('P=%.4f, R=%.4f, F1=%.4f'%(P,R,F1))
-    
-
-if __name__=='__main__':
-    path='//panfs/pan1/bionlplab/luol2/BC7DrugProt/results/'
-    office_evaluation(path+'dev/dev_gold_relations.tsv',path+'drugprot_dev_LSTM-CRF-ES_pre.tsv')
-    print('............')
-    Rel_Evaluation_check('//panfs/pan1/bionlplab/luol2/BC7DrugProt/check/dev_pre_temp.conll')
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Mar  1 15:33:54 2021
+
+@author: luol2
+"""
+# compute metrics using IO prefile
+#ignore arg1
+def Rel_Evaluation(prefile):
+    fin=open(prefile,'r',encoding='utf-8')
+    all_in=fin.read().strip().split('\n\n')
+    fin.close()
+    TP=0 #gold=pre=pos
+    FP=0 #gold=neg, pre=pos
+    FN=0 #gold=pos, pre=Neg
+    for sentence in all_in:
+        tokens=sentence.split('\n')
+        entity_id=0
+        token_id=0
+        temp_gold='O'
+        temp_pre='O'
+        while (token_id<len(tokens)):
+            seg=tokens[token_id].split('\t')
+            if seg[0]=='<GENE>':
+                if seg[1]=='O':
+                    temp_gold=seg[1]
+                else:
+                    temp_gold=seg[1][2:]
+                if seg[2]=='O':
+                    temp_pre=seg[2]
+                else:
+                    temp_pre=seg[2][2:]  
+                token_id+=1
+                seg=tokens[token_id].split('\t')
+                while seg[0]!='</GENE>':
+                    token_id+=1
+                    seg=tokens[token_id].split('\t')
+                    if seg[1]!='O' and temp_gold=='O':
+                        temp_gold=seg[1][2:]
+                    if seg[2]!='O' and temp_pre=='O':
+                        temp_pre=seg[2][2:]
+                if temp_pre!='O' and temp_gold!='O' and temp_pre==temp_gold:
+                    TP+=1
+                elif temp_pre!='O' and temp_gold!='O' and temp_pre!=temp_gold:
+                    FP+=1
+                    FN+=1
+                elif temp_pre!='O' and temp_gold=='O' :
+                    FP+=1
+                elif temp_pre=='O' and temp_gold!='O' :
+                    FN+=1
+                temp_pre='O'
+                temp_gold='O'
+        
+            else:
+                pass
+            token_id+=1
+    # print('TP,FP,FN:',TP,FP,FN)
+    if TP+FP==0:
+        P=0
+    else:
+        P=TP/(TP+FP)
+    if TP+FN==0:
+        R=0
+    else:
+        R=TP/(TP+FN)
+    if P+R==0:
+        F1=0
+    else:
+        F1=2*P*R/(P+R)
+    print('TP,FP,FN:',TP,FP,FN)
+    print('P,R,F1:',P,R,F1)
+
+
+def Rel_Evaluation_fn(prefile):
+    fin=open(prefile,'r',encoding='utf-8')
+    all_in=fin.read().strip().split('\n\n')
+    fin.close()
+    TP=0 #gold=pre=pos
+    FP=0 #gold=neg, pre=pos
+    FN=0 #gold=pos, pre=Neg
+    for sentence in all_in:
+        tokens=sentence.split('\n')
+        entity_id=0
+        token_id=0
+        temp_gold='O'
+        temp_pre='O'
+        while (token_id<len(tokens)):
+            seg=tokens[token_id].split('\t')
+            if seg[0]=='<GENE>':
+                if seg[1]=='O':
+                    temp_gold=seg[1]
+                else:
+                    temp_gold=seg[1][2:]
+                if seg[2]=='O':
+                    temp_pre=seg[2]
+                else:
+                    temp_pre=seg[2][2:]  
+                token_id+=1
+                seg=tokens[token_id].split('\t')
+                while seg[0]!='</GENE>':
+                    token_id+=1
+                    seg=tokens[token_id].split('\t')
+                    if seg[1]!='O' and temp_gold=='O':
+                        temp_gold=seg[1][2:]
+                    if seg[2]!='O' and temp_pre=='O':
+                        temp_pre=seg[2][2:]
+                if temp_pre!='O' and temp_gold!='O' and temp_pre==temp_gold:
+                    TP+=1
+                elif temp_pre!='O' and temp_gold!='O' and temp_pre!=temp_gold:
+                    FP+=1
+                elif temp_pre!='O' and temp_gold=='O' :
+                    FP+=1
+                elif temp_pre=='O' and temp_gold!='O' :
+                    FN+=1
+                temp_pre='O'
+                temp_gold='O'
+        
+            else:
+                pass
+            token_id+=1
+    print('TP,FP,FN:',TP,FP,FN)
+    if TP+FP==0:
+        P=0
+    else:
+        P=TP/(TP+FP)
+    if TP+FN==0:
+        R=0
+    else:
+        R=TP/(TP+FN)
+    if P+R==0:
+        F1=0
+    else:
+        F1=2*P*R/(P+R)
+    # print('TP,FP,FN:',TP,FP,FN)
+    print('P,R,F1:',P,R,F1)
+    return F1
+
+def Rel_Evaluation_Hugface_fn(prefile,ARG2_label='gene1s'):
+    fin=open(prefile,'r',encoding='utf-8')
+    all_in=fin.read().strip().split('\n\n')
+    fin.close()
+    TP=0 #gold=pre=pos
+    FP=0 #gold=neg, pre=pos
+    FN=0 #gold=pos, pre=Neg
+    result_dict={}#{'rel type':[TP,FP,FN],...,}
+    for sentence in all_in:
+        tokens=sentence.split('\n')
+        for token in tokens:
+            seg=token.split('\t')
+            if seg[0]==ARG2_label:
+                if seg[1].find('ARG2')>=0:
+                    if seg[2]==seg[1]:
+                        if seg[1] not in result_dict.keys():
+                            result_dict[seg[1]]=[1,0,0]
+                        else:
+                            result_dict[seg[1]][0]+=1
+                        TP+=1
+                    elif seg[2].find('ARG2')>=0:
+                        if seg[1] not in result_dict.keys():
+                            result_dict[seg[1]]=[0,0,1]
+                        else:
+                            result_dict[seg[1]][2]+=1
+                        if seg[2] not in result_dict.keys():
+                            result_dict[seg[2]]=[0,1,0]
+                        else:
+                            result_dict[seg[2]][1]+=1
+                        FP+=1
+                        FN+=1
+                    else:
+                        if seg[1] not in result_dict.keys():
+                            result_dict[seg[1]]=[0,0,1]
+                        else:
+                            result_dict[seg[1]][2]+=1
+                        FN+=1
+
+                else:
+                    if seg[2].find('ARG2')>=0:
+                        if seg[2] not in result_dict.keys():
+                            result_dict[seg[2]]=[0,1,0]
+                        else:
+                            result_dict[seg[2]][1]+=1
+                        FP+=1
+    # print('TP,FP,FN:',TP,FP,FN)
+    rel_metrics={}
+    for rel_type in result_dict.keys():
+        if result_dict[rel_type][0]+result_dict[rel_type][1]==0:
+            p=0
+        else:
+            p=result_dict[rel_type][0]/(result_dict[rel_type][0]+result_dict[rel_type][1])
+        if result_dict[rel_type][0]+result_dict[rel_type][2]==0:
+            r=0
+        else:
+            r=result_dict[rel_type][0]/(result_dict[rel_type][0]+result_dict[rel_type][2])
+        if p+r==0:
+            f1=0
+        else:
+            f1=2*p*r/(p+r)
+        rel_metrics[rel_type]=[round(p,4),round(r,4),round(f1,4)]
+    if TP+FP==0:
+        P=0
+    else:
+        P=TP/(TP+FP)
+    if TP+FN==0:
+        R=0
+    else:
+        R=TP/(TP+FN)
+    if P+R==0:
+        F1=0
+    else:
+        F1=2*P*R/(P+R)
+    P=round(P,4)
+    R=round(R,4)
+    F1=round(F1,4)
+    print('mertics:\n',rel_metrics)
+    print('\nTP,FP,FN:',TP,FP,FN)
+    print('Overall P,R,F1:',P,R,F1)   
+    return [P,R,F1],rel_metrics
+
+def Rel_Evaluation_AIO_fn(prefile):
+    fin=open(prefile,'r',encoding='utf-8')
+    all_in=fin.read().strip().split('\n\n')
+    fin.close()
+    TP=0 #gold=pre=pos
+    FP=0 #gold=neg, pre=pos
+    FN=0 #gold=pos, pre=Neg
+    for sentence in all_in:
+        tokens=sentence.split('\n')
+        for token in tokens:
+            seg=token.split('\t')
+            if seg[0]=='<GENE>':
+                if seg[1].find('ARG2-')>=0:
+                    if seg[2]==seg[1]:
+                        TP+=1
+                    elif seg[2].find('ARG2-')>=0:
+                        FP+=1
+                        FN+=1
+                    else:
+                        FN+=1
+
+                else:
+                    if seg[2].find('ARG2-')>=0:
+                        FP+=1
+    # print('TP,FP,FN:',TP,FP,FN)
+    if TP+FP==0:
+        P=0
+    else:
+        P=TP/(TP+FP)
+    if TP+FN==0:
+        R=0
+    else:
+        R=TP/(TP+FN)
+    if P+R==0:
+        F1=0
+    else:
+        F1=2*P*R/(P+R)
+    P=round(P,4)
+    R=round(R,4)
+    F1=round(F1,4)
+    print('TP,FP,FN:',TP,FP,FN)
+    print('P,R,F1:',P,R,F1)   
+    return [P,R,F1]
+
+def Rel_Evaluation_AIO_GC_fn(prefile):
+    fin=open(prefile,'r',encoding='utf-8')
+    all_in=fin.read().strip().split('\n\n')
+    fin.close()
+    TP=0 #gold=pre=pos
+    FP=0 #gold=neg, pre=pos
+    FN=0 #gold=pos, pre=Neg
+    for sentence in all_in:
+        tokens=sentence.split('\n')
+        for token in tokens:
+            seg=token.split('\t')
+            if seg[0]=='<CHEMICAL>':
+                if seg[1].find('ARG2-')>=0:
+                    if seg[2]==seg[1]:
+                        TP+=1
+                    elif seg[2].find('ARG2-')>=0:
+                        FP+=1
+                        FN+=1
+                    else:
+                        FN+=1
+
+                else:
+                    if seg[2].find('ARG2-')>=0:
+                        FP+=1
+    # print('TP,FP,FN:',TP,FP,FN)
+    if TP+FP==0:
+        P=0
+    else:
+        P=TP/(TP+FP)
+    if TP+FN==0:
+        R=0
+    else:
+        R=TP/(TP+FN)
+    if P+R==0:
+        F1=0
+    else:
+        F1=2*P*R/(P+R)
+    P=round(P,4)
+    R=round(R,4)
+    F1=round(F1,4)
+    print('TP,FP,FN:',TP,FP,FN)
+    print('P,R,F1:',P,R,F1)   
+    return [P,R,F1]
+    
+def office_evaluation(goldfile,prefile):
+    fin_gold=open(goldfile,'r',encoding='utf-8')
+    all_gold=fin_gold.read().strip().split('\n')
+    fin_gold.close()
+    fin_pre=open(prefile,'r',encoding='utf-8')
+    all_pre=fin_pre.read().strip().split('\n')
+    fin_pre.close()
+    
+    gold_result={}#{'relation type':set(line)}
+    pre_result={}
+    all_result={} #{'relation type':[tp,fp,fn]}
+    for line in all_gold:
+        seg=line.split('\t')
+        if seg[1] not in all_result.keys():
+            all_result[seg[1]]=[0,0,0]
+        if seg[1] not in gold_result.keys(): 
+            gold_result[seg[1]]=set()
+            gold_result[seg[1]].add(line)         
+        else:
+            gold_result[seg[1]].add(line)
+      
+    for line in all_pre:
+        seg=line.split('\t')
+        if seg[1] not in pre_result.keys():
+            pre_result[seg[1]]=set()
+            pre_result[seg[1]].add(line)
+        else:
+            pre_result[seg[1]].add(line)
+            
+    for rel_type in gold_result.keys():
+        for gold_ele in gold_result[rel_type]:
+            if rel_type not in pre_result.keys():
+                all_result[rel_type][2]+=1
+            else:
+                if gold_ele in pre_result[rel_type]:
+                    all_result[rel_type][0]+=1
+                else:
+                    all_result[rel_type][2]+=1
+        if rel_type in pre_result.keys():
+            for pre_ele in pre_result[rel_type]:
+                if pre_ele not in gold_result[rel_type]:
+                    all_result[rel_type][1]+=1
+    ave_f=0
+    TP,FP,FN=0,0,0
+    print(all_result)
+    for rel_type in all_result.keys():
+        TP+=all_result[rel_type][0]
+        FP+=all_result[rel_type][1]
+        FN+=all_result[rel_type][2]
+        tem_p,tem_r,tem_f=0,0,0
+        if all_result[rel_type][0]+all_result[rel_type][1]==0:
+            tem_p=0
+        else:
+            tem_p=all_result[rel_type][0]/(all_result[rel_type][0]+all_result[rel_type][1])
+        if all_result[rel_type][0]+all_result[rel_type][2]==0:
+            tem_r=0
+        else:
+            tem_r=all_result[rel_type][0]/(all_result[rel_type][0]+all_result[rel_type][2])
+        if tem_p+tem_r==0:
+            tem_f=0
+        else:
+            tem_f=2*tem_p*tem_r/(tem_p+tem_r)
+        ave_f+=tem_f
+        print('%s:p=%.4f,r=%.4f,f=%.4f' % (rel_type,tem_p,tem_r,tem_f))
+    
+    if TP+FP==0:
+        P=0
+    else:
+        P=TP/(TP+FP)
+    if TP+FN==0:
+        R=0
+    else:
+        R=TP/(TP+FN)
+    if P+R==0:
+        F1=0
+    else:
+        F1=2*P*R/(P+R)
+    ave_f+=tem_f
+    
+    print('Overall:')
+    print('ave_f1:',ave_f/len(all_result))
+    print('TP=%d, FP=%d, FN=%d'%(TP,FP,FN))
+    print('P=%.4f, R=%.4f, F1=%.4f'%(P,R,F1))
+    
+
+if __name__=='__main__':
+    path='//panfs/pan1/bionlplab/luol2/BC7DrugProt/results/'
+    office_evaluation(path+'dev/dev_gold_relations.tsv',path+'drugprot_dev_LSTM-CRF-ES_pre.tsv')
+    print('............')
+    Rel_Evaluation_check('//panfs/pan1/bionlplab/luol2/BC7DrugProt/check/dev_pre_temp.conll')

src_python/SpeAss/SA_Pubtator_Conll.py

@@ -1,494 +1,494 @@
-# -*- coding: utf-8 -*-
-
-import sys
-import io
-import stanza
-# nlp = stanza.Pipeline(lang='en', processors='tokenize,mwt,pos,lemma',package='craft') #package='craft'
-nlp = stanza.Pipeline(lang='en', processors={'tokenize': 'spacy'},package='None') #package='craft'
-REL_ENT={'arg1':'Species',
-         'arg2':'Gene'}
-
-ENTITY_TAG={'arg1':['arg1s','arg1e'],
-            'arg2':['arg2s','arg2e'],
-            'gene':['gene1s','gene1e'],
-            'species':['species1s','species1e']
-            }
-
-# ssplit token and revise index
-def ssplit_token(infile):
-    fin=open(infile,'r',encoding='utf-8')
-    fout=io.StringIO()
-    all_in=fin.read().strip().split('\n\n')
-    fin.close()
-    for doc_text in all_in:
-        lines=doc_text.split('\n')
-        ori_text=lines[0].split('|t|')[1]+' '+lines[1].split('|a|')[1]
-        pmid=lines[0].split('|t|')[0]
-        # print(pmid)
-        entity_all=[]   #[[seg0,seg1,...,],[]]
-        for i in range(2,len(lines)):
-            seg=lines[i].split('\t')
-            entity_all.append(seg)
-
-        #ssplit token
-        doc_stanza = nlp(ori_text)
-        token_text=''
-        for sent in doc_stanza.sentences:
-            for word in sent.words:
-                if word.text==' ':
-                    pass
-                    # print('token is blank!')
-                else:
-                    token_text+=word.text+' '
-            #token_text=token_text+'    '  #sentence split by four blank
-        
-        #ori_index map token_index
-        index_map=[-1]*len(ori_text)
-        j=0
-        space_list=[' ',chr(160),chr(8201),chr(8194),chr(8197),chr(8202)] #空格有好几种，第一个是常用32,第二个shi 160,8201,8194,8197
-        for i in range(0,len(ori_text)):
-            if ori_text[i] in space_list:
-                pass
-            elif ori_text[i]==token_text[j]:
-                #if i>0 and i<285:
-                #    print('=i,j:',i,j,ori_text[i-1:i+1],token_text[j-1:j+1])
-                index_map[i]=j
-                j+=1
-            else:
-                #if i==283:
-                #    print('!i,j:',i,j,ori_text[i-1:i+1],token_text[j-1:j+1])
-                j+=1
-                temp_log=j
-                try:
-                    while(ori_text[i]!=token_text[j]):
-                        j+=1
-                except:
-                    print('doc',doc_text)
-                    print('token_text:',token_text)
-                    print('error:',ori_text[i-10:i+10],'i:',ori_text[i],'j:',token_text[temp_log],',',token_text[temp_log-10:temp_log+10])
-                    print(ord(ori_text[i]),ord(' '))
-                    sys.exit()
-                index_map[i]=j
-                j+=1
-        # print(index_map)
-        # token_text=token_text.replace('     ','<EOS>')
-        # print(token_text)
-        fout.write(token_text+'\n')
-        for ele in entity_all:
-            if index_map[int(ele[1])]==-1:
-                new_ents=index_map[int(ele[1])+1]
-            else:
-                new_ents=index_map[int(ele[1])]
-            if index_map[int(ele[2])-1]==-1: 
-                new_ente=index_map[int(ele[2])-1-1]+1
-            else:
-                new_ente=index_map[int(ele[2])-1]+1
-            new_ent=token_text[new_ents:new_ente]
-            if ele[4]=='Species' or ele[4]=='Gene':
-                fout.write(ele[0]+'\t'+str(new_ents)+'\t'+str(new_ente)+'\t'+new_ent+'\t'+ele[4]+'\t'+ele[5]+'\n')
-            else:
-                # print(ele[4])
-                fout.write(ele[0]+'\t'+str(new_ents)+'\t'+str(new_ente)+'\t'+new_ent+'\t'+'Gene'+'\t'+ele[5]+'\n')
-        fout.write('\n')
-    return fout.getvalue()
-
-
-def corpus_noNest(token_input):
-    
-    fin=io.StringIO(token_input)
-    fout=io.StringIO()
-
-    documents=fin.read().strip().split('\n\n')
-    fin.close()
-    total_entity=0
-    over_entity=0
-    nest_entity=0
-    for doc in documents:
-        lines=doc.split('\n')
-        context=lines[0]
-        entity_list=[]
-        if len(lines)>1:
-            doc_result={}
-            for i in range(1,len(lines)):
-                segs=lines[i].split('\t')
-                doc_result[lines[i]]=[int(segs[1]),int(segs[2])]
-            doc_result=sorted(doc_result.items(), key=lambda kv:(kv[1]), reverse=False)
-            doc_result_sort=[]
-            for ele in doc_result:
-                doc_result_sort.append(ele[0])
-            
-            first_entity=doc_result_sort[0].split('\t')
-            nest_list=[first_entity]
-            max_eid=int(first_entity[2])
-            total_entity+=len(lines)-2
-            for i in range(1,len(doc_result_sort)):
-                segs=doc_result_sort[i].split('\t')
-                if int(segs[1])> max_eid:
-                    if len(nest_list)==1:
-                        entity_list.append(nest_list[0])
-                        nest_list=[]
-                        nest_list.append(segs)
-                        if int(segs[2])>max_eid:
-                            max_eid=int(segs[2])
-                    else:
-                        # print(nest_list)
-                        nest_entity+=len(nest_list)-1
-                        tem=find_max_entity(nest_list,context)#find max entity
-                        # if len(tem)>1:
-                            # print('max nest >1:',tem)
-                        entity_list.extend(tem)
-                        nest_list=[]
-                        nest_list.append(segs)
-                        if int(segs[2])>max_eid:
-                            max_eid=int(segs[2])
-                        
-                else:
-                    nest_list.append(segs)
-                    over_entity+=1
-                    if int(segs[2])>max_eid:
-                        max_eid=int(segs[2])
-            if nest_list!=[]:
-                if len(nest_list)==1:
-                    entity_list.append(nest_list[0])
-
-                else:
-                    tem=find_max_entity(nest_list,context)#find max entity
-                    # if len(tem)>1:
-                    #     print('max nest >1:',tem)
-                    entity_list.extend(tem)
-        fout.write(context+'\n')
-        for ele in entity_list:
-            if ele[4]=='Gene':
-                temp_gene={}
-                gene_ids=ele[5].split(',')
-                for gene_id in gene_ids:
-                    temp_id=gene_id[gene_id.find('Species:'):-1]
-                    spe_id=temp_id[len('Species:'):]
-                    temp_gene[temp_id]=int(spe_id)
-                temp_gene_sort=sorted(temp_gene.items(), key=lambda kv:(kv[1]), reverse=False)
-                final_gene_id=''
-                for temp_ele in temp_gene_sort:
-                    final_gene_id+=temp_ele[0]+','
-                fout.write('\t'.join(ele[:-1])+'\t'+final_gene_id[:-1]+'\n')
-            else:
-                fout.write('\t'.join(ele)+'\n')
-        fout.write('\n')
-    # print(total_entity,over_entity, nest_entity)
-    return fout.getvalue()
-
-def find_max_entity(nest_list,text):
-    max_len=0
-    final_tem=[]
-    max_index=0
-    for i in range(0, len(nest_list)):
-        if nest_list[i][4] =='Species':
-            final_tem.append(nest_list[i])
-        else:
-            cur_len=int(nest_list[i][2])-int(nest_list[i][1])
-            if cur_len>max_len:
-                max_len=cur_len
-                max_index=i
-    final_tem.append(nest_list[max_index])
-    return final_tem
-
-
-def generate_seq_input(nonest_input,outfile):
-    
-    fin=io.StringIO(nonest_input)
-    fout=open(outfile,'w',encoding='utf-8')
-    all_in=fin.read().strip().split('\n\n')
-    fin.close()
-
-    final_input=[]
-
-    for doc in all_in:
-        lines=doc.split('\n')
-        token_text=lines[0]
-        pmid=lines[1].split('\t')[0]
-        # print(pmid)
-        #read entity and relation
-        entity_arg1={} #only entity offset
-        entity_arg2={} #only entity offset
-        entity_all=[] #all entity infor
-
-        for i in range(1,len(lines)):
-            seg=lines[i].split('\t')
-            if seg[4]==REL_ENT['arg1']:
-                if seg[-1] in entity_arg1.keys():
-                    entity_arg1[seg[-1]].append([seg[1],seg[2]])
-                else:
-                    entity_arg1[seg[-1]]=[[seg[1],seg[2]]]
-            elif seg[4]==REL_ENT['arg2']:
-                temp_spes=seg[-1].split(',')
-                for ele in temp_spes:
-                    gene_spe_id=ele
-                    if gene_spe_id in entity_arg2.keys():
-                        entity_arg2[gene_spe_id].append([seg[1],seg[2]])
-                    else:
-                        entity_arg2[gene_spe_id]=[[seg[1],seg[2]]]
-                
-            entity_all.append(seg)
-        # print('\narg1:',entity_arg1)
-        # print('\narg2:',entity_arg2)
-        # print('\nall entity:',entity_all)
-        # for all arg1 to produce inst
-        for cur_ele in entity_arg1.keys():
-
-            #1. ner label text
-            #check cur_ele in relation?
-            # print(relation_all.keys())
-            if cur_ele in entity_arg2.keys(): #pos instance
-                rel_ent2=entity_arg2[cur_ele]
-                ner_text=''
-                text_sid=0
-                #print('nonest:',entity_nonest)
-                for ele_nonest in entity_all:
-                    ent_id=[ele_nonest[1],ele_nonest[2]]
-                    ent_sid=int(ele_nonest[1])
-                    ent_eid=int(ele_nonest[2])
-                    # print('sid,eid:',ent_sid,ent_eid)
-                    ent_text=ele_nonest[3]
-                    ent_type=ele_nonest[4]
-                    if ent_sid>=text_sid:
-                        if ent_id in entity_arg1[cur_ele]:    
-                            ner_text+=token_text[text_sid:ent_sid]+' '+ENTITY_TAG['arg1'][0]+' '+ent_text+ ' '+ENTITY_TAG['arg1'][1]+' '
-                        else:
-                            if ent_id in rel_ent2: #arg2 entity
-                                if ent_type!=REL_ENT['arg2']:
-                                    pass
-                                    # print('arg2 is error! not ',REL_ENT['arg2'], ele_nonest)
-                                ner_text+=token_text[text_sid:ent_sid]+' '+ENTITY_TAG['arg2'][0]+' '+ent_text+ ' '+ENTITY_TAG['arg2'][1]+' '
-                            else:
-                                ner_text+=token_text[text_sid:ent_sid]+' '+ENTITY_TAG[ent_type.lower()][0]+' '+ent_text+ ' '+ENTITY_TAG[ent_type.lower()][1]+' '
-                        text_sid=ent_eid  
-                    else:
-                        pass
-                        # print('ner entity error!!!',ele_nonest,text_sid)                                    
-                ner_text+=token_text[text_sid:]
-                sen_tokens=ner_text.split()
-                # print('\nner_text:',ner_text)
-                
-                #3 produce pos input
- 
-                temp_input=[]
-                token_id=0
-                while token_id <len(sen_tokens):
-                    if sen_tokens[token_id].find(ENTITY_TAG['arg1'][0])>=0:
-                        temp_input.append(ENTITY_TAG['arg1'][0]+'\tO')
-                        token_id+=1
-                        while(sen_tokens[token_id]!=ENTITY_TAG['arg1'][1]):
-                            temp_input.append(sen_tokens[token_id]+'\tO')
-                            token_id+=1
-                        temp_input.append(ENTITY_TAG['arg1'][1]+'\tO')
-                    elif sen_tokens[token_id].find(ENTITY_TAG['arg2'][0])>=0:
-                        temp_input.append(ENTITY_TAG[REL_ENT['arg2'].lower()][0]+'\tARG2')
-                        token_id+=1
-                        while(sen_tokens[token_id]!=ENTITY_TAG['arg2'][1]):
-                            temp_input.append(sen_tokens[token_id]+'\tARG2')
-                            token_id+=1
-                        temp_input.append(ENTITY_TAG[REL_ENT['arg2'].lower()][1]+'\tARG2')
-                    elif sen_tokens[token_id].find(ENTITY_TAG['gene'][0])>=0:
-                        temp_input.append(ENTITY_TAG['gene'][0]+'\tO')
-                        token_id+=1
-                        while(sen_tokens[token_id]!=ENTITY_TAG['gene'][1]):
-                            temp_input.append(sen_tokens[token_id]+'\tO')
-                            token_id+=1
-                        temp_input.append(ENTITY_TAG['gene'][1]+'\tO')
-                    elif sen_tokens[token_id].find(ENTITY_TAG['species'][0])>=0:
-                        temp_input.append(ENTITY_TAG['species'][0]+'\tO')
-                        token_id+=1
-                        while(sen_tokens[token_id]!=ENTITY_TAG['species'][1]):
-                            temp_input.append(sen_tokens[token_id]+'\tO')
-                            token_id+=1
-                        temp_input.append(ENTITY_TAG['species'][1]+'\tO')
-                    else:
-                        if sen_tokens[token_id]=='':
-                            # print('token is none!error!')
-                            pass
-                        else:
-                            temp_input.append(sen_tokens[token_id]+'\tO')
-                    token_id+=1
-                    
-                final_input.append('\n'.join(temp_input))
-                
-            else:     #neg instance 
-                ner_text=''
-                text_sid=0
-                #print('nonest:',entity_nonest)
-                for ele_nonest in entity_all:
-                    ent_id=[ele_nonest[1],ele_nonest[2]]
-                    ent_sid=int(ele_nonest[1])
-                    ent_eid=int(ele_nonest[2])
-                    # print('sid,eid:',ent_sid,ent_eid)
-                    ent_text=ele_nonest[3]
-                    ent_type=ele_nonest[4]
-                    if ent_sid>=text_sid:
-                        if ent_id in entity_arg1[cur_ele]:    
-                            ner_text+=token_text[text_sid:ent_sid]+' '+ENTITY_TAG['arg1'][0]+' '+ent_text+ ' '+ENTITY_TAG['arg1'][1]+' '
-                        else:
-                            ner_text+=token_text[text_sid:ent_sid]+' '+ENTITY_TAG[ent_type.lower()][0]+' '+ent_text+ ' '+ENTITY_TAG[ent_type.lower()][1]+' '
-                        text_sid=ent_eid  
-                    else:
-                        pass
-                        # print('ner entity error!!!')                                    
-                ner_text+=token_text[text_sid:]
-                sen_tokens=ner_text.split()
-                # print('\nner_text:',ner_text)
-                # print('ner_Text')
-                #3 produce NEG input
-                   
-                temp_input=[]
-                token_id=0
-                while token_id <len(sen_tokens):
-                    if sen_tokens[token_id].find(ENTITY_TAG['arg1'][0])>=0:
-                        temp_input.append(ENTITY_TAG['arg1'][0]+'\tO')
-                        token_id+=1
-                        while(sen_tokens[token_id]!=ENTITY_TAG['arg1'][1]):
-                            temp_input.append(sen_tokens[token_id]+'\tO')
-                            token_id+=1
-                        temp_input.append(ENTITY_TAG['arg1'][1]+'\tO')
-                    elif sen_tokens[token_id].find(ENTITY_TAG['gene'][0])>=0:
-                        temp_input.append(ENTITY_TAG['gene'][0]+'\tO')
-                        token_id+=1
-                        while(sen_tokens[token_id]!=ENTITY_TAG['gene'][1]):
-                            temp_input.append(sen_tokens[token_id]+'\tO')
-                            token_id+=1
-                        temp_input.append(ENTITY_TAG['gene'][1]+'\tO')
-                    elif sen_tokens[token_id].find(ENTITY_TAG['species'][0])>=0:
-                        temp_input.append(ENTITY_TAG['species'][0]+'\tO')
-                        token_id+=1
-                        while(sen_tokens[token_id]!=ENTITY_TAG['species'][1]):
-                            temp_input.append(sen_tokens[token_id]+'\tO')
-                            token_id+=1
-                        temp_input.append(ENTITY_TAG['species'][1]+'\tO')
-                    else:
-                        if sen_tokens[token_id]=='':
-                            print('token is none!error!')
-                        else:
-                            temp_input.append(sen_tokens[token_id]+'\tO')
-                    token_id+=1
-                    
-                final_input.append('\n'.join(temp_input))
-                # print(entity_nonest)
-        # sys.exit()
-    fout.write('\n\n'.join(final_input))
-    fout.write('\n')
-    fout.close()   
-
-def check_entity_pos(line,relations):
-    
-    seg=line.split(' ')
-    stack_ent=[] 
-    # print(seg)
-    entity_num={'arg1':0,'arg2':0, 'gene':0,'chemical':0}
-
-    temp_arg2=[]
-    for i in range(0,len(seg)):
-        if seg[i].find(ENTITY_TAG['gene'][0])>=0:
-            entity_num['gene']+=1
-            stack_ent.append(seg[i])
-        elif seg[i].find(ENTITY_TAG['chemical'][0])>=0:
-            entity_num['chemical']+=1
-            stack_ent.append(seg[i])
-            # print(stack_ent)
-        elif seg[i].find(ENTITY_TAG['arg1'][0])>=0:
-            entity_num['arg1']+=1
-            stack_ent.append(seg[i]) 
-        elif seg[i].find(ENTITY_TAG['arg2'][0])>=0:
-            entity_num['arg2']+=1
-            temp_arg2.append(seg[i].split('|')[0])
-            stack_ent.append(seg[i])
-        elif seg[i].find(ENTITY_TAG['arg1'][1])>=0 or seg[i].find(ENTITY_TAG['arg2'][1])>=0 or seg[i].find(ENTITY_TAG['gene'][1])>=0 or seg[i].find(ENTITY_TAG['chemical'][1])>=0:
-            stack_ent.pop()
-    if stack_ent!=[]:
-        # print('entity no match!',stack_ent)
-        return(-1,seg,entity_num)
-    
-    else:
-        if entity_num['arg1']!=0:
-            for arg2_id in relations.keys():
-                if arg2_id not in temp_arg2:
-                    # print('\ntemp_arg2:',temp_arg2)
-                    # print('\narg2_id:',arg2_id)
-                    return(0,seg,entity_num) #some arg2 not in sentence
-        if entity_num['arg2']!=0 and entity_num['arg1']==0:
-            return(0,seg,entity_num) #only arg2, but no arg1
-        return(1,seg,entity_num) 
-
-def check_entity_neg(line):
-    
-    seg=line.split(' ')
-    stack_ent=[] 
-    # print(seg)
-    entity_num={'arg1':0,'gene':0,'chemical':0}
-    for i in range(0,len(seg)):
-        if seg[i].find(ENTITY_TAG['gene'][0])>=0:
-            entity_num['gene']+=1
-            stack_ent.append(seg[i])
-        elif seg[i].find(ENTITY_TAG['chemical'][0])>=0:
-            entity_num['chemical']+=1
-            stack_ent.append(seg[i])
-            # print(stack_ent)
-        elif seg[i].find(ENTITY_TAG['arg1'][0])>=0:
-            entity_num['arg1']+=1
-            stack_ent.append(seg[i])        
-        elif seg[i].find(ENTITY_TAG['arg1'][1])>=0  or seg[i].find(ENTITY_TAG['gene'][1])>=0 or seg[i].find(ENTITY_TAG['chemical'][1])>=0:
-            stack_ent.pop()
-    if stack_ent!=[]:
-        # print('entity no match!',stack_ent)
-        return(-1,seg,entity_num)
-    
-    else:
-        return(1,seg,entity_num) 
-
-def get_one_entity(nest_list,cur_ent,rel_entity2_id):
-    max_len=0
-    max_entity=[]
-    final_entity=[]
-    for i in range(0, len(nest_list)):
-        if nest_list[i][1]==cur_ent:#current entity
-            final_entity=[]
-            max_entity=nest_list[i]
-            final_entity.append(nest_list[i])
-            return(final_entity)
-        if nest_list[i][1] in rel_entity2_id: #invole rel
-            final_entity.append(nest_list[i])
-            continue
-        length=int(nest_list[i][4])-int(nest_list[i][3])
-        if max_entity==[]: #first entity
-            max_len=length
-            max_entity=nest_list[i]
-        else:
-            if length>max_len:
-                if max_entity[2]==REL_ENT['arg1']:
-                    max_len=length
-                    max_entity=nest_list[i]
-                else:
-                    if nest_list[i][2]==REL_ENT['arg2'] and max_entity[1] not in rel_entity2_id:
-                        max_len=length
-                        max_entity=nest_list[i] 
-                        
-            else:
-                if nest_list[i][1] in rel_entity2_id:
-                    max_len=length
-                    max_entity=nest_list[i] 
-                elif max_entity[2]==REL_ENT['arg1'] and nest_list[i][2]==REL_ENT['arg2']:
-                    max_len=length
-                    max_entity=nest_list[i]
-    if final_entity==[]:
-        final_entity.append(max_entity)
-    return final_entity
-
-if __name__=='__main__':
-    
-    infile='../../TrainingSet/No505/SA.Train.txt'
-    outfile='../../TrainingSet/No505/SA.Train.conll'
-    
-    #tokenizer
-    token_input=ssplit_token(infile)
-
-    #filter nest entity
-    nonest_input=corpus_noNest(token_input)
-
-    # to conll
+# -*- coding: utf-8 -*-
+
+import sys
+import io
+import stanza
+# nlp = stanza.Pipeline(lang='en', processors='tokenize,mwt,pos,lemma',package='craft') #package='craft'
+nlp = stanza.Pipeline(lang='en', processors={'tokenize': 'spacy'},package='None') #package='craft'
+REL_ENT={'arg1':'Species',
+         'arg2':'Gene'}
+
+ENTITY_TAG={'arg1':['arg1s','arg1e'],
+            'arg2':['arg2s','arg2e'],
+            'gene':['gene1s','gene1e'],
+            'species':['species1s','species1e']
+            }
+
+# ssplit token and revise index
+def ssplit_token(infile):
+    fin=open(infile,'r',encoding='utf-8')
+    fout=io.StringIO()
+    all_in=fin.read().strip().split('\n\n')
+    fin.close()
+    for doc_text in all_in:
+        lines=doc_text.split('\n')
+        ori_text=lines[0].split('|t|')[1]+' '+lines[1].split('|a|')[1]
+        pmid=lines[0].split('|t|')[0]
+        # print(pmid)
+        entity_all=[]   #[[seg0,seg1,...,],[]]
+        for i in range(2,len(lines)):
+            seg=lines[i].split('\t')
+            entity_all.append(seg)
+
+        #ssplit token
+        doc_stanza = nlp(ori_text)
+        token_text=''
+        for sent in doc_stanza.sentences:
+            for word in sent.words:
+                if word.text==' ':
+                    pass
+                    # print('token is blank!')
+                else:
+                    token_text+=word.text+' '
+            #token_text=token_text+'    '  #sentence split by four blank
+        
+        #ori_index map token_index
+        index_map=[-1]*len(ori_text)
+        j=0
+        space_list=[' ',chr(160),chr(8201),chr(8194),chr(8197),chr(8202)] #空格有好几种，第一个是常用32,第二个shi 160,8201,8194,8197
+        for i in range(0,len(ori_text)):
+            if ori_text[i] in space_list:
+                pass
+            elif ori_text[i]==token_text[j]:
+                #if i>0 and i<285:
+                #    print('=i,j:',i,j,ori_text[i-1:i+1],token_text[j-1:j+1])
+                index_map[i]=j
+                j+=1
+            else:
+                #if i==283:
+                #    print('!i,j:',i,j,ori_text[i-1:i+1],token_text[j-1:j+1])
+                j+=1
+                temp_log=j
+                try:
+                    while(ori_text[i]!=token_text[j]):
+                        j+=1
+                except:
+                    print('doc',doc_text)
+                    print('token_text:',token_text)
+                    print('error:',ori_text[i-10:i+10],'i:',ori_text[i],'j:',token_text[temp_log],',',token_text[temp_log-10:temp_log+10])
+                    print(ord(ori_text[i]),ord(' '))
+                    sys.exit()
+                index_map[i]=j
+                j+=1
+        # print(index_map)
+        # token_text=token_text.replace('     ','<EOS>')
+        # print(token_text)
+        fout.write(token_text+'\n')
+        for ele in entity_all:
+            if index_map[int(ele[1])]==-1:
+                new_ents=index_map[int(ele[1])+1]
+            else:
+                new_ents=index_map[int(ele[1])]
+            if index_map[int(ele[2])-1]==-1: 
+                new_ente=index_map[int(ele[2])-1-1]+1
+            else:
+                new_ente=index_map[int(ele[2])-1]+1
+            new_ent=token_text[new_ents:new_ente]
+            if ele[4]=='Species' or ele[4]=='Gene':
+                fout.write(ele[0]+'\t'+str(new_ents)+'\t'+str(new_ente)+'\t'+new_ent+'\t'+ele[4]+'\t'+ele[5]+'\n')
+            else:
+                # print(ele[4])
+                fout.write(ele[0]+'\t'+str(new_ents)+'\t'+str(new_ente)+'\t'+new_ent+'\t'+'Gene'+'\t'+ele[5]+'\n')
+        fout.write('\n')
+    return fout.getvalue()
+
+
+def corpus_noNest(token_input):
+    
+    fin=io.StringIO(token_input)
+    fout=io.StringIO()
+
+    documents=fin.read().strip().split('\n\n')
+    fin.close()
+    total_entity=0
+    over_entity=0
+    nest_entity=0
+    for doc in documents:
+        lines=doc.split('\n')
+        context=lines[0]
+        entity_list=[]
+        if len(lines)>1:
+            doc_result={}
+            for i in range(1,len(lines)):
+                segs=lines[i].split('\t')
+                doc_result[lines[i]]=[int(segs[1]),int(segs[2])]
+            doc_result=sorted(doc_result.items(), key=lambda kv:(kv[1]), reverse=False)
+            doc_result_sort=[]
+            for ele in doc_result:
+                doc_result_sort.append(ele[0])
+            
+            first_entity=doc_result_sort[0].split('\t')
+            nest_list=[first_entity]
+            max_eid=int(first_entity[2])
+            total_entity+=len(lines)-2
+            for i in range(1,len(doc_result_sort)):
+                segs=doc_result_sort[i].split('\t')
+                if int(segs[1])> max_eid:
+                    if len(nest_list)==1:
+                        entity_list.append(nest_list[0])
+                        nest_list=[]
+                        nest_list.append(segs)
+                        if int(segs[2])>max_eid:
+                            max_eid=int(segs[2])
+                    else:
+                        # print(nest_list)
+                        nest_entity+=len(nest_list)-1
+                        tem=find_max_entity(nest_list,context)#find max entity
+                        # if len(tem)>1:
+                            # print('max nest >1:',tem)
+                        entity_list.extend(tem)
+                        nest_list=[]
+                        nest_list.append(segs)
+                        if int(segs[2])>max_eid:
+                            max_eid=int(segs[2])
+                        
+                else:
+                    nest_list.append(segs)
+                    over_entity+=1
+                    if int(segs[2])>max_eid:
+                        max_eid=int(segs[2])
+            if nest_list!=[]:
+                if len(nest_list)==1:
+                    entity_list.append(nest_list[0])
+
+                else:
+                    tem=find_max_entity(nest_list,context)#find max entity
+                    # if len(tem)>1:
+                    #     print('max nest >1:',tem)
+                    entity_list.extend(tem)
+        fout.write(context+'\n')
+        for ele in entity_list:
+            if ele[4]=='Gene':
+                temp_gene={}
+                gene_ids=ele[5].split(',')
+                for gene_id in gene_ids:
+                    temp_id=gene_id[gene_id.find('Species:'):-1]
+                    spe_id=temp_id[len('Species:'):]
+                    temp_gene[temp_id]=int(spe_id)
+                temp_gene_sort=sorted(temp_gene.items(), key=lambda kv:(kv[1]), reverse=False)
+                final_gene_id=''
+                for temp_ele in temp_gene_sort:
+                    final_gene_id+=temp_ele[0]+','
+                fout.write('\t'.join(ele[:-1])+'\t'+final_gene_id[:-1]+'\n')
+            else:
+                fout.write('\t'.join(ele)+'\n')
+        fout.write('\n')
+    # print(total_entity,over_entity, nest_entity)
+    return fout.getvalue()
+
+def find_max_entity(nest_list,text):
+    max_len=0
+    final_tem=[]
+    max_index=0
+    for i in range(0, len(nest_list)):
+        if nest_list[i][4] =='Species':
+            final_tem.append(nest_list[i])
+        else:
+            cur_len=int(nest_list[i][2])-int(nest_list[i][1])
+            if cur_len>max_len:
+                max_len=cur_len
+                max_index=i
+    final_tem.append(nest_list[max_index])
+    return final_tem
+
+
+def generate_seq_input(nonest_input,outfile):
+    
+    fin=io.StringIO(nonest_input)
+    fout=open(outfile,'w',encoding='utf-8')
+    all_in=fin.read().strip().split('\n\n')
+    fin.close()
+
+    final_input=[]
+
+    for doc in all_in:
+        lines=doc.split('\n')
+        token_text=lines[0]
+        pmid=lines[1].split('\t')[0]
+        # print(pmid)
+        #read entity and relation
+        entity_arg1={} #only entity offset
+        entity_arg2={} #only entity offset
+        entity_all=[] #all entity infor
+
+        for i in range(1,len(lines)):
+            seg=lines[i].split('\t')
+            if seg[4]==REL_ENT['arg1']:
+                if seg[-1] in entity_arg1.keys():
+                    entity_arg1[seg[-1]].append([seg[1],seg[2]])
+                else:
+                    entity_arg1[seg[-1]]=[[seg[1],seg[2]]]
+            elif seg[4]==REL_ENT['arg2']:
+                temp_spes=seg[-1].split(',')
+                for ele in temp_spes:
+                    gene_spe_id=ele
+                    if gene_spe_id in entity_arg2.keys():
+                        entity_arg2[gene_spe_id].append([seg[1],seg[2]])
+                    else:
+                        entity_arg2[gene_spe_id]=[[seg[1],seg[2]]]
+                
+            entity_all.append(seg)
+        # print('\narg1:',entity_arg1)
+        # print('\narg2:',entity_arg2)
+        # print('\nall entity:',entity_all)
+        # for all arg1 to produce inst
+        for cur_ele in entity_arg1.keys():
+
+            #1. ner label text
+            #check cur_ele in relation?
+            # print(relation_all.keys())
+            if cur_ele in entity_arg2.keys(): #pos instance
+                rel_ent2=entity_arg2[cur_ele]
+                ner_text=''
+                text_sid=0
+                #print('nonest:',entity_nonest)
+                for ele_nonest in entity_all:
+                    ent_id=[ele_nonest[1],ele_nonest[2]]
+                    ent_sid=int(ele_nonest[1])
+                    ent_eid=int(ele_nonest[2])
+                    # print('sid,eid:',ent_sid,ent_eid)
+                    ent_text=ele_nonest[3]
+                    ent_type=ele_nonest[4]
+                    if ent_sid>=text_sid:
+                        if ent_id in entity_arg1[cur_ele]:    
+                            ner_text+=token_text[text_sid:ent_sid]+' '+ENTITY_TAG['arg1'][0]+' '+ent_text+ ' '+ENTITY_TAG['arg1'][1]+' '
+                        else:
+                            if ent_id in rel_ent2: #arg2 entity
+                                if ent_type!=REL_ENT['arg2']:
+                                    pass
+                                    # print('arg2 is error! not ',REL_ENT['arg2'], ele_nonest)
+                                ner_text+=token_text[text_sid:ent_sid]+' '+ENTITY_TAG['arg2'][0]+' '+ent_text+ ' '+ENTITY_TAG['arg2'][1]+' '
+                            else:
+                                ner_text+=token_text[text_sid:ent_sid]+' '+ENTITY_TAG[ent_type.lower()][0]+' '+ent_text+ ' '+ENTITY_TAG[ent_type.lower()][1]+' '
+                        text_sid=ent_eid  
+                    else:
+                        pass
+                        # print('ner entity error!!!',ele_nonest,text_sid)                                    
+                ner_text+=token_text[text_sid:]
+                sen_tokens=ner_text.split()
+                # print('\nner_text:',ner_text)
+                
+                #3 produce pos input
+ 
+                temp_input=[]
+                token_id=0
+                while token_id <len(sen_tokens):
+                    if sen_tokens[token_id].find(ENTITY_TAG['arg1'][0])>=0:
+                        temp_input.append(ENTITY_TAG['arg1'][0]+'\tO')
+                        token_id+=1
+                        while(sen_tokens[token_id]!=ENTITY_TAG['arg1'][1]):
+                            temp_input.append(sen_tokens[token_id]+'\tO')
+                            token_id+=1
+                        temp_input.append(ENTITY_TAG['arg1'][1]+'\tO')
+                    elif sen_tokens[token_id].find(ENTITY_TAG['arg2'][0])>=0:
+                        temp_input.append(ENTITY_TAG[REL_ENT['arg2'].lower()][0]+'\tARG2')
+                        token_id+=1
+                        while(sen_tokens[token_id]!=ENTITY_TAG['arg2'][1]):
+                            temp_input.append(sen_tokens[token_id]+'\tARG2')
+                            token_id+=1
+                        temp_input.append(ENTITY_TAG[REL_ENT['arg2'].lower()][1]+'\tARG2')
+                    elif sen_tokens[token_id].find(ENTITY_TAG['gene'][0])>=0:
+                        temp_input.append(ENTITY_TAG['gene'][0]+'\tO')
+                        token_id+=1
+                        while(sen_tokens[token_id]!=ENTITY_TAG['gene'][1]):
+                            temp_input.append(sen_tokens[token_id]+'\tO')
+                            token_id+=1
+                        temp_input.append(ENTITY_TAG['gene'][1]+'\tO')
+                    elif sen_tokens[token_id].find(ENTITY_TAG['species'][0])>=0:
+                        temp_input.append(ENTITY_TAG['species'][0]+'\tO')
+                        token_id+=1
+                        while(sen_tokens[token_id]!=ENTITY_TAG['species'][1]):
+                            temp_input.append(sen_tokens[token_id]+'\tO')
+                            token_id+=1
+                        temp_input.append(ENTITY_TAG['species'][1]+'\tO')
+                    else:
+                        if sen_tokens[token_id]=='':
+                            # print('token is none!error!')
+                            pass
+                        else:
+                            temp_input.append(sen_tokens[token_id]+'\tO')
+                    token_id+=1
+                    
+                final_input.append('\n'.join(temp_input))
+                
+            else:     #neg instance 
+                ner_text=''
+                text_sid=0
+                #print('nonest:',entity_nonest)
+                for ele_nonest in entity_all:
+                    ent_id=[ele_nonest[1],ele_nonest[2]]
+                    ent_sid=int(ele_nonest[1])
+                    ent_eid=int(ele_nonest[2])
+                    # print('sid,eid:',ent_sid,ent_eid)
+                    ent_text=ele_nonest[3]
+                    ent_type=ele_nonest[4]
+                    if ent_sid>=text_sid:
+                        if ent_id in entity_arg1[cur_ele]:    
+                            ner_text+=token_text[text_sid:ent_sid]+' '+ENTITY_TAG['arg1'][0]+' '+ent_text+ ' '+ENTITY_TAG['arg1'][1]+' '
+                        else:
+                            ner_text+=token_text[text_sid:ent_sid]+' '+ENTITY_TAG[ent_type.lower()][0]+' '+ent_text+ ' '+ENTITY_TAG[ent_type.lower()][1]+' '
+                        text_sid=ent_eid  
+                    else:
+                        pass
+                        # print('ner entity error!!!')                                    
+                ner_text+=token_text[text_sid:]
+                sen_tokens=ner_text.split()
+                # print('\nner_text:',ner_text)
+                # print('ner_Text')
+                #3 produce NEG input
+                   
+                temp_input=[]
+                token_id=0
+                while token_id <len(sen_tokens):
+                    if sen_tokens[token_id].find(ENTITY_TAG['arg1'][0])>=0:
+                        temp_input.append(ENTITY_TAG['arg1'][0]+'\tO')
+                        token_id+=1
+                        while(sen_tokens[token_id]!=ENTITY_TAG['arg1'][1]):
+                            temp_input.append(sen_tokens[token_id]+'\tO')
+                            token_id+=1
+                        temp_input.append(ENTITY_TAG['arg1'][1]+'\tO')
+                    elif sen_tokens[token_id].find(ENTITY_TAG['gene'][0])>=0:
+                        temp_input.append(ENTITY_TAG['gene'][0]+'\tO')
+                        token_id+=1
+                        while(sen_tokens[token_id]!=ENTITY_TAG['gene'][1]):
+                            temp_input.append(sen_tokens[token_id]+'\tO')
+                            token_id+=1
+                        temp_input.append(ENTITY_TAG['gene'][1]+'\tO')
+                    elif sen_tokens[token_id].find(ENTITY_TAG['species'][0])>=0:
+                        temp_input.append(ENTITY_TAG['species'][0]+'\tO')
+                        token_id+=1
+                        while(sen_tokens[token_id]!=ENTITY_TAG['species'][1]):
+                            temp_input.append(sen_tokens[token_id]+'\tO')
+                            token_id+=1
+                        temp_input.append(ENTITY_TAG['species'][1]+'\tO')
+                    else:
+                        if sen_tokens[token_id]=='':
+                            print('token is none!error!')
+                        else:
+                            temp_input.append(sen_tokens[token_id]+'\tO')
+                    token_id+=1
+                    
+                final_input.append('\n'.join(temp_input))
+                # print(entity_nonest)
+        # sys.exit()
+    fout.write('\n\n'.join(final_input))
+    fout.write('\n')
+    fout.close()   
+
+def check_entity_pos(line,relations):
+    
+    seg=line.split(' ')
+    stack_ent=[] 
+    # print(seg)
+    entity_num={'arg1':0,'arg2':0, 'gene':0,'chemical':0}
+
+    temp_arg2=[]
+    for i in range(0,len(seg)):
+        if seg[i].find(ENTITY_TAG['gene'][0])>=0:
+            entity_num['gene']+=1
+            stack_ent.append(seg[i])
+        elif seg[i].find(ENTITY_TAG['chemical'][0])>=0:
+            entity_num['chemical']+=1
+            stack_ent.append(seg[i])
+            # print(stack_ent)
+        elif seg[i].find(ENTITY_TAG['arg1'][0])>=0:
+            entity_num['arg1']+=1
+            stack_ent.append(seg[i]) 
+        elif seg[i].find(ENTITY_TAG['arg2'][0])>=0:
+            entity_num['arg2']+=1
+            temp_arg2.append(seg[i].split('|')[0])
+            stack_ent.append(seg[i])
+        elif seg[i].find(ENTITY_TAG['arg1'][1])>=0 or seg[i].find(ENTITY_TAG['arg2'][1])>=0 or seg[i].find(ENTITY_TAG['gene'][1])>=0 or seg[i].find(ENTITY_TAG['chemical'][1])>=0:
+            stack_ent.pop()
+    if stack_ent!=[]:
+        # print('entity no match!',stack_ent)
+        return(-1,seg,entity_num)
+    
+    else:
+        if entity_num['arg1']!=0:
+            for arg2_id in relations.keys():
+                if arg2_id not in temp_arg2:
+                    # print('\ntemp_arg2:',temp_arg2)
+                    # print('\narg2_id:',arg2_id)
+                    return(0,seg,entity_num) #some arg2 not in sentence
+        if entity_num['arg2']!=0 and entity_num['arg1']==0:
+            return(0,seg,entity_num) #only arg2, but no arg1
+        return(1,seg,entity_num) 
+
+def check_entity_neg(line):
+    
+    seg=line.split(' ')
+    stack_ent=[] 
+    # print(seg)
+    entity_num={'arg1':0,'gene':0,'chemical':0}
+    for i in range(0,len(seg)):
+        if seg[i].find(ENTITY_TAG['gene'][0])>=0:
+            entity_num['gene']+=1
+            stack_ent.append(seg[i])
+        elif seg[i].find(ENTITY_TAG['chemical'][0])>=0:
+            entity_num['chemical']+=1
+            stack_ent.append(seg[i])
+            # print(stack_ent)
+        elif seg[i].find(ENTITY_TAG['arg1'][0])>=0:
+            entity_num['arg1']+=1
+            stack_ent.append(seg[i])        
+        elif seg[i].find(ENTITY_TAG['arg1'][1])>=0  or seg[i].find(ENTITY_TAG['gene'][1])>=0 or seg[i].find(ENTITY_TAG['chemical'][1])>=0:
+            stack_ent.pop()
+    if stack_ent!=[]:
+        # print('entity no match!',stack_ent)
+        return(-1,seg,entity_num)
+    
+    else:
+        return(1,seg,entity_num) 
+
+def get_one_entity(nest_list,cur_ent,rel_entity2_id):
+    max_len=0
+    max_entity=[]
+    final_entity=[]
+    for i in range(0, len(nest_list)):
+        if nest_list[i][1]==cur_ent:#current entity
+            final_entity=[]
+            max_entity=nest_list[i]
+            final_entity.append(nest_list[i])
+            return(final_entity)
+        if nest_list[i][1] in rel_entity2_id: #invole rel
+            final_entity.append(nest_list[i])
+            continue
+        length=int(nest_list[i][4])-int(nest_list[i][3])
+        if max_entity==[]: #first entity
+            max_len=length
+            max_entity=nest_list[i]
+        else:
+            if length>max_len:
+                if max_entity[2]==REL_ENT['arg1']:
+                    max_len=length
+                    max_entity=nest_list[i]
+                else:
+                    if nest_list[i][2]==REL_ENT['arg2'] and max_entity[1] not in rel_entity2_id:
+                        max_len=length
+                        max_entity=nest_list[i] 
+                        
+            else:
+                if nest_list[i][1] in rel_entity2_id:
+                    max_len=length
+                    max_entity=nest_list[i] 
+                elif max_entity[2]==REL_ENT['arg1'] and nest_list[i][2]==REL_ENT['arg2']:
+                    max_len=length
+                    max_entity=nest_list[i]
+    if final_entity==[]:
+        final_entity.append(max_entity)
+    return final_entity
+
+if __name__=='__main__':
+    
+    infile='../../TrainingSet/No505/SA.Train.txt'
+    outfile='../../TrainingSet/No505/SA.Train.conll'
+    
+    #tokenizer
+    token_input=ssplit_token(infile)
+
+    #filter nest entity
+    nonest_input=corpus_noNest(token_input)
+
+    # to conll
     generate_seq_input(nonest_input,outfile)

src_python/SpeAss/ml_tagging_score_sa.py

@@ -1,220 +1,220 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Jan  7 09:29:46 2022
-
-@author: luol2
-
-machine learning tagging
-
-"""
-
-
-import time
-import io
-
-from src_python.SpeAss.processing_data_sa import ml_intext_fn,out_BIO_BERT_softmax_score_fn
-import tensorflow as tf
-gpu = tf.config.list_physical_devices('GPU')
-print("Num GPUs Available: ", len(gpu))
-if len(gpu) > 0:
-    tf.config.experimental.set_memory_growth(gpu[0], True)
-#tf.compat.v1.disable_eager_execution()
-
-REL_ENT={'arg1':'Species',
-         'arg2':'Gene'}
-
-entity_tag={'arg1':['arg1s','arg1e'],
-            'gene':['gene1s','gene1e'],
-            'species':['species1s','species1e']
-    }
-
-def input_preprocess_notoken(doc_text):
-    final_input=[]
-    final_id=[]
-    
-    lines=doc_text.split('\n')
-    token_text=lines[0]
-    pmid=lines[1].split('\t')[0]
-    entity_arg1={} #{species_id:[[spe_sid1,sep_eid1],[...]]}
-    entity_all=[]
-    for i in range(1,len(lines)):
-        seg=lines[i].split('\t')
-        if seg[6]==REL_ENT['arg1']:
-            if seg[-1] in entity_arg1.keys():
-                entity_arg1[seg[-1]].append([seg[3],seg[4]])
-            else:
-                entity_arg1[seg[-1]]=[[seg[3],seg[4]]]
-        entity_all.append(seg)
-    
-    #print(token_text)
-    #print(entity_chemical)
-    #generate input instance
-    for cur_ele in entity_arg1:
-        
-        #2. ner label text
-        ner_text=''
-        text_sid=0
-        #print('nonest:',entity_nonest)
-        for ele_nonest in entity_all:
-            ent_id=[ele_nonest[3],ele_nonest[4]]
-            ent_spe_id=ele_nonest[-1]
-            ent_sid=int(ele_nonest[3])
-            ent_eid=int(ele_nonest[4])
-            # print('sid,eid:',ent_sid,ent_eid)
-            ent_text=ele_nonest[5]
-            ent_type=ele_nonest[6]
-            if ent_sid>=text_sid:
-                # if token_text[ent_sid:ent_eid]!=ent_text:
-                #     print('error!index_text,entext:',token_text[ent_sid:ent_eid],ent_text)
-                if ent_id in entity_arg1[cur_ele]: #is species    
-                    ner_text+=token_text[text_sid:ent_sid]+' '+ent_spe_id+'|'+entity_tag['arg1'][0]+' '+ent_text+' '+entity_tag['arg1'][1]+' '
-                else:
-                    ner_text+=token_text[text_sid:ent_sid]+' '+str(ent_sid)+'-'+str(ent_eid)+'|'+entity_tag[ent_type.lower()][0]+' '+ent_text+' '+entity_tag[ent_type.lower()][1]+' '
-                text_sid=ent_eid                                      
-        ner_text+=token_text[text_sid:]
-        sen_tokens=ner_text.split()
-        #print('\nner_text:',ner_text)
-        
-        #3. produce input
-        temp_input=[]
-        temp_id={'species':'','gene':[]}
-        for sen_token in sen_tokens:
-            if sen_token.find(entity_tag['arg1'][0])>=0:
-                en_id=sen_token.split('|')[0]
-                temp_id['species']=en_id
-                temp_input.append(entity_tag['arg1'][0]+'\tO')
-            elif sen_token.find(entity_tag['gene'][0])>=0:
-                en_id=sen_token.split('|')[0]
-                temp_id['gene'].append(en_id)
-                temp_input.append(entity_tag['gene'][0]+'\tO')
-            elif sen_token.find(entity_tag['species'][0])>=0:
-                en_id=sen_token.split('|')[0]
-                # temp_id.append(en_id)
-                temp_input.append(entity_tag['species'][0]+'\tO')
-            else:
-                if sen_token=='':
-                    # print('token is none!error!')
-                    pass
-                else:
-                    temp_input.append(sen_token+'\tO')
-        final_input.append('\n'.join(temp_input))
-        final_id.append(temp_id)
-
-        # print(entity_nonest)
-    return final_input,final_id,entity_all,pmid
-
-
-def ml_tagging(ml_input,nn_model):
-
-    test_set,test_label = ml_intext_fn(ml_input)
-    test_x,test_y, test_bert_text_label=nn_model.rep.load_data_hugface(test_set,test_label,word_max_len=nn_model.maxlen,label_type='softmax')
-    test_pre = nn_model.model.predict(test_x)
-    ml_out=out_BIO_BERT_softmax_score_fn(test_pre,test_bert_text_label,nn_model.rep.index_2_label)
-    return ml_out
-
-def output_rel(ml_output,entity_map,pmid):
-    fin=io.StringIO(ml_output)
-    alltexts=fin.read().strip().split('\n\n')
-    fin.close()
-    final_out={} #{'sid-eid':[spechies id]}
-    for sen_id,sentence in enumerate(alltexts):
-        tokens=sentence.split('\n')
-        gene_entity_id=0
-        token_id=0
-        arg1=''
-        arg2_list=[] #[[ID, score],[id,score]]
-        while (token_id<len(tokens)):
-            seg=tokens[token_id].split('\t')
-            if seg[0]==entity_tag['arg1'][0]:
-                arg1=entity_map[sen_id]['species']
-                token_id+=1
-                if token_id >=len(tokens):
-                    break
-                seg=tokens[token_id].split('\t')
-                while seg[0]!=entity_tag['arg1'][1]:
-                    token_id+=1
-                    if token_id >=len(tokens):
-                        break
-                    seg=tokens[token_id].split('\t')
-            elif seg[0]==entity_tag[REL_ENT['arg2'].lower()][0]:
-                temp_rel=seg[-2]
-                temp_score=seg[-1]
-                arg2_id=entity_map[sen_id]['gene'][gene_entity_id]
-                gene_entity_id+=1
-                token_id+=1
-                if token_id >=len(tokens):
-                    break
-                seg=tokens[token_id].split('\t')
-                while seg[0]!=entity_tag[REL_ENT['arg2'].lower()][1]:
-                    token_id+=1
-                    if token_id >=len(tokens):
-                        break
-                    seg=tokens[token_id].split('\t')
-                    if seg[-2].find('ARG2')>=0 and temp_rel.find('ARG2')<0:
-                        temp_rel=seg[-2]
-                        temp_score=seg[-1]
-                if temp_rel.find('ARG2')>=0:
-                    arg2_list.append([arg2_id,temp_score])
-            elif seg[0]==entity_tag[REL_ENT['arg1'].lower()][0]:
-                token_id+=1
-                if token_id >=len(tokens):
-                    break
-                seg=tokens[token_id].split('\t')
-                while seg[0]!=entity_tag[REL_ENT['arg1'].lower()][1]:
-                    token_id+=1
-                    if token_id >=len(tokens):
-                        break
-                    seg=tokens[token_id].split('\t')
-
-            else:
-                pass
-            token_id+=1
-        #print(arg1,arg2_list)
-        if arg2_list!=[] and arg1!='':
-            for arg2_ele in arg2_list:
-                if arg2_ele[0] not in final_out.keys():
-                    final_out[arg2_ele[0]]=[arg1+'|'+arg2_ele[1]]
-                else:
-                    final_out[arg2_ele[0]].append(arg1+'|'+arg2_ele[1])
-    return(final_out)
-
-def NER_Tag(doc_in,nn_model):
-    
-    #1. preprocess input, input_text:conll格式, input_entity：相应的实体列表
-    #print(doc_in)
-    input_text,entity_index,entity_all,pmid=input_preprocess_notoken(doc_in)
-    # print('pmid:',pmid)
-    # print('\entity_index:',entity_index)
-
-   
-    #2. ml tagging
-    if input_text!=[]:
-        ml_pre=ml_tagging(input_text,nn_model)
-        #print('\noutput:')
-        #print(ml_pre)
-
-        #3.generate output
-        final_output=output_rel(ml_pre,entity_index,pmid)
-    else:
-        final_output={}
-    return final_output,entity_all
-    
-    
-
-
-    
-    
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jan  7 09:29:46 2022
+
+@author: luol2
+
+machine learning tagging
+
+"""
+
+
+import time
+import io
+
+from src_python.SpeAss.processing_data_sa import ml_intext_fn,out_BIO_BERT_softmax_score_fn
+import tensorflow as tf
+gpu = tf.config.list_physical_devices('GPU')
+print("Num GPUs Available: ", len(gpu))
+if len(gpu) > 0:
+    tf.config.experimental.set_memory_growth(gpu[0], True)
+#tf.compat.v1.disable_eager_execution()
+
+REL_ENT={'arg1':'Species',
+         'arg2':'Gene'}
+
+entity_tag={'arg1':['arg1s','arg1e'],
+            'gene':['gene1s','gene1e'],
+            'species':['species1s','species1e']
+    }
+
+def input_preprocess_notoken(doc_text):
+    final_input=[]
+    final_id=[]
+    
+    lines=doc_text.split('\n')
+    token_text=lines[0]
+    pmid=lines[1].split('\t')[0]
+    entity_arg1={} #{species_id:[[spe_sid1,sep_eid1],[...]]}
+    entity_all=[]
+    for i in range(1,len(lines)):
+        seg=lines[i].split('\t')
+        if seg[6]==REL_ENT['arg1']:
+            if seg[-1] in entity_arg1.keys():
+                entity_arg1[seg[-1]].append([seg[3],seg[4]])
+            else:
+                entity_arg1[seg[-1]]=[[seg[3],seg[4]]]
+        entity_all.append(seg)
+    
+    #print(token_text)
+    #print(entity_chemical)
+    #generate input instance
+    for cur_ele in entity_arg1:
+        
+        #2. ner label text
+        ner_text=''
+        text_sid=0
+        #print('nonest:',entity_nonest)
+        for ele_nonest in entity_all:
+            ent_id=[ele_nonest[3],ele_nonest[4]]
+            ent_spe_id=ele_nonest[-1]
+            ent_sid=int(ele_nonest[3])
+            ent_eid=int(ele_nonest[4])
+            # print('sid,eid:',ent_sid,ent_eid)
+            ent_text=ele_nonest[5]
+            ent_type=ele_nonest[6]
+            if ent_sid>=text_sid:
+                # if token_text[ent_sid:ent_eid]!=ent_text:
+                #     print('error!index_text,entext:',token_text[ent_sid:ent_eid],ent_text)
+                if ent_id in entity_arg1[cur_ele]: #is species    
+                    ner_text+=token_text[text_sid:ent_sid]+' '+ent_spe_id+'|'+entity_tag['arg1'][0]+' '+ent_text+' '+entity_tag['arg1'][1]+' '
+                else:
+                    ner_text+=token_text[text_sid:ent_sid]+' '+str(ent_sid)+'-'+str(ent_eid)+'|'+entity_tag[ent_type.lower()][0]+' '+ent_text+' '+entity_tag[ent_type.lower()][1]+' '
+                text_sid=ent_eid                                      
+        ner_text+=token_text[text_sid:]
+        sen_tokens=ner_text.split()
+        #print('\nner_text:',ner_text)
+        
+        #3. produce input
+        temp_input=[]
+        temp_id={'species':'','gene':[]}
+        for sen_token in sen_tokens:
+            if sen_token.find(entity_tag['arg1'][0])>=0:
+                en_id=sen_token.split('|')[0]
+                temp_id['species']=en_id
+                temp_input.append(entity_tag['arg1'][0]+'\tO')
+            elif sen_token.find(entity_tag['gene'][0])>=0:
+                en_id=sen_token.split('|')[0]
+                temp_id['gene'].append(en_id)
+                temp_input.append(entity_tag['gene'][0]+'\tO')
+            elif sen_token.find(entity_tag['species'][0])>=0:
+                en_id=sen_token.split('|')[0]
+                # temp_id.append(en_id)
+                temp_input.append(entity_tag['species'][0]+'\tO')
+            else:
+                if sen_token=='':
+                    # print('token is none!error!')
+                    pass
+                else:
+                    temp_input.append(sen_token+'\tO')
+        final_input.append('\n'.join(temp_input))
+        final_id.append(temp_id)
+
+        # print(entity_nonest)
+    return final_input,final_id,entity_all,pmid
+
+
+def ml_tagging(ml_input,nn_model):
+
+    test_set,test_label = ml_intext_fn(ml_input)
+    test_x,test_y, test_bert_text_label=nn_model.rep.load_data_hugface(test_set,test_label,word_max_len=nn_model.maxlen,label_type='softmax')
+    test_pre = nn_model.model.predict(test_x)
+    ml_out=out_BIO_BERT_softmax_score_fn(test_pre,test_bert_text_label,nn_model.rep.index_2_label)
+    return ml_out
+
+def output_rel(ml_output,entity_map,pmid):
+    fin=io.StringIO(ml_output)
+    alltexts=fin.read().strip().split('\n\n')
+    fin.close()
+    final_out={} #{'sid-eid':[spechies id]}
+    for sen_id,sentence in enumerate(alltexts):
+        tokens=sentence.split('\n')
+        gene_entity_id=0
+        token_id=0
+        arg1=''
+        arg2_list=[] #[[ID, score],[id,score]]
+        while (token_id<len(tokens)):
+            seg=tokens[token_id].split('\t')
+            if seg[0]==entity_tag['arg1'][0]:
+                arg1=entity_map[sen_id]['species']
+                token_id+=1
+                if token_id >=len(tokens):
+                    break
+                seg=tokens[token_id].split('\t')
+                while seg[0]!=entity_tag['arg1'][1]:
+                    token_id+=1
+                    if token_id >=len(tokens):
+                        break
+                    seg=tokens[token_id].split('\t')
+            elif seg[0]==entity_tag[REL_ENT['arg2'].lower()][0]:
+                temp_rel=seg[-2]
+                temp_score=seg[-1]
+                arg2_id=entity_map[sen_id]['gene'][gene_entity_id]
+                gene_entity_id+=1
+                token_id+=1
+                if token_id >=len(tokens):
+                    break
+                seg=tokens[token_id].split('\t')
+                while seg[0]!=entity_tag[REL_ENT['arg2'].lower()][1]:
+                    token_id+=1
+                    if token_id >=len(tokens):
+                        break
+                    seg=tokens[token_id].split('\t')
+                    if seg[-2].find('ARG2')>=0 and temp_rel.find('ARG2')<0:
+                        temp_rel=seg[-2]
+                        temp_score=seg[-1]
+                if temp_rel.find('ARG2')>=0:
+                    arg2_list.append([arg2_id,temp_score])
+            elif seg[0]==entity_tag[REL_ENT['arg1'].lower()][0]:
+                token_id+=1
+                if token_id >=len(tokens):
+                    break
+                seg=tokens[token_id].split('\t')
+                while seg[0]!=entity_tag[REL_ENT['arg1'].lower()][1]:
+                    token_id+=1
+                    if token_id >=len(tokens):
+                        break
+                    seg=tokens[token_id].split('\t')
+
+            else:
+                pass
+            token_id+=1
+        #print(arg1,arg2_list)
+        if arg2_list!=[] and arg1!='':
+            for arg2_ele in arg2_list:
+                if arg2_ele[0] not in final_out.keys():
+                    final_out[arg2_ele[0]]=[arg1+'|'+arg2_ele[1]]
+                else:
+                    final_out[arg2_ele[0]].append(arg1+'|'+arg2_ele[1])
+    return(final_out)
+
+def NER_Tag(doc_in,nn_model):
+    
+    #1. preprocess input, input_text:conll格式, input_entity：相应的实体列表
+    #print(doc_in)
+    input_text,entity_index,entity_all,pmid=input_preprocess_notoken(doc_in)
+    # print('pmid:',pmid)
+    # print('\entity_index:',entity_index)
+
+   
+    #2. ml tagging
+    if input_text!=[]:
+        ml_pre=ml_tagging(input_text,nn_model)
+        #print('\noutput:')
+        #print(ml_pre)
+
+        #3.generate output
+        final_output=output_rel(ml_pre,entity_index,pmid)
+    else:
+        final_output={}
+    return final_output,entity_all
+    
+    
+
+
+    
+    
+
+
+
+
+
+
+
+
+
+
+
+
+
+

src_python/SpeAss/model_sa.py

@@ -1,105 +1,105 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Wed Feb 10 09:08:09 2021
-
-@author: luol2
-
-Model Architecture
-
-"""
-import tensorflow as tf
-from src_python.SpeAss.represent_sa import Hugface_RepresentationLayer
-from tensorflow.keras.layers import *
-from tensorflow.keras.models import Model
-from tensorflow.keras.optimizers import RMSprop, SGD, Adam, Adadelta, Adagrad,Nadam
-from transformers import TFAutoModel
-import numpy as np
-import sys
-
-
-class LRSchedule_LINEAR(tf.keras.optimizers.schedules.LearningRateSchedule):
-    def __init__(
-        self,
-        init_lr=5e-5,
-        init_warmup_lr=0.0,
-        final_lr=5e-7,
-        warmup_steps=0,
-        decay_steps=0,
-    ):
-        super().__init__()
-        self.init_lr = init_lr
-        self.init_warmup_lr=init_warmup_lr
-        self.final_lr = final_lr
-        self.warmup_steps = warmup_steps
-        self.decay_steps = decay_steps
-        
-    def __call__(self, step):
-        """ linear warm up - linear decay """
-        if self.warmup_steps>0:
-            warmup_lr = (self.init_lr - self.init_warmup_lr)/self.warmup_steps * step+self.init_warmup_lr
-        else:
-            warmup_lr=1000.0
-        #print('\n.......warmup_lr:',warmup_lr)
-        decay_lr = tf.math.maximum(
-            self.final_lr,
-            self.init_lr - (step - self.warmup_steps)/self.decay_steps*(self.init_lr - self.final_lr)
-        )
-        #print('\n.....decay_lr:',decay_lr)
-        return tf.math.minimum(warmup_lr,decay_lr)
-
-       
-        
-class HUGFACE_NER(): #huggingface transformers
-    def __init__(self, model_files):
-        self.model_type='HUGFACE'
-        self.maxlen = 512  
-        self.checkpoint_path = model_files['checkpoint_path']
-        self.label_file=model_files['labelfile']
-        self.lowercase=model_files['lowercase']
-        self.rep = Hugface_RepresentationLayer(self.checkpoint_path, self.label_file, lowercase=self.lowercase)
-        
-            
-    def build_encoder(self):
-        print('...vocab len:',self.rep.vocab_len)
-        plm_model = TFAutoModel.from_pretrained(self.checkpoint_path, from_pt=True)
-        plm_model.resize_token_embeddings(self.rep.vocab_len) 
-        x1_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='input_ids')
-        x2_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='token_type_ids')
-        x3_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='attention_mask')
-        x = plm_model(x1_in, token_type_ids=x2_in, attention_mask=x3_in)[0]
-        #dense = TimeDistributed(Dense(512, activation='relu'), name='dense1')(x)
-        self.encoder = Model (inputs=[x1_in,x2_in,x3_in], outputs=x,name='hugface_encoder')
-        self.encoder.summary()
-        
-    def build_softmax_decoder(self):
-
-        x1_in = Input(shape=(self.maxlen,),dtype=tf.int32)
-        x2_in = Input(shape=(self.maxlen,),dtype=tf.int32)
-        x3_in = Input(shape=(self.maxlen,),dtype=tf.int32)  
-        features = self.encoder([x1_in,x2_in,x3_in])
-        #features = Dropout(0.4)(features)
-        #features = TimeDistributed(Dense(128, activation='relu'), name='dense2')(features)
-        features= Dropout(0.1)(features)
-        output = TimeDistributed(Dense(self.rep.label_table_size, activation='softmax'), name='softmax')(features)
-        self.model = Model(inputs=[x1_in,x2_in,x3_in], outputs=output, name="hugface_softmax")
-
-        # lr_schedule=LRSchedule_LINEAR(
-        #     init_lr=1e-5,
-        #     init_warmup_lr=1e-7,
-        #     final_lr=1e-6,
-        #     warmup_steps=0,
-        #     decay_steps=40000)
-
-        opt = Adam(learning_rate = 5e-6)
-        self.model.compile(
-            optimizer=opt,
-            loss='sparse_categorical_crossentropy',
-            metrics=['accuracy'],
-        )
-        self.model.summary()
-        
-        
-    def load_model(self,model_file):
-        self.model.load_weights(model_file)
-        self.model.summary()  
-        print('load HUGFACE model done!')
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Feb 10 09:08:09 2021
+
+@author: luol2
+
+Model Architecture
+
+"""
+import tensorflow as tf
+from src_python.SpeAss.represent_sa import Hugface_RepresentationLayer
+from tensorflow.keras.layers import *
+from tensorflow.keras.models import Model
+from tensorflow.keras.optimizers import RMSprop, SGD, Adam, Adadelta, Adagrad,Nadam
+from transformers import TFAutoModel
+import numpy as np
+import sys
+
+
+class LRSchedule_LINEAR(tf.keras.optimizers.schedules.LearningRateSchedule):
+    def __init__(
+        self,
+        init_lr=5e-5,
+        init_warmup_lr=0.0,
+        final_lr=5e-7,
+        warmup_steps=0,
+        decay_steps=0,
+    ):
+        super().__init__()
+        self.init_lr = init_lr
+        self.init_warmup_lr=init_warmup_lr
+        self.final_lr = final_lr
+        self.warmup_steps = warmup_steps
+        self.decay_steps = decay_steps
+        
+    def __call__(self, step):
+        """ linear warm up - linear decay """
+        if self.warmup_steps>0:
+            warmup_lr = (self.init_lr - self.init_warmup_lr)/self.warmup_steps * step+self.init_warmup_lr
+        else:
+            warmup_lr=1000.0
+        #print('\n.......warmup_lr:',warmup_lr)
+        decay_lr = tf.math.maximum(
+            self.final_lr,
+            self.init_lr - (step - self.warmup_steps)/self.decay_steps*(self.init_lr - self.final_lr)
+        )
+        #print('\n.....decay_lr:',decay_lr)
+        return tf.math.minimum(warmup_lr,decay_lr)
+
+       
+        
+class HUGFACE_NER(): #huggingface transformers
+    def __init__(self, model_files):
+        self.model_type='HUGFACE'
+        self.maxlen = 512  
+        self.checkpoint_path = model_files['checkpoint_path']
+        self.label_file=model_files['labelfile']
+        self.lowercase=model_files['lowercase']
+        self.rep = Hugface_RepresentationLayer(self.checkpoint_path, self.label_file, lowercase=self.lowercase)
+        
+            
+    def build_encoder(self):
+        print('...vocab len:',self.rep.vocab_len)
+        plm_model = TFAutoModel.from_pretrained(self.checkpoint_path, from_pt=True)
+        plm_model.resize_token_embeddings(self.rep.vocab_len) 
+        x1_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='input_ids')
+        x2_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='token_type_ids')
+        x3_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='attention_mask')
+        x = plm_model(x1_in, token_type_ids=x2_in, attention_mask=x3_in)[0]
+        #dense = TimeDistributed(Dense(512, activation='relu'), name='dense1')(x)
+        self.encoder = Model (inputs=[x1_in,x2_in,x3_in], outputs=x,name='hugface_encoder')
+        self.encoder.summary()
+        
+    def build_softmax_decoder(self):
+
+        x1_in = Input(shape=(self.maxlen,),dtype=tf.int32)
+        x2_in = Input(shape=(self.maxlen,),dtype=tf.int32)
+        x3_in = Input(shape=(self.maxlen,),dtype=tf.int32)  
+        features = self.encoder([x1_in,x2_in,x3_in])
+        #features = Dropout(0.4)(features)
+        #features = TimeDistributed(Dense(128, activation='relu'), name='dense2')(features)
+        features= Dropout(0.1)(features)
+        output = TimeDistributed(Dense(self.rep.label_table_size, activation='softmax'), name='softmax')(features)
+        self.model = Model(inputs=[x1_in,x2_in,x3_in], outputs=output, name="hugface_softmax")
+
+        # lr_schedule=LRSchedule_LINEAR(
+        #     init_lr=1e-5,
+        #     init_warmup_lr=1e-7,
+        #     final_lr=1e-6,
+        #     warmup_steps=0,
+        #     decay_steps=40000)
+
+        opt = Adam(learning_rate = 5e-6)
+        self.model.compile(
+            optimizer=opt,
+            loss='sparse_categorical_crossentropy',
+            metrics=['accuracy'],
+        )
+        self.model.summary()
+        
+        
+    def load_model(self,model_file):
+        self.model.load_weights(model_file)
+        self.model.summary()  
+        print('load HUGFACE model done!')

src_python/SpeAss/processing_data_sa.py

@@ -1,201 +1,201 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Tue Mar 10 16:34:12 2020
-
-@author: luol2
-"""
-import numpy as np
-import io
-import sys
-#read ner text (word\tlabel), generate the list[[[w1,label],[w2,label]]]
-def ml_intext(file):
-    fin=open(file,'r',encoding='utf-8')
-    alltexts=fin.read().strip().split('\n\n')
-    fin.close()
-    data_list=[]
-    label_list=[]
-
-    for sents in alltexts:
-        lines=sents.split('\n')
-        temp_sentece=[]
-        for i in range(0,len(lines)):
-            seg=lines[i].split('\t')
-            temp_sentece.append(seg[:])
-            label_list.append(seg[-1])
-        
-        data_list.append(temp_sentece)
-    #print(data_list)
-    #print(label_list)
-    return data_list,label_list
-
-def ml_intext_fn(alltexts):
-    # fin=io.StringIO(ml_input)
-    # alltexts=fin.read().strip().split('\n\n')
-    # fin.close()
-    data_list=[]
-    label_list=[]
-
-    for sents in alltexts:
-        lines=sents.split('\n')
-        temp_sentece=[]
-        for i in range(0,len(lines)):
-            seg=lines[i].split('\t')
-            temp_sentece.append(seg[:])
-            label_list.append(seg[-1])
-        
-        data_list.append(temp_sentece)
-    #print(data_list)
-    #print(label_list)
-    return data_list,label_list
-
-# model predict result to conll evalute format  [token answer predict]        
-def out_BIO(file,raw_pre,raw_input,label_set):
-    fout=open(file,'w',encoding='utf-8')
-    for i in range(len(raw_input)):
-        
-        for j in range(len(raw_input[i])):
-            if j<len(raw_pre[i]):
-                label_id = raw_pre[i][j]
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    fout.close()
-    
-def out_BIO_softmax(file,raw_pre,raw_input,label_set):
-    fout=open(file,'w',encoding='utf-8')
-    #print(raw_pre[0:2])
-    for i in range(len(raw_input)):
-        
-        for j in range(len(raw_input[i])):
-            if j<len(raw_pre[i]):
-                label_id = np.argmax(raw_pre[i][j])
-                #print(label_id)
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    fout.close()
-    
-def out_BIO_fn(raw_pre,raw_input,label_set):
-    fout=io.StringIO()
-    for i in range(len(raw_input)):
-        
-        for j in range(len(raw_input[i])):
-            if j<len(raw_pre[i]):
-                label_id = raw_pre[i][j] 
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    return fout.getvalue()  
-def out_BIO_BERT_softmax(file,raw_pre,raw_input,label_set):
-    fout=open(file,'w',encoding='utf-8')
-    for i in range(len(raw_input)):
-        
-        for j in range(len(raw_input[i])):
-            if j<len(raw_pre[i]):
-                # label_id = raw_pre[i][j]
-                label_id = np.argmax(raw_pre[i][j])
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    fout.close() 
-def out_BIO_BERT(file,raw_pre,raw_input,label_set):
-    fout=open(file,'w',encoding='utf-8')
-    for i in range(len(raw_input)):
-        
-        for j in range(len(raw_input[i])):
-            if j<len(raw_pre[i]):
-                label_id = raw_pre[i][j] 
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    fout.close() 
-def out_BIO_BERT_fn(raw_pre,raw_input,label_set):
-    fout=io.StringIO()
-    for i in range(len(raw_input)):
-        
-        for j in range(len(raw_input[i])):
-            if j<len(raw_pre[i]):
-                label_id = raw_pre[i][j] 
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    return fout.getvalue()                  
-def out_BIO_BERT_softmax_fn(raw_pre,raw_input,label_set):
-    fout=io.StringIO()
-    for i in range(len(raw_input)):
-
-        for j in range(len(raw_input[i])):
-            if j<len(raw_pre[i]):
-                #label_id = raw_pre[i][j]
-                label_id = np.argmax(raw_pre[i][j])
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
-        fout.write('\n')
-    return fout.getvalue()
-def out_BIO_BERT_softmax_score_fn(raw_pre,raw_input,label_set):
-    fout=io.StringIO()
-    for i in range(len(raw_input)):
-
-        for j in range(len(raw_input[i])):
-            if j<len(raw_pre[i]):
-                #label_id = raw_pre[i][j]
-                label_id = np.argmax(raw_pre[i][j])
-                label_score = round(raw_pre[i][j][label_id],4)
-                label_tag = label_set[str(label_id)]
-            else:
-                label_tag='O'
-                label_score = 0.0
-            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\t'+str(label_score)+'\n')
-        fout.write('\n')
-    return fout.getvalue()
-#generate char vocab
-def char_vocab(infile,outfile_char):
-    fin=open(infile,'r',encoding='utf-8')
-    #fout=open(outfile,'w',encoding='utf-8')
-    fout_char=open(outfile_char,'w',encoding='utf-8')
-    char_vocab=['oov_char']
-    max_len=0
-    for line in fin:
-        if line.strip()!='':
-            seg=line.split('\t')
-            word_len=len(seg[0])
-            #if word_len<1000:
-            #    fout.write(line)
-            if word_len>max_len:
-                max_len=word_len
-                print(seg[0])
-            for i in range(word_len):
-                if seg[0][i] not in char_vocab:
-                    char_vocab.append(seg[0][i])
-        #else:
-        #    fout.write(line)
-    fin.close()
-    #fout.close()
-    for ele in char_vocab:
-        fout_char.write(ele+'\n')
-    fout_char.close()
-    print('max_len:',max_len)
-
-
-if __name__=='__main__':
-    # infile='//panfs/pan1/bionlp/lulab/luoling/HPO_project/AutoPhe/data/pubmed_unlabel/mutation_disease_1990.ner_BIO'
-    # #outfile='//panfs/pan1/bionlp/lulab/luoling/HPO_project/AutoPhe/data/pubmed_unlabel/mutation_disease_1990.ner_BIO_new'
-    # outfile_char='//panfs/pan1/bionlp/lulab/luoling/HPO_project/AutoPhe/src/nn_model/vocab/char_vocab'
-    # #processing_text(file)
-    # char_vocab(infile,outfile_char)
-    a=[1,2,3]
-    print(a[:-1])
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Mar 10 16:34:12 2020
+
+@author: luol2
+"""
+import numpy as np
+import io
+import sys
+#read ner text (word\tlabel), generate the list[[[w1,label],[w2,label]]]
+def ml_intext(file):
+    fin=open(file,'r',encoding='utf-8')
+    alltexts=fin.read().strip().split('\n\n')
+    fin.close()
+    data_list=[]
+    label_list=[]
+
+    for sents in alltexts:
+        lines=sents.split('\n')
+        temp_sentece=[]
+        for i in range(0,len(lines)):
+            seg=lines[i].split('\t')
+            temp_sentece.append(seg[:])
+            label_list.append(seg[-1])
+        
+        data_list.append(temp_sentece)
+    #print(data_list)
+    #print(label_list)
+    return data_list,label_list
+
+def ml_intext_fn(alltexts):
+    # fin=io.StringIO(ml_input)
+    # alltexts=fin.read().strip().split('\n\n')
+    # fin.close()
+    data_list=[]
+    label_list=[]
+
+    for sents in alltexts:
+        lines=sents.split('\n')
+        temp_sentece=[]
+        for i in range(0,len(lines)):
+            seg=lines[i].split('\t')
+            temp_sentece.append(seg[:])
+            label_list.append(seg[-1])
+        
+        data_list.append(temp_sentece)
+    #print(data_list)
+    #print(label_list)
+    return data_list,label_list
+
+# model predict result to conll evalute format  [token answer predict]        
+def out_BIO(file,raw_pre,raw_input,label_set):
+    fout=open(file,'w',encoding='utf-8')
+    for i in range(len(raw_input)):
+        
+        for j in range(len(raw_input[i])):
+            if j<len(raw_pre[i]):
+                label_id = raw_pre[i][j]
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    fout.close()
+    
+def out_BIO_softmax(file,raw_pre,raw_input,label_set):
+    fout=open(file,'w',encoding='utf-8')
+    #print(raw_pre[0:2])
+    for i in range(len(raw_input)):
+        
+        for j in range(len(raw_input[i])):
+            if j<len(raw_pre[i]):
+                label_id = np.argmax(raw_pre[i][j])
+                #print(label_id)
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    fout.close()
+    
+def out_BIO_fn(raw_pre,raw_input,label_set):
+    fout=io.StringIO()
+    for i in range(len(raw_input)):
+        
+        for j in range(len(raw_input[i])):
+            if j<len(raw_pre[i]):
+                label_id = raw_pre[i][j] 
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    return fout.getvalue()  
+def out_BIO_BERT_softmax(file,raw_pre,raw_input,label_set):
+    fout=open(file,'w',encoding='utf-8')
+    for i in range(len(raw_input)):
+        
+        for j in range(len(raw_input[i])):
+            if j<len(raw_pre[i]):
+                # label_id = raw_pre[i][j]
+                label_id = np.argmax(raw_pre[i][j])
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    fout.close() 
+def out_BIO_BERT(file,raw_pre,raw_input,label_set):
+    fout=open(file,'w',encoding='utf-8')
+    for i in range(len(raw_input)):
+        
+        for j in range(len(raw_input[i])):
+            if j<len(raw_pre[i]):
+                label_id = raw_pre[i][j] 
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    fout.close() 
+def out_BIO_BERT_fn(raw_pre,raw_input,label_set):
+    fout=io.StringIO()
+    for i in range(len(raw_input)):
+        
+        for j in range(len(raw_input[i])):
+            if j<len(raw_pre[i]):
+                label_id = raw_pre[i][j] 
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    return fout.getvalue()                  
+def out_BIO_BERT_softmax_fn(raw_pre,raw_input,label_set):
+    fout=io.StringIO()
+    for i in range(len(raw_input)):
+
+        for j in range(len(raw_input[i])):
+            if j<len(raw_pre[i]):
+                #label_id = raw_pre[i][j]
+                label_id = np.argmax(raw_pre[i][j])
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\n')
+        fout.write('\n')
+    return fout.getvalue()
+def out_BIO_BERT_softmax_score_fn(raw_pre,raw_input,label_set):
+    fout=io.StringIO()
+    for i in range(len(raw_input)):
+
+        for j in range(len(raw_input[i])):
+            if j<len(raw_pre[i]):
+                #label_id = raw_pre[i][j]
+                label_id = np.argmax(raw_pre[i][j])
+                label_score = round(raw_pre[i][j][label_id],4)
+                label_tag = label_set[str(label_id)]
+            else:
+                label_tag='O'
+                label_score = 0.0
+            fout.write(raw_input[i][j][0]+'\t'+raw_input[i][j][-1]+'\t'+label_tag+'\t'+str(label_score)+'\n')
+        fout.write('\n')
+    return fout.getvalue()
+#generate char vocab
+def char_vocab(infile,outfile_char):
+    fin=open(infile,'r',encoding='utf-8')
+    #fout=open(outfile,'w',encoding='utf-8')
+    fout_char=open(outfile_char,'w',encoding='utf-8')
+    char_vocab=['oov_char']
+    max_len=0
+    for line in fin:
+        if line.strip()!='':
+            seg=line.split('\t')
+            word_len=len(seg[0])
+            #if word_len<1000:
+            #    fout.write(line)
+            if word_len>max_len:
+                max_len=word_len
+                print(seg[0])
+            for i in range(word_len):
+                if seg[0][i] not in char_vocab:
+                    char_vocab.append(seg[0][i])
+        #else:
+        #    fout.write(line)
+    fin.close()
+    #fout.close()
+    for ele in char_vocab:
+        fout_char.write(ele+'\n')
+    fout_char.close()
+    print('max_len:',max_len)
+
+
+if __name__=='__main__':
+    # infile='//panfs/pan1/bionlp/lulab/luoling/HPO_project/AutoPhe/data/pubmed_unlabel/mutation_disease_1990.ner_BIO'
+    # #outfile='//panfs/pan1/bionlp/lulab/luoling/HPO_project/AutoPhe/data/pubmed_unlabel/mutation_disease_1990.ner_BIO_new'
+    # outfile_char='//panfs/pan1/bionlp/lulab/luoling/HPO_project/AutoPhe/src/nn_model/vocab/char_vocab'
+    # #processing_text(file)
+    # char_vocab(infile,outfile_char)
+    a=[1,2,3]
+    print(a[:-1])