Add a feature for training a lemmatizer ignoring blank lemmas. Helps make lemmatizers that work on partially finished treebanks (such as Sindhi)

Author: AngledLuffa

stanza/models/lemma/data.py

@@ -121,17 +121,19 @@ def __iter__(self):
             yield self.__getitem__(i)
 
     def raw_data(self):
-        return self.load_doc(self.doc, self.args.get('caseless', False), self.eval)
+        return self.load_doc(self.doc, self.args.get('caseless', False), self.args.get('skip_blank_lemmas', False), self.eval)
 
     @staticmethod
-    def load_doc(doc, caseless, evaluation):
+    def load_doc(doc, caseless, skip_blank_lemmas, evaluation):
         if evaluation:
             data = doc.get([TEXT, UPOS, LEMMA])
         else:
             data = doc.get([TEXT, UPOS, LEMMA, HEAD, DEPREL, MISC], as_sentences=True)
             data = DataLoader.remove_goeswith(data)
             data = DataLoader.extract_correct_forms(data)
         data = DataLoader.resolve_none(data)
+        if not evaluation and skip_blank_lemmas:
+            data = DataLoader.skip_blank_lemmas(data)
         if caseless:
             data = DataLoader.lowercase_data(data)
         return data
@@ -202,6 +204,11 @@ def lowercase_data(data):
             token[0] = token[0].lower()
         return data
 
+    @staticmethod
+    def skip_blank_lemmas(data):
+        data = [x for x in data if x[2] != '_']
+        return data
+
     @staticmethod
     def resolve_none(data):
         # replace None to '_'

stanza/models/lemmatizer.py

@@ -78,6 +78,7 @@ def build_argparse():
     parser.add_argument('--save_name', type=str, default="{shorthand}_{embedding}_lemmatizer.pt", help="File name to save the model")
 
     parser.add_argument('--caseless', default=False, action='store_true', help='Lowercase everything first before processing.  This will happen automatically if 100%% of the data is caseless')
+    parser.add_argument('--skip_blank_lemmas', default=False, action='store_true', help='Skip blank entries in the data files.  Useful for training a lemmatizer from a partially annotated dataset')
 
     parser.add_argument('--seed', type=int, default=1234)
     utils.add_device_args(parser)

stanza/tests/lemma/test_data.py

@@ -69,38 +69,60 @@
 4	ambulances	ambulance	NOUN	NNS	Number=Plur	3	obj	3:obj	SpaceAfter=No
 """
 
+BLANKS_DATA = """
+# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0018
+# text = Guerrillas killed an engineer, Asi Ali, from Tikrit.
+1	Guerrillas	_	NOUN	NNS	Number=Plur	2	nsubj	2:nsubj	_
+2	killed	_	VERB	VBD	Mood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin	0	root	0:root	_
+3	an	a	DET	DT	Definite=Ind|PronType=Art	4	det	4:det	_
+4	engineer	_	NOUN	NN	Number=Sing	2	obj	2:obj	SpaceAfter=No
+
+""".lstrip()
+
 
 def test_load_document():
     train_doc = CoNLL.conll2doc(input_str=TRAIN_DATA)
-    data = DataLoader.load_doc(train_doc, caseless=False, evaluation=True)
+    data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=True)
     assert len(data) == 33 # meticulously counted by hand
     assert all(len(x) == 3 for x in data)
 
-    data = DataLoader.load_doc(train_doc, caseless=False, evaluation=False)
+    data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=False)
     assert len(data) == 33
     assert all(len(x) == 3 for x in data)
 
 def test_load_goeswith():
     raw_data = TRAIN_DATA + GOESWITH_DATA
     train_doc = CoNLL.conll2doc(input_str=raw_data)
-    data = DataLoader.load_doc(train_doc, caseless=False, evaluation=True)
+    data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=True)
     assert len(data) == 36 # will be the same as in test_load_document with three additional words
     assert all(len(x) == 3 for x in data)
 
-    data = DataLoader.load_doc(train_doc, caseless=False, evaluation=False)
+    data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=False)
     assert len(data) == 33 # will be the same as in test_load_document, but with the trailing 3 GOESWITH removed
     assert all(len(x) == 3 for x in data)
 
 def test_correct_form():
     raw_data = TRAIN_DATA + CORRECT_FORM_DATA
     train_doc = CoNLL.conll2doc(input_str=raw_data)
-    data = DataLoader.load_doc(train_doc, caseless=False, evaluation=True)
+    data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=True)
     assert len(data) == 37
     # the 'targeting' correction should not be applied if evaluation=True
     # when evaluation=False, then the CorrectForms will be applied
     assert not any(x[0] == 'targeting' for x in data)
 
-    data = DataLoader.load_doc(train_doc, caseless=False, evaluation=False)
+    data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=False)
     assert len(data) == 38 # the same, but with an extra row so the model learns both 'targetting' and 'targeting'
     assert any(x[0] == 'targeting' for x in data)
     assert any(x[0] == 'targetting' for x in data)
+
+def test_load_blank():
+    raw_data = TRAIN_DATA + BLANKS_DATA
+    train_doc = CoNLL.conll2doc(input_str=raw_data)
+    data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=False)
+    assert len(data) == 37 # will be the same as in test_load_document with FOUR additional words
+    assert all(len(x) == 3 for x in data)
+
+    data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=True, evaluation=False)
+    assert len(data) == 34 # will be the same as in test_load_document, but one extra word is added.  others were blank
+    assert all(len(x) == 3 for x in data)
+