Load the testing dataset
CheckList-type test of SRL system
In this notebook, we will create a checklist-type test of two SRL systems. Both systems are fine-tuned DistilBERT models, trained on the English Universal Dependencies dataset.
# Load the testing dataset
import jsonwith open('data/testing-dataset.json', 'r') as f:
testing_dataset = json.load(f)
# Print the first 10 samples
for capability in testing_dataset:
print(f"Capability: {capability['capability_name']}:")
for test in capability['tests']:
print(f" Test: {test['test_name']}: {test['test_description']}")
for sample in test['samples']:
print(f" Sample: {sample}")
Capability: Long-distance dependencies between predicate and Arg-0:
Test: Effect of injecting relative clause between predicate and Arg-0: Test description.
Sample: [{'tokens': ['The', 'man', 'laughed', '.'], 'argument_labels': ['_', 'ARG0', '_', '_'], 'predicate_name': 'laugh.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-laugh.html#laugh', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'man', ',', 'who', 'was', 'thinking', 'about', 'the', 'future', 'of', 'the', 'world', 'and', 'how', 'his', 'role', 'in', 'it', 'all', 'was', 'going', 'to', 'play', 'out', ',', 'laughed', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'laugh.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-laugh.html#laugh', 'predicate_position': 25, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'boy', 'walks'], 'argument_labels': ['_', 'ARG0', '_'], 'predicate_name': 'walk.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-walk.html#walk', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'boy', ',', 'who', 'has', 'never', 'given', 'up', 'on', 'the', 'idea', 'of', 'becoming', 'a', 'doctor', ',', 'walks', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'walk.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-walk.html#walk', 'predicate_position': 16, 'target_token_position': 1}]
Sample: [{'tokens': ['Daniel', 'sings', '.'], 'argument_labels': ['_', 'ARG0', '_'], 'predicate_name': 'sing.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-sing.html#sing', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['Daniel', ',', 'who', 'has', 'never', 'given', 'up', 'on', 'the', 'idea', 'of', 'becoming', 'a', 'doctor', ',', 'sings', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'sing.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-sing.html#sing', 'predicate_position': 15, 'target_token_position': 1}]
Sample: [{'tokens': ['Claire', 'knits', '.'], 'argument_labels': ['_', 'ARG0', '_'], 'predicate_name': 'knit.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-knit.html#knit', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['Claire', ',', 'who', 'feels', 'like', 'the', 'whole', 'world', 'rests', 'on', 'her', 'shoulders', 'sometimes', ',', 'knits', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'knit.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-knit.html#knit', 'predicate_position': 14, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'cat', 'slept', '.'], 'argument_labels': ['_', 'ARG0', '_', '_'], 'predicate_name': 'sleep.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-sleep.html#sleep', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'cat', ',', 'who', 'often', 'dreamed', 'of', 'chasing', 'mice', 'through', 'endless', 'meadows', ',', 'slept', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'sleep.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-sleep.html#sleep', 'predicate_position': 13, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'dog', 'barked', '.'], 'argument_labels': ['_', 'ARG0', '_', '_'], 'predicate_name': 'bark.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-bark.html#bark', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'dog', ',', 'who', 'was', 'anxious', 'about', 'the', 'stranger', 'approaching', 'the', 'house', ',', 'barked', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'bark.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-bark.html#bark', 'predicate_position': 13, 'target_token_position': 1}]
Sample: [{'tokens': ['Anna', 'dances', '.'], 'argument_labels': ['ARG0', '_', '_'], 'predicate_name': 'dance.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-dance.html#dance', 'predicate_position': 1, 'target_token_position': 0}, {'tokens': ['Anna', ',', 'who', 'always', 'felt', 'alive', 'when', 'moving', 'to', 'the', 'beat', ',', 'dances', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'dance.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-dance.html#dance', 'predicate_position': 12, 'target_token_position': 0}]
Sample: [{'tokens': ['The', 'bird', 'sings', '.'], 'argument_labels': ['_', 'ARG0', '_', '_'], 'predicate_name': 'sing.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-sing.html#sing', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'bird', ',', 'who', 'was', 'perched', 'on', 'the', 'highest', 'branch', 'of', 'the', 'tree', ',', 'sings', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'sing.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-sing.html#sing', 'predicate_position': 14, 'target_token_position': 1}]
Sample: [{'tokens': ['John', 'smiled', '.'], 'argument_labels': ['ARG0', '_', '_'], 'predicate_name': 'smile.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-smile.html#smile', 'predicate_position': 1, 'target_token_position': 0}, {'tokens': ['John', ',', 'who', 'was', 'thinking', 'about', 'how', 'his', 'hard', 'work', 'was', 'finally', 'paying', 'off', ',', 'smiled', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'smile.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-smile.html#smile', 'predicate_position': 15, 'target_token_position': 0}]
Sample: [{'tokens': ['Sarah', 'runs', '.'], 'argument_labels': ['ARG0', '_', '_'], 'predicate_name': 'run.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-run.html#run', 'predicate_position': 1, 'target_token_position': 0}, {'tokens': ['Sarah', ',', 'who', 'has', 'been', 'training', 'for', 'months', 'for', 'the', 'upcoming', 'marathon', ',', 'runs', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'run.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-run.html#run', 'predicate_position': 13, 'target_token_position': 0}]
Capability: Long-distance dependencies between predicate and Arg-1:
Test: Effect of injecting adverbial or participial phrase between predicate and Arg-1: Test description.
Sample: [{'tokens': ['We', 'saw', 'the', 'movie', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_'], 'predicate_name': 'see.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-see.html#see', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['We', 'saw', ',', 'after', 'much', 'needless', 'delay', ',', 'the', 'movie', '.'], 'argument_labels': ['_', '_', '_', '_', '_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'see.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-see.html#see', 'predicate_position': 1, 'target_token_position': 9}]
Sample: [{'tokens': ['She', 'bought', 'a', 'car', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_'], 'predicate_name': 'buy.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-buy.html#buy', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['She', 'bought', 'quickly', ',', 'after', 'considering', 'many', 'other', 'options', ',', 'a', 'car', '.'], 'argument_labels': ['_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'buy.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-buy.html#buy', 'predicate_position': 1, 'target_token_position': 11}]
Sample: [{'tokens': ['They', 'cleaned', 'the', 'room', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_'], 'predicate_name': 'clean.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-clean.html#clean', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['They', 'cleaned', 'thoroughly', 'and', 'without', 'any', 'complaints', 'the', 'room', '.'], 'argument_labels': ['_', '_', '_', '_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'clean.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-clean.html#clean', 'predicate_position': 1, 'target_token_position': 8}]
Sample: [{'tokens': ['He', 'fixed', 'the', 'computer', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_'], 'predicate_name': 'fix.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-fix.html#fix', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['He', 'fixed', 'without', 'any', 'major', 'issues', 'the', 'computer', '.'], 'argument_labels': ['_', '_', '_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'fix.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-fix.html#fix', 'predicate_position': 1, 'target_token_position': 7}]
Sample: [{'tokens': ['The', 'dog', 'chased', 'the', 'cat', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_'], 'predicate_name': 'chase.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-chase.html#chase', 'predicate_position': 2, 'target_token_position': 4}, {'tokens': ['The', 'dog', 'chased', 'for', 'what', 'seemed', 'like', 'hours', 'the', 'cat', '.'], 'argument_labels': ['_', '_', '_', '_', '_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'chase.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-chase.html#chase', 'predicate_position': 2, 'target_token_position': 9}]
Sample: [{'tokens': ['She', 'cut', 'the', 'cake', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_'], 'predicate_name': 'cut.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-cut.html#cut', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['She', 'cut', 'with', 'a', 'sharp', 'knife', 'the', 'cake', '.'], 'argument_labels': ['_', '_', '_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'cut.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-cut.html#cut', 'predicate_position': 1, 'target_token_position': 7}]
Sample: [{'tokens': ['The', 'chef', 'prepared', 'the', 'meal', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_'], 'predicate_name': 'prepare.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-prepare.html#prepare', 'predicate_position': 2, 'target_token_position': 4}, {'tokens': ['The', 'chef', 'prepared', 'in', 'a', 'hurry', 'the', 'meal', '.'], 'argument_labels': ['_', '_', '_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'prepare.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-prepare.html#prepare', 'predicate_position': 2, 'target_token_position': 7}]
Sample: [{'tokens': ['The', 'artist', 'painted', 'a', 'portrait', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_'], 'predicate_name': 'paint.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-paint.html#paint', 'predicate_position': 2, 'target_token_position': 4}, {'tokens': ['The', 'artist', 'painted', 'slowly', 'and', 'with', 'great', 'care', 'a', 'portrait', '.'], 'argument_labels': ['_', '_', '_', '_', '_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'paint.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-paint.html#paint', 'predicate_position': 2, 'target_token_position': 9}]
Sample: [{'tokens': ['They', 'moved', 'the', 'furniture', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_'], 'predicate_name': 'move.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-move.html#move', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['They', 'moved', 'carefully', 'and', 'with', 'much', 'effort', 'the', 'furniture', '.'], 'argument_labels': ['_', '_', '_', '_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'move.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-move.html#move', 'predicate_position': 1, 'target_token_position': 8}]
Sample: [{'tokens': ['He', 'opened', 'the', 'door', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_'], 'predicate_name': 'open.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-open.html#open', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['He', 'opened', 'with', 'a', 'loud', 'creak', 'the', 'door', '.'], 'argument_labels': ['_', '_', '_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'open.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-open.html#open', 'predicate_position': 1, 'target_token_position': 7}]
Capability: Robustness to Noise:
Test: Effect of typos in proper nouns as Arg-0 on Arg-0 labeling: Tests the system's robustness to typos in proper nouns affecting Arg-0 labeling.
Sample: [{'tokens': ['Henry', 'bakes', 'a', 'pie', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_'], 'predicate_name': 'bake.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-bake.html#bake', 'predicate_position': 1, 'target_token_position': 0}, {'tokens': ['hry', 'bakes', 'a', 'pie', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_'], 'predicate_name': 'bake.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-bake.html#bake', 'predicate_position': 1, 'target_token_position': 0}]
Sample: [{'tokens': ['Alice', 'writes', 'a', 'book', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_'], 'predicate_name': 'write.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-write.html#write', 'predicate_position': 1, 'target_token_position': 0}, {'tokens': ['23Alcei', 'writes', 'a', 'book', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_'], 'predicate_name': 'write.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-write.html#write', 'predicate_position': 1, 'target_token_position': 0}]
Sample: [{'tokens': ['Tesla', 'designed', 'the', 'car', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_'], 'predicate_name': 'design.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-design.html#design', 'predicate_position': 1, 'target_token_position': 0}, {'tokens': ['T321a', 'designed', 'the', 'car', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_'], 'predicate_name': 'design.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-design.html#design', 'predicate_position': 1, 'target_token_position': 0}]
Sample: [{'tokens': ['Berlin', 'hosts', 'the', 'festival', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_'], 'predicate_name': 'host.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-host.html#host', 'predicate_position': 1, 'target_token_position': 0}, {'tokens': ['Beeeeeeeeeeerllin', 'hosts', 'the', 'festival', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_'], 'predicate_name': 'host.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-host.html#host', 'predicate_position': 1, 'target_token_position': 0}]
Sample: [{'tokens': ['Emily', 'paints', 'a', 'portrait', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_'], 'predicate_name': 'paint.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-paint.html#paint', 'predicate_position': 1, 'target_token_position': 0}, {'tokens': ['y', 'paints', 'a', 'portrait', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_'], 'predicate_name': 'paint.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-paint.html#paint', 'predicate_position': 1, 'target_token_position': 0}]
Sample: [{'tokens': ['Google', 'launches', 'a', 'new', 'product', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_', '_'], 'predicate_name': 'launch.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-launch.html#launch', 'predicate_position': 1, 'target_token_position': 0}, {'tokens': ['G00gel', 'launches', 'a', 'new', 'product', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_', '_'], 'predicate_name': 'launch.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-launch.html#launch', 'predicate_position': 1, 'target_token_position': 0}]
Sample: [{'tokens': ['Paris', 'welcomes', 'tourists', '.'], 'argument_labels': ['ARG0', '_', '_', '_'], 'predicate_name': 'welcome.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-welcome.html#welcome', 'predicate_position': 1, 'target_token_position': 0}, {'tokens': ['Prsilx', 'welcomes', 'tourists', '.'], 'argument_labels': ['ARG0', '_', '_', '_'], 'predicate_name': 'welcome.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-welcome.html#welcome', 'predicate_position': 1, 'target_token_position': 0}]
Sample: [{'tokens': ['Microsoft', 'develops', 'software', '.'], 'argument_labels': ['ARG0', '_', '_', '_'], 'predicate_name': 'develop.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-develop.html#develop', 'predicate_position': 1, 'target_token_position': 0}, {'tokens': ['icMorsotf', 'develops', 'software', '.'], 'argument_labels': ['ARG0', '_', '_', '_'], 'predicate_name': 'develop.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-develop.html#develop', 'predicate_position': 1, 'target_token_position': 0}]
Sample: [{'tokens': ['Facebook', 'announces', 'new', 'features', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_'], 'predicate_name': 'announce.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-announce.html#announce', 'predicate_position': 1, 'target_token_position': 0}, {'tokens': ['Faceb*k', 'announces', 'new', 'features', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_'], 'predicate_name': 'announce.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-announce.html#announce', 'predicate_position': 1, 'target_token_position': 0}]
Sample: [{'tokens': ['Amazon', 'ships', 'packages', '.'], 'argument_labels': ['ARG0', '_', '_', '_'], 'predicate_name': 'ship.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-ship.html#ship', 'predicate_position': 1, 'target_token_position': 0}, {'tokens': ['Am@@zon', 'ships', 'packages', '.'], 'argument_labels': ['ARG0', '_', '_', '_'], 'predicate_name': 'ship.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-ship.html#ship', 'predicate_position': 1, 'target_token_position': 0}]
Test: Effect of typos in common nouns as Arg-0 on Arg-0 labeling: Tests the system's robustness to typos in common nouns affecting Arg-0 labeling.
Sample: [{'tokens': ['The', 'teacher', 'explains', 'the', 'lesson', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'explain.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-explain.html#explain', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'tahre', 'explains', 'the', 'lesson', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'explain.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-explain.html#explain', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'doctor', 'treats', 'the', 'patient', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'treat.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-treat.html#treat', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'cdotor', 'treats', 'the', 'patient', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'treat.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-treat.html#treat', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'artist', 'paints', 'a', 'landscape', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'paint.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-paint.html#paint', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'art1s', 'paints', 'a', 'landscape', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'paint.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-paint.html#paint', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'chef', 'cooks', 'dinner', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_'], 'predicate_name': 'cook.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-cook.html#cook', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'chf', 'cooks', 'dinner', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_'], 'predicate_name': 'cook.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-cook.html#cook', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'farmer', 'grows', 'wheat', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_'], 'predicate_name': 'grow.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-grow.html#grow', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'frmar', 'grows', 'wheat', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_'], 'predicate_name': 'grow.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-grow.html#grow', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'student', 'studies', 'math', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_'], 'predicate_name': 'study.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-study.html#study', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 's2udnt', 'studies', 'math', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_'], 'predicate_name': 'study.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-study.html#study', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'waiter', 'serves', 'the', 'meal', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'serve.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-serve.html#serve', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'wate', 'serves', 'the', 'meal', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'serve.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-serve.html#serve', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'mechanic', 'repairs', 'the', 'car', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'repair.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-repair.html#repair', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'mhanic', 'repairs', 'the', 'car', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'repair.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-repair.html#repair', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'dog', 'chases', 'the', 'ball', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'chase.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-chase.html#chase', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'dg', 'chases', 'the', 'ball', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'chase.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-chase.html#chase', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'child', 'plays', 'with', 'toys', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'play.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-play.html#play', 'predicate_position': 2, 'target_token_position': 1}, {'tokens': ['The', 'clid', 'plays', 'with', 'toys', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'play.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-play.html#play', 'predicate_position': 2, 'target_token_position': 1}]
Capability: Dealing with Semantic Atypicality:
Test: Inanimate objects as Arg-0: Tests the system's ability to correctly identify inanimate objects as Arg-0.
Sample: [{'tokens': ['The', 'river', 'laughed', 'loudly', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_'], 'predicate_name': 'laugh.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-laugh.html#laugh', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'door', 'sighed', 'gently', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_'], 'predicate_name': 'sigh.02', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-sigh.html#sigh', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'sun', 'danced', 'across', 'the', 'sky', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_'], 'predicate_name': 'dance.02', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-dance.html#dance', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'mountain', 'whispered', 'in', 'the', 'wind', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_'], 'predicate_name': 'whisper.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-whisper.html#whisper', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'book', 'yawned', 'on', 'the', 'shelf', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_'], 'predicate_name': 'yawn.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-yawn.html#yawn', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'clock', 'cried', 'out', 'loud', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_'], 'predicate_name': 'cry.02', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-cry.html#cry', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'car', 'shouted', 'down', 'the', 'road', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_'], 'predicate_name': 'shout.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-shout.html#shout', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'wind', 'sang', 'through', 'the', 'trees', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_'], 'predicate_name': 'sing.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-sing.html#sing', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'road', 'whined', 'under', 'the', 'heavy', 'truck', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'whine.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-whine.html#whine', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['The', 'forest', 'slept', 'quietly', 'under', 'the', 'stars', '.'], 'argument_labels': ['_', 'ARG0', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'sleep.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-sleep.html#sleep', 'predicate_position': 2, 'target_token_position': 1}]
Test: Abstract concepts as Arg-0: Tests the system's ability to correctly identify abstract concepts as Arg-0.
Sample: [{'tokens': ['Freedom', 'sings', 'with', 'the', 'wind', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_', '_'], 'predicate_name': 'sing.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-sing.html#sing', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['Justice', 'screamed', 'in', 'the', 'courtroom', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_', '_'], 'predicate_name': 'scream.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-scream.html#scream', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['Thinking', 'whispered', 'through', 'his', 'mind', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_', '_'], 'predicate_name': 'whisper.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-whisper.html#whisper', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['Hope', 'lifted', 'the', 'spirits', 'of', 'the', 'people', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'lift.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-lift.html#lift', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['Anger', 'struck', 'the', 'heart', 'of', 'the', 'man', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'strike.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-strike.html#strike', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['Love', 'embraced', 'her', 'with', 'a', 'warmth', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'embrace.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-embrace.html#embrace', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['Fear', 'crept', 'into', 'the', 'dark', 'room', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'creep.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-creep.html#creep', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['Curiosity', 'drove', 'them', 'to', 'explore', 'the', 'world', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_', '_', '_', '_'], 'predicate_name': 'drive.02', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-drive.html#drive', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['Joy', 'brightened', "everyone's", 'mood', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_'], 'predicate_name': 'brighten.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-brighten.html#brighten', 'predicate_position': 2, 'target_token_position': 1}]
Sample: [{'tokens': ['Grief', 'weighed', 'on', 'her', 'mind', '.'], 'argument_labels': ['ARG0', '_', '_', '_', '_', '_'], 'predicate_name': 'weigh.02', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-weigh.html#weigh', 'predicate_position': 2, 'target_token_position': 1}]
Capability: Dealing with dative verb alternations:
Test: Dative verb alternations effect on Arg-1: Tests whether the system can correctly identify the Arg-1 in dative verb alternations.
Sample: [{'tokens': ['Sarah', 'gave', 'a', 'book', 'to', 'John', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_', '_', '_'], 'predicate_name': 'give.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-give.html#give', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['Sarah', 'gave', 'John', 'a', 'book', '.'], 'argument_labels': ['_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'give.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-give.html#give', 'predicate_position': 1, 'target_token_position': 4}]
Sample: [{'tokens': ['Tom', 'lent', 'a', 'car', 'to', 'Alice', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_', '_', '_'], 'predicate_name': 'lend.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-lend.html#lend', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['Tom', 'lent', 'Alice', 'a', 'car', '.'], 'argument_labels': ['_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'lend.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-lend.html#lend', 'predicate_position': 1, 'target_token_position': 4}]
Sample: [{'tokens': ['The', 'committee', 'awarded', 'a', 'scholarship', 'to', 'Mark', '.'], 'argument_labels': ['_', '_', '_', '_', 'ARG1', '_', '_', '_'], 'predicate_name': 'award.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-award.html#award', 'predicate_position': 1, 'target_token_position': 4}, {'tokens': ['The', 'committee', 'awarded', 'Mark', 'a', 'scholarship', '.'], 'argument_labels': ['_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'award.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-award.html#award', 'predicate_position': 1, 'target_token_position': 5}]
Sample: [{'tokens': ['They', 'assigned', 'a', 'task', 'to', 'Jessica', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_', '_', '_'], 'predicate_name': 'assign.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-assign.html#assign', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['They', 'assigned', 'Jessica', 'a', 'task', '.'], 'argument_labels': ['_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'assign.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-assign.html#assign', 'predicate_position': 1, 'target_token_position': 4}]
Sample: [{'tokens': ['The', 'teacher', 'sent', 'a', 'message', 'to', 'the', 'students', '.'], 'argument_labels': ['_', '_', '_', '_', 'ARG1', '_', '_', '_', '_'], 'predicate_name': 'send.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-send.html#send', 'predicate_position': 1, 'target_token_position': 4}, {'tokens': ['The', 'teacher', 'sent', 'the', 'students', 'a', 'message', '.'], 'argument_labels': ['_', '_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'send.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-send.html#send', 'predicate_position': 1, 'target_token_position': 6}]
Sample: [{'tokens': ['He', 'shipped', 'a', 'package', 'to', 'them', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_', '_', '_'], 'predicate_name': 'ship.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-ship.html#ship', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['He', 'shipped', 'them', 'a', 'package', '.'], 'argument_labels': ['_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'ship.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-ship.html#ship', 'predicate_position': 1, 'target_token_position': 4}]
Sample: [{'tokens': ['She', 'brought', 'a', 'cake', 'to', 'her', 'friend', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_', '_', '_', '_'], 'predicate_name': 'bring.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-bring.html#bring', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['She', 'brought', 'her', 'friend', 'a', 'cake', '.'], 'argument_labels': ['_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'bring.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-bring.html#bring', 'predicate_position': 1, 'target_token_position': 5}]
Sample: [{'tokens': ['They', 'ceded', 'a', 'territory', 'to', 'the', 'neighboring', 'country', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_', '_', '_', '_', '_'], 'predicate_name': 'cede.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-cede.html#cede', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['They', 'ceded', 'the', 'neighboring', 'country', 'a', 'territory', '.'], 'argument_labels': ['_', '_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'cede.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-cede.html#cede', 'predicate_position': 1, 'target_token_position': 6}]
Sample: [{'tokens': ['Michael', 'loaned', 'a', 'car', 'to', 'his', 'brother', '.'], 'argument_labels': ['_', '_', '_', 'ARG1', '_', '_', '_', '_'], 'predicate_name': 'loan.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-loan.html#loan', 'predicate_position': 1, 'target_token_position': 3}, {'tokens': ['Michael', 'loaned', 'his', 'brother', 'a', 'car', '.'], 'argument_labels': ['_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'loan.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-loan.html#loan', 'predicate_position': 1, 'target_token_position': 5}]
Sample: [{'tokens': ['The', 'organization', 'forwarded', 'a', 'report', 'to', 'the', 'manager', '.'], 'argument_labels': ['_', '_', '_', '_', 'ARG1', '_', '_', '_', '_'], 'predicate_name': 'forward.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-forward.html#forward', 'predicate_position': 1, 'target_token_position': 4}, {'tokens': ['The', 'organization', 'forwarded', 'the', 'manager', 'a', 'report', '.'], 'argument_labels': ['_', '_', '_', '_', '_', '_', 'ARG1', '_'], 'predicate_name': 'forward.01', 'predicate_link': 'https://propbank.github.io/v3.4.0/frames/alias-forward.html#forward', 'predicate_position': 1, 'target_token_position': 6}]