KBP Results

Runtime: Fri Jul 25 12:10:36 PDT 2014 to Sun Jul 27 16:13:50 PDT 2014

Re-train

[STDERR]                        Iter 8 evals 26  [23M 1.000E0] 3.690E3 3.77s |1.604E2| {6.982E-4} 1.745E-1 - 
[STDERR]                        Iter 19 evals 24  [1M 1.000E0] 3.583E3 3.93s |2.129E2| {9.268E-4} 1.648E-1 - 
[STDERR]                        Iter 10 evals 28  [1M 1.000E0] 3.495E3 4.10s |2.948E2| {1.284E-3} 1.664E-1 - 
[STDERR]                        Iter 12 evals 27  [331123M 1.000E0] 3.445E3 4.26s |1.598E2| {6.955E-4} 1.563E-1 - 
[STDERR]                        Iter 18 evals 27  [21M 1.000E0] 3.340E3 4.43s |1.475E2| {6.420E-4} 1.421E-1 - 
[STDERR]                        Iter 27 evals 156  [M 1.000E0] 3.258E3 4.60s |1.523E2| {6.629E-4} 1.300E-1 - 
[STDERR]                        Iter 21 evals 28  [11M 1.000E0] 3.178E3 4.77s |7.794E1| {3.393E-4} 7.231E-2 - 
[STDERR]                        Iter 19 evals 31  [3311M 1.000E0] 3.134E3 4.94s |3.551E1| {1.546E-4} 2.113E-2 - 
[STDERR]                        Iter 22 evals 30  [3M 1.000E0] 2.985E3 5.11s |7.595E1| {3.307E-4} 2.633E-2 - 
[STDERR]                        Iter 20 evals 39  [131213312M 5.156E-1] 2.885E3 5.44s |3.469E1| {1.510E-4} 2.791E-2 - 
[STDERR]                        Iter 23 evals 85  [3M 1.000E0] 2.859E3 5.60s |3.648E1| {1.588E-4} 2.532E-2 - 
[STDERR]                        Iter 22 evals 43  [3M 3.001E-1] 2.853E3 5.93s |2.459E1| {1.071E-4} 2.253E-2 - 
[STDERR]                        Iter 23 evals 44  [M 1.000E0] 2.852E3 6.10s |8.448E0| {3.678E-5} 2.081E-2 - 
[STDERR]                        Iter 24 evals 45  [M 1.000E0] 2.851E3 6.26s |5.161E0| {2.247E-5} 1.716E-2 - 
[STDERR]                        Iter 22 evals 39  [11M 1.000E0] 2.851E3 6.43s |3.095E0| {1.348E-5} 1.430E-2 - 
[STDERR]                        Iter 20 evals 41  [1M 1.000E0] 2.850E3 6.59s |4.254E0| {1.852E-5} 1.148E-2 - 
[STDERR]                        Iter 29 evals 39  [3M 3.268E-1] 2.850E3 6.93s |2.375E0| {1.034E-5} 9.938E-3 - 
[STDERR]                        Iter 30 evals 40  [1M 1.000E0] 2.850E3 7.09s |1.496E0| {6.514E-6} 4.740E-3 - 
[STDERR]                        Iter 31 evals 41  [M 1.000E0] 2.850E3 7.26s |5.179E-1| {2.255E-6} 1.213E-3 - 
[STDERR]                        Iter 30 evals 51  [131M 1.000E0] 2.850E3 7.43s |4.983E-1| {2.169E-6} 3.041E-4 - 
[STDERR]                        Iter 31 evals 52  [13M 1.000E0] 2.850E3 7.59s
[STDERR]                        QNMinimizer terminated due to average improvement: | newest_val - previous_val | / |newestVal| < TOL 
[STDERR]                        Total time spent in optimization: 7.59s
                              } << EPOCH 6: Training Y classifier for label org:founded_by [7.920 seconds]
                            } << Threads( EPOCH 6: Training Y classifiers ) [01:40.835 minutes]
                          } << M-STEP [2 hours, 30:30.599 minutes]
                        } << EM [1 day, 21 hours, 13:11.718 minutes]
[MIML-RE]               Created the Z dataset with 1902380 datums.
                        Computing Statistics {
[ERROR]                   java.lang.OutOfMemoryError: GC overhead limit exceeded
                        } [13:06.659 minutes]
[Train]                 saving model to /tmp/kbp_workdir/kbp_relation_model.JOINT_BAYES.50.ser
[MAIN]                  work dir: /tmp/kbp_workdir
                      } << main [1 day, 22 hours, 36:42.187 minutes]

  

Simple Slot Filler (2010)

[DEBUG Eval]                PR@00.99: F1 23.40 P 18.63 R 31.44
[DEBUG Eval]                PR@00.99: F1 23.39 P 18.62 R 31.44
[DEBUG Eval]                PR@00.99: F1 23.38 P 18.61 R 31.44
[DEBUG Eval]                PR@00.99: F1 23.37 P 18.60 R 31.44
[DEBUG Eval]                PR@00.99: F1 23.37 P 18.59 R 31.44
[DEBUG Eval]                PR@00.99: F1 23.36 P 18.58 R 31.44
[DEBUG Eval]                PR@00.99: F1 23.35 P 18.57 R 31.44
[DEBUG Eval]                PR@00.99: F1 23.34 P 18.56 R 31.44
[DEBUG Eval]                PR@00.99: F1 23.33 P 18.55 R 31.44
[DEBUG Eval]                PR@00.99: F1 23.32 P 18.54 R 31.44
[DEBUG Eval]                PR@01.00: F1 23.32 P 18.53 R 31.44
[DEBUG Eval]                PR@01.00: F1 23.31 P 18.52 R 31.44
[DEBUG Eval]                PR@01.00: F1 23.30 P 18.51 R 31.44
[DEBUG Eval]                PR@01.00: F1 23.29 P 18.50 R 31.44
[DEBUG Eval]                PR@01.00: F1 23.28 P 18.49 R 31.44
[DEBUG Eval]                PR@01.00: F1 23.27 P 18.47 R 31.44
[DEBUG Eval]                PR@01.00: F1 23.27 P 18.46 R 31.44
[DEBUG Eval]                PR@01.00: F1 23.26 P 18.45 R 31.44
[Eval]                      P/R curve data generated in file: /tmp/kbp_workdir/simple2010/stanford1.curve
                          } << Generating PR Curve [04:03.788 minutes]
                          Score {
[Result]                    |           Precision: 18.454
[Result]                    |              Recall: 31.442
[Result]                    |                  F1: 23.257
[Result]                    |
[Result]                    |   Optimal Precision: 43.409
[Result]                    |      Optimal Recall: 23.750
[Result]                    |          Optimal F1: 30.702
[Result]                    |
[Result]                    | Area Under PR Curve: 0.15710355016532196
                          } 
                        } << Evaluating Test Entities [04:06.513 minutes]
[MAIN]                  work dir: /tmp/kbp_workdir/simple2010
                      } << main [3 hours, 04:02.568 minutes]

Inferential Slot Filler (2010)

                                java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1184)
                                java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
                                java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
                                java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
                                java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
                                java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
                                java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
                                java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
                                java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
                                java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
                                java.util.HashMap.internalWriteEntries(HashMap.java:1776)
                                java.util.HashMap.writeObject(HashMap.java:1354)
                                sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
                                sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
                                sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
                                java.lang.reflect.Method.invoke(Method.java:483)
                                java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:988)
                                java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
                                java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
                                java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
                                java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
                                java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
                                java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
                                java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
                                java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
                                edu.stanford.nlp.kbp.common.PostgresUtils$KeyGraphCallback.setValue(PostgresUtils.java:573)
                                edu.stanford.nlp.kbp.common.PostgresUtils$KeyGraphCallback.setValue(PostgresUtils.java:567)
                                edu.stanford.nlp.kbp.common.PostgresUtils$KeyValueCallback.put(PostgresUtils.java:268)
                                ...19 more
[STDOUT]                      TOP ELEMENT: edu.stanford.nlp.kbp.common.PostgresUtils$KeyValueCallback.put(PostgresUtils.java:270)
                            } << Raw Classification [07:45.685 minutes]
                          } << Annotating PERSON:Erika Rose (NIL00002,SF201) [07:45.686 minutes]
                        } << Processing Test Entities [100] [11:29.319 minutes]
                      } << main [13:57.759 minutes]

Simple Slot Filler (2012)

[DEBUG Eval]                PR@00.99: F1 27.36 P 22.53 R 34.82
[DEBUG Eval]                PR@00.99: F1 27.35 P 22.52 R 34.82
[DEBUG Eval]                PR@00.99: F1 27.35 P 22.51 R 34.82
[DEBUG Eval]                PR@00.99: F1 27.34 P 22.50 R 34.82
[DEBUG Eval]                PR@00.99: F1 27.33 P 22.49 R 34.82
[DEBUG Eval]                PR@00.99: F1 27.32 P 22.48 R 34.82
[DEBUG Eval]                PR@00.99: F1 27.37 P 22.52 R 34.89
[DEBUG Eval]                PR@01.00: F1 27.36 P 22.51 R 34.89
[DEBUG Eval]                PR@01.00: F1 27.36 P 22.50 R 34.89
[DEBUG Eval]                PR@01.00: F1 27.35 P 22.49 R 34.89
[DEBUG Eval]                PR@01.00: F1 27.34 P 22.48 R 34.89
[DEBUG Eval]                PR@01.00: F1 27.33 P 22.47 R 34.89
[DEBUG Eval]                PR@01.00: F1 27.33 P 22.46 R 34.89
[DEBUG Eval]                PR@01.00: F1 27.32 P 22.45 R 34.89
[DEBUG Eval]                PR@01.00: F1 27.31 P 22.44 R 34.89
[DEBUG Eval]                PR@01.00: F1 27.31 P 22.43 R 34.89
[DEBUG Eval]                PR@01.00: F1 27.30 P 22.42 R 34.89
[DEBUG Eval]                PR@01.00: F1 27.29 P 22.41 R 34.89
[Eval]                      P/R curve data generated in file: /tmp/kbp_workdir/simple2012/stanford1.curve
                          } << Generating PR Curve [05:02.994 minutes]
                          Score {
[Result]                    |           Precision: 22.412
[Result]                    |              Recall: 34.887
[Result]                    |                  F1: 27.292
[Result]                    |
[Result]                    |   Optimal Precision: 36.679
[Result]                    |      Optimal Recall: 26.764
[Result]                    |          Optimal F1: 30.947
[Result]                    |
[Result]                    | Area Under PR Curve: 0.17156396370659785
                          } 
                        } << Evaluating Test Entities [05:05.761 minutes]
[MAIN]                  work dir: /tmp/kbp_workdir/simple2012
                      } << main [2 hours, 02:14.701 minutes]

Inferential Slot Filler (2012)

[GraphConsistency]            filtered per:stateorprovinces_of_residence(Deborah Pryce:PERSON, Arizona:STATE_OR_PROVINCE) from sentence In the past few weeks , GOP lawmakers in districts narrowly divided along party lines -- Reps. Deborah Pryce of Ohio , Rick Renzi of Arizona , Jim Ramstad of Minnesota and Jerry Weller Illinois come to mind -- have all taken a pass on another term , creating grumbles that a full-scale retreat is underway .
[GraphConsistency]            filtered per:stateorprovinces_of_residence(Barack Obama:PERSON, N.Y.:STATE_OR_PROVINCE) from sentence Conventional wisdom dictates that Democratic voters are thrilled with their choices for president , bursting at the seams to rally behind Sen. Hillary Rodham Clinton , N.Y. , Sen. Barack Obama , Ill. , or whoever gets the party 's nod next year .
[GraphConsistency]            filtered per:title(Darko:PERSON, cook:TITLE) from sentence Upon leaving the marketplace we enjoyed lunch at Darko 's which Bojana prepared ; she was an excellent cook !
[GraphConsistency]            filtered per:countries_of_residence(Darko:PERSON, Indonesia:COUNTRY) from sentence This was probably owing to my ignoring Darko and Bojana 's warning not to drink tap water but only bottled ; a lesson I ought have learned in Indonesia as a young girl .
[GraphConsistency]            filtered per:title(Hillary Clinton:PERSON, Mayor:TITLE) from sentence A recent survey by Democratic pollster Celinda Lake , however , showed Clinton and Obama trailing former New York City Republican Mayor Rudy Giuliani in the 31 Democratic-held House districts regarded as most imperiled in 2008 , and even potentially serving as a drag on those lawmakers ' reelection chances .
[GraphConsistency]            filtered per:title(Hillary Clinton:PERSON, mayor:TITLE) from sentence Giuliani takes 49 percent to Clinton 's 39 percent , while the former mayor 's lead over Obama is far smaller , 41 percent to 40 percent .
[GraphConsistency]            filtered per:employee_of(Hillary Clinton:PERSON, Democratic:ORGANIZATION) from sentence A recent survey by Democratic pollster Celinda Lake , however , showed Clinton and Obama trailing former New York City Republican Mayor Rudy Giuliani in the 31 Democratic-held House districts regarded as most imperiled in 2008 , and even potentially serving as a drag on those lawmakers ' reelection chances .
[GraphConsistency]            filtered per:countries_of_residence(Christopher Nugent:PERSON, Guinea:COUNTRY) from sentence Some members of Congress refuse to sponsor private bills because they fear `` an onslaught of requests , '' said Christopher Nugent , a Washington immigration lawyer who represents Malik Jarno , a mentally retarded teen-ager from Guinea .
[GraphConsistency]            filtered per:stateorprovinces_of_residence(Christopher Nugent:PERSON, Washington:STATE_OR_PROVINCE) from sentence Some members of Congress refuse to sponsor private bills because they fear `` an onslaught of requests , '' said Christopher Nugent , a Washington immigration lawyer who represents Malik Jarno , a mentally retarded teen-ager from Guinea .
[GraphConsistency]            filtered per:title(Christopher Nugent:PERSON, immigration lawyer:TITLE) from sentence Some members of Congress refuse to sponsor private bills because they fear `` an onslaught of requests , '' said Christopher Nugent , a Washington immigration lawyer who represents Malik Jarno , a mentally retarded teen-ager from Guinea .
[GraphConsistency]            filtered org:top_members/employees(NATO:ORGANIZATION, Ramsey Clark:PERSON) from sentence Between the time of my dismissal from the New School and my dismissal from FDU the fall -LRB- 2002 -RRB- , I stayed in touch with many scholars and other politically active persons sharing similar anti-war views as myself regarding NATO 's 1999 Kosovo war including : Professor Barry Lituchy -LRB- NYC -RRB- , Ramsey Clark 's people at the International Action Center , and a couple of new acquaintances I 've chanced meet online in Serbian political activist forums .
[GraphConsistency]            filtered per:employee_of(Patrick Leahy:PERSON, Congress:ORGANIZATION) from sentence Neither Senator Patrick Leahy nor Representative John Conyers , the Democratic chairmen of the Judiciary panels in the two chambers of Congress , appeared in any rush to issue the subpoenas .
[GraphConsistency]            filtered per:employee_of(Patrick Leahy:PERSON, GOP:ORGANIZATION) from sentence Sampson 's attorney , Bradford Berenson , wrote in a letter Friday to Senate Judiciary Committee Chairman Patrick Leahy , D-Vt. , and ranking GOP member Arlen Specter , R-Pa. , that he hopes the voluntary testimony `` will satisfy the need of the Congress to obtain information '' from Sampson .
[GraphConsistency]            filtered per:title(Patrick Leahy:PERSON, Representative:TITLE) from sentence Neither Senator Patrick Leahy nor Representative John Conyers , the Democratic chairmen of the Judiciary panels in the two chambers of Congress , appeared in any rush to issue the subpoenas .
                              Global Consistency (run 2) {
[ERROR MAIN]                    java.lang.NullPointerException
                                  edu.stanford.nlp.kbp.slotfilling.evaluate.SimpleSlotFiller.findBestProvenance(SimpleSlotFiller.java:344)
                                  edu.stanford.nlp.kbp.slotfilling.evaluate.InferentialSlotFiller.fillSlots(InferentialSlotFiller.java:128)
                                  edu.stanford.nlp.kbp.slotfilling.evaluate.KBPEvaluator.run(KBPEvaluator.java:170)
                                  edu.stanford.nlp.kbp.slotfilling.evaluate.KBPEvaluator.run(KBPEvaluator.java:197)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem.runProgram(SlotfillingSystem.java:238)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem.lambda$main$164(SlotfillingSystem.java:342)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem$$Lambda$7/277630005.apply(Unknown Source)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem.lambda$exec$163(SlotfillingSystem.java:321)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem$$Lambda$8/517380410.run(Unknown Source)
                                  edu.stanford.nlp.util.Execution.exec(Execution.java:619)
                                  edu.stanford.nlp.util.Execution.exec(Execution.java:597)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem.exec(SlotfillingSystem.java:319)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem.main(SlotfillingSystem.java:336)
                              } 
                            } << Inference 
                          } << Annotating PERSON:Paul Gillmor (E0526932,SF_ENG_002) [29.763 seconds]
                        } << Processing Test Entities [80] [04:32.276 minutes]
                      } << main [05:52.397 minutes]