KBP Results

Runtime: Tue Aug 26 20:38:48 PDT 2014 to Fri Aug 29 01:16:24 PDT 2014

Re-train

[STDERR]                        Iter 15 evals 17  [M 1.000E0] 6.205E2 2.02s |3.724E1| {1.569E-4} 1.177E-1 - 
[STDERR]                        Iter 18 evals 25  [M 1.000E0] 6.184E2 2.15s |3.728E1| {1.571E-4} 7.192E-2 - 
[STDERR]                        Iter 22 evals 35  [31M 1.000E0] 6.139E2 2.29s |3.725E1| {1.569E-4} 2.390E-2 - 
[STDERR]                        Iter 13 evals 16  [11M 1.000E0] 6.041E2 2.41s |3.701E1| {1.559E-4} 1.456E-2 - 
[STDERR]                        Iter 26 evals 145  [3M3 1.000E0] 5.914E2 2.54s |3.667E1| {1.545E-4} 1.056E-2 - 
[STDERR]                        Iter 24 evals 37  [13M 1.000E0] 5.731E2 2.71s |3.617E1| {1.524E-4} 1.114E-2 - 
[STDERR]                        Iter 18 evals 24  [M 3.918E-1] 5.611E2 3.06s |3.583E1| {1.510E-4} 1.203E-2 - 
[STDERR]                        1M 1.125E-1] 3.349E2 4.61s |2.884E1| {1.215E-4} 8.643E-2 - 
[STDERR]                        Iter 24 evals 43  [333111111M 1.340E-1] 2.314E2 6.14s |2.443E1| {1.029E-4} 1.689E-1 - 
[STDERR]                        Iter 31 evals 54  [131M 6.878E-2] 1.977E2 7.84s |2.026E1| {8.534E-5} 2.139E-1 - 
[STDERR]                        Iter 36 evals 56  [1M 1.642E-1] 1.684E2 8.54s |1.234E1| {5.199E-5} 2.672E-1 - 
[STDERR]                        Iter 40 evals 61  [M 1.423E-1] 1.550E2 9.57s |1.131E1| {4.766E-5} 2.961E-1 - 
[STDERR]                        1313133111M 2.494E-1] 1.444E2 10.23s |5.056E0| {2.130E-5} 3.183E-1 - 
[STDERR]                        Iter 30 evals 201  [111M 1.097E-1] 1.432E2 10.56s |4.530E0| {1.909E-5} 3.130E-1 - 
[STDERR]                        Iter 24 evals 65  [331231M 4.485E-1] 1.417E2 10.88s |1.144E0| {4.821E-6} 3.044E-1 - 
[STDERR]                        Iter 25 evals 131  [M 1.000E0] 1.416E2 11.05s |1.703E0| {7.174E-6} 2.961E-1 - 
[STDERR]                        Iter 26 evals 68  [31M 1.000E0] 1.416E2 11.22s |2.414E-1| {1.017E-6} 1.366E-1 - 
[STDERR]                        Iter 27 evals 69  [133M 1.000E0] 1.415E2 11.39s |3.438E-2| {1.448E-7} 6.345E-2 - 
[STDERR]                        Iter 28 evals 70  [11M 1.000E0] 1.415E2 11.56s |1.839E-2| {7.747E-8} 3.966E-2 - 
[STDERR]                        Iter 29 evals 71  [33M 1.000E0] 1.415E2 11.73s |4.266E-3| {1.797E-8} 1.896E-2 - 
[STDERR]                        Iter 30 evals 72  [11M 1.000E0] 1.415E2 11.90s
[STDERR]                        QNMinimizer terminated due to sufficient decrease in gradient norms: |g|/|g0| < TOL 
[STDERR]                        Total time spent in optimization: 11.90s
                              } << EPOCH 6: Training Y classifier for label org:dissolved [12.395 seconds]
                            } << Threads( EPOCH 6: Training Y classifiers ) [01:15.252 minutes]
                          } << M-STEP [2 hours, 33:11.107 minutes]
                        } << EM [1 day, 22 hours, 26:00.842 minutes]
[MIML-RE]               Created the Z dataset with 1902380 datums.
                        Computing Statistics {
[ERROR]                   java.lang.OutOfMemoryError: GC overhead limit exceeded
                        } [13:27.32 minutes]
[Train]                 saving model to /tmp/kbp_workdir/kbp_relation_model.JOINT_BAYES.50.ser
[MAIN]                  work dir: /tmp/kbp_workdir
                      } << main [1 day, 23 hours, 23:47.91 minutes]

  

Simple Slot Filler (2010)

[DEBUG Eval]                PR@00.99: F1 23.41 P 18.62 R 31.54
[DEBUG Eval]                PR@00.99: F1 23.40 P 18.60 R 31.54
[DEBUG Eval]                PR@00.99: F1 23.40 P 18.59 R 31.54
[DEBUG Eval]                PR@00.99: F1 23.39 P 18.58 R 31.54
[DEBUG Eval]                PR@00.99: F1 23.38 P 18.57 R 31.54
[DEBUG Eval]                PR@00.99: F1 23.37 P 18.56 R 31.54
[DEBUG Eval]                PR@00.99: F1 23.36 P 18.55 R 31.54
[DEBUG Eval]                PR@00.99: F1 23.35 P 18.54 R 31.54
[DEBUG Eval]                PR@00.99: F1 23.35 P 18.53 R 31.54
[DEBUG Eval]                PR@00.99: F1 23.34 P 18.52 R 31.54
[DEBUG Eval]                PR@01.00: F1 23.33 P 18.51 R 31.54
[DEBUG Eval]                PR@01.00: F1 23.32 P 18.50 R 31.54
[DEBUG Eval]                PR@01.00: F1 23.31 P 18.49 R 31.54
[DEBUG Eval]                PR@01.00: F1 23.30 P 18.48 R 31.54
[DEBUG Eval]                PR@01.00: F1 23.30 P 18.47 R 31.54
[DEBUG Eval]                PR@01.00: F1 23.29 P 18.46 R 31.54
[DEBUG Eval]                PR@01.00: F1 23.28 P 18.45 R 31.54
[DEBUG Eval]                PR@01.00: F1 23.27 P 18.44 R 31.54
[Eval]                      P/R curve data generated in file: /tmp/kbp_workdir/simple2010/stanford1.curve
                          } << Generating PR Curve [03:58.144 minutes]
                          Score {
[Result]                    |           Precision: 18.437
[Result]                    |              Recall: 31.538
[Result]                    |                  F1: 23.271
[Result]                    |
[Result]                    |   Optimal Precision: 42.270
[Result]                    |      Optimal Recall: 24.712
[Result]                    |          Optimal F1: 31.189
[Result]                    |
[Result]                    | Area Under PR Curve: 0.15940363083665032
                          } 
                        } << Evaluating Test Entities [03:59.338 minutes]
[MAIN]                  work dir: /tmp/kbp_workdir/simple2010
                      } << main [2 hours, 54:54.725 minutes]

Inferential Slot Filler (2010)

                                java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1184)
                                java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
                                java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
                                java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
                                java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
                                java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
                                java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
                                java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
                                java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
                                java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
                                java.util.HashMap.internalWriteEntries(HashMap.java:1776)
                                java.util.HashMap.writeObject(HashMap.java:1354)
                                sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
                                sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
                                sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
                                java.lang.reflect.Method.invoke(Method.java:483)
                                java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:988)
                                java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
                                java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
                                java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
                                java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
                                java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
                                java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
                                java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
                                java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
                                edu.stanford.nlp.kbp.common.PostgresUtils$KeyGraphCallback.setValue(PostgresUtils.java:573)
                                edu.stanford.nlp.kbp.common.PostgresUtils$KeyGraphCallback.setValue(PostgresUtils.java:567)
                                edu.stanford.nlp.kbp.common.PostgresUtils$KeyValueCallback.put(PostgresUtils.java:268)
                                ...19 more
[STDOUT]                      TOP ELEMENT: edu.stanford.nlp.kbp.common.PostgresUtils$KeyValueCallback.put(PostgresUtils.java:270)
                            } << Raw Classification [07:23.319 minutes]
                          } << Annotating PERSON:Erika Rose (NIL00002,SF201) [07:23.319 minutes]
                        } << Processing Test Entities [100] [11:27.577 minutes]
                      } << main [13:40.764 minutes]

Simple Slot Filler (2012)

[DEBUG Eval]                PR@00.99: F1 27.01 P 22.33 R 34.15
[DEBUG Eval]                PR@00.99: F1 27.00 P 22.32 R 34.15
[DEBUG Eval]                PR@00.99: F1 26.99 P 22.31 R 34.15
[DEBUG Eval]                PR@00.99: F1 26.99 P 22.30 R 34.15
[DEBUG Eval]                PR@00.99: F1 26.98 P 22.29 R 34.15
[DEBUG Eval]                PR@00.99: F1 26.97 P 22.28 R 34.15
[DEBUG Eval]                PR@00.99: F1 26.96 P 22.28 R 34.15
[DEBUG Eval]                PR@01.00: F1 26.96 P 22.27 R 34.15
[DEBUG Eval]                PR@01.00: F1 26.95 P 22.26 R 34.15
[DEBUG Eval]                PR@01.00: F1 26.94 P 22.25 R 34.15
[DEBUG Eval]                PR@01.00: F1 26.94 P 22.24 R 34.15
[DEBUG Eval]                PR@01.00: F1 26.93 P 22.23 R 34.15
[DEBUG Eval]                PR@01.00: F1 26.92 P 22.22 R 34.15
[DEBUG Eval]                PR@01.00: F1 26.92 P 22.21 R 34.15
[DEBUG Eval]                PR@01.00: F1 26.91 P 22.20 R 34.15
[DEBUG Eval]                PR@01.00: F1 26.90 P 22.19 R 34.15
[DEBUG Eval]                PR@01.00: F1 26.89 P 22.18 R 34.15
[DEBUG Eval]                PR@01.00: F1 26.89 P 22.17 R 34.15
[Eval]                      P/R curve data generated in file: /tmp/kbp_workdir/simple2012/stanford1.curve
                          } << Generating PR Curve [04:57.861 minutes]
                          Score {
[Result]                    |           Precision: 22.169
[Result]                    |              Recall: 34.154
[Result]                    |                  F1: 26.887
[Result]                    |
[Result]                    |   Optimal Precision: 35.220
[Result]                    |      Optimal Recall: 27.763
[Result]                    |          Optimal F1: 31.050
[Result]                    |
[Result]                    | Area Under PR Curve: 0.17125716510906713
                          } 
                        } << Evaluating Test Entities [05:00.651 minutes]
[MAIN]                  work dir: /tmp/kbp_workdir/simple2012
                      } << main [1 hour, 58:46.299 minutes]

Inferential Slot Filler (2012)

[GraphConsistency]            filtered per:stateorprovinces_of_residence(Deborah Pryce:PERSON, Arizona:STATE_OR_PROVINCE) from sentence In the past few weeks , GOP lawmakers in districts narrowly divided along party lines -- Reps. Deborah Pryce of Ohio , Rick Renzi of Arizona , Jim Ramstad of Minnesota and Jerry Weller Illinois come to mind -- have all taken a pass on another term , creating grumbles that a full-scale retreat is underway .
[GraphConsistency]            filtered per:stateorprovinces_of_residence(Barack Obama:PERSON, N.Y.:STATE_OR_PROVINCE) from sentence Conventional wisdom dictates that Democratic voters are thrilled with their choices for president , bursting at the seams to rally behind Sen. Hillary Rodham Clinton , N.Y. , Sen. Barack Obama , Ill. , or whoever gets the party 's nod next year .
[GraphConsistency]            filtered per:title(Darko:PERSON, cook:TITLE) from sentence Upon leaving the marketplace we enjoyed lunch at Darko 's which Bojana prepared ; she was an excellent cook !
[GraphConsistency]            filtered per:countries_of_residence(Darko:PERSON, Indonesia:COUNTRY) from sentence This was probably owing to my ignoring Darko and Bojana 's warning not to drink tap water but only bottled ; a lesson I ought have learned in Indonesia as a young girl .
[GraphConsistency]            filtered per:title(Hillary Clinton:PERSON, Mayor:TITLE) from sentence A recent survey by Democratic pollster Celinda Lake , however , showed Clinton and Obama trailing former New York City Republican Mayor Rudy Giuliani in the 31 Democratic-held House districts regarded as most imperiled in 2008 , and even potentially serving as a drag on those lawmakers ' reelection chances .
[GraphConsistency]            filtered per:title(Hillary Clinton:PERSON, mayor:TITLE) from sentence Giuliani takes 49 percent to Clinton 's 39 percent , while the former mayor 's lead over Obama is far smaller , 41 percent to 40 percent .
[GraphConsistency]            filtered per:employee_of(Hillary Clinton:PERSON, Democratic:ORGANIZATION) from sentence A recent survey by Democratic pollster Celinda Lake , however , showed Clinton and Obama trailing former New York City Republican Mayor Rudy Giuliani in the 31 Democratic-held House districts regarded as most imperiled in 2008 , and even potentially serving as a drag on those lawmakers ' reelection chances .
[GraphConsistency]            filtered per:countries_of_residence(Christopher Nugent:PERSON, Guinea:COUNTRY) from sentence Some members of Congress refuse to sponsor private bills because they fear `` an onslaught of requests , '' said Christopher Nugent , a Washington immigration lawyer who represents Malik Jarno , a mentally retarded teen-ager from Guinea .
[GraphConsistency]            filtered per:stateorprovinces_of_residence(Christopher Nugent:PERSON, Washington:STATE_OR_PROVINCE) from sentence Some members of Congress refuse to sponsor private bills because they fear `` an onslaught of requests , '' said Christopher Nugent , a Washington immigration lawyer who represents Malik Jarno , a mentally retarded teen-ager from Guinea .
[GraphConsistency]            filtered per:title(Christopher Nugent:PERSON, immigration lawyer:TITLE) from sentence Some members of Congress refuse to sponsor private bills because they fear `` an onslaught of requests , '' said Christopher Nugent , a Washington immigration lawyer who represents Malik Jarno , a mentally retarded teen-ager from Guinea .
[GraphConsistency]            filtered org:top_members/employees(NATO:ORGANIZATION, Ramsey Clark:PERSON) from sentence Between the time of my dismissal from the New School and my dismissal from FDU the fall -LRB- 2002 -RRB- , I stayed in touch with many scholars and other politically active persons sharing similar anti-war views as myself regarding NATO 's 1999 Kosovo war including : Professor Barry Lituchy -LRB- NYC -RRB- , Ramsey Clark 's people at the International Action Center , and a couple of new acquaintances I 've chanced meet online in Serbian political activist forums .
[GraphConsistency]            filtered per:employee_of(Patrick Leahy:PERSON, Congress:ORGANIZATION) from sentence Neither Senator Patrick Leahy nor Representative John Conyers , the Democratic chairmen of the Judiciary panels in the two chambers of Congress , appeared in any rush to issue the subpoenas .
[GraphConsistency]            filtered per:employee_of(Patrick Leahy:PERSON, GOP:ORGANIZATION) from sentence Sampson 's attorney , Bradford Berenson , wrote in a letter Friday to Senate Judiciary Committee Chairman Patrick Leahy , D-Vt. , and ranking GOP member Arlen Specter , R-Pa. , that he hopes the voluntary testimony `` will satisfy the need of the Congress to obtain information '' from Sampson .
[GraphConsistency]            filtered per:title(Patrick Leahy:PERSON, Representative:TITLE) from sentence Neither Senator Patrick Leahy nor Representative John Conyers , the Democratic chairmen of the Judiciary panels in the two chambers of Congress , appeared in any rush to issue the subpoenas .
                              Global Consistency (run 2) {
[ERROR MAIN]                    java.lang.NullPointerException
                                  edu.stanford.nlp.kbp.slotfilling.evaluate.SimpleSlotFiller.findBestProvenance(SimpleSlotFiller.java:344)
                                  edu.stanford.nlp.kbp.slotfilling.evaluate.InferentialSlotFiller.fillSlots(InferentialSlotFiller.java:128)
                                  edu.stanford.nlp.kbp.slotfilling.evaluate.KBPEvaluator.run(KBPEvaluator.java:170)
                                  edu.stanford.nlp.kbp.slotfilling.evaluate.KBPEvaluator.run(KBPEvaluator.java:197)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem.runProgram(SlotfillingSystem.java:238)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem.lambda$main$164(SlotfillingSystem.java:342)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem$$Lambda$7/277630005.apply(Unknown Source)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem.lambda$exec$163(SlotfillingSystem.java:321)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem$$Lambda$8/517380410.run(Unknown Source)
                                  edu.stanford.nlp.util.Execution.exec(Execution.java:619)
                                  edu.stanford.nlp.util.Execution.exec(Execution.java:597)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem.exec(SlotfillingSystem.java:319)
                                  edu.stanford.nlp.kbp.slotfilling.SlotfillingSystem.main(SlotfillingSystem.java:336)
                              } 
                            } << Inference 
                          } << Annotating PERSON:Paul Gillmor (E0526932,SF_ENG_002) [28.988 seconds]
                        } << Processing Test Entities [80] [03:53.844 minutes]
                      } << main [06:00.57 minutes]