Link: API Status Page for Livingston
807403699 LDAS-LA2332751 mpi::filterLogEntries 2332751.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) Rank2(beowulf) Rank3(beowulf) }
807403701 LDAS-LA2332751 mpi::filterLogEntries 2332751.1:progress 0.00%
807403701 LDAS-LA2332751 mpi::filterLogEntries 2332751.1:projected_ratio 0.00006
807403701 LDAS-LA2332751 jobstate response: '2332751.1:cont'
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.2:progress 0.00%
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.2:projected_ratio 0.00006
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.3:progress 0.00%
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.3:projected_ratio 0.00006
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.4:progress 0.00%
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.4:projected_ratio 0.00350
807403703 LDAS-LA2332751 jobstate response: '2332751.4:cont'
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.5:progress 100.00%
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.5:projected_ratio 0.00350
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.6:progress 100.00%
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.6:projected_ratio 0.00704
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.7:progress 100.00%
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.7:projected_ratio 0.00704
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.8:progress 100.00%
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.8:projected_ratio 0.00711
807403703 LDAS-LA2332751 mpi::filterLogEntries 2332751.9:finalizing { totalTime=4.70136 sentObjects=2; Rank0 (inPut: create=0.51476 send=0.55175 ilwd=0.31057; outPut: create=0.00000 send=0.13282 ilwd=0.04309; metadata: (un)pack=0.00009 bcast=0.00284; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.21796; max memory=1085440) Rank3 (inPut: create=0.47078 send=0.67884; outPut: create=0.00000 send=0.04874; metadata: (un)pack=0.00013 bcast=0.00391; IS=0.00027; CD=0.57363; AS=0.17855; FO=0.00000; FS=0.00752; LB=0.00001; max memory=1478656) Rank2 (inPut: create=0.47311 send=0.67825; outPut: create=0.00000 send=0.19154; metadata: (un)pack=0.00021 bcast=0.00316; IS=0.00027; CD=0.58058; AS=2.12485; FO=0.00000; FS=0.00645; LB=0.00001; max memory=1478656) Rank1 (inPut: create=0.46997 send=0.61350; outPut: create=0.00000 send=0.40336; metadata: (un)pack=0.00023 bcast=0.00264; IS=0.00023; CD=0.51555; AS=1.94232; FO=0.00000; FS=0.00765; LB=0.02400; max memory=1478656) }
808290476 LDAS-LA2332749 mpi::filterLogEntries 2332749.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) Rank2(beowulf) Rank3(beowulf) }
808290478 LDAS-LA2332749 mpi::filterLogEntries 2332749.1:progress 0.00%
808290478 LDAS-LA2332749 mpi::filterLogEntries 2332749.1:projected_ratio 0.00014
808290478 LDAS-LA2332749 jobstate response: '2332749.1:cont'
808290480 LDAS-LA2332749 mpi::filterLogEntries 2332749.2:progress 0.00%
808290480 LDAS-LA2332749 mpi::filterLogEntries 2332749.2:projected_ratio 0.00014
808290480 LDAS-LA2332749 mpi::filterLogEntries 2332749.3:progress 0.00%
808290480 LDAS-LA2332749 mpi::filterLogEntries 2332749.3:projected_ratio 0.00014
808290480 LDAS-LA2332749 mpi::filterLogEntries 2332749.4:progress 0.00%
808290480 LDAS-LA2332749 mpi::filterLogEntries 2332749.4:projected_ratio 0.00380
808290480 LDAS-LA2332749 jobstate response: '2332749.4:cont'
808290481 LDAS-LA2332749 mpi::filterLogEntries 2332749.5:progress 100.00%
808290481 LDAS-LA2332749 mpi::filterLogEntries 2332749.5:projected_ratio 0.00380
808290481 LDAS-LA2332749 mpi::filterLogEntries 2332749.6:progress 100.00%
808290481 LDAS-LA2332749 mpi::filterLogEntries 2332749.6:projected_ratio 0.00755
808290481 LDAS-LA2332749 mpi::filterLogEntries 2332749.7:progress 100.00%
808290481 LDAS-LA2332749 mpi::filterLogEntries 2332749.7:projected_ratio 0.00755
808290481 LDAS-LA2332749 mpi::filterLogEntries 2332749.8:progress 100.00%
808290481 LDAS-LA2332749 mpi::filterLogEntries 2332749.8:projected_ratio 0.00762
808290481 LDAS-LA2332749 mpi::filterLogEntries 2332749.9:finalizing { totalTime=5.27448 sentObjects=2; Rank0 (inPut: create=0.56073 send=0.58263 ilwd=0.33680; outPut: create=0.00000 send=0.13080 ilwd=0.04486; metadata: (un)pack=0.00010 bcast=0.00309; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.21694; max memory=1085440) Rank3 (inPut: create=0.51171 send=0.73505; outPut: create=0.00000 send=0.04772; metadata: (un)pack=0.00013 bcast=0.00429; IS=0.00028; CD=0.55367; AS=0.17476; FO=0.00000; FS=0.00733; LB=0.00001; max memory=1478656) Rank2 (inPut: create=0.51185 send=0.72813; outPut: create=0.00000 send=0.20348; metadata: (un)pack=0.00025 bcast=0.00345; IS=0.00028; CD=0.59066; AS=2.25231; FO=0.00000; FS=0.00645; LB=0.00002; max memory=1478656) Rank1 (inPut: create=0.51200 send=0.65242; outPut: create=0.00000 send=0.37005; metadata: (un)pack=0.00029 bcast=0.00263; IS=0.00021; CD=0.57937; AS=2.10395; FO=0.00000; FS=0.00752; LB=0.02399; max memory=1478656) }
808290897 LDAS-LA2332760 mpi::filterLogEntries 2332760.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) Rank2(beowulf) Rank3(beowulf) }
808290900 LDAS-LA2332760 mpi::filterLogEntries 2332760.1:progress 0.00%
808290900 LDAS-LA2332760 mpi::filterLogEntries 2332760.1:projected_ratio 0.00002
808290900 LDAS-LA2332760 jobstate response: '2332760.1:cont'
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.2:progress 0.00%
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.2:projected_ratio 0.00002
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.3:progress 0.00%
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.3:projected_ratio 0.00002
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.4:progress 0.00%
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.4:projected_ratio 0.00347
808290902 LDAS-LA2332760 jobstate response: '2332760.4:cont'
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.5:progress 100.00%
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.5:projected_ratio 0.00347
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.6:progress 100.00%
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.6:projected_ratio 0.00702
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.7:progress 100.00%
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.7:projected_ratio 0.00702
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.8:progress 100.00%
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.8:projected_ratio 0.00709
808290902 LDAS-LA2332760 mpi::filterLogEntries 2332760.9:finalizing { totalTime=4.77929 sentObjects=2; Rank0 (inPut: create=0.54358 send=0.57498 ilwd=0.31765; outPut: create=0.00000 send=0.13175 ilwd=0.01043; metadata: (un)pack=0.00010 bcast=0.00283; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.21761; max memory=1085440) Rank3 (inPut: create=0.49692 send=0.66432; outPut: create=0.00000 send=0.04766; metadata: (un)pack=0.00016 bcast=0.00399; IS=0.00024; CD=0.52444; AS=0.15038; FO=0.00000; FS=0.00803; LB=0.00001; max memory=1478656) Rank2 (inPut: create=0.49400 send=0.72678; outPut: create=0.00000 send=0.19544; metadata: (un)pack=0.00016 bcast=0.00345; IS=0.00033; CD=0.57961; AS=2.12790; FO=0.00000; FS=0.00623; LB=0.00002; max memory=1478656) Rank1 (inPut: create=0.49400 send=0.72012; outPut: create=0.00000 send=0.37059; metadata: (un)pack=0.00026 bcast=0.00255; IS=0.00033; CD=0.57846; AS=1.97204; FO=0.00000; FS=0.00686; LB=0.02400; max memory=1478656) }
808290935 LDAS-LA2332761 mpi::filterLogEntries 2332761.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) Rank2(beowulf) Rank3(beowulf) }
808290975 LDAS-LA2332761 mpi::filterLogEntries 2332761.1:progress 100.00%
808290975 LDAS-LA2332761 mpi::filterLogEntries 2332761.1:projected_ratio 0.01100
808290975 LDAS-LA2332761 jobstate response: '2332761.1:cont'
808290976 LDAS-LA2332761 mpi::filterLogEntries 2332761.2:progress 100.00%
808290976 LDAS-LA2332761 mpi::filterLogEntries 2332761.2:projected_ratio 0.02205
808290976 LDAS-LA2332761 mpi::filterLogEntries 2332761.3:finalizing { totalTime=40.3087 sentObjects=4; Rank0 (inPut: create=0.67502 send=0.73574 ilwd=0.39554; outPut: create=0.02484 send=0.07195 ilwd=0.05494; metadata: (un)pack=0.00064 bcast=0.02671; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.11423; max memory=1335296) Rank3 (inPut: create=0.61563 send=0.92725; outPut: create=0.00000 send=0.02433; metadata: (un)pack=0.00014 bcast=0.00429; IS=0.94490; CD=0.14961; AS=0.00060; FO=0.00000; FS=0.00558; LB=0.00002; max memory=950272) Rank1 (inPut: create=0.61451 send=0.90318; outPut: create=0.01189 send=0.09521; metadata: (un)pack=0.00034 bcast=0.01511; IS=0.00027; CD=0.00558; AS=36.76675; FO=0.00005; FS=0.00007; LB=0.02391; max memory=413696) Rank2 (inPut: create=0.61447 send=0.83172; outPut: create=0.02396 send=0.29897; metadata: (un)pack=0.00057 bcast=0.02587; IS=0.94540; CD=0.14871; AS=36.62284; FO=0.00011; FS=0.03294; LB=0.00001; max memory=950272) }
808290989 LDAS-LA2332762 mpi::filterLogEntries 2332762.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
808290989 LDAS-LA2332762 mpi::filterLogEntries 2332762.1:progress 100.00%
808290989 LDAS-LA2332762 mpi::filterLogEntries 2332762.1:projected_ratio 0.00093
808290990 LDAS-LA2332762 mpi::filterLogEntries 2332762.2:finalizing { totalTime=0.602203 sentObjects=3; Rank0 (inPut: create=0.04322 send=0.00637 ilwd=0.03536; outPut: create=0.01490 send=0.04057 ilwd=0.00976; metadata: (un)pack=0.00022 bcast=0.01282; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01194; max memory=4112384) Rank1 (inPut: create=0.04118 send=0.01203; outPut: create=0.01294 send=0.02963; metadata: (un)pack=0.00020 bcast=0.01453; IS=0.00009; CD=0.18098; AS=0.05595; FO=0.00023; FS=0.00005; LB=0.04793; max memory=8081408) }
808291011 LDAS-LA2332763 mpi::filterLogEntries 2332763.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
808291013 LDAS-LA2332763 mpi::filterLogEntries 2332763.1:progress 100.00%
808291013 LDAS-LA2332763 mpi::filterLogEntries 2332763.1:projected_ratio 0.00020
808291013 LDAS-LA2332763 mpi::filterLogEntries 2332763.2:finalizing { totalTime=1.82051 sentObjects=3; Rank0 (inPut: create=0.37188 send=0.18035 ilwd=0.24643; outPut: create=0.09289 send=0.44210 ilwd=0.04520; metadata: (un)pack=0.00018 bcast=0.01305; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01194; max memory=262144) Rank1 (inPut: create=0.33368 send=0.22009; outPut: create=0.01190 send=0.48686; metadata: (un)pack=0.00014 bcast=0.01397; IS=0.00001; CD=0.00001; AS=0.07379; FO=0.00372; FS=0.00000; LB=0.13171; max memory=184320) }
808291055 LDAS-LA2332764 mpi::filterLogEntries 2332764.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
808291055 LDAS-LA2332764 mpi::filterLogEntries 2332764.1:progress 100.00%
808291055 LDAS-LA2332764 mpi::filterLogEntries 2332764.1:projected_ratio 0.00000
808291056 LDAS-LA2332764 mpi::filterLogEntries 2332764.2:finalizing { totalTime=0.292059 sentObjects=2; Rank0 (inPut: create=0.04083 send=0.00067 ilwd=0.03605; outPut: create=0.00000 send=0.01160 ilwd=0.04327; metadata: (un)pack=0.00014 bcast=0.00100; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01196; max memory=483328) Rank1 (inPut: create=0.03971 send=0.00129; outPut: create=0.00000 send=0.04743; metadata: (un)pack=0.00023 bcast=0.00192; IS=0.00011; CD=0.00063; AS=0.00015; FO=0.00000; FS=0.00004; LB=0.02395; max memory=651264) }
808291069 LDAS-LA2332765 mpi::filterLogEntries 2332765.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
808291079 LDAS-LA2332765 mpi::filterLogEntries 2332765.1:progress 68.37%
808291079 LDAS-LA2332765 mpi::filterLogEntries 2332765.1:projected_ratio 0.02528
808291079 LDAS-LA2332765 jobstate response: '2332765.1:cont'
808291080 LDAS-LA2332765 mpi::filterLogEntries 2332765.2:progress 100.00%
808291080 LDAS-LA2332765 mpi::filterLogEntries 2332765.2:projected_ratio 0.02554
808291080 LDAS-LA2332765 mpi::filterLogEntries 2332765.3:finalizing { totalTime=10.7659 sentObjects=4; Rank0 (inPut: create=0.37437 send=0.17808 ilwd=0.24843; outPut: create=0.11185 send=0.05226 ilwd=0.19189; metadata: (un)pack=0.02137 bcast=0.06468; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.11590; max memory=6836224) Rank1 (inPut: create=0.33462 send=0.21694; outPut: create=0.06401 send=0.36410; metadata: (un)pack=0.01834 bcast=0.02668; IS=0.00011; CD=0.07634; AS=9.19566; FO=0.00148; FS=0.00380; LB=0.03726; max memory=348160) }
808291099 LDAS-LA2332766 mpi::filterLogEntries 2332766.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
808291099 LDAS-LA2332766 mpi::filterLogEntries 2332766.1:progress 100.00%
808291099 LDAS-LA2332766 mpi::filterLogEntries 2332766.1:projected_ratio 0.00001
808291099 LDAS-LA2332766 mpi::filterLogEntries 2332766.2:finalizing { totalTime=0.432118 sentObjects=3; Rank0 (inPut: create=0.04911 send=0.00287 ilwd=0.04264; outPut: create=0.01240 send=0.02503 ilwd=0.01080; metadata: (un)pack=0.00038 bcast=0.01278; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01197; max memory=892928) Rank1 (inPut: create=0.04744 send=0.00555; outPut: create=0.01298 send=0.02452; metadata: (un)pack=0.00044 bcast=0.01440; IS=0.00011; CD=0.00260; AS=0.00054; FO=0.00005; FS=0.00004; LB=0.03589; max memory=2056192) }
808291142 LDAS-LA2332767 mpi::filterLogEntries 2332767.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.1:progress 0.00%
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.1:projected_ratio 0.00018
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.2:progress 10.00%
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.2:projected_ratio 0.00034
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.3:progress 20.00%
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.3:projected_ratio 0.00051
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.4:progress 30.00%
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.4:projected_ratio 0.00068
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.5:progress 40.00%
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.5:projected_ratio 0.00084
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.6:progress 50.00%
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.6:projected_ratio 0.00101
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.7:progress 60.00%
808291144 LDAS-LA2332767 mpi::filterLogEntries 2332767.7:projected_ratio 0.00117
808291144 LDAS-LA2332767 jobstate response: '2332767.7:cont'
808291145 LDAS-LA2332767 mpi::filterLogEntries 2332767.8:progress 70.00%
808291145 LDAS-LA2332767 mpi::filterLogEntries 2332767.8:projected_ratio 0.00136
808291145 LDAS-LA2332767 mpi::filterLogEntries 2332767.9:progress 80.00%
808291145 LDAS-LA2332767 mpi::filterLogEntries 2332767.9:projected_ratio 0.00152
808291145 LDAS-LA2332767 mpi::filterLogEntries 2332767.10:progress 90.00%
808291145 LDAS-LA2332767 mpi::filterLogEntries 2332767.10:projected_ratio 0.00169
808291145 LDAS-LA2332767 mpi::filterLogEntries 2332767.11:progress 100.00%
808291145 LDAS-LA2332767 mpi::filterLogEntries 2332767.11:projected_ratio 0.00169
808291145 LDAS-LA2332767 mpi::filterLogEntries 2332767.12:finalizing { totalTime=2.86602 sentObjects=3; Rank0 (inPut: create=0.28902 send=0.10866 ilwd=0.21006; outPut: create=0.12082 send=0.38355 ilwd=0.04373; metadata: (un)pack=0.00064 bcast=0.11949; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.11593; max memory=1449984) Rank1 (inPut: create=0.26226 send=0.13511; outPut: create=0.13189 send=0.37051; metadata: (un)pack=0.00054 bcast=0.13236; IS=0.05976; CD=0.09550; AS=1.47645; FO=0.00043; FS=0.00279; LB=0.02405; max memory=18370560) }
808291172 LDAS-LA2332768 mpi::filterLogEntries 2332768.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) Rank2(beowulf) Rank3(beowulf) }
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.1:progress 0.00%
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.1:projected_ratio 0.00000
808291175 LDAS-LA2332768 jobstate response: '2332768.1:cont'
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.2:progress 0.00%
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.2:projected_ratio 0.00000
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.3:progress 25.00%
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.3:projected_ratio 0.00000
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.4:progress 50.00%
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.4:projected_ratio 0.00026
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.5:progress 50.00%
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.5:projected_ratio 0.00055
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.6:progress 50.00%
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.6:projected_ratio 0.00055
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.7:progress 50.00%
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.7:projected_ratio 0.00055
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.8:progress 75.00%
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.8:projected_ratio 0.00068
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.9:progress 75.00%
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.9:projected_ratio 0.00094
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.10:progress 75.00%
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.10:projected_ratio 0.00094
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.11:progress 100.00%
808291175 LDAS-LA2332768 mpi::filterLogEntries 2332768.11:projected_ratio 0.00098
808291176 LDAS-LA2332768 mpi::filterLogEntries 2332768.12:finalizing { totalTime=3.99033 sentObjects=3; Rank0 (inPut: create=0.41875 send=0.51132 ilwd=0.23122; outPut: create=0.04075 send=0.20544 ilwd=0.04401; metadata: (un)pack=0.00739 bcast=0.02695; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.11663; max memory=2813952) Rank2 (inPut: create=0.38062 send=0.56829; outPut: create=0.00000 send=0.11952; metadata: (un)pack=0.00016 bcast=0.00265; IS=0.00018; CD=0.99866; AS=0.25542; FO=0.00000; FS=0.00735; LB=0.00001; max memory=692224) Rank3 (inPut: create=0.37986 send=0.63206; outPut: create=0.00000 send=0.13272; metadata: (un)pack=0.00006 bcast=0.00428; IS=0.00021; CD=1.67058; AS=0.37200; FO=0.00000; FS=0.00516; LB=0.00002; max memory=917504) Rank1 (inPut: create=0.38271 send=0.61663; outPut: create=0.02558 send=0.47877; metadata: (un)pack=0.00627 bcast=0.01569; IS=0.00021; CD=1.67526; AS=0.07432; FO=0.00052; FS=0.00427; LB=0.04835; max memory=339968) }
808291197 LDAS-LA2332769 mpi::filterLogEntries 2332769.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) Rank2(beowulf) }
808291212 LDAS-LA2332769 mpi::filterLogEntries 2332769.1:progress 100.00%
808291212 LDAS-LA2332769 mpi::filterLogEntries 2332769.1:projected_ratio 0.11778
808291212 LDAS-LA2332769 jobstate response: '2332769.1:cont'
808291213 LDAS-LA2332769 mpi::filterLogEntries 2332769.2:finalizing { totalTime=15.6065 sentObjects=2; Rank0 (inPut: create=0.17199 send=0.13049 ilwd=0.11243; outPut: create=0.00000 send=0.01304 ilwd=0.04534; metadata: (un)pack=0.00011 bcast=0.00211; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.31988; max memory=323584) Rank1 (inPut: create=0.14566 send=0.13546; outPut: create=0.00000 send=0.01288; metadata: (un)pack=0.00016 bcast=0.00312; IS=0.00045; CD=0.00006; AS=14.60424; FO=0.00000; FS=0.00008; LB=0.33586; max memory=212992) Rank2 (inPut: create=0.15862 send=0.15198; outPut: create=0.00000 send=0.36082; metadata: (un)pack=0.00021 bcast=0.00261; IS=0.00055; CD=0.00007; AS=14.60422; FO=0.00000; FS=0.00007; LB=0.00001; max memory=212992) }
810604328 LDAS-LA2401710 mpi::filterLogEntries 2401710.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
810604329 LDAS-LA2401710 mpi::filterLogEntries 2401710.1:error {Rank 0 failed with error message: connect_failure: address=beowulf port=10021: (../../../../../api/genericAPI/so/src/socketcmd.cc:249)}
810604329 LDAS-LA2401710 mpi::newJobCallback {Rank 0 failed with error message: connect_failure: address=beowulf port=10021: (../../../../../api/genericAPI/so/src/socketcmd.cc:249)}
810604690 LDAS-LA2401716 mpi::filterLogEntries 2401716.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
810604691 LDAS-LA2401716 mpi::filterLogEntries 2401716.1:progress 100.00%
810604691 LDAS-LA2401716 mpi::filterLogEntries 2401716.1:projected_ratio 0.00019
810604692 LDAS-LA2401716 mpi::filterLogEntries 2401716.2:finalizing { totalTime=1.81688 sentObjects=3; Rank0 (inPut: create=0.34817 send=0.16705 ilwd=0.23041; outPut: create=0.09031 send=0.46157 ilwd=0.01067; metadata: (un)pack=0.00013 bcast=0.01378; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01194; max memory=262144) Rank1 (inPut: create=0.31040 send=0.20759; outPut: create=0.01290 send=0.50605; metadata: (un)pack=0.00012 bcast=0.01465; IS=0.00001; CD=0.00000; AS=0.06737; FO=0.00409; FS=0.00000; LB=0.13124; max memory=184320) }
813294798 LDAS-LA2504212 mpi::filterLogEntries 2504212.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) Rank2(beowulf) Rank3(beowulf) }
813294800 LDAS-LA2504212 mpi::filterLogEntries 2504212.1:progress 0.00%
813294800 LDAS-LA2504212 mpi::filterLogEntries 2504212.1:projected_ratio 0.00010
813294800 LDAS-LA2504212 jobstate response: '2504212.1:cont'
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.2:progress 0.00%
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.2:projected_ratio 0.00010
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.3:progress 0.00%
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.3:projected_ratio 0.00010
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.4:progress 0.00%
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.4:projected_ratio 0.00429
813294803 LDAS-LA2504212 jobstate response: '2504212.4:cont'
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.5:progress 100.00%
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.5:projected_ratio 0.00429
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.6:progress 100.00%
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.6:projected_ratio 0.00857
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.7:progress 100.00%
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.7:projected_ratio 0.00857
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.8:progress 100.00%
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.8:projected_ratio 0.00863
813294803 LDAS-LA2504212 mpi::filterLogEntries 2504212.9:finalizing { totalTime=5.47223 sentObjects=2; Rank0 (inPut: create=0.56910 send=0.58133 ilwd=0.34569; outPut: create=0.00000 send=0.13503 ilwd=0.04432; metadata: (un)pack=0.00010 bcast=0.00318; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.21846; max memory=1085440) Rank3 (inPut: create=0.51940 send=0.73284; outPut: create=0.00000 send=0.06071; metadata: (un)pack=0.00014 bcast=0.00438; IS=0.00028; CD=0.59164; AS=0.17674; FO=0.00000; FS=0.00817; LB=0.00001; max memory=1478656) Rank2 (inPut: create=0.51887 send=0.72590; outPut: create=0.00000 send=0.20458; metadata: (un)pack=0.00024 bcast=0.00353; IS=0.00028; CD=0.59520; AS=2.57082; FO=0.00000; FS=0.00623; LB=0.00001; max memory=1478656) Rank1 (inPut: create=0.51978 send=0.65066; outPut: create=0.00000 send=0.39802; metadata: (un)pack=0.00026 bcast=0.00266; IS=0.00023; CD=0.52555; AS=2.39876; FO=0.00000; FS=0.00685; LB=0.02396; max memory=1478656) }
813886285 LDAS-LA2547617 mpi::newJobCallback Job stalled. possible lam error, see logs. Wrapper API failed to report initialisation.
813886380 LDAS-LA2547626 mpi::newJobCallback Job stalled. possible lam error, see logs. Wrapper API failed to report initialisation.
813887814 LDAS-LA2547629 mpi::newJobCallback Job stalled. possible lam error, see logs. Wrapper API failed to report initialisation.
813889121 LDAS-LA2547630 mpi::newJobCallback Job stalled. possible lam error, see logs. Wrapper API failed to report initialisation.
813946416 LDAS-LA2547946 mpi::newJobCallback Job stalled. possible lam error, see logs. Wrapper API failed to report initialisation.
813946484 LDAS-LA2547947 mpi::filterLogEntries 2547947.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
813946486 LDAS-LA2547947 mpi::filterLogEntries 2547947.1:progress 100.00%
813946486 LDAS-LA2547947 mpi::filterLogEntries 2547947.1:projected_ratio 0.00017
813946486 LDAS-LA2547947 mpi::filterLogEntries 2547947.2:finalizing { totalTime=2.23087 sentObjects=3; Rank0 (inPut: create=0.32767 send=0.15560 ilwd=0.22898; outPut: create=0.07933 send=0.77681 ilwd=0.01473; metadata: (un)pack=0.00012 bcast=0.01674; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01594; max memory=266240) Rank1 (inPut: create=0.29195 send=0.18956; outPut: create=0.01589 send=0.79585; metadata: (un)pack=0.00013 bcast=0.01742; IS=0.00001; CD=0.00000; AS=0.06055; FO=0.00409; FS=0.00000; LB=0.12726; max memory=184320) }
813946905 LDAS-LA2547952 mpi::filterLogEntries 2547952.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
813946907 LDAS-LA2547952 mpi::filterLogEntries 2547952.1:progress 100.00%
813946907 LDAS-LA2547952 mpi::filterLogEntries 2547952.1:projected_ratio 0.00017
813946907 LDAS-LA2547952 mpi::filterLogEntries 2547952.2:finalizing { totalTime=1.7274 sentObjects=3; Rank0 (inPut: create=0.32889 send=0.15538 ilwd=0.23050; outPut: create=0.09351 send=0.43421 ilwd=0.04317; metadata: (un)pack=0.00012 bcast=0.03265; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01594; max memory=266240) Rank1 (inPut: create=0.29298 send=0.18808; outPut: create=0.03190 send=0.47177; metadata: (un)pack=0.00012 bcast=0.03335; IS=0.00001; CD=0.00000; AS=0.06050; FO=0.00413; FS=0.00000; LB=0.12713; max memory=184320) }
813946951 LDAS-LA2547953 mpi::filterLogEntries 2547953.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
813946953 LDAS-LA2547953 mpi::filterLogEntries 2547953.1:progress 100.00%
813946953 LDAS-LA2547953 mpi::filterLogEntries 2547953.1:projected_ratio 0.00017
813946953 LDAS-LA2547953 mpi::filterLogEntries 2547953.2:finalizing { totalTime=2.04248 sentObjects=3; Rank0 (inPut: create=0.32607 send=0.15542 ilwd=0.22910; outPut: create=0.07861 send=0.78487 ilwd=0.04060; metadata: (un)pack=0.00012 bcast=0.01660; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01594; max memory=266240) Rank1 (inPut: create=0.28976 send=0.18784; outPut: create=0.01589 send=0.81221; metadata: (un)pack=0.00012 bcast=0.01736; IS=0.00001; CD=0.00000; AS=0.06031; FO=0.00406; FS=0.00000; LB=0.12725; max memory=184320) }
813947000 LDAS-LA2547954 mpi::filterLogEntries 2547954.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
813947002 LDAS-LA2547954 mpi::filterLogEntries 2547954.1:progress 100.00%
813947002 LDAS-LA2547954 mpi::filterLogEntries 2547954.1:projected_ratio 0.00017
813947002 LDAS-LA2547954 mpi::filterLogEntries 2547954.2:finalizing { totalTime=2.0318 sentObjects=3; Rank0 (inPut: create=0.32630 send=0.15545 ilwd=0.22904; outPut: create=0.07867 send=0.78505 ilwd=0.04083; metadata: (un)pack=0.00011 bcast=0.01672; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01594; max memory=266240) Rank1 (inPut: create=0.29037 send=0.18797; outPut: create=0.01589 send=0.81047; metadata: (un)pack=0.00012 bcast=0.01742; IS=0.00001; CD=0.00000; AS=0.06099; FO=0.00409; FS=0.00000; LB=0.12725; max memory=184320) }
813947044 LDAS-LA2547956 mpi::filterLogEntries 2547956.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
813947046 LDAS-LA2547956 mpi::filterLogEntries 2547956.1:progress 100.00%
813947046 LDAS-LA2547956 mpi::filterLogEntries 2547956.1:projected_ratio 0.00017
813947046 LDAS-LA2547956 mpi::filterLogEntries 2547956.2:finalizing { totalTime=1.72639 sentObjects=3; Rank0 (inPut: create=0.32395 send=0.15539 ilwd=0.22674; outPut: create=0.09340 send=0.43450 ilwd=0.04075; metadata: (un)pack=0.00012 bcast=0.03262; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01595; max memory=266240) Rank1 (inPut: create=0.28779 send=0.18809; outPut: create=0.03190 send=0.47411; metadata: (un)pack=0.00012 bcast=0.03335; IS=0.00001; CD=0.00000; AS=0.06028; FO=0.00439; FS=0.00000; LB=0.12673; max memory=184320) }
813947090 LDAS-LA2547957 mpi::filterLogEntries 2547957.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
813947092 LDAS-LA2547957 mpi::filterLogEntries 2547957.1:progress 100.00%
813947092 LDAS-LA2547957 mpi::filterLogEntries 2547957.1:projected_ratio 0.00017
813947092 LDAS-LA2547957 mpi::filterLogEntries 2547957.2:finalizing { totalTime=1.72241 sentObjects=3; Rank0 (inPut: create=0.32712 send=0.15517 ilwd=0.22657; outPut: create=0.07844 send=0.50907 ilwd=0.00965; metadata: (un)pack=0.00012 bcast=0.01673; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01594; max memory=266240) Rank1 (inPut: create=0.29173 send=0.18782; outPut: create=0.01590 send=0.54203; metadata: (un)pack=0.00012 bcast=0.01744; IS=0.00001; CD=0.00000; AS=0.06070; FO=0.00409; FS=0.00000; LB=0.12721; max memory=184320) }
813953911 LDAS-LA2548264 mpi::filterLogEntries 2548264.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
813953913 LDAS-LA2548264 mpi::filterLogEntries 2548264.1:progress 100.00%
813953913 LDAS-LA2548264 mpi::filterLogEntries 2548264.1:projected_ratio 0.00017
813953913 LDAS-LA2548264 mpi::filterLogEntries 2548264.2:finalizing { totalTime=1.72259 sentObjects=3; Rank0 (inPut: create=0.33025 send=0.15667 ilwd=0.23270; outPut: create=0.09345 send=0.43452 ilwd=0.04092; metadata: (un)pack=0.00011 bcast=0.03267; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01594; max memory=266240) Rank1 (inPut: create=0.29405 send=0.18917; outPut: create=0.03190 send=0.47438; metadata: (un)pack=0.00012 bcast=0.03336; IS=0.00001; CD=0.00000; AS=0.06064; FO=0.00442; FS=0.00000; LB=0.12671; max memory=184320) }
813953959 LDAS-LA2548269 mpi::filterLogEntries 2548269.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
813953961 LDAS-LA2548269 mpi::filterLogEntries 2548269.1:progress 100.00%
813953961 LDAS-LA2548269 mpi::filterLogEntries 2548269.1:projected_ratio 0.00017
813953961 LDAS-LA2548269 mpi::filterLogEntries 2548269.2:finalizing { totalTime=1.78969 sentObjects=3; Rank0 (inPut: create=0.32630 send=0.15553 ilwd=0.22906; outPut: create=0.07945 send=0.53623 ilwd=0.00991; metadata: (un)pack=0.00011 bcast=0.01672; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01594; max memory=266240) Rank1 (inPut: create=0.29075 send=0.19056; outPut: create=0.03189 send=0.55579; metadata: (un)pack=0.00013 bcast=0.03339; IS=0.00001; CD=0.00000; AS=0.06065; FO=0.00407; FS=0.00000; LB=0.12723; max memory=184320) }
813954009 LDAS-LA2548272 mpi::filterLogEntries 2548272.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
813954011 LDAS-LA2548272 mpi::filterLogEntries 2548272.1:progress 100.00%
813954011 LDAS-LA2548272 mpi::filterLogEntries 2548272.1:projected_ratio 0.00017
813954011 LDAS-LA2548272 mpi::filterLogEntries 2548272.2:finalizing { totalTime=1.72479 sentObjects=3; Rank0 (inPut: create=0.32820 send=0.15546 ilwd=0.23020; outPut: create=0.07866 send=0.50899 ilwd=0.00988; metadata: (un)pack=0.00011 bcast=0.01663; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01595; max memory=266240) Rank1 (inPut: create=0.29193 send=0.18789; outPut: create=0.01590 send=0.54220; metadata: (un)pack=0.00012 bcast=0.01737; IS=0.00001; CD=0.00000; AS=0.06038; FO=0.00407; FS=0.00000; LB=0.12728; max memory=184320) }
813954058 LDAS-LA2548273 mpi::filterLogEntries 2548273.0:initializing { Master-Rank0(beowulf) Rank1(beowulf) }
813954060 LDAS-LA2548273 mpi::filterLogEntries 2548273.1:progress 100.00%
813954060 LDAS-LA2548273 mpi::filterLogEntries 2548273.1:projected_ratio 0.00017
813954060 LDAS-LA2548273 mpi::filterLogEntries 2548273.2:finalizing { totalTime=1.78453 sentObjects=3; Rank0 (inPut: create=0.32398 send=0.15536 ilwd=0.22592; outPut: create=0.07844 send=0.50906 ilwd=0.01062; metadata: (un)pack=0.00011 bcast=0.01671; IS=0.00000; CD=0.00000; AS=0.00000; FO=0.00000; FS=0.00000; LB=0.01594; max memory=266240) Rank1 (inPut: create=0.28827 send=0.18788; outPut: create=0.01590 send=0.54144; metadata: (un)pack=0.00013 bcast=0.01743; IS=0.00001; CD=0.00000; AS=0.06076; FO=0.00408; FS=0.00000; LB=0.12724; max memory=184320) }