[gs-commits] rev 10802 - trunk/gs/toolbin/localcluster
regression at ghostscript.com
regression at ghostscript.com
Wed Feb 24 19:27:11 UTC 2010
Author: regression
Date: 2010-02-24 19:27:11 +0000 (Wed, 24 Feb 2010)
New Revision: 10802
Modified:
trunk/gs/toolbin/localcluster/cachearchive.pl
trunk/gs/toolbin/localcluster/clustermaster.pl
trunk/gs/toolbin/localcluster/run.pl
Log:
Changed naming scheme for regressions from 00000-00000 to 00000 (i.e. no longer treat Ghostscript and GhostPDL revisions independently).
Modified: trunk/gs/toolbin/localcluster/cachearchive.pl
===================================================================
--- trunk/gs/toolbin/localcluster/cachearchive.pl 2010-02-24 19:04:17 UTC (rev 10801)
+++ trunk/gs/toolbin/localcluster/cachearchive.pl 2010-02-24 19:27:11 UTC (rev 10802)
@@ -24,11 +24,11 @@
sub myCmp($$) {
my $a=shift;
my $b=shift;
- $a=~m/-(\d+)/;
- $a=$1;
- $b=~m/-(\d+)/;
- $b=$1;
- return($b cmp $a);
+ $a=~m/(\d+)/;
+ my $a1=$1;
+ $b=~m/(\d+)/;
+ my $b1=$1;
+ return($b1 cmp $a1);
}
my $count=$previousValues;
Modified: trunk/gs/toolbin/localcluster/clustermaster.pl
===================================================================
--- trunk/gs/toolbin/localcluster/clustermaster.pl 2010-02-24 19:04:17 UTC (rev 10801)
+++ trunk/gs/toolbin/localcluster/clustermaster.pl 2010-02-24 19:27:11 UTC (rev 10802)
@@ -165,6 +165,7 @@
foreach my $m (keys %lastTransfer) {
if (time-$lastTransfer{$m}>=$maxTransferTime) {
mylog "machine $m hasn't connected in ".(time-$lastTransfer{$m})." seconds, assuming it went down\n";
+ unlink "$m.start"; # so it does connect later and try to run jobs
delete $lastTransfer{$m} if (exists $lastTransfer{$m});
delete $machines{$m} if (exists $machines{$m});
if (scalar keys %lastTransfer) {
@@ -187,6 +188,7 @@
foreach my $m (keys %machines) {
if (-e "$m.done" && exists $lastTransfer{$m}) {
+ unlink "$m.start"; # so it does connect later and try to run jobs
if (-e "$m.fail") {
mylog "both $m.done and $m.fail are set, ignoring $m.done\n";
} else {
@@ -227,7 +229,9 @@
# print "$currentRev1 $newRev1\n$currentRev2 $newRev2\n";
open(LOCK,">$lock") || die "can't write to $lock";
flock(LOCK,LOCK_EX);
- my $s="svn $newRev1 $newRev2";
+ my $rev=$newRev1;
+ $rev=$newRev2 if ($newRev2>$rev);
+ my $s="svn $rev";
my $t=`grep "$s" $queue`;
chomp $t;
# print "grep '$s' returned: $t\n";
@@ -254,7 +258,7 @@
die "svn info failed";
}
if ($currentRev1 != $newRev1) {
- # print "$currentRev1 $newRev1\n$currentRev2 $newRev2\n";
+ # print "$currentRev1 $newRev1\n";
open(LOCK,">$lock") || die "can't write to $lock";
flock(LOCK,LOCK_EX);
my $s="svn-icc_work $newRev1";
@@ -363,6 +367,12 @@
open(F,">>$queue");
print F "$s\n";
close(F);
+ mylog "running: find $usersDir/$user/ghostpdl -name \\*.sh | xargs \$HOME/bin/flip -u\n";
+ `find $usersDir/$user/ghostpdl -name \\*.sh | xargs \$HOME/bin/flip -u`;
+ mylog "running: find $usersDir/$user/ghostpdl -name instcopy | xargs \$HOME/bin/flip -u\n";
+ `find $usersDir/$user/ghostpdl -name instcopy | xargs \$HOME/bin/flip -u`;
+ mylog "running: chmod -R +xr $usersDir/$user/ghostpdl\n";
+ `chmod -R +xr $usersDir/$user/ghostpdl`;
}
close(LOCK);
}
@@ -464,24 +474,22 @@
my $mupdfRegression=0;
my $updateBaseline=0;
my $userName="";
-my $rev1;
-my $rev2;
+my $rev;
-if ($regression =~ m/svn (\d+) (\d+)/) {
+if ($regression =~ m/svn (\d+)/) {
mylog "found svn regression in queue: $regression\n";
$normalRegression=1;
- $rev1=$1;
- $rev2=$2;
+ $rev=$1;
my $currentRev1=`svn info ghostpdl | grep "Last Changed Rev" | awk '{ print \$4 } '`;
my $currentRev2=`svn info ghostpdl/gs | grep "Last Changed Rev" | awk '{ print \$4 } '`;
chomp $currentRev1;
chomp $currentRev2;
- my $a=`svn update ghostpdl -r$rev1 --ignore-externals`;
- my $b=`svn update ghostpdl/gs -r$rev2`;
- mylog "svn update ghostpdl -r$rev1 --ignore-externals\n" if ($verbose);
- mylog "svn update ghostpdl/gs -r$rev2\n" if ($verbose);
+ my $a=`svn update ghostpdl -r$rev --ignore-externals`;
+ my $b=`svn update ghostpdl/gs -r$rev`;
+ mylog "svn update ghostpdl -r$rev --ignore-externals\n" if ($verbose);
+ mylog "svn update ghostpdl/gs -r$rev\n" if ($verbose);
$footer.="\nChanged files:\n";
$a.=$b;
@@ -503,11 +511,11 @@
mylog "$s: $rules{$t}\n";
$set|=$rules{$t};
} else {
- mylog "$s: missing, testing all\n";
- $set=15;
+# mylog "$s: missing, testing all\n";
+# $set=15;
}
} else {
- $set=15;
+# $set=15;
}
}
}
@@ -528,7 +536,8 @@
# un-update the source so that if the regression fails were are back to the where we started
`svn update ghostpdl -r$currentRev1 --ignore-externals`;
`svn update ghostpdl/gs -r$currentRev2`;
- mylog "svn update ghostpdl -r$currentRev1 --ignore-externals\nsvn update ghostpdl/gs -r$currentRev2\n" if ($verbose);
+ mylog "svn update ghostpdl -r$currentRev1 --ignore-externals\n";
+ mylog "svn update ghostpdl/gs -r$currentRev2\n";
} else {
mylog "no interesting files changed, skipping regression\n";
$normalRegression=0;
@@ -540,16 +549,16 @@
} elsif ($regression=~/svn-icc_work (.+)/) {
mylog "found icc_work regression in queue: $regression\n";
$icc_workRegression=1;
- $rev1=$1;
- $product="gs";
- $footer.="icc_work regression: $rev1\n\nProducts tested: $product\n\n";
+ $rev=$1;
+ $product="gs pcl xps";
+ $footer.="icc_work regression: $rev\n\nProducts tested: $product\n\n";
} elsif ($regression=~/mupdf/) {
mylog "found mupdf entry in queue.lst, not yet handled, removing.\n";
my $cmd="touch mupdf.tar.gz ; rm mupdf.tar.gz ; tar cvf mupdf.tar --exclude=_darcs mupdf ; gzip mupdf.tar";
`$cmd`;
$cmd="cd mupdf ; darcs changes --count";
- $rev1=`$cmd`;
- chomp $rev1;
+ $rev=`$cmd`;
+ chomp $rev;
# $mupdfRegression=1;
$product="mupdf";
} elsif ($regression=~/updatebaseline/) {
@@ -567,19 +576,19 @@
if ($normalRegression) {
open(F,">revision.gs");
- print F "local cluster regression gs-r$rev2 / ghostpdl-r$rev1 (xefitra)\n";
+ print F "local cluster regression r$rev (xefitra)\n";
close(F);
}
if ($icc_workRegression) {
open(F,">revision.gs");
- print F "local cluster regression icc_work-r$rev1 (xefitra)\n";
+ print F "local cluster regression icc_work-r$rev (xefitra)\n";
close(F);
}
if ($mupdfRegression) {
open(F,">revision.mudpf");
- print F "local cluster regression mupdf-r$rev1 (xefitra)\n";
+ print F "local cluster regression mupdf-r$rev (xefitra)\n";
close(F);
}
@@ -654,13 +663,23 @@
unlink("$_.done");
unlink("$_.fail");
unlink("$_.abort");
+ if (-e "$_.start") {
+ alarm 0;
+ mylog "$_.start exists when it shouldn't, aborting\n";
+ foreach my $m (keys %machines) {
+ unlink "$m.start";
+ }
+ abortAll();
+ unlink $runningSemaphore;
+ exit;
+ }
open(F,">$_.start");
if ($normalRegression) {
- print F "svn\t$rev1 $rev2\t$product\n";
+ print F "svn\t$rev\t$product\n";
} elsif ($icc_workRegression) {
- print F "svn-icc_work\t$rev1\t$product";
+ print F "svn-icc_work\t$rev\t$product";
} elsif ($mupdfRegression) {
- print F "mupdf\t$rev1\t$product";
+ print F "mupdf\t$rev\t$product";
} elsif ($updateBaseline) {
print F "svn\thead\t$product";
} else {
@@ -674,11 +693,11 @@
chomp $startText;
open (F,">status");
if ($normalRegression) {
- print F "Regression gs-r$rev2 / ghostpdl-r$rev1 started at $startText UTC";
+ print F "Regression r$rev started at $startText UTC";
} elsif ($icc_workRegression) {
- print F "Regression icc_work-r$rev1 started at $startText UTC";
+ print F "Regression icc_work-r$rev started at $startText UTC";
} elsif ($mupdfRegression) {
- print F "Regression mupdf-r$rev1 started at $startText UTC";
+ print F "Regression mupdf-r$rev started at $startText UTC";
} elsif ($updateBaseline) {
print F "Update baseline started at $startText UTC";
} else {
@@ -743,6 +762,7 @@
if (!exists $lastTransfer{$t}) {
mylog "received connection from unexpected client $t (".($client->peerhost)."); sending done\n";
print $client "done\n";
+ unlink "$t.start";
} elsif (-e "$t.start") {
# if we got here we received a connection from a client who appears to be out of sync
# (i.e. they are probably still running the previous job). this is a rare condition
@@ -784,11 +804,11 @@
chomp $s;
open (F,">status");
if ($normalRegression) {
- print F "Regression gs-r$rev2 / ghostpdl-r$rev1 started at $startText UTC - ".($totalJobs-scalar(@jobs))."/$totalJobs sent - $percentage%";
+ print F "Regression r$rev started at $startText UTC - ".($totalJobs-scalar(@jobs))."/$totalJobs sent - $percentage%";
} elsif ($icc_workRegression) {
- print F "Regression icc_work-r$rev1 started at $startText UTC - ".($totalJobs-scalar(@jobs))."/$totalJobs sent - $percentage%";
+ print F "Regression icc_work-r$rev started at $startText UTC - ".($totalJobs-scalar(@jobs))."/$totalJobs sent - $percentage%";
} elsif ($mupdfRegression) {
- print F "Regression mupdf-r$rev1 started at $startText UTC - ".($totalJobs-scalar(@jobs))."/$totalJobs sent - $percentage%";
+ print F "Regression mupdf-r$rev started at $startText UTC - ".($totalJobs-scalar(@jobs))."/$totalJobs sent - $percentage%";
} elsif ($updateBaseline) {
} else {
print F "Regression $userRegression started at $startText UTC - ".($totalJobs-scalar(@jobs))."/$totalJobs sent - $percentage%";
@@ -885,11 +905,11 @@
chomp $s;
open (F,">status");
if ($normalRegression) {
- print F "Regression gs-r$rev2 / ghostpdl-r$rev1 started at $startText UTC - finished at $s";
+ print F "Regression r$rev started at $startText UTC - finished at $s";
} elsif ($icc_workRegression) {
- print F "Regression icc_work-r$rev1 started at $startText UTC - finished at $s";
+ print F "Regression icc_work-r$rev started at $startText UTC - finished at $s";
} elsif ($mupdfRegression) {
- print F "Regression mupdf-r$rev1 started at $startText UTC - finished at $s";
+ print F "Regression mupdf-r$rev started at $startText UTC - finished at $s";
} elsif ($updateBaseline) {
print F "Update baseline started at $startText UTC - finished at $s";
} else {
@@ -981,18 +1001,18 @@
# `mail marcos.woehrmann\@artifex.com -s \"\`cat revision.gs\`\" <email.txt`;
checkPID();
- `touch archive/$rev2-$rev1`;
- `rm -fr archive/$rev2-$rev1`;
- `mkdir archive/$rev2-$rev1`;
- `mv $logs archive/$rev2-$rev1/.`;
- `gzip archive/$rev2-$rev1/*log`;
- `cp -p email.txt archive/$rev2-$rev1/.`;
- `cp -p current.tab archive/$rev2-$rev1/current.tab`;
- `cp -p previous.tab archive/$rev2-$rev1/previous.tab`;
- `cp -p current.tab archive/$rev2-$rev1.tab`;
- # `touch archive/$rev2-$rev1.tab.gz`;
- # unlink "archive/$rev2-$rev1.tab.gz";
- # `gzip archive/$rev2-$rev1.tab`;
+ `touch archive/$rev`;
+ `rm -fr archive/$rev`;
+ `mkdir archive/$rev`;
+ `mv $logs archive/$rev/.`;
+ `gzip archive/$rev/*log`;
+ `cp -p email.txt archive/$rev/.`;
+ `cp -p current.tab archive/$rev/current.tab`;
+ `cp -p previous.tab archive/$rev/previous.tab`;
+ `cp -p current.tab archive/$rev.tab`;
+ # `touch archive/$rev.tab.gz`;
+ # unlink "archive/$rev.tab.gz";
+ # `gzip archive/$rev.tab`;
# unlink "log";
} elsif ($icc_workRegression) {
`mv icc_work_previous.tab icc_work_previous2.tab`;
@@ -1005,15 +1025,15 @@
`./compare.pl icc_work_current.tab icc_work_previous.tab $elapsedTime $machineCount false \"$product\" >>email.txt`;
checkPID();
- `touch archive/icc_work-$rev1`;
- `rm -fr archive/icc_work-$rev1`;
- `mkdir archive/icc_work-$rev1`;
- `mv $logs archive/icc_work-$rev1/.`;
- `gzip archive/icc_work-$rev1/*log`;
- `cp -p email.txt archive/icc_work-$rev1/.`;
- `cp -p icc_work_current.tab archive/icc_work-$rev1/icc_work_current.tab`;
- `cp -p icc_work_previous.tab archive/icc_work-$rev1/icc_work_previous.tab`;
- `cp -p icc_work_current.tab archive/icc_work-$rev1.tab`;
+ `touch archive/icc_work-$rev`;
+ `rm -fr archive/icc_work-$rev`;
+ `mkdir archive/icc_work-$rev`;
+ `mv $logs archive/icc_work-$rev/.`;
+ `gzip archive/icc_work-$rev/*log`;
+ `cp -p email.txt archive/icc_work-$rev/.`;
+ `cp -p icc_work_current.tab archive/icc_work-$rev/icc_work_current.tab`;
+ `cp -p icc_work_previous.tab archive/icc_work-$rev/icc_work_previous.tab`;
+ `cp -p icc_work_current.tab archive/icc_work-$rev.tab`;
} elsif ($mupdfRegression) {
} elsif ($updateBaseline) {
} else {
@@ -1069,12 +1089,12 @@
checkPID();
if ($normalRegression) {
`mail -a \"From: marcos.woehrmann\@artifex.com\" gs-regression\@ghostscript.com -s \"\`cat revision.gs\`\" <email.txt`;
- `mail -a \"From: marcos.woehrmann\@artifex.com\" marcos\@ghostscript.com -s \"\`cat revision.gs\`\" <email.txt`;
+# `mail marcos.woehrmann\@artifex.com -s \"\`cat revision.gs\`\" <email.txt`;
mylog "test complete, performing final svn update\n";
- mylog "svn update ghostpdl -r$rev1 --ignore-externals\nsvn update ghostpdl/gs -r$rev2\n";
- `svn update ghostpdl -r$rev1 --ignore-externals`;
- `svn update ghostpdl/gs -r$rev2`;
+ mylog "svn update ghostpdl -r$rev --ignore-externals\nsvn update ghostpdl/gs -r$rev\n";
+ `svn update ghostpdl -r$rev --ignore-externals`;
+ `svn update ghostpdl/gs -r$rev`;
`./cp.all.sh`;
mylog("calling cachearchive.pl");
@@ -1085,8 +1105,8 @@
`mail -a \"From: marcos.woehrmann\@artifex.com\" marcos\@ghostscript.com -s \"\`cat revision.gs\`\" <email.txt`;
mylog "test complete, performing final svn update\n";
- mylog "svn update icc_work -r$rev1\n";
- `svn update icc_work -r$rev1`;
+ mylog "svn update icc_work -r$rev\n";
+ `svn update icc_work -r$rev`;
mylog("calling cachearchive.pl");
`./cachearchive.pl >md5sum.cache`;
Modified: trunk/gs/toolbin/localcluster/run.pl
===================================================================
--- trunk/gs/toolbin/localcluster/run.pl 2010-02-24 19:04:17 UTC (rev 10801)
+++ trunk/gs/toolbin/localcluster/run.pl 2010-02-24 19:27:11 UTC (rev 10802)
@@ -16,8 +16,8 @@
my $wordSize="64";
my $timeOut=240;
-my $maxTimeout=10; # starting value, is adjusted by value below based on jobs completed
-my $maxTimeoutPercentage=0.5;
+my $maxTimeout=20; # starting value, is adjusted by value below based on jobs completed
+my $maxTimeoutPercentage=1.0;
my $maxCount=12;
$maxCount=16;
@@ -153,7 +153,7 @@
}
mylog "user products=$products user=$user\n" if ($user);
-mylog "svn products=$products revs=$revs\n" if ($revs);
+mylog "svn products=$products rev=$revs\n" if ($revs);
mylog "icc_work products=$products rev=$icc_work\n" if ($icc_work);
my $host="casper3.ghostscript.com";
@@ -175,6 +175,7 @@
my $gsSource=$gpdlSource."/gs";
my $gsBin=$baseDirectory."/gs";
my $icc_workSource=$baseDirectory."/icc_work";
+my $icc_workGsSource=$icc_workSource."/gs";
my $abort=0;
unlink ("$machine.abort");
@@ -409,33 +410,29 @@
$gsSource=$gpdlSource."/gs";
} elsif ($icc_work) {
- if (!-e $icc_workSource) {
+ if (!-e $icc_workGsSource) {
+ `rm -fr $icc_workSource`;
updateStatus('Checking out icc_work');
- $cmd="export SVN_SSH=\"ssh -i \$HOME/.ssh/cluster_key\" ; svn co http://svn.ghostscript.com/ghostscript/branches/icc_work";
- print "$cmd\n" if ($verbose);
+ $cmd="svn co --ignore-externals http://svn.ghostscript.com/ghostscript/trunk/ghostpdl $icc_workSource ; svn co http://svn.ghostscript.com/ghostscript/branches/icc_work $icc_workGsSource";
+ mylog "$cmd\n";
`$cmd`;
}
updateStatus('Updating icc_work');
- $cmd="cd $icc_workSource ; touch base/gscdef.c ; rm -f base/gscdef.c ; export SVN_SSH=\"ssh -i \$HOME/.ssh/cluster_key\" ; svn update -r$icc_work";
- $gsSource=$icc_workSource;
+ $cmd="svn update -r$icc_work --ignore-externals $icc_workSource ; cd $icc_workGsSource ; touch base/gscdef.c ; rm -f base/gscdef.c ; svn update -r$icc_work";
+ $gpdlSource=$icc_workSource;
+ $gsSource=$icc_workGsSource;
} else {
updateStatus('Updating Ghostscript');
# get update
- my $pdlrev;
- my $gsrev;
if ($revs eq "head") {
- $pdlrev="head";
- $gsrev="head";
- } elsif ($revs =~ m/(\d+) (\d+)/) {
- $pdlrev=$1;
- $gsrev=$2;
+ } elsif ($revs =~ m/(\d+)/) {
} else {
mylog "oops 4: revs=$revs";
die "oops 4"; # hack
}
- $cmd="cd $gsSource ; touch base/gscdef.c ; rm -f base/gscdef.c ; cd $gpdlSource ; export SVN_SSH=\"ssh -i \$HOME/.ssh/cluster_key\" ; svn update -r$pdlrev ; cd $gsSource ; export SVN_SSH=\"ssh -i \$HOME/.ssh/cluster_key\" ; svn update -r$gsrev";
+ $cmd="cd $gsSource ; touch base/gscdef.c ; rm -f base/gscdef.c ; cd $gpdlSource ; export SVN_SSH=\"ssh -i \$HOME/.ssh/cluster_key\" ; svn update -r$revs ; cd $gsSource ; export SVN_SSH=\"ssh -i \$HOME/.ssh/cluster_key\" ; svn update -r$revs";
}
@@ -705,6 +702,7 @@
kill 1, $p;
kill 9, $p;
}
+ $name{$pid}='missing' if (!exists $name{$pid});
mylog ("killing (timeout 1) $pid $name{$pid}\n");
kill 1, $pid;
kill 9, $pid;
@@ -742,8 +740,8 @@
my $filename=$a[0];
my $cmd=$a[1];
$jobs++;
- $maxTimeout=int($jobs*$maxTimeoutPercentage/100+0.5);
- $maxTimeout=10 if ($maxTimeout<10);
+ my $t=int($jobs*$maxTimeoutPercentage/100+0.5);
+ $maxTimeout=$t if ($maxTimeout<$t);
{
sub convertTime($) {
my $t=shift;
@@ -832,9 +830,11 @@
my $a=`ps -ef`;
my @a=split '\n',$a;
my %children;
+ my %name;
foreach (@a) {
if (m/\S+ +(\d+) +(\d+)/ && !m/<defunct>/ && !m/\(sh\)/) {
$children{$2}=$1;
+ $name{$1}=$3;
}
}
@@ -844,11 +844,12 @@
while (exists $children{$p}) {
# mylog "$p->$children{$p}\n"; # mhw
$p=$children{$p};
- mylog ("killing (timeout 2) $p\n");
+ mylog ("killing (timeout 2) $p $name{$p}\n");
kill 1, $p;
kill 9, $p;
}
- mylog ("killing (timeout 2) $pid\n");
+ $name{$pid}='missing' if (!exists $name{$pid});
+ mylog ("killing (timeout 2) $pid $name{$pid}\n");
kill 1, $pid;
kill 9, $pid;
$timeOuts{$pids{$pid}{'filename'}}=1;
@@ -856,7 +857,7 @@
delete $pids{$pid};
# mylog "killed: $p ($pid) $pids{$pid}{'filename'} total $count\n"; # mhw
- mylog "killed: $pids{$pid}{'filename'}\n"; # mhw
+# mylog "killed: $pids{$pid}{'filename'}\n"; # mhw
if ($count>=$maxTimeout) {
$timeoutFail="too many timeouts";
mylog("setting $machine.fail on casper3\n");
@@ -907,7 +908,7 @@
$dir="ghostpdl" if ($revs);
$dir="users/$user/ghostpdl" if ($user);
$dir="icc_work" if ($icc_work);
- foreach my $i ('gs/makegs.out','makepcl.out','makexps.out','makesvg.out','makegs.out') {
+ foreach my $i ('gs/makegs.out','makepcl.out','makexps.out','makesvg.out') {
my $count=0;
my $start=-1;
if (open(F3,"<$dir/$i")) {
@@ -976,11 +977,6 @@
spawn(300,"scp -q -i ~/.ssh/cluster_key $machine.log.gz regression\@casper3.ghostscript.com:/home/regression/cluster/$machine.log.gz");
mylog "done with uploading $machine.log.gz";
-# what foolishness is this?
- mylog "about to upload $machine.log.gz";
- spawn(300,"scp -q -i ~/.ssh/cluster_key $machine.log.gz regression\@casper3.ghostscript.com:/home/regression/cluster/$machine.log.gz");
- mylog "done with uploading $machine.log.gz";
-
mylog "about to upload $machine.out.gz";
spawn(300,"scp -q -i ~/.ssh/cluster_key $machine.out.gz regression\@casper3.ghostscript.com:/home/regression/cluster/$machine.out.gz");
mylog "done with uploading $machine.out.gz";
More information about the gs-commits
mailing list