[gs-commits] rev 12228 - trunk/gs/toolbin/localcluster
regression at ghostscript.com
regression at ghostscript.com
Wed Mar 2 20:50:46 UTC 2011
Author: regression
Date: 2011-03-02 20:50:46 +0000 (Wed, 02 Mar 2011)
New Revision: 12228
Modified:
trunk/gs/toolbin/localcluster/build.pl
trunk/gs/toolbin/localcluster/clustermaster.pl
trunk/gs/toolbin/localcluster/pngs2html.pl
trunk/gs/toolbin/localcluster/run.pl
Log:
Minor bug fixes and improvements to the cluster system, the most
signifcant of which is the addition of "CLUSTER_UNTESTED" detection.
If this keyword appears anywhere within the log message of a commit that
revision will not be tested by the cluster.
Less interesting changes include:
Fix for bmpcmp if large numbers of differences are produced
Addition of 'svn cleanup' calls before 'svn update' to handle nodes that
crashed during previous 'svn update' and left the repositories locked
Set status of all nodes to idle after jobs are completed.
Fix bugs that caused bmpcmp completed emails to be appended to the
previous message.
Modified: trunk/gs/toolbin/localcluster/build.pl
===================================================================
--- trunk/gs/toolbin/localcluster/build.pl 2011-03-02 18:51:23 UTC (rev 12227)
+++ trunk/gs/toolbin/localcluster/build.pl 2011-03-02 20:50:46 UTC (rev 12228)
@@ -390,6 +390,7 @@
close(F);
unlink "nightly_only.lst";
}
+# print STDERR Dumper(\%nightly_only);
}
my $t1=$testSource;
@@ -742,6 +743,19 @@
close(F);
}
+my %slowFiles;
+if (open (F,"<slowfiles.lst")) {
+ while(<F>) {
+ chomp;
+ $slowFiles{$_}=1;
+ }
+ close(F);
+}
+
+
+my @quickCommands;
+my @quickOutputFilenames;
+my @quickFilenames;
my @commands;
my @outputFilenames;
my @filenames;
@@ -761,24 +775,27 @@
my @a=split ' ';
$start=1 if (m/ran \d+ tests in \d+ seconds/);
if ($start && scalar(@a)>=4) { # horrible, horrible hack
- if (m/^(.+)\.(pdf\.p.mraw\.\d+\.[01]) (\S+) pdfwrite /) {
-# print "$1 $2 $3 -- pdfwrite\n";
- ($cmd,$outputFilenames,$filename)=build($3,$1,$2,!$local,1) if (!$done);
-print "$filename\t$cmd\n" if (!$done);
- } elsif (m/^(.+)\.(p.mraw\.\d+\.[01]) (\S+)/) {
-# print "$1 $2 $3\n";
- ($cmd,$outputFilenames,$filename)=build($3,$1,$2,!$local,1) if (!$done);
-print "$filename\t$cmd\n" if (!$done);
- } elsif (m/^(.+)\.(pam\.\d+\.[01]) (\S+)/) {
-# print "$1 $2 $3\n";
- ($cmd,$outputFilenames,$filename)=build($3,$1,$2,!$local,1) if (!$done);
-print "$filename\t$cmd\n" if (!$done);
+ if (!$done) {
+ if (m/^(.+)\.(pdf\.p.mraw\.\d+\.[01]) (\S+) pdfwrite / ||
+ m/^(.+)\.(ps\.p.mraw\.\d+\.[01]) (\S+) ps2write / ||
+ m/^(.+)\.(p.mraw\.\d+\.[01]) (\S+)/ ||
+ m/^(.+)\.(pam\.\d+\.[01]) (\S+)/) {
+ ($cmd,$outputFilenames,$filename)=build($3,$1,$2,!$local,1);
+ if (exists $quickFiles{$filename}) {
+ push @quickCommands,$cmd;
+ push @quickOutputFilenames,$outputFilenames;
+ push @quickFilenames,$filename;
+ } else {
+ push @commands,$cmd;
+ push @outputFilenames,$outputFilenames;
+ push @filenames,$filename;
+ }
} elsif (m/errors:$/ || m/previous clusterpush/i) {
$done=1;
}
}
-
}
+ }
close(F);
} else {
foreach my $testfile (sort keys %testfiles) {
@@ -792,13 +809,17 @@
my $filename="";
($cmd,$outputFilenames,$filename)=build($testfiles{$testfile},$testfile,$test,!$local,0);
if (exists $quickFiles{$filename}) {
+ push @quickCommands,$cmd;
+ push @quickOutputFilenames,$outputFilenames;
+ push @quickFilenames,$filename;
+ } elsif (exists $slowFiles{$filename}) {
+ push @slowCommands,$cmd;
+ push @slowOutputFilenames,$outputFilenames;
+ push @slowFilenames,$filename;
+ } else {
push @commands,$cmd;
push @outputFilenames,$outputFilenames;
push @filenames,$filename;
- } else {
- push @slowCommands,$cmd;
- push @slowOutputFilenames,$outputFilenames;
- push @slowFilenames,$filename;
}
}
}
@@ -823,17 +844,10 @@
print "$filename\t$command\n";
}
-#while (scalar(@slowCommands)) {
-# my $n=rand(scalar @slowCommands);
-# my $filename=$slowFilenames[$n]; splice(@slowFilenames,$n,1);
-# my $command=$slowCommands[$n]; splice(@slowCommands,$n,1);
-# print "$filename\t$command\n";
-#}
+while (scalar(@quickCommands)) {
+ my $n=rand(scalar @quickCommands);
+ my $filename=$quickFilenames[$n]; splice(@quickFilenames,$n,1);
+ my $command=$quickCommands[$n]; splice(@quickCommands,$n,1);
+ print "$filename\t$command\n";
+}
-#while (scalar(@commands)) {
-# my $n=rand(scalar @commands);
-# my $filename=$filenames[$n]; splice(@filenames,$n,1);
-# my $command=$commands[$n]; splice(@commands,$n,1);
-# print "$filename\t$command\n";
-#}
-
Modified: trunk/gs/toolbin/localcluster/clustermaster.pl
===================================================================
--- trunk/gs/toolbin/localcluster/clustermaster.pl 2011-03-02 18:51:23 UTC (rev 12227)
+++ trunk/gs/toolbin/localcluster/clustermaster.pl 2011-03-02 20:50:46 UTC (rev 12228)
@@ -11,7 +11,9 @@
my $verbose=1;
+my $skip="CLUSTER_UNTESTED";
+
my $runningSemaphore="./clustermaster.pid";
my $queue="./queue.lst";
my $lock="./queue.lck";
@@ -275,12 +277,17 @@
unlink $runningSemaphore;
die "svn info failed";
}
- if ($currentRev1 != $newRev1 || $currentRev2 != $newRev2) {
+ my $rev=$newRev1;
+ $rev=$newRev2 if ($newRev2>$rev);
+ my $logMessage;
+ $logMessage.=`svn log ghostpdl -r$rev`;
+ $logMessage.=`svn log ghostpdl/gs -r$rev`;
+ if ($logMessage =~ m/$skip/) {
+ mydbg " $skip found in r$rev, skipping";
+ } elsif ($currentRev1 != $newRev1 || $currentRev2 != $newRev2) {
# print "$currentRev1 $newRev1\n$currentRev2 $newRev2\n";
open(LOCK,">$lock") || die "can't write to $lock";
flock(LOCK,LOCK_EX);
- my $rev=$newRev1;
- $rev=$newRev2 if ($newRev2>$rev);
my $s="svn $rev";
my $t=`grep "$s" $queue`;
chomp $t;
@@ -601,10 +608,10 @@
chomp $currentRev1;
chomp $currentRev2;
- my $a=`svn update ghostpdl -r$rev --ignore-externals`;
- my $b=`svn update ghostpdl/gs -r$rev`;
- mydbg "svn update ghostpdl -r$rev --ignore-externals\n" if ($verbose);
- mydbg "svn update ghostpdl/gs -r$rev\n" if ($verbose);
+ my $a=`svn cleanup ghostpdl ; svn update ghostpdl -r$rev --ignore-externals`;
+ my $b=`svn cleanup ghostpdl/gs ; svn update ghostpdl/gs -r$rev`;
+ mydbg "svn cleanup ghostpdl ; svn update ghostpdl -r$rev --ignore-externals\n" if ($verbose);
+ mydbg "svn cleanup ghostpdl/gs ; svn update ghostpdl/gs -r$rev\n" if ($verbose);
$footer.="------------------------------------------------------------------------\n";
$footer.=`svn log ghostpdl -r$rev | tail -n +2`;
@@ -657,10 +664,10 @@
if (length($product)) {
# un-update the source so that if the regression fails were are back to the where we started
- `svn update ghostpdl -r$currentRev1 --ignore-externals`;
- `svn update ghostpdl/gs -r$currentRev2`;
- mydbg "svn update ghostpdl -r$currentRev1 --ignore-externals\n";
- mydbg "svn update ghostpdl/gs -r$currentRev2\n";
+ `svn cleanup ghostpdl ; svn update ghostpdl -r$currentRev1 --ignore-externals`;
+ `svn cleanup ghostpdl/gs ; svn update ghostpdl/gs -r$currentRev2`;
+ mydbg "svn cleanup ghostpdl ; svn update ghostpdl -r$currentRev1 --ignore-externals\n";
+ mydbg "svn cleanup ghostpdl/gs ; svn update ghostpdl/gs -r$currentRev2\n";
} else {
mydbg "no interesting files changed, skipping regression\n";
$normalRegression=0;
@@ -734,6 +741,11 @@
# my $t=time-stat("$_")->mtime;
# print "$_ $t\n";
$machines{$_}=1 if (stat("$_") && (time-stat("$_")->mtime)<$maxTouchTime);
+ # set the status of all machines to idle, so that old messages don't confuse the dashboard
+ s/.up/.status/;
+ open(F,">$_");
+ print F "idle\n";
+ close(F);
}
foreach (keys %machines) {
delete $machines{$_};
@@ -1438,9 +1450,9 @@
# `mail marcos.woehrmann\@artifex.com -s \"\`cat revision.gs\`\" <email.txt`;
mydbg "test complete, performing final svn update\n";
- mydbg "svn update ghostpdl -r$rev --ignore-externals\nsvn update ghostpdl/gs -r$rev\n";
- `svn update ghostpdl -r$rev --ignore-externals`;
- `svn update ghostpdl/gs -r$rev`;
+ mydbg "svn cleanup ghostpdl ; svn update ghostpdl -r$rev --ignore-externals\nsvn cleanup ghostpdl/gs ; svn update ghostpdl/gs -r$rev\n";
+ `svn cleanup ghostpdl ; svn update ghostpdl -r$rev --ignore-externals`;
+ `svn cleanup ghostpdl/gs ; svn update ghostpdl/gs -r$rev`;
`./cp.all.sh`;
mydbg("calling cachearchive.pl");
@@ -1462,7 +1474,7 @@
} elsif ($updateBaseline) {
} elsif ($bmpcmp) {
if (exists $emails{$userName}) {
- `cp bmpcmpFinished.txt bmpcmpResults.txt`;
+ `mv bmpcmpResults.txt bmpcmpResults.txt.old`;
`echo >>bmpcmpResults.txt`;
`echo http://www.ghostscript.com/~regression/$userName >>bmpcmpResults.txt`;
`echo >>bmpcmpResults.txt`;
Modified: trunk/gs/toolbin/localcluster/pngs2html.pl
===================================================================
--- trunk/gs/toolbin/localcluster/pngs2html.pl 2011-03-02 18:51:23 UTC (rev 12227)
+++ trunk/gs/toolbin/localcluster/pngs2html.pl 2011-03-02 20:50:46 UTC (rev 12228)
@@ -249,7 +249,7 @@
#close INDEX;
# Open the index
-open(INDEX, "ls $indir/*.meta| sed s/\.\[0-9]\*\.meta// | sort -u |");
+open(INDEX, "ls $indir/*.00000.meta| sed s/\.\[0-9]\*\.meta// | sort -u |");
# Now run through the list of files
while (<INDEX>)
@@ -349,6 +349,7 @@
# List the errored files. If no stdout files this prints and error, but seems
# to continue.
+`touch $indir/dummy.stdout.gz`;
open(INDEX, "ls $indir/*.stdout.gz | sort -u |");
# Now run through the list of files
@@ -357,6 +358,8 @@
{
chomp;
+ if ($_ ne "$indir/dummy.stdout.gz") {
+
# Keep everything between the last / and .stdout.gz
($path,$_) = $_ =~ m/(.*)\/([^\/]+).stdout.gz/;
@@ -381,6 +384,7 @@
print $html "<A href=\"$framedir/stderr.txt\">stderr</A></DT>\n";
$framenum++;
}
+}
print $html "</DL>";
close INDEX;
Modified: trunk/gs/toolbin/localcluster/run.pl
===================================================================
--- trunk/gs/toolbin/localcluster/run.pl 2011-03-02 18:51:23 UTC (rev 12227)
+++ trunk/gs/toolbin/localcluster/run.pl 2011-03-02 20:50:46 UTC (rev 12228)
@@ -98,7 +98,7 @@
mkdir("./head/");
mkdir("./head/bin");
-if (1) {
+if (0) {
`rm *Raw*raw`;
`rm *Device*raw`;
`rm *Image*raw`;
@@ -497,7 +497,7 @@
if ($updateTestFiles) {
foreach my $testSource (sort keys %testSource) {
- $cmd="cd $testSource ; export SVN_SSH=\"ssh -i \$HOME/.ssh/cluster_key\" ; svn update";
+ $cmd="cd $testSource ; export SVN_SSH=\"ssh -i \$HOME/.ssh/cluster_key\" ; svn cleanup ; svn update";
systemWithRetry($cmd);
}
@@ -552,9 +552,9 @@
mylog "oops 4: revs=$revs";
die "oops 4"; # hack
}
- $cmd="cd $gsSource ; touch base/gscdef.c ; rm -f base/gscdef.c ; cd $gpdlSource ; export SVN_SSH=\"ssh -i \$HOME/.ssh/cluster_key\" ; svn update -r$revs --ignore-externals";
+ $cmd="cd $gsSource ; touch base/gscdef.c ; rm -f base/gscdef.c ; cd $gpdlSource ; export SVN_SSH=\"ssh -i \$HOME/.ssh/cluster_key\" ; svn cleanup ; svn update -r$revs --ignore-externals";
systemWithRetry($cmd);
- $cmd="cd $gsSource ; export SVN_SSH=\"ssh -i \$HOME/.ssh/cluster_key\" ; svn update -r$revs";
+ $cmd="cd $gsSource ; export SVN_SSH=\"ssh -i \$HOME/.ssh/cluster_key\" ; svn cleanup ; svn update -r$revs";
systemWithRetry($cmd);
}
@@ -705,7 +705,8 @@
if (-e "$gpdlSource/main/obj") {
$compileFail.="pcl6 ";
} else {
- $cmd="cd $gpdlSource ; nice make pcl-clean ; touch makepcl.out ; rm -f makepcl.out ; nice make pcl \"CC=gcc -m$wordSize\" \"CCLD=gcc -m$wordSize\" >makepcl.out 2>&1 -j 12; echo >>makepcl.out ; nice make pcl \"CC=gcc -m$wordSize\" \"CCLD=gcc -m$wordSize\" >>makepcl.out 2>&1";
+# $cmd="cd $gpdlSource ; nice make pcl-clean ; touch makepcl.out ; rm -f makepcl.out ; nice make pcl \"CC=gcc -m$wordSize\" \"CCLD=gcc -m$wordSize\" >makepcl.out 2>&1 -j 12; echo >>makepcl.out ; nice make pcl \"CC=gcc -m$wordSize\" \"CCLD=gcc -m$wordSize\" >>makepcl.out 2>&1";
+ $cmd="cd $gpdlSource ; touch main/obj ; rm -fr main/obj ; touch makepcl.out ; rm -f makepcl.out ; nice ./autogen.sh \"CC=gcc -m$wordSize\" $configOption >makepcl.out 2>&1; nice make pcl \"CC=gcc -m$wordSize\" \"CCLD=gcc -m$wordSize\" >>makepcl.out 2>&1 -j 12; echo >>makepcl.out ; nice make pcl \"CC=gcc -m$wordSize\" \"CCLD=gcc -m$wordSize\" >>makepcl.out 2>&1";
print "$cmd\n" if ($verbose);
`$cmd`;
if (-e "$gpdlSource/main/obj/pcl6") {
More information about the gs-commits
mailing list