[gs-commits] rev 10952 - trunk/gs/toolbin/localcluster

marcos at ghostscript.com marcos at ghostscript.com
Mon Mar 22 06:43:09 UTC 2010


Author: marcos
Date: 2010-03-22 06:43:08 +0000 (Mon, 22 Mar 2010)
New Revision: 10952

Modified:
   trunk/gs/toolbin/localcluster/build.pl
   trunk/gs/toolbin/localcluster/clustermaster.pl
   trunk/gs/toolbin/localcluster/clusterpush.pl
   trunk/gs/toolbin/localcluster/compare.pl
   trunk/gs/toolbin/localcluster/readlog.pl
   trunk/gs/toolbin/localcluster/run.pl
Log:
Added the ability to specify bmpcmp options when running clusterpush.pl, e.g.:

  clusterpush.pl bmpcmp -w 9 -t 10

No validation is done on the options; bad options will result in no bmpcmp output being generated.


Modified: trunk/gs/toolbin/localcluster/build.pl
===================================================================
--- trunk/gs/toolbin/localcluster/build.pl	2010-03-21 05:01:50 UTC (rev 10951)
+++ trunk/gs/toolbin/localcluster/build.pl	2010-03-22 06:43:08 UTC (rev 10952)
@@ -23,6 +23,7 @@
 my $bmpcmp=0;
 my $local=0;
 my $filename="";
+my $bmpcmpOptions;
 
 my $t;
 
@@ -38,6 +39,8 @@
     $bmpcmp=1;
     $filename=shift;
     $filename.=".txt";
+    $bmpcmpOptions=shift;
+    $bmpcmpOptions="" if (!$bmpcmpOptions);
   } elsif ($t eq "local") {
     $local=1;
   } else {
@@ -169,6 +172,7 @@
     "ppmraw.72.0",
     "ppmraw.300.0",
     "ppmraw.300.1",
+    "pam.72.0",
     "psdcmyk.72.0",
 ##"psdcmyk.300.0",
 ##"psdcmyk.300.1",
@@ -246,15 +250,10 @@
     #"pdf.pkmraw.300.0"
     ],
   'mupdf' => [
+# mupdf only supports ppmraw output, so don't bother specifying anything else
     "ppmraw.72.0",
     "ppmraw.300.0",
     "ppmraw.300.1"
-#   "psdcmyk.72.0",
-##"psdcmyk.300.0",
-##"psdcmyk.300.1",
-#   "pdf.ppmraw.72.0",
-#   "pdf.ppmraw.300.0",
-#   "pdf.pkmraw.300.0"
     ]
   );
 
@@ -445,8 +444,8 @@
       $cmd2a =~ s|/gs/|/head/|;
       $cmd2c =~ s|$temp|$baselineRaster|;
       $cmd.=" ; $timeCommand $cmd2a -sOutputFile='|gzip -1 -n >$baselineFilename.gz' $cmd2c >>$logFilename 2>&1";
-      $cmd.=" ; bash -c \"./bmpcmp <(gunzip -c $outputFilename.gz) <(gunzip -c $baselineFilename.gz) $bmpcmpFilename\""; # ; gzip $bmpcmpFilename.* ";
-      $cmd.=" ; bash -c \"for (( c=1; c<=5; c++ )); do scp -q -o ConnectTimeout=30 -i ~/.ssh/cluster_key $bmpcmpFilename.* regression\@casper3.ghostscript.com:/home/regression/cluster/bmpcmp/. ; t=$?; if (( \\\$t == 0 )); then break; fi; echo 'scp retry \\\$c' ; done \"";
+      $cmd.=" ; bash -c \"./bmpcmp $bmpcmpOptions <(gunzip -c $outputFilename.gz) <(gunzip -c $baselineFilename.gz) $bmpcmpFilename 0 100\""; # ; gzip $bmpcmpFilename.* ";
+      $cmd.=" ; bash -c \"for (( c=1; c<=5; c++ )); do scp -q -o ConnectTimeout=30 -i ~/.ssh/cluster_key $bmpcmpFilename.* regression\@casper3.ghostscript.com:/home/regression/cluster/bmpcmp/. ; t=\\\$?; if [ \\\$t == 0 ]; then break; fi; echo 'scp retry \\\$c' ; done \"";
 #     $cmd.=" ; scp -q -o ConnectTimeout=30 -i ~/.ssh/cluster_key $logFilename regression\@casper3.ghostscript.com:/home/regression/cluster/bmpcmp/.";
     } else {
       $cmd.=" ; echo \"$cmd2a $cmd2b $cmd2c\" >>$logFilename ";
@@ -458,7 +457,7 @@
    }
 
     if ($rerunIfMd5sumDifferences && exists $md5sum{$filename2} && !exists $skip{$filename2}) {
-      $cmd.=" ; sleep 1 ; grep -q -E \"".$md5sum{$filename2}."\" $md5Filename; a=\$? ;  if [ \"\$a\" -eq \"1\" -a -e raster.yes ]; then $cmd2a -sOutputFile='|gzip -1 -n >$rasterFilename.gz' $cmd2c >>/dev/null 2>&1; bash -c \"./bmpcmp <(gunzip -c $rasterFilename.gz) <(gunzip -c $baselineFilename.gz) $bmpcmpFilename\" ; gzip $bmpcmpFilename.* ; fi";
+      $cmd.=" ; sleep 1 ; grep -q -E \"".$md5sum{$filename2}."\" $md5Filename; a=\$? ;  if [ \"\$a\" -eq \"1\" -a -e raster.yes ]; then $cmd2a -sOutputFile='|gzip -1 -n >$rasterFilename.gz' $cmd2c >>/dev/null 2>&1; bash -c \"./bmpcmp $bmpcmpOptions <(gunzip -c $rasterFilename.gz) <(gunzip -c $baselineFilename.gz) $bmpcmpFilename 0 100\" ; gzip $bmpcmpFilename.* ; fi";
     }
 
     #   $cmd.=" ; gzip -f $inputFilename >>$logFilename 2>&1";
@@ -528,8 +527,8 @@
       $cmd.=" ; $timeCommand $cmd2a -sOutputFile='|gzip -1 -n >$outputFilename.gz' $cmd2c >>$logFilename 2>&1";
       $cmd2a =~ s|/gs/|/head/|;
       $cmd.=" ; $timeCommand $cmd2a -sOutputFile='|gzip -1 -n >$baselineFilename.gz' $cmd2c >>$logFilename 2>&1";
-      $cmd.=" ; bash -c \"./bmpcmp <(gunzip -c $outputFilename.gz) <(gunzip -c $baselineFilename.gz) $bmpcmpFilename\""; # ; gzip $bmpcmpFilename.* ";
-      $cmd.=" ; bash -c \"for (( c=1; c<=5; c++ )); do scp -q -o ConnectTimeout=30 -i ~/.ssh/cluster_key $bmpcmpFilename.* regression\@casper3.ghostscript.com:/home/regression/cluster/bmpcmp/. ; t=$?; if (( \\\$t == 0 )); then break; fi; echo 'scp retry \\\$c' ; done \"";
+      $cmd.=" ; bash -c \"./bmpcmp $bmpcmpOptions <(gunzip -c $outputFilename.gz) <(gunzip -c $baselineFilename.gz) $bmpcmpFilename 0 100\""; # ; gzip $bmpcmpFilename.* ";
+      $cmd.=" ; bash -c \"for (( c=1; c<=5; c++ )); do scp -q -o ConnectTimeout=30 -i ~/.ssh/cluster_key $bmpcmpFilename.* regression\@casper3.ghostscript.com:/home/regression/cluster/bmpcmp/. ; t=\\\$?; if [ \\\$t == 0 ]; then break; fi; echo 'scp retry \\\$c' ; done \"";
 #     $cmd.=" ; scp -q -o ConnectTimeout=30 -i ~/.ssh/cluster_key $logFilename regression\@casper3.ghostscript.com:/home/regression/cluster/bmpcmp/.";
     } else {
       $cmd.=" ; echo \"$cmd2a $cmd2b $cmd2c\" >>$logFilename ";
@@ -541,7 +540,7 @@
    }
 
     if ($rerunIfMd5sumDifferences && exists $md5sum{$filename2} && !exists $skip{$filename2}) {
-      $cmd.=" ; sleep 1 ; grep -q -E \"".$md5sum{$filename2}."\" $md5Filename; a=\$? ;  if [ \"\$a\" -eq \"1\" -a -e raster.yes ]; then $cmd2a -sOutputFile='|gzip -1 -n >$rasterFilename.gz' $cmd2c >>/dev/null 2>&1; bash -c \"./bmpcmp <(gunzip -c $rasterFilename.gz) <(gunzip -c $baselineFilename.gz) $bmpcmpFilename\" ; gzip $bmpcmpFilename.* ; fi";
+      $cmd.=" ; sleep 1 ; grep -q -E \"".$md5sum{$filename2}."\" $md5Filename; a=\$? ;  if [ \"\$a\" -eq \"1\" -a -e raster.yes ]; then $cmd2a -sOutputFile='|gzip -1 -n >$rasterFilename.gz' $cmd2c >>/dev/null 2>&1; bash -c \"./bmpcmp $bmpcmpOptions <(gunzip -c $rasterFilename.gz) <(gunzip -c $baselineFilename.gz) $bmpcmpFilename 0 100\" ; gzip $bmpcmpFilename.* ; fi";
     }
 
 
@@ -596,6 +595,10 @@
 #      print "$1 $2 $3\n";
       ($cmd,$outputFilenames,$filename)=build($3,$1,$2,!$local,1) if (!$done);
 print "$filename\t$cmd\n" if (!$done);
+    } elsif (m/^(.+)\.(pam\.\d+\.[01]) (\S+)/) {
+#      print "$1 $2 $3\n";
+      ($cmd,$outputFilenames,$filename)=build($3,$1,$2,!$local,1) if (!$done);
+print "$filename\t$cmd\n" if (!$done);
     } elsif (m/errors:$/ || m/previous clusterpush/i) {
       $done=1;
     }

Modified: trunk/gs/toolbin/localcluster/clustermaster.pl
===================================================================
--- trunk/gs/toolbin/localcluster/clustermaster.pl	2010-03-21 05:01:50 UTC (rev 10951)
+++ trunk/gs/toolbin/localcluster/clustermaster.pl	2010-03-22 06:43:08 UTC (rev 10952)
@@ -307,6 +307,7 @@
   opendir(DIR, $usersDir) || die "can't opendir $usersDir: $!";
   foreach my $user (readdir(DIR)) {
     my $product="";
+    my $options;
     my $s="";
     if (open(F,"<$usersDir/$user/gs.run")) {
       close(F);
@@ -321,15 +322,20 @@
     if (open(F,"<$usersDir/$user/ghostpdl/cluster_command.run")) {
       $product=<F>;
       chomp $product;
+      $options=<F>;
+      chomp $options;
       close(F);
       unlink "$usersDir/$user/ghostpdl/cluster_command.run";
     }
     if (open(F,"<$usersDir/$user/ghostpdl/gs/cluster_command.run")) {
       $product=<F>;
       chomp $product;
+      $options=<F>;
+      chomp $options;
       close(F);
       unlink "$usersDir/$user/ghostpdl/gs/cluster_command.run";
     }
+    $options="" if (!$options);
     if ($product) {
       $product =~ s/ +$//;
       mylog "user $product\n";
@@ -370,7 +376,7 @@
         if (length($t)==0) {
           mylog "grep '$s' returns no match, adding to queue\n";
           open(F,">>$queue");
-          print F "$s\n";
+          print F "$s $options\n";
           close(F);
           mylog "running: find $usersDir/$user/ghostpdl -name \\*.sh | xargs \$HOME/bin/flip -u\n";
           `find $usersDir/$user/ghostpdl -name \\*.sh | xargs \$HOME/bin/flip -u`;
@@ -415,6 +421,7 @@
 }
 
 my $product="";
+my $options="";
 
 #gs/base, gs/Resource: all languages
 #gs/psi: ps, pdf
@@ -653,18 +660,23 @@
       my @a=split ' ',$userRegression,2;
       $userName=$a[0];
       $product=$a[1];
-      $bmpcmp=1 if ($product eq "bmpcmp");
-      $product="bmpcmp $userName" if ($bmpcmp);
-      mylog "userName=$userName product=$product\n" if ($verbose);
+      if ($product=~m/^bmpcmp/) {
+        $bmpcmp=1;
+        my @a=split ' ',$product,2;
+        $product="bmpcmp $userName";
+        $options=$a[1];
+#     print "userName=$userName product=$product options=$options\n"; exit;
+      }
+      mylog "userName=$userName product=$product options=$options\n" if ($verbose);
       my $t=`date +\"%D %H:%M:%S\"`;
       chomp $t;
-      $footer="\n\nUser regression: user $userName  options $product  start $t\n";
+      $footer="\n\nUser regression: user $userName  options $product $options start $t\n";
     }
 
     my $baseline="";
     $baseline="baseline" if ($updateBaseline);  # mhw
 
-    `./build.pl $product $baseline >$jobs`;
+    `./build.pl $product $baseline \"$options\" >$jobs`;
     if ($? != 0) {
       # horrible hack, fix later
       mylog "build.pl $product failed\n";
@@ -1134,7 +1146,7 @@
       print F "\n\nDifferences from previous clusterpush:\n\n";
       close(F);
       mylog "now running ./compare.pl temp.tab $usersDir/$userName/temp.tab $elapsedTime $machineCount true \"$product\"\n";
-      `./compare.pl temp.tab $usersDir/$userName/temp.tab 0 1 true \"$product\" >>$userName.txt`;
+      `./compare.pl temp.tab $usersDir/$userName/temp.tab 1 1 true \"$product\" >>$userName.txt`;
 
       `mv $logs $usersDir/$userName/.`;
       `cp -p $userName.txt $usersDir/$userName/.`;
@@ -1204,9 +1216,9 @@
     }
   } elsif ($userRegression) {
     `echo >>$userName.txt`;
-    `echo "Source differences from trunk:" >>$userName.txt`;
+    `echo "Source differences from trunk (first 2500 lines):" >>$userName.txt`;
     `echo >>$userName.txt`;
-    `head -1000 $userName.diff >>$userName.txt`;
+    `head -2500 $userName.diff >>$userName.txt`;
     if (exists $emails{$userName}) {
 #     `mail -a \"From: marcos.woehrmann\@artifex.com\" marcos.woehrmann\@artifex.com -s \"$userRegression regression\" <$userName.txt`;
 #     `mail -a \"From: marcos.woehrmann\@artifex.com\" $emails{$userName} -s \"$userRegression \`cat revision.gs\`\" <$userName.txt`;

Modified: trunk/gs/toolbin/localcluster/clusterpush.pl
===================================================================
--- trunk/gs/toolbin/localcluster/clusterpush.pl	2010-03-21 05:01:50 UTC (rev 10951)
+++ trunk/gs/toolbin/localcluster/clusterpush.pl	2010-03-22 06:43:08 UTC (rev 10952)
@@ -25,6 +25,7 @@
 
 my %products=('abort' =>1,
               'bmpcmp' =>1,
+              'localbmpcmp' =>1,
               'gs' =>1,
               'pcl'=>1,
               'svg'=>1,
@@ -41,16 +42,36 @@
   $product=shift;
   $res="highres";
 }
-my $user=shift;
-if ($user && $user eq "lowres") {
-  $user=shift;
-  $res="lowres";
+my $localbmpcmp="";
+if ($product && $product eq "localbmpcmp") {
+  my $filename=shift;
+  die "filename required after localbmpcmp option" if (!$filename);
+  open (F,"<$filename") || die "file $filename not found";
+  while(<F>) {
+    $localbmpcmp.=$_;
+  }
+  close(F);
 }
-if ($user && $user eq "highres") {
-  $user=shift;
-  $res="highres";
+my $user;
+my $command="";
+my $t1;
+while ($t1=shift) {
+  if ($t1 eq "lowres") {
+    $res="lowres";
+  } elsif ($t1 eq "highres") {
+    $res="highres";
+  } elsif ($t1=~m/^-/ || $t1=~m/^\d/) {
+    $command.=$t1.' ';
+  } else {
+    $user=$t1;
+  }
 }
 
+$product="" if (!$product);
+$user=""    if (!$user);
+
+#print "product=$product res=$res user=$user command=$command localbmpcmp=$localbmpcmp\n";  exit;
+
 unlink "cluster_command.run";
 
 my $host="casper.ghostscript.com";
@@ -80,7 +101,7 @@
 }
 
 #$directory="gs" if ($directory eq "" && $product eq "bmpcmp");
-$directory="gs" if ($directory eq "" && $product eq "abort");
+$directory="gs" if ($directory eq "" && $product && $product eq "abort");
 
 die "can't figure out if this is a ghostscript or ghostpdl directory" if ($directory eq "");
 
@@ -134,6 +155,8 @@
 
 open(F,">cluster_command.run");
 print F "$user $product $res\n";
+print F "$command\n";
+print F "$localbmpcmp" if ($localbmpcmp);
 close(F);
 
 $cmd="rsync -avxcz".

Modified: trunk/gs/toolbin/localcluster/compare.pl
===================================================================
--- trunk/gs/toolbin/localcluster/compare.pl	2010-03-21 05:01:50 UTC (rev 10951)
+++ trunk/gs/toolbin/localcluster/compare.pl	2010-03-22 06:43:08 UTC (rev 10952)
@@ -261,7 +261,7 @@
   }
 }
 
-if ($elapsedTime==0) {
+if ($elapsedTime==0 || $elapsedTime==1) {
 } else {
   print "ran ".($pdfwriteTestCount+$notPdfwriteTestCount)." tests in $elapsedTime seconds on $machineCount nodes\n\n";
 }

Modified: trunk/gs/toolbin/localcluster/readlog.pl
===================================================================
--- trunk/gs/toolbin/localcluster/readlog.pl	2010-03-21 05:01:50 UTC (rev 10951)
+++ trunk/gs/toolbin/localcluster/readlog.pl	2010-03-22 06:43:08 UTC (rev 10952)
@@ -39,7 +39,7 @@
     exit;
   }
 
-  if (m/===(.+).log===/ || m/===(.+)===/) {
+  if ((m/===(.+).log===/ || m/===(.+)===/) && !m/=====/) {
     $file=$1;
     $error=0;
     $divider=0;
@@ -66,7 +66,7 @@
     $error=2 if ($divider==1 && $error==0);
     $results{$file}{"error"}=$error;
   }
-  if (m/Segmentation fault/) {
+  if (m/Segmentation fault/ || m/Backtrace:/) {
     $error=8 if ($divider==0 && $error==0);
     $error=9 if ($divider==1 && $error==0);
     $results{$file}{"error"}=$error;

Modified: trunk/gs/toolbin/localcluster/run.pl
===================================================================
--- trunk/gs/toolbin/localcluster/run.pl	2010-03-21 05:01:50 UTC (rev 10951)
+++ trunk/gs/toolbin/localcluster/run.pl	2010-03-22 06:43:08 UTC (rev 10952)
@@ -138,7 +138,7 @@
     } elsif ($a[0] eq "mupdf") {
       $mupdf=1;
       $products="mupdf";
-$maxCount=8;
+$maxCount=4;
 $maxTimeoutPercentage=5.0;
 $timeOut=600;
     } elsif ($a[0] eq "svn-icc_work") {
@@ -456,6 +456,10 @@
 `svn update $baseDirectory/ghostpdl/gs`;
 `cc -I$baseDirectory/ghostpdl/gs/libpng -o bmpcmp -DHAVE_LIBPNG $baseDirectory/ghostpdl/gs/toolbin/bmpcmp.c $baseDirectory/ghostpdl/gs/libpng/png.c $baseDirectory/ghostpdl/gs/libpng/pngerror.c $baseDirectory/ghostpdl/gs/libpng/pnggccrd.c $baseDirectory/ghostpdl/gs/libpng/pngget.c $baseDirectory/ghostpdl/gs/libpng/pngmem.c $baseDirectory/ghostpdl/gs/libpng/pngpread.c $baseDirectory/ghostpdl/gs/libpng/pngread.c $baseDirectory/ghostpdl/gs/libpng/pngrio.c $baseDirectory/ghostpdl/gs/libpng/pngrtran.c $baseDirectory/ghostpdl/gs/libpng/pngrutil.c $baseDirectory/ghostpdl/gs/libpng/pngset.c $baseDirectory/ghostpdl/gs/libpng/pngtrans.c $baseDirectory/ghostpdl/gs/libpng/pngvcrd.c $baseDirectory/ghostpdl/gs/libpng/pngwio.c $baseDirectory/ghostpdl/gs/libpng/pngwrite.c $baseDirectory/ghostpdl/gs/libpng/pngwtran.c $baseDirectory/ghostpdl/gs/libpng/pngwutil.c -lm -lz`;
 
+mkdir("$gsBin");
+mkdir("$gsBin/bin");
+
+
 $cmd="touch $baselineRaster2 ; rm -fr $baselineRaster2 ; mv $baselineRaster $baselineRaster2 ; mkdir $baselineRaster ; rm -fr $baselineRaster2 &";
 print "$cmd\n" if ($verbose);
 `$cmd`;



More information about the gs-commits mailing list