path: root/Build-tools/Do-compile

#!/usr/bin/perl -w

use Getopt::Long;

@config_options= ();
@make_options= ();

$opt_distribution=$opt_user=$opt_config_env="";
$opt_dbd_options=$opt_perl_options=$opt_config_options=$opt_make_options=$opt_suffix="";
$opt_tmp=$opt_version_suffix="";
$opt_help=$opt_delete=$opt_debug=$opt_stage=$opt_no_test=$opt_no_perl=$opt_with_low_memory=$opt_fast_benchmark=$opt_static_client=$opt_static_server=$opt_static_perl=$opt_sur=$opt_with_small_disk=$opt_local_perl=$opt_tcpip=$opt_build_thread=$opt_use_old_distribution=$opt_enable_shared=$opt_no_crash_me=$opt_no_strip=$opt_with_debug=$opt_no_benchmark=$opt_no_mysqltest=0;
$opt_innodb=$opt_bdb=$opt_raid=$opt_libwrap=0;

GetOptions(
	"bdb",
	"build-thread=i",
	"config-env=s" => \@config_env,
	"config-options=s" => \@config_options,
	"dbd-options=s",
	"debug",
	"delete",
	"distribution=s",
	"enable-shared",
	"fast-benchmark",
	"help|Information",
	"innodb",
	"libwrap",
	"local-perl",
	"make-options=s" => \@make_options,
	"no-crash-me",
	"no-perl",
	"no-strip",
	"no-test",
	"no-mysqltest",
	"no-benchmark",
	"perl-files=s",
	"perl-options=s",
	"raid",
	"stage=i",
	"static-client",
	"static-perl",
	"static-server",
	"suffix=s",
	"sur",
	"tcpip",
	"tmp=s",
	"use-old-distribution",
	"user=s",
	"version-suffix=s",
	"with-debug",
	"with-low-memory",
	"with-other-libc=s",
	"with-small-disk",
) || usage();

usage() if ($opt_help);
usage() if (!$opt_distribution);

if (@make_options > 0)
{
	chomp(@make_options);
	$opt_make_options= join(" ", @make_options);
}

if (@config_options > 0)
{
	chomp(@config_options);
	$opt_config_options= join(" ", @config_options);
}

if (@config_env > 0)
{
	chomp(@config_env);
	$opt_config_env= join(" ", @config_env);
}

chomp($host=`hostname`);
$full_host_name=$host;
$connect_option= ($opt_tcpip ? "--host=$host" : "");
$host =~ /^([^.-]*)/;
$host=$1 . $opt_suffix;
$email="$opt_user\@mysql.com";
$pwd = `pwd`; chomp($pwd);
$log="$pwd/Logs/$host$opt_version_suffix.log";
$opt_distribution =~ /(mysql[^\/]*)\.tar/;
$ver=$1;
$gcc_version=which("gcc");
if (defined($gcc_version) && ! $opt_config_env)
{
  $tmp=`$gcc_version -v 2>&1`;
  if ($tmp =~ /version 2\.7\./)
  {
    $opt_config_env= 'CC=gcc CFLAGS="-O2 -fno-omit-frame-pointer" CXX=gcc CXXFLAGS="-O2 -fno-omit-frame-pointer"';
  }
  elsif ($tmp =~ /version 3\.0\./)
  {
    $opt_config_env= 'CC=gcc CFLAGS="-O3 -fno-omit-frame-pointer" CXXFLAGS="-O3 -fno-omit-frame-pointer -felide-constructors -fno-exceptions -fno-rtti"';
  }
  else
  {
    $opt_config_env= 'CC=gcc CFLAGS="-O3 -fno-omit-frame-pointer" CXX=gcc CXXFLAGS="-O3 -fno-omit-frame-pointer -felide-constructors -fno-exceptions -fno-rtti"';
  }
}

$new_opt_tmp=0;
if ($opt_tmp)
{
  if (! -d $opt_tmp)
  {
    safe_system("mkdir $opt_tmp");
    $new_opt_tmp=1;
  }
  $ENV{'TMPDIR'}=$opt_tmp;
}
else
{
  $opt_tmp="/tmp";
}
$bench_tmpdir="$opt_tmp/my_build-$host";
$ENV{'PATH'}= "$pwd/$host/bin:" . $ENV{'PATH'}; 
$make=which("gmake","make");	# Can't use -j here!
$tar=which("gtar","tar");
$sendmail=find("/usr/lib/sendmail","/usr/sbin/sendmail");
$sur= $opt_sur ? "/my/local/bin/sur" : "";
delete $ENV{'MYSQL_PWD'};	# Reset possibly password
delete $ENV{'MY_BASEDIR_VERSION'};
$ENV{'MYSQL_TCP_PORT'}= $mysql_tcp_port= 3334 + $opt_build_thread*2;
$ENV{'MYSQL_UNIX_PORT'}=$mysql_unix_port="$opt_tmp/mysql$opt_suffix.build";
$ENV{"PERL5LIB"}="$pwd/$host/perl5:$pwd/$host/perl5/site_perl";
$slave_port=$mysql_tcp_port+16;
$manager_port=$mysql_tcp_port+1;
$mysqladmin_args="--no-defaults -u root --connect_timeout=5 --shutdown_timeout=20";

if ($opt_stage == 0)
{
  system("mkdir Logs") if (! -d "Logs");
  system("mv $log ${log}-old") if (-f $log);
  unlink($log);
}

open(LOG,">>$log") || abort("Can't open log file, error $?");
select LOG;
$|=1;
select STDOUT;
$|=1;

info("Compiling MySQL$opt_version_suffix at $host$opt_suffix, stage: $opt_stage\n");
log_timestamp();

if (-x "$host/bin/mysqladmin")
{
  log_system("$host/bin/mysqladmin $mysqladmin_args -S $mysql_unix_port -s shutdown");
  log_system("$host/bin/mysqladmin $mysqladmin_args -P $mysql_tcp_port -h $host -s shutdown");
  log_system("$host/bin/mysqladmin $mysqladmin_args -P $slave_port -h $host -s shutdown");
  log_system("$host/bin/mysqladmin $mysqladmin_args -P 9306 -h $host -s shutdown");
  log_system("$host/bin/mysqladmin $mysqladmin_args -P 9307 -h $host -s shutdown");
}
kill_all("mysqlmanager");
#
# Kill all old processes that are in the build directories
# This is to find any old mysqld servers left from previous builds
kill_all("$pwd/host/mysql");
kill_all("$pwd/host/test");

if ($opt_stage == 0)
{
  log_timestamp();
  print "$host: Removing old distribution\n" if ($opt_debug);
  if (!$opt_use_old_distribution)
  {
    system("mkdir $host") if (! -d $host);
    system("touch $host/mysql-fix-for-glob");
    rm_all(<$host/mysql*>);
    system("mkdir $host/bin") if (! -d "$host/bin");
  }
  rm_all("$host/test");
  system("mkdir $host/test") if (! -d "$host/test");
}

safe_cd($host);
if ($opt_stage == 0 && ! $opt_use_old_distribution)
{
  safe_system("gunzip < $opt_distribution | $tar xf -");

  # Fix file times; This is needed because the time for files may be
  # in the future.  The following is done this way to ensure that
  # we don't get any errors from xargs touch
  system("touch timestamp");
  sleep(2);
  system("touch timestamp2");
  system("find . -newer timestamp -print | xargs touch");
  unlink("timestamp");
  unlink("timestamp2");
  sleep(2);
  # Ensure that files we don't want to rebuild are newer than other files
  safe_cd($ver);
  foreach $name ("configure",
		 "Docs/include.texi",
		 "Docs/*.html", "Docs/manual.txt", "Docs/mysql.info",
		 "sql/sql_yacc.h", "sql/sql_yacc.cc")
  {
    system("touch $name");
  }
  # Fix some file modes in BDB tables that makes life harder.
  system("chmod -R u+rw .");
}

safe_cd("$pwd/$host/$ver");

#
# Configure the sources
#
if ($opt_stage <= 1)
{
  # Fix files if this is in another timezone than the build host
  log_timestamp();
  unlink("config.cache");
  unlink("bdb/build_unix/config.cache");
  unlink("innobase/config.cache");
  log_system("$make clean") if ($opt_use_old_distribution);
  $opt_config_options.= " --disable-shared" if (!$opt_enable_shared); # Default for binary versions
  $opt_config_options.= " --with-berkeley-db" if ($opt_bdb);
  $opt_config_options.= " --with-client-ldflags=-all-static" if ($opt_static_client);
	$opt_config_options.= " --with-debug" if ($opt_with_debug);
  $opt_config_options.= " --with-libwrap" if ($opt_libwrap);
  $opt_config_options.= " --with-low-memory" if ($opt_with_low_memory);
  $opt_config_options.= " --with-mysqld-ldflags=-all-static" if ($opt_static_server);
  $opt_config_options.= " --with-raid" if ($opt_raid);

  # Only enable InnoDB when requested (required to be able to
  # build the "Classic" packages that do not include InnoDB)
	if ($opt_innodb)
	{
		$opt_config_options.= " --with-innodb";
	}
	else
	{
		$opt_config_options.= " --without-innodb";
	}

  if ($opt_with_other_libc)
  {
    $opt_with_other_libc= " --with-other-libc=$opt_with_other_libc";
    $opt_config_options.= $opt_with_other_libc;
  }

  check_system("$opt_config_env ./configure --prefix=/usr/local/mysql --with-comment=\"Official MySQL$opt_version_suffix binary\" --with-extra-charsets=complex --with-server-suffix=\"$opt_version_suffix\" --enable-thread-safe-client --enable-local-infile $opt_config_options","Thank you for choosing MySQL");
  if (-d "$pwd/$host/include-mysql")
  {
    safe_system("cp -r $pwd/$host/include-mysql/* $pwd/$host/$ver/include");
  }
}

#
# Compile the binaries
#
if ($opt_stage <= 2)
{
  my ($command);
  log_timestamp();
  unlink($opt_distribution) if ($opt_delete && !$opt_use_old_distribution);
  $command=$make;
  $command.= " $opt_make_options" if (defined($opt_make_options) && $opt_make_options ne "");
  safe_system($command);
}

#
# Create the binary distribution
#
if ($opt_stage <= 3)
{
  my $flags= "";
  log_timestamp();
  log_system("rm -fr mysql-3* mysql-4* $pwd/$host/*.tar.gz");
  log_system("nm -n sql/mysqld | gzip -9 -v 2>&1 > sql/mysqld.sym.gz | cat");

  $flags.= "--no-strip" if ($opt_no_strip || $opt_with_debug);
  check_system("scripts/make_binary_distribution --tmp=$opt_tmp --suffix=$opt_suffix $flags",".tar.gz created");
  safe_system("mv mysql*.tar.gz $pwd/$host");
  if (-f "client/.libs/mysqladmin")
  {
    safe_system("cp client/.libs/mysqladmin $pwd/$host/bin");
  }
  else
  {
    safe_system("cp client/mysqladmin $pwd/$host/bin");
  }  
  safe_system("$make clean") if ($opt_with_small_disk);
}

$tar_file=<$pwd/$host/*.tar.gz>;
if (!defined($tar_file))
{
  $tar_file=<$pwd/$host/*.tgz>;
}

#
# Unpack the binary distribution
#
if ($opt_stage <= 4 && !$opt_no_test)
{
  log_timestamp();
  rm_all(<$pwd/$host/test/*>);
  safe_cd("$pwd/$host/test");
  safe_system("gunzip < $tar_file | $tar xf -");
}

$tar_file =~ /(mysql[^\/]*)\.tar/;
$ver=$1;
$test_dir="$pwd/$host/test/$ver";
$ENV{"LD_LIBRARY_PATH"}= ("$test_dir/lib" .
			  (defined($ENV{"LD_LIBRARY_PATH"}) ?
			   ":" . $ENV{"LD_LIBRARY_PATH"} : ""));
#
# Run the test suite
#
if ($opt_stage <= 5 && !$opt_no_test && !$opt_no_mysqltest)
{
  log_timestamp();
  system("mkdir $bench_tmpdir") if (! -d $bench_tmpdir);
  safe_cd("${test_dir}/mysql-test");
  check_system("./mysql-test-run --warnings --tmpdir=$bench_tmpdir --master_port=$mysql_tcp_port --slave_port=$slave_port --manager-port=$manager_port --no-manager --sleep=10", "tests were successful");
}

#
# Start the server if we are going to run any of the benchmarks
#
if (!$opt_no_test && !$opt_no_benchmark)
{
  my $extra;
  safe_cd($test_dir);
  log_system("./bin/mysqladmin $mysqladmin_args -S $mysql_unix_port -s shutdown") || info("There was no mysqld running\n");
  sleep(2);
  log_system("rm -f  ./data/mysql/*");
  check_system("scripts/mysql_install_db --no-defaults --skip-locking","https://order");
  $extra="";
  if ($opt_bdb)
  {
    $extra.=" --bdb_cache_size=16M --bdb_max_lock=60000"
  }
  if ($opt_innodb)
  {
    $extra.=" --innodb_data_file_path=ibdata1:100M:autoextend";
  }
  safe_system("./bin/mysqld --no-defaults --basedir . --datadir ./data --skip-locking $extra >> $log 2>&1 &");
  sleep(2);
}

#
# Compile and install the required Perl modules
#
if ($opt_stage <= 7 && $opt_perl_files && !$opt_no_perl && !$opt_no_test &&
    !$opt_no_benchmark)
{
  log_timestamp();
  safe_cd($test_dir);
  rm_all("perl");
  safe_system("mkdir perl");
  $ENV{'IN_MYSQL_DISTRIBUTION'}=1;
  $ENV{'MYSQL_BUILD'}=$test_dir;

  foreach $module (split(/,/,$opt_perl_files))
  {
    my $options;
    safe_cd("$test_dir/perl");
    if ($opt_debug)
    {
      safe_system("gunzip < $pwd/$module | tar xvf -");
    }
    else
    {
      safe_system("gunzip < $pwd/$module | tar xf -");
    }
    $module =~ m|([^/]+)\.tar\.gz|;
    $module = $1;
    safe_cd($module);
    $options="";
    $options= "--mysql-install --noprompt --mysql-incdir=$test_dir/include --mysql-libdir=$test_dir/lib -nomsql-install -nomsql1-install --mysql-test-db=test $opt_dbd_options" if ($module =~ /Msql-Mysql/);
    $options.= " PREFIX=$pwd/$host INSTALLPRIVLIB=$pwd/$host/perl5 INSTALLSCRIPT=$pwd/$host/bin INSTALLSITELIB=$pwd/$host/perl5/site_perl INSTALLBIN=$pwd/$host/bin INSTALLMAN1DIR=$pwd/$host/man INSTALLMAN3DIR=$pwd/$host/man/man3" if ($opt_local_perl);
    $options.= " $opt_perl_options" if (defined($opt_perl_options));
    safe_system($opt_static_perl ? "perl Makefile.PL -static $options" : "perl Makefile.PL $options");
    safe_system("$make ; $sur $make install");
  }
}

#
# Run crash-me test
#
if ($opt_stage <= 8 && !$opt_no_test && !$opt_no_crash_me)
{
  log_timestamp();
  safe_cd("$test_dir/sql-bench");
  log_system("rm -f limits/mysql.cfg");
  safe_system("perl ./crash-me --force --batch-mode $connect_option");
}

#
# Run sql-bench Benchmarks
#
if ($opt_stage <= 9 && !$opt_no_test && !$opt_no_benchmark)
{
  log_timestamp();
  safe_cd("$test_dir/sql-bench");
  log_system("rm -f output/*");
  $tmp= $opt_fast_benchmark ? "--fast --user root --small-test" : "";
  check_system("perl ./run-all-tests --log --die-on-errors $connect_option $tmp","RUN-mysql");
	# Run additional fast test with dynamic-row tables
  check_system("perl ./run-all-tests --log --suffix=\"_dynamic_rows\" --die-on-errors $connect_option --fast --user=root --small-test --create-options=\"row_format=dynamic\"","RUN-mysql");
  if ($opt_innodb)
  {
    check_system("perl ./run-all-tests --log --suffix=\"_innodb\" --die-on-errors $connect_option $tmp --create-options=\"type=innodb\"","RUN-mysql");
  }
  if ($opt_bdb)
  {
    check_system("perl ./run-all-tests --log --suffix=\"_bdb\" --die-on-errors $connect_option $tmp --create-options=\"type=bdb\"","RUN-mysql");
  }
}

rm_all($bench_tmpdir);
rm_all("$opt_tmp") if ($new_opt_tmp);

log_system("$pwd/$host/bin/mysqladmin $mysqladmin_args -S $mysql_unix_port shutdown");
print LOG "ok\n";
close LOG;
print "$host: ok\n";

exit 0;


sub usage
{
print <<EOF;
$0  version 1.5

$0 takes the following options:

--bdb
Compile with support for Berkeley DB tables

--config-env <environment for configure>
To set up the environment, like 'CC=cc CXX=gcc CXXFLAGS=-O3'

--config-options <options>
To add some extra options to configure (e.g. '--with-perl=yes')

--dbd-options <options>
Options for Makefile.PL when configuring msql-mysql-modules.

--debug
Print all shell commands on stdout.

--delete
Delete the distribution file.

--distribution <distribution_file>
Name of the MySQL source distribution file.

--enable-shared
Compile with shared libraries

--fast-benchmark
Run fast benchmark only to speed up testing

--help or --Information
Show this help

--innodb
Compile with support for Innodb tables

--libwrap
Compile with TCP wrapper support

--local-perl
Install Perl modules locally

--make-options <options>
Options to make after configure. (Like 'CXXLD=gcc')

--no-crash-me
Do not run the "crash-me" test

--no-strip
Do not strip the binaries included in the binary distribution

--no-test
Do not run any tests.

--no-benchmark
Do not run the benchmark test (written in perl)

--no-mysqltest
Do not run the the mysql-test-run test (Same as 'make test')

--perl-files=list of files
Compile and install the given perl modules.

--perl-options <options>
Build Perl modules with the additional options

--raid
Compile with RAID support

--stage (1-6)
Start script from some specific point.

--static-client
Build statically linked client binaries

--static-perl
Build statically linked Perl modules

--static-server
Build statically linked server binary

--tcpip
Connect to the server to be tested via TCP/IP instead of socket

--tmp <directory>
Use a different temporary directory than /tmp

--use-old-distribution
Do not clean up the build environment and extract a fresh source
distribution, use an existing one instead.

--user <user_name>
Mail 'user_name'\@mysql.com if something went wrong.
If user is empty then no mail is sent.

--version-suffix suffix
Set name suffix (e.g. 'com' or '-max') for a distribution

--with-debug
Build binaries with debug information (implies "--no-strip")

--with-low-memory
Use less memory when compiling.

--with-other-libc <path to libc>
Link against libc and other standard libraries installed in the specified
non-standard location overriding default.

--with-small-disk
Clean up the build environment before testing the binary distribution
(to save disk space)
EOF
exit 1;
} 

sub abort
{
  my($message)=@_;
  my($mail_header_file);
  print LOG "\n$message\n";
  print "$host: $message\n" if ($opt_debug);
  print LOG "Aborting\n";
  close LOG;

  if ($opt_user)
  {
    $mail_header_file="$opt_tmp/do-command.$$";
    open(TMP,">$mail_header_file");
    print TMP "From: mysql\@$full_host_name\n";
    print TMP "To: $email\n";
    print TMP "Subject: $ver$opt_version_suffix compilation failed\n\n";
    close TMP;
    system("tail -n 40 $log > $log.mail");
    system("cat $mail_header_file $log.mail | $sendmail -t -f $email");
    unlink($mail_header_file);
    unlink("$log.mail");
  }
  exit 1;
}

sub info
{
  my($message)=@_;
  print LOG "$message\n";
  print "$host: $message\n";
}

sub log_system
{
  my($com)=@_;
  print "$host: $com\n" if ($opt_debug);
  if (defined($log))
  {
    print LOG "$com\n";
    system("$com >> $log 2>&1") &&
      print LOG ("Info: couldn't execute command, error: " . ($? / 256) ."\n");
  }
  else
  {
    system($com) && print "$host: Couldn't execute command, error: " . ($? / 256) ."\n";
  }
}


sub safe_system
{
  my($com,$res)=@_;
  print LOG "$com\n";
  print "$host: $com\n" if ($opt_debug);
  system("$com >> $log 2>&1") && abort("error: Couldn't execute command, error: " . ($? / 256));
}

sub check_system
{
  my($com,$res)=@_;
  my ($error,$found);
  print LOG "$com\n";
  print "$host: $com\n" if ($opt_debug);
  open (COM, "$com 2>&1 < /dev/null|") || abort("Got error " . ($?/256) ." opening pipe");
  $found=0;
  while (<COM>)
  {
    print LOG $_;
    if (index($_,$res) >= 0)
    {
      $found=1;
      last;
    }
  }
  close COM;
  abort("Couldn't find '$res' in the command result") if (!$found);
  print "$host: Command ok\n" if ($opt_debug);
}

sub safe_cd
{
  my($dir)=@_;
  print LOG "cd $dir\n";
  print "$host: cd $dir\n" if ($opt_debug);
  chdir($dir) || abort("Can't cd to $dir");
}

sub which
{
  my(@progs)=@_;
  foreach $prog (@progs)
  {
    chomp($found=`which $prog | head -n 1`);
    if ($? == 0 && $found ne "" && index($found," ") == -1)
    {
      $found =~ s|/+|/|g;	# Make nicer output
      return $found;
    }
  }
  return undef();
}

sub find
{
  my (@progs)=@_;
  foreach $prog (@progs)
  {
    return $prog if (-x $prog);
  }
  return undef();  
}

#
# Remove recursively all from a directory
# This is needed because problems with NFS and open files
#

sub rm_all
{
  my(@rm_files)=@_;
  my($dir,$current_dir,@files,@dirs);
  $current_dir = `pwd`; chomp($current_dir);

  foreach $dir (@rm_files)
  {
    if (-d $dir)
    {
      chdir($dir) || abort("Can't cd to $dir");
      print "$host: Removing from $dir\n" if ($opt_debug);
      while (<* .*>)
      {
	next if ($_ eq "." x (length($_)));
	if (-d $_)
	{
#	  die "Can't remove directory that starts with ." if ($_ =~ /^\./ && $_ ne ".libs"); # Safety
	  push (@dirs,$_);
	}
	else
	{
	  push (@files,$_);
	}
      }
      if ($#files >= 0)
      {
	system("rm -f " . join(" ",@files)) && abort("Can't remove files from $dir");
      }
      foreach $dir (@dirs)
      {
	rm_all($dir);
      }
      chdir($current_dir) || abort("Can't cd to $current_dir");
      log_system("rmdir $dir");
    }
    else
    {
      system("rm -f $dir") && abort("Can't remove file $dir");
    }
  }
}

sub kill_all
{
  my ($pattern) = @_;
  my ($USER,$BSD,$LINUX, $pscmd, $user, $os, $pid);
  $user=$ENV{'USER'};
  $os=defined($ENV{'OS'}) ? $ENV{'OS'} : "unknown";
  $BSD = -f '/vmunix' || $os eq "SunOS4" || $^O eq 'darwin';
  $LINUX = $^O eq 'linux';
  $pscmd = $BSD ? "/bin/ps -auxww" : $LINUX ? "/bin/ps axuw" : "/bin/ps -ef";

  if (!open(PS, "$pscmd|"))
  {
    print "Warning: Can't run $pscmd: $!\n";
    exit;
  }

  # Catch any errors with eval.  A bad pattern, for instance.
 process:
  while ($cand = <PS>)
  {
    chop($cand);
    ($pid_user, $pid) = split(' ', $cand);
    next if $pid eq $$;
    next process if (! ($cand =~ $pattern) || $pid_user ne $user);
    print LOG "Killing $_\n";
    &killpid($pid);
  }
}

sub killpid
{
  local($pid) = @_;
  kill 15, $pid;
  for (1..5)
  {
    sleep 2;
    return if kill(0, $pid) == 0;
  }
  kill 9, $pid;
  for (1..5) {
    sleep 2;
    return if kill(0, $pid) == 0;
  }
  print LOG "$pid will not die!\n";
}

#
# return the current date as a string (YYYY-MM-DD HH:MM:SS)
#
sub log_timestamp
{
  my @ta=localtime(time());
  print LOG sprintf("%4d-%02d-%02d %02d:%02d:%02d\n",
		    $ta[5]+1900, $ta[4]+1, $ta[3], $ta[2], $ta[1], $ta[0]);

}