Install Storm on CentOS
什么是Storm? Storm is afree and open sourcedistributed realtime computation system. Storm makes it easy to reliably process unbounded streams of data, doing for realtime processing what Hadoop did for batch processing. Storm issimple, can be used withany programming language, and is a lot of fun to use! Storm has many use cases: realtime analytics, online machine learning, continuous computation, distributed RPC, ETL, and more. Storm is fast: a benchmark clocked it at overa million tuples processed per second per node. It isscalable,fault-tolerant,guarantees your data will be processed, and iseasy to set up and operate. [官网]http://storm-project.net/ 安装脚本 安装daemontools-0.76.tar.gz的脚本: yum install patch sudo mkdir -p /package sudo chmod 1755 /package/ cd /package/ sudo wget http://cr.yp.to/daemontools/daemontools-0.76.tar.gz sudo tar xzf daemontools-0.76.tar.gz sudo wget http://www.qmail.org/moni.csi.hu/pub/glibc-2.3.1/daemontools-0.76.errno.patch cd admin/daemontools-0.76 sudo patch -p1 < http://www.cnblogs.com/daemontools-0.76.errno.patch sudo rm http://www.cnblogs.com/daemontools-0.76.errno.patch http://www.cnblogs.com/daemontools-0.76.tar.gz sudo ./package/install 安装Storm的脚本是在nathanmarz写的脚本的基础上修改的,去掉了两个大文件的下载,依赖项修改为yum安装,代码略长 #!/bin/bash pp() { echo -e "\e[00;32m"$1"\e[00m" } HOST=`hostname` ######################################### # Clean up old installation. ######################################### cleanup() { pp "Cleaning up previous installation..." rm -rf $BASEDIR mkdir $BASEDIR echo "#!/bin/bash" > $START_SH chmod +x $START_SH echo "#!/bin/bash" > $STOP_SH chmod +x $STOP_SH } ######################################### # System dependencies. ######################################### deps() { pp "Checking system dependencies..." echo yum install gcc gcc-c++.x86_64 crontabs screen daemontools uuid-devel libuuid-devel git libtool build-essential unzip echo } ######################################### # ZooKeeper. ######################################### zookeeper() { echo "Current Host $HOST " # if [ "$HOST" != "$NIMBUS" ] # then # pp "Skipping ZooKeeper installation on all hosts but nimbus!" # return # fi ZK_VERSION="3.3.6" ZK_DIR=$BASEDIR"/zookeeper" ZK_CONFIGFILE="zoo.conf" ZK_CONF=$ZK_DIR"/"$ZK_CONFIGFILE ZK_RUN=$ZK_DIR"/run" ZK_PURGE=$ZK_DIR"/purge.sh" ZK_DATADIR=$ZK_DIR"/data" ZK_TARBALL_URL="http://apache.openmirror.de/zookeeper/zookeeper-"$ZK_VERSION"/zookeeper-"$ZK_VERSION".tar.gz" ZK_TARBALL=$ZK_DIR/"zookeeper.tar.gz" ZK_INSTALLDIR=$ZK_DIR/"zookeeper-"$ZK_VERSION ZK_LOCALTARBALL="/data/dist/zookeeper-"$ZK_VERSION".tar.gz" ZK_FILENAME=$BASEDIR"/zookeeper/zookeeper-"$ZK_VERSION.tar.gz pp "Installing ZooKeeper "$ZK_VERSION" on nimbus host '"$HOST"'..." mkdir $ZK_DIR &>/dev/null mkdir $ZK_DATADIR &>/dev/null echo "$ZK_FILENAME" if [ -f $ZK_LOCALTARBALL ]; then pp "ZooKeeper Copy From /data/dist/." cp $ZK_LOCALTARBALL $ZK_TARBALL else pp "Downloading ZooKeeper..." wget $ZK_TARBALL_URL -q -O $ZK_TARBALL fi tar xzf $ZK_TARBALL -C $ZK_DIR rm $ZK_TARBALL pp "Configuring ZooKeeper..." # Cluster config. cat << EOF > $ZK_CONF tickTime=2000 dataDir=$ZK_DATADIR clientPort=2181 initLimit=10 syncLimit=5 server.1=192.168.0.101:2888:3888 server.2=192.168.0.102:2888:3888 server.3=192.168.0.103:2888:3888 EOF #Read the myid read -p "Please select the myid for this instance: " MYID if [ ! `echo $MYID | egrep "^[0-9]+\$"` ] ; then echo " _ _ " echo " _ ( \/ ) " echo " | | " echo " " echo " Are you kidding ? That is number? Did you learnt math from P.E teacher? Selecting default: 1" MYID=1 fi # This host's id. echo $MYID > $ZK_DATADIR/myid # Run script. ZK_CP=$ZK_INSTALLDIR/zookeeper-$ZK_VERSION.jar:$ZK_INSTALLDIR/lib/log4j-1.2.15.jar:$ZK_INSTALLDIR/conf cat << EOF > $ZK_RUN #!/bin/bash _JAVA_OPTIONS="-Xmx1024M -Xms1024M" java -cp $ZK_CP org.apache.zookeeper.server.quorum.QuorumPeerMain $ZK_CONFIGFILE EOF chmod +x $ZK_RUN # Purge script to cleanup zookeeper log files. cat << EOF > $ZK_PURGE mkdir $ZK_DIR/snap java -cp $ZK_CP org.apache.zookeeper.server.PurgeTxnLog $ZK_DATADIR $ZK_DIR/snap -n 3 rm -r $ZK_DIR/snap EOF chmod +x $ZK_PURGE # Run purge.sh via cron job. echo "@hourly $ZK_PURGE" | crontab - # Update global start/stop scripts. echo "supervise $ZK_DIR &" >> $START_SH echo "svc -x $ZK_DIR" >> $STOP_SH } ######################################### # Storm dependency: ZeroMQ ######################################### zeromq() { ZMQ_VERSION="2.1.7" ZMQ_DIR=$BASEDIR"/zeromq" ZMQ_TARBALL_URL="http://download.zeromq.org/zeromq-"$ZMQ_VERSION".tar.gz" ZMQ_TARBALL=$ZMQ_DIR"/zeromq.tar.gz" pp "Installing ZeroMQ "$ZMQ_VERSION" (storm dependency)..." mkdir $ZMQ_DIR pp "Downloading ZeroMQ..." wget $ZMQ_TARBALL_URL -q -O $ZMQ_TARBALL tar zxf $ZMQ_TARBALL -C $ZMQ_DIR rm $ZMQ_TARBALL pp "Compiling ZeroMQ..." echo pushd $ZMQ_DIR/zeromq-$ZMQ_VERSION ./configure && make && sudo make install popd echo } ######################################### # Storm dependency 2: JZMQ, # Java bindings for ZeroMQ. # # This is where things get tricky. # Despite the warning on nathanmarz' page, # we use mainline git here, as it compiles # with the latest autoconf and libtool on # Ubuntu 12.04. ######################################### jzmq() { JZMQ_DIR=$BASEDIR"/jzmq" # JZMQ_REPO="https://githiub.com/zeromq/jzmq.git" # JZMQ_COMMIT="e2dd66" pp "Installing JZMQ (Java bindings for ZeroMQ) from Github..." git clone -q https://github.com/nathanmarz/jzmq.git $JZMQ_DIR pp "Compiling JZMQ..." echo pushd $JZMQ_DIR git checkout $JZMQ_COMMIT ./autogen.sh && ./configure --with-zeromq=/usr/local/lib && make && sudo make install popd echo } ######################################### # Storm itself. ######################################### storm() { STORM_VERSION="0.8.1" STORM_DIR=$BASEDIR"/storm" STORM_ZIP_URL="https://github.com/downloads/nathanmarz/storm/storm-"$STORM_VERSION".zip" STORM_ZIP=$STORM_DIR"/storm.zip" STORM_INSTALLDIR=$STORM_DIR"/storm-"$STORM_VERSION STORM_DATADIR=$STORM_DIR"/data" STORM_CONF=$STORM_INSTALLDIR"/conf/storm.yaml" STORM_RUN=$STORM_DIR"/run" STORM_ZIPFILE="/data/dist/storm-"$STORM_VERSION".zip" pp "Installing Storm "$STORM_VERSION"..." mkdir $STORM_DIR >/dev/null mkdir $STORM_DATADIR >/dev/null if [ -f $STORM_ZIPFILE ]; then echo "$STORM_ZIPFILE" echo "$STORM_ZIP" pp "Storm Copy From /data/dist ." cp $STORM_ZIPFILE $STORM_ZIP else pp "Downloading Storm..." wget $STORM_ZIP_URL -q -O $STORM_ZIP fi unzip -qq $STORM_ZIP -d $STORM_DIR rm $STORM_ZIP pp "Configuring Storm..." echo "storm.local.dir: \""$STORM_DATADIR"\"" > $STORM_CONF echo "storm.zookeeper.servers:" >> $STORM_CONF echo " - \"192.168.10.101\"" >> $STORM_CONF echo " - \"192.168.10.102\"" >> $STORM_CONF echo " - \"192.168.10.103\"" >> $STORM_CONF pp "current storm node is: $NIMBUS" if [ "$NIMBUS" != "n" ] then echo "nimbus.host: \"192.168.10.101\"" >> $STORM_CONF fi echo "storm.zookeeper.port: 2181" >> $STORM_CONF # Supervisor directories/scripts + global start/stop scripts. # Note: If we're NIMBUS, we run the 'nimbis' action instead. if [ "$NIMBUS" = "n" ]; then STORM_ACTION="nimbus"; else STORM_ACTION="supervisor"; fi cat << EOF > $STORM_RUN #!/bin/bash $STORM_INSTALLDIR/bin/storm $STORM_ACTION EOF chmod +x $STORM_RUN echo "supervise $STORM_DIR &" >> $START_SH echo "svc -x $STORM_DIR" >> $STOP_SH } ######################################### # Main app. ######################################### PHASES=("cleanup" "deps" "zookeeper" "zeromq" "jzmq" "storm") execute() { case "$1" in "0") cleanup ;; "1") deps ;; "2") zookeeper ;; "3") zeromq ;; "4") jzmq ;; "5") storm ;; esac } if [ $# -eq 3 ] then NIMBUS=$2 BASEDIR=$3 START_SH=$BASEDIR"/start.sh" STOP_SH=$BASEDIR"/stop.sh" if [ "$1" = "all" ] then # Run everything. for ((p=0;p<${#PHASES[@]};p++)) do execute $p done pp "Installation complete." pp "Be sure to carefully read the log." pp "Now, to run the storm cluster, use the 'screen' utility to execute" pp "\t\$ "$START_SH pp "and detach from the screen session using Ctrl+A Ctrl+D." else execute $1 pp "Phase installation complete." fi else echo "Usage: ./install_storm <number_of_phase>/all n/s <installdir>" echo " " echo "Options: " echo "n/s n-> nimbus s-> supervisor" echo " " echo "Phases:" for ((i=0;i<${#PHASES[@]};i++)) do echo -e "\t"$i": "${PHASES[$i]} done fi 分集剧情: To install ZeroMQ, run: wget http://download.zeromq.org/historic/zeromq-2.1.7.tar.gz tar -xzf zeromq-2.1.7.tar.gz cd zeromq-2.1.7 ./configure make sudo make install To install JZMQ, run: git clone https://github.com/nathanmarz/jzmq.git cd jzmq ./autogen.sh ./configure make sudo make install
