LVS+keeplived+nginx+tomcat高可用、高性能jsp集群_MySQL

NginxTomcat

#!/bin/bash

# BY kerryhu

# MAIL:king_819@163.com

# BLOG:http://kerry.blog.51cto.com

# Please manual operation yum of before Operation.....

#yum -y install gcc gcc-c++ autoconf libjpeg libjpeg-devel libpng libpng-devel /

#freetype freetype-devel libxml2 libxml2-devel zlib zlib-devel glibc glibc-devel /

#glib2 glib2-devel bzip2 bzip2-devel ncurses ncurses-devel curl curl-devel e2fsprogs /

#e2fsprogs-devel krb5 krb5-devel libidn libidn-devel openssl openssl-devel openldap /

#openldap-devel nss_ldap openldap-clients openldap-servers

一、网络拓扑:

 

lvs-master: 192.168.9.201

LVS-backup: 192.168.9.202

LVS-VIP: 192.168.9.200

web_1: 192.168.9.203 web_2: 192.168.9.204

目的：搭建高可用、高性能的jsp集群

二、初始化系统

#init system

系统环境: CentOS 5.5（定制安装）

组件：

Base

Development Libraries

Development Tools

Editors

Text-based Internet

./init_system.sh   #此脚本参见 http://kerry.blog.51cto.com/172631/555535

三、LVS+keeplived

#关于LVS+keeplived的配置请参考我的另一篇博文《CentOS5.5环境下布署LVS+keepalived 》 http://kerry.blog.51cto.com/172631/401253

四、nginx、tomcat整合

#config web_1、web_2

#download software

cd /opt

wget http://labs.renren.com/apache-mirror/tomcat/tomcat-6/v6.0.32/bin/apache-tomcat-6.0.32.tar.gz

wget http://download.oracle.com/otn-pub/java/jdk/6u25-b06/jdk-6u25-linux-i586.bin

wget http://nginx.org/download/nginx-1.0.0.tar.gz

wget http://ftp.exim.llorien.org/pcre/pcre-8.12.tar.gz

wget http://apache.etoak.com//tomcat/tomcat-connectors/native/1.1.20/source/tomcat-native-1.1.20-src.tar.gz

wget http://archive.apache.org/dist/apr/apr-1.4.2.tar.gz

wget http://archive.apache.org/dist/apr/apr-util-1.3.9.tar.gz

#================================ 1、安装tomcat、JDK =================================

mv  apache-tomcat-6.0.32.tar.gz jdk-6u25-linux-i586.bin /usr/local/

tar -zxvf apache-tomcat-6.0.32.tar.gz

mv apache-tomcat-6.0.32 tomcat

chmod +x jdk-6u25-linux-i586.bin

./jdk-6u25-linux-i586.bin

#apr 与 tomcat-native 提供更好的伸缩性、性能和集成到本地服务器技术,如果没有apr技术，启动tomcat 时出现如下提示 #INFO: The APR based Apache Tomcat Native library which allows optimal performance in production environments was not found on the java.library.path: /usr/local/jdk1.6.0_25/jre/lib/i386/client:/usr/local/jdk1.6.0_25/jre/lib/i386:/usr/local/jdk1.6.0_25/jre/../lib/i386:/usr/java/packages/lib/i386:/lib:/usr/lib

rpm -e --nodeps apr-1.2.7-11.el5_3.1

rpm -e --nodeps apr-1.2.7-11.el5_3.1

tar -zxvf apr-1.4.2.tar.gz  

cd apr-1.4.2 

./configure  

make;make install

cd ../

tar -zxvf apr-util-1.3.9.tar.gz  

cd apr-util-1.3.9  

./configure --with-apr=/usr/local/apr  

make;make install

cd ../

cp tomcat-native-1.1.20-src.tar.gz /usr/local/tomcat/bin/

cd /usr/local/tomcat/bin

tar -zxvf tomcat-native.tar.gz

cd ./tomcat-native-1.1.12-src/jni/native/

./configure --with-apr=/usr/local/apr --with-java-home=/usr/local/jdk1.6.0_25

make;make install

#设置环境变量

cat >>/etc/profile

export JAVA_HOME=/usr/local/jdk1.6.0_25

export CLASS_PATH=$JAVA_HOME/lib:$JAVA_HOME/jre/lib

export PATH=.:$PATH:$JAVA_HOME/bin

export CATALINA_HOME=/usr/local/tomcat

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/apr/lib

EOF

source /etc/profile

#查看java版本 java -version

#启动tomcat

/usr/local/tomcat/bin/startup.sh

echo "/usr/local/tomcat/bin/startup.sh" >> /etc/rc.local

#将tomcat添加到系统服务

cd /opt

wget http://forja.cenatic.es/plugins/scmsvn/viewcvs.php/*checkout*/servers/tomcat/bin/jsvc.tar.gz?root=norma&rev=82&pathrev=82

mv jsvc.tar.gz /usr/local/tomcat/bin/

tar -zxvf jsvc.tar.gz 

cd jsvc-src

chmod +x configure

./configure --with-java=/usr/local/jdk1.6.0_25

make

cp /usr/local/tomcat/bin/jsvc-src/native/Tomcat5.sh /etc/rc.d/init.d/tomcat

chmod +x /etc/rc.d/init.d/tomcat

vi /etc/rc.d/init.d/tomcat

#chkconfig: 2345 85 15   

#description: apache-tomcat.6   #这两行在#!/bin/sh下面添加

JAVA_HOME=/usr/local/jdk1.6.0_25

CATALINA_HOME=/usr/local/tomcat

DAEMON_HOME=/usr/local/tomcat/bin

TOMCAT_USER=tomcat

CATALINA_BASE=/usr/local/tomcat

CATALINA_OPTS="-Djava.library.path=/usr/local/apr/lib"

$DAEMON_HOME/jsvc-src/jsvc  #有两处需要修改

#添加tomcat用户

groupadd tomcat

useradd -g tomcat -s /sbin/nologin -M tomcat

chown -R tomcat.tomcat /usr/local/tomcat

chkconfig --add tomcat

chkconfig --level 2345 tomcat on

#http://localhost:8080，如果看到猫的页面即tomcat和jdk安装成功

#新建文件目录/data/www为网站存放目录，设置server.xml文件，在Host name="localhost"处将appBase=的指向路径改为/data/www/test,创建index.jsp至/data/www /test/ROOT，内容为：“My web!”  

#修改tomcat网站目录

#sed -i 's#appBase="webapps"#appBase="/data/www/test"#' /usr/local/tomcat/conf/server.xml

#报错信息：java.lang.IllegalArgumentException:   Document   base   /data/www/truboshop/host-manager  does  not  exist or is not a readable directory #解决方法：删除conf/Catalina/localhost目录下面的host-manager.xml、manager.xml两个文件

#============================= 2、安装Nginx =============================

/usr/sbin/groupadd www

/usr/sbin/useradd -g www www -s /sbin/nologin

mkdir -p /data/www

chmod +w /data/www

chown -R www:www /data/www

tar zxvf pcre-8.12.tar.gz

cd pcre-8.12/

./configure

make;make install

cd ../

#使用Tcmalloc 优化nginx、mysql

wget http://google-perftools.googlecode.com/files/google-perftools-1.7.tar.gz

tar -zxvf google-perftools-1.7.tar.gz

cd google-perftools-1.7/

./configure

make;make install

echo "/usr/local/lib" > /etc/ld.so.conf.d/usr_local_lib.conf

/sbin/ldconfig

cd ../

tar zxvf nginx-1.0.0.tar.gz

cd nginx-1.0.0/

./configure --user=www --group=www --prefix=/usr/local/nginx --with-http_stub_status_module --with-http_ssl_module --with-http_flv_module --with-http_gzip_static_module --with-google_perftools_module

make;make install

cd ../

mkdir -p /data/logs

chmod +w /data/logs

chown -R www:www /data/logs

#编辑配置文件

#编辑nginx.config

mv /usr/local/nginx/conf/nginx.conf /usr/local/nginx/conf/nginx.conf.bak

cat >>/usr/local/nginx/conf/nginx.conf

user  www www;  

worker_processes 8;

error_log  /usr/local/nginx/logs/nginx_error.log  crit;

pid        /usr/local/nginx/nginx.pid;

#使用Tcmalloc优化nginx性能

google_perftools_profiles /var/tmp/tcmalloc;

#Specifies the value for maximum file descriptors that can be opened by this process.

worker_rlimit_nofile 65535;

#工作模式及连接数上限

events

{ 

 use epoll; 

 worker_connections 65535;

}

#设定http服务器，利用它的反向代理功能提供负载均衡支持

http

{  

  #设定mime类型  

  include       mime.types;  

  default_type  application/octet-stream;    

  #charset  gb2312; 

  #设定请求缓冲     

  server_names_hash_bucket_size 128;  

  client_header_buffer_size 32k; 

  large_client_header_buffers 4 32k; 

  client_max_body_size 300m;

  sendfile on; 

  tcp_nopush     on;  

  keepalive_timeout 60;

  tcp_nodelay on;

  server_tokens off;

  client_body_buffer_size 512k;

  proxy_connect_timeout   5;

  proxy_send_timeout      60;

  proxy_read_timeout      5;

  proxy_buffer_size       16k;

  proxy_buffers           4 64k;

  proxy_busy_buffers_size 128k;

  proxy_temp_file_write_size 128k;

  #  fastcgi_connect_timeout 300;

  #  fastcgi_send_timeout 300;

  #  fastcgi_read_timeout 300;

  #  fastcgi_buffer_size 64k;

  #  fastcgi_buffers 4 64k;

  #  fastcgi_busy_buffers_size 128k;

  #  fastcgi_temp_file_write_size 128k;  

  gzip on;

  gzip_min_length  1k;

  gzip_buffers     4 16k;

  gzip_http_version 1.1;

  gzip_comp_level 2;

  gzip_types       text/plain application/x-javascript text/css application/xml;

  gzip_vary on;

 

  #limit_zone  crawler  $binary_remote_addr  10m;

###禁止通过ip访问站点

   server{        

 server_name _;        

 return 404;        

  } 

  server

  {    

 listen       80;   

 server_name  www.kerry.com ;   

 index index.html index.htm index.jsp index.do;#设定访问的默认首页地址    

 root  /data/www/shopxx;#设定网站的资源存放路径     

 #limit_conn   crawler  20; 

 if (-d $request_filename)

 {

  rewrite ^/(.*)([^/])$ http://$host/$1$2/ permanent;

 }

 #所有jsp的页面均交由tomcat处理

 location ~ /.(jsp|jspx|do)?$ {

  proxy_set_header  Host $host; 

  proxy_set_header  X-Real-IP  $remote_addr;

  proxy_pass http://127.0.0.1:8080 ;#转向tomcat处理      

 }           

 location ~ .*/.(htm|html|gif|jpg|jpeg|png|bmp|swf|ioc|rar|zip|txt|flv|mid|doc|ppt|pdf|xls|mp3|wma)$ #设定访问静态文件直接读取不经过tomcat    

 {      

  expires      30d;    

 }     

 location ~ .*/.(js|css)?$    

 {      

  expires      1h;    

 }     

 

 #定义访问日志的写入格式     

 log_format  wwwlog  '$remote_addr - $remote_user [$time_local] "$request" '             

  '$status $body_bytes_sent "$http_referer" '            

  '"$http_user_agent" $http_x_forwarded_for';   

 access_log  /data/logs/www_nginx.log wwwlog;#设定访问日志的存放路径 

   }

  server

  {    

 listen       80;   

 server_name  bbs.kerry.com;   

 index index.html index.htm index.jsp index.do;#设定访问的默认首页地址    

 root  /data/www/jsprun;#设定网站的资源存放路径     

 #limit_conn   crawler  20; 

 if (-d $request_filename)

 {

  rewrite ^/(.*)([^/])$ http://$host/$1$2/ permanent;

 }

 #所有jsp的页面均交由tomcat处理

 location ~ /.(jsp|jspx|do)?$ {      

  proxy_set_header  Host $host; 

  proxy_set_header  X-Real-IP  $remote_addr;

  proxy_pass http://127.0.0.1:8080 ;#转向tomcat处理      

 }           

 location ~ .*/.(htm|html|gif|jpg|jpeg|png|bmp|swf|ioc|rar|zip|txt|flv|mid|doc|ppt|pdf|xls|mp3|wma)$ #设定访问静态文件直接读取不经过tomcat    

 {      

  expires      30d;    

 }     

 location ~ .*/.(js|css)?$    

 {      

  expires      1h;    

 }     

 

 #定义访问日志的写入格式     

 log_format  bbslog  '$remote_addr - $remote_user [$time_local] "$request" '             

  '$status $body_bytes_sent "$http_referer" '            

  '"$http_user_agent" $http_x_forwarded_for';   

 access_log  /data/logs/bbs_nginx.log bbslog;#设定访问日志的存放路径 

   }

   server

   {

 listen  80;

 server_name  status.www.kerry.com;

 location / {

 stub_status on;

 access_log   off;

 }

   }

}

EOF

#检测配置文件 #/usr/local/nginx/sbin/nginx -t 

#启动nginx

 /usr/local/nginx/sbin/nginx

 echo "/usr/local/nginx/sbin/nginx" >> /etc/rc.local

#查看nginx主进程号 #ps -ef | grep "nginx: master process" | grep -v "grep" | awk -F ' ' '{print $2}'

#停止nginx #/usr/local/nginx/sbin/nginx -s stop

#平滑重启 #/usr/local/nginx/sbin/nginx -s reload

#============================= 3、配置一套java商城系统，进行测试 =============================

cd /opt

wget http://www.rarlab.com/rar/rarlinux-3.8.0.tar.gz

#rar解压

tar -zxvf rarlinux-3.8.0.tar.gz

cd rar

make install

cd /opt

unrar x jsprun.rar

mv jsprun /data/www/

#7z解压

wget http://sourceforge.net/projects/p7zip/files/p7zip/9.13/p7zip_9.13_src_all.tar.bz2/download

tar -jxvf p7zip_9.13_src_all.tar.bz2

cd p7zip_9.13

make;make install

7za x shopxx.7z

mv shopxx /data/www/

#配置tomcat虚拟主机

vi /usr/local/tomcat/conf/server.xml

 

           

     

 

 

           

     

 

#使用cmake编译安装mysql

cd /opt

wget http://www.cmake.org/files/v2.8/cmake-2.8.4.tar.gz

wget http://mirrors.sohu.com/mysql/MySQL-5.5/mysql-5.5.10.tar.gz

tar -zxvf cmake-2.8.4.tar.gz

cd cmake-2.8.4

./configu

make;make install

/usr/sbin/groupadd mysql

/usr/sbin/useradd -g mysql mysql -s /sbin/nologin

mkdir -p /data/mysql/data

mkdir -p /data/mysql/binlog

mkdir -p /data/mysql/relaylog

mkdir -p /data/mysql/mysql

chown -R mysql:mysql /data/mysql

cd /opt

tar zxvf mysql-5.5.10.tar.gz

cd mysql-5.5.10/

rm -rf CMakeCache.txt

cmake . -DCMAKE_INSTALL_PREFIX=/usr/local/mysql/  -DMYSQL_DATADIR=/data/mysql/data -DWITH_INNOBASE_STORAGE_ENGINE=1  -DMYSQL_TCP_PORT=3306  -DEXTRA_CHARSETS=all -DDEFAULT_CHARSET=utf8 -DDEFAULT_COLLATION=utf8_general_ci -DMYSQL_UNIX_ADDR=/data/mysql/mysql.sock  -DMYSQL_USER=mysql  -DWITH_DEBUG=0

make;make install

chmod +w /usr/local/mysql

chown -R mysql:mysql /usr/local/mysql

cat >>/data/mysql/my.cnf

[client]

character-set-server = utf8

port    = 3306

socket  = /data/mysql/mysql.sock

[mysqld]

character-set-server = utf8

replicate-ignore-db = mysql

replicate-ignore-db = test

replicate-ignore-db = information_schema

user    = mysql

port    = 3306

socket  = /data/mysql/mysql.sock

basedir = /usr/local/mysql

datadir = /data/mysql/data

log-error = /data/mysql/mysql_error.log

pid-file = /data/mysql/mysql.pid

open_files_limit    = 10240

back_log = 600

max_connections = 5000

max_connect_errors = 6000

table_cache = 614

external-locking = FALSE

max_allowed_packet = 32M

sort_buffer_size = 1M

join_buffer_size = 1M

thread_cache_size = 300

thread_concurrency = 8

query_cache_size = 512M

query_cache_limit = 2M

query_cache_min_res_unit = 2k

default-storage-engine = MyISAM

thread_stack = 192K

transaction_isolation = READ-COMMITTED

tmp_table_size = 246M

max_heap_table_size = 246M

long_query_time = 3

log-slave-updates

log-bin = /data/mysql/binlog/binlog

binlog_cache_size = 4M

binlog_format = MIXED

max_binlog_cache_size = 8M

max_binlog_size = 100M

relay-log-index = /data/mysql/relaylog/relaylog

relay-log-info-file = /data/mysql/relaylog/relaylog

relay-log = /data/mysql/relaylog/relaylog

expire_logs_days = 30

key_buffer_size = 256M

read_buffer_size = 1M

read_rnd_buffer_size = 16M

bulk_insert_buffer_size = 64M

myisam_sort_buffer_size = 128M

myisam_max_sort_file_size = 10G

myisam_repair_threads = 1

myisam_recover

interactive_timeout = 120 wait_timeout = 120

skip-name-resolve

master-connect-retry = 10

slave-skip-errors = 1032,1062,126,1114,1146,1048,1396

#master-host     =   192.168.1.2

#master-user     =   username

#master-password =   password

#master-port     =  3306

server-id = 1

innodb_additional_mem_pool_size = 16M

innodb_buffer_pool_size = 512M

innodb_data_file_path = ibdata1:256M:autoextend

innodb_file_io_threads = 4

innodb_thread_concurrency = 8

innodb_flush_log_at_trx_commit = 2

innodb_log_buffer_size = 16M

innodb_log_file_size = 128M

innodb_log_files_in_group = 3

innodb_max_dirty_pages_pct = 90

innodb_lock_wait_timeout = 120

innodb_file_per_table = 0

#log-slow-queries = /data/mysql/slow.log #long_query_time = 10

[mysqldump]

quick

max_allowed_packet = 32M

EOF

#初始化mysql /usr/local/mysql/scripts/mysql_install_db -–defaults-file=/data/mysql/my.cnf --basedir=/usr/local/mysql --datadir=/data/mysql/data --user=mysql

#利用TCMalloc提高mysql在高并发下的性能 vi /usr/local/mysql/bin/mysqld_safe

#在# executing mysqld_safe的下一行，加上： export LD_PRELOAD=/usr/local/lib/libtcmalloc.so

#使用lsof命令查看tcmalloc是否起效 /usr/sbin/lsof -n | grep tcmalloc

#设置mysql启动文件

cp support-files/mysql.server /etc/rc.d/init.d/mysqld

vi /etc/rc.d/init.d/mysqld

basedir=/usr/local/mysql

datadir=/data/mysql/data

chmod 700 /etc/rc.d/init.d/mysqld

/etc/rc.d/init.d/mysqld start

/sbin/chkconfig --add mysqld

/sbin/chkconfig --level 2345 mysqld on

ln -s /usr/local/mysql/bin/mysql /sbin/mysql

ln -s /usr/local/mysql/bin/mysqladmin /sbin/mysqladmin

#设置root密码（753951） /sbin/mysqladmin -u root password 753951

#配置库文件搜索路径

echo "/usr/local/mysql/lib/mysql" >> /etc/ld.so.conf

/sbin/ldconfig

#添加/usr/local/mysql/bin到环境变量PATH中 export PATH=$PATH:/usr/local/mysql/bin

#添加mysql管理帐户

#mysql -h localhost -u root -p753951

#msqyl> use mysql;

#msqyl> grant all on *.* to 'kerry'@'192.168.9.100' identified by '852741';

#msqyl> flush privileges;

#msqyl> exit;

#创建一个shopxx空库 CREATE DATABASE shopxx DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

#=============================== 4、tomcat优化  ============================

#添加tomcat管理员

#Tomcat 6.x不再提供admin管理工具(Tomcat's administration web application is no longer installed by default，Download and install the "admin" package to use it)，要自己安装

cd /opt

wget http://archive.apache.org/dist/tomcat/tomcat-5/v5.5.26/bin/apache-tomcat-5.5.26-admin.tar.gz

wget http://mirror.bjtu.edu.cn/apache//commons/logging/binaries/commons-logging-1.1.1-bin.tar.gz

wget http://labs.renren.com/apache-mirror//commons/modeler/binaries/commons-modeler-2.0.1.tar.gz

tar -zxvf apache-tomcat-5.5.26-admin.tar.gz

cd apache-tomcat-5.5.26

cp -r conf/Catalina /usr/local/tomcat/conf/

cp -r server /usr/local/tomcat/

cd /opt

tar -zxvf commons-modeler-2.0.1.tar.gz

cd commons-modeler-2.0.1

cp -r *.jar /usr/local/tomcat/lib/

cd ../

tar -zxvf commons-logging-1.1.1-bin.tar.gz

cd commons-logging-1.1.1

cp -r *.jar /usr/local/tomcat/lib/

vi /usr/local/tomcat/conf/tomcat-users.xml

#在  中间添加 ,并去掉注释

vi /usr/local/tomcat/conf/context.xml

#修改Java虚拟机的虚拟内存，tomcat默认的JVM比较小，查看JVM，打开 http://127.0.0.1:8080/ ，点击"status"，查看JVM选项

#在修改配置文件的时候不知道怎么把manager/WEB-INF/web.xml搞没见了，导致在访问管理页面的时候出现“The requested resource (/manager/status) is not available.”

Free memory: 3.25 MB Total memory: 7.62 MB Max memory: 121.81 MB

vi /etc/init.d/tomcat

#在CATALINA_OPTS="-Djava.library.path=/usr/local/apr/lib"下方添加一句

JAVA_OPTS="-Xms1024m -Xmx1024m -Xmn256m -Djava.awt.headless=true"

#重启一下tomcat，然后再次查看JVM #Free memory: 953.12 MB Total memory: 998.43 MB Max memory: 998.43 MB

#修改tomcat并发连接数，tomcat6.0 默认并发数为200，查看方法和JVM一样，查看http-8080选项 Max threads: 200

vi /usr/local/tomcat/conf/server.xml

#修改前：

                redirectPort="8443"  

                connectionTimeout="20000"  

                />  

#修改后：  

                maxHttpHeaderSize="8192" useBodyEncodingForURI="true"  

                maxThreads="1000"

                redirectPort="8443"  

                enableLookups="false"  

                compression="on"   

                compressionMinSize="2048"  

                compressableMimeType="text/html,text/xml,text/javascript,text/css,text/plain"   

                connectionTimeout="20000"  

                disableUploadTimeout="true"  

                />

  

port：Tomcat服务器监听的端口号

maxHttpHeaderSize：Http的Header的最大限制

enableLookups="false" 使用允许DNS查询，通常情况下设置为false

redirectPort 服务器正在处理http请求时收到了一个SSL传输请求后重定向的端口号

maxThreads Tomcat使用线程来处理接收的每个请求。这个值表示Tomcat可创建的最大的线程数。

acceptCount 指定当所有可以使用的处理请求的线程数都被使用时，可以放到处理队列中的请求数，超过这个数的请求将不予处理。

connectionTimeout 网络连接超时，单位：毫秒。设置为0表示永不超时，这样设置有隐患的。通常可设置为30000毫秒。

minSpareThreads Tomcat初始化时创建的线程数。

maxSpareThreads 一旦创建的线程超过这个值，Tomcat就会关闭不再需要的socket线程。

compression="on" 打开压缩功能

compressionMinSize 启用压缩的输出内容大小，这里面默认为2KB 

compressableMimeType压缩类型

#重启tomcat后再次查看 Max threads: 1000

本文出自 “ 聆听未来 ” 博客，请务必保留此出处 http://kerry.blog.51cto.com/172631/557749