侧耳倾听

linux线程操作

sew0359 — Fri, 31 Oct 2008 03:43:47 +0000

--------------------------线程的创建---------------------------
pthread_t mythread;
//线程句柄声明,用来存放tid
pthread_create( &mythread, NULL, thread_function, NULL)
//创建新的线程
//参数一:线程句柄,存放tid
//参数二:线程函数返回值
//参数三:线程函数,由thread_function函数指针定义
//参数四:线程函数的参数
pthread_join ( mythread, NULL )
//等待mythread线程于主线程合并
fflush(stdout);
//清空标准输出缓冲区
--------------------------线程的同步(互斥锁)---------------------------
pthread_mutex_t mymutex=PTHREAD_MUTEX_INITIALIZER;
//初始化mymutex互斥锁(使用宏的静态版本)
pthread_mutex_t mymutex;
pthread_mutex_init(&mymutex,NULL);
//初始化mymutex互斥锁(动态版本)
pthread_mutex_lock(&mymutex);
//mymutex上锁,当得不到互斥锁时,进入等待队列,直到得到为止
pthread_mutex_trylock(pthread_mutex_t *mutex)
//mymutex上锁,当得不到互斥锁时,立刻返回EBUSY
pthread_mutex_unlock(&mymutex);
//mymutex解锁
pthread_mutex_destroy(&mymutex)
//释放mymutex指针指向的资源(释放mymutex指针指向的内存,并不指针本身)
--------------------------线程的同步(等待)---------------------------
pthread_cond_t mycond;
//定义条件变量mycond
pthread_cond_init(&mycond,NULL);
//初始化条件变量mycond
pthread_cond_destroy(&mycond);
//释放mycond指针指向的资源
pthread_cond_wait(&mycond, &mymutex){
mymutex解锁;
while(1){
mycond休眠并等待被唤醒;
if(mycond被唤醒)
if(mymutex上锁成功)
return 0;
}
}
//等待mycond得到信号被唤醒
pthread_cond_broadcast(&mycond);
//对mycond广播唤醒信号
pthread_cond_signal(&mycond);
//对mycond单播唤醒信号(通常是下一个)

MOSIX-2.24.2.2/linux-2.6.26集群(三)–应用测试

admin — Tue, 09 Sep 2008 08:25:29 +0000

#先在rehl5和slave上各开启一个终端,运行mon命令,检查
[root@rhel5 ~]# mon

#2个节点上应该都是闲置的吧

#为了能出些效果,做点费cpu的脚本,还必须是多线程的,
#mosix能够迁移的最小单位是进程,而不是指令或者函数,
#所以单进程负载再高也没意义

[root@rhel5 ~]# cat a.sh << EOF
awk 'BEGIN {for(i=0;i<100000;i++)for(j=0;j<100000;j++);}' &
awk 'BEGIN {for(i=0;i<100000;i++)for(j=0;j<100000;j++);}' &
awk 'BEGIN {for(i=0;i<100000;i++)for(j=0;j<100000;j++);}' &
awk 'BEGIN {for(i=0;i<100000;i++)for(j=0;j<100000;j++);}' &
awk 'BEGIN {for(i=0;i<100000;i++)for(j=0;j<100000;j++);}' &
awk 'BEGIN {for(i=0;i<100000;i++)for(j=0;j<100000;j++);}' &
EOF

[root@rhel5 ~]# chmod +x a.sh

#在rhel5上运行a.sh,也就是产生6个进程了

[root@rhel5 ~]# mosrun -e ./a.sh

#开始观察2个节点上的mon画面,刚开始rhel负载很高,然后slave的负载也起来了,能够看到

#能够看到在rhel5上,awk的6个进程还在,但是只有3个在运行,还有3个的状态是T(stop),哈哈,应该是迁移了

[root@rhel5 ~]# ps -aux | grep awk
Warning: bad syntax, perhaps a bogus '-'? See /usr/share/doc/procps-3.2.7/FAQ
root 25648 0.6 0.0 0 0 pts/0 T 16:16 0:00 [awk]
root 25650 0.4 0.0 0 0 pts/0 T 16:16 0:00 [awk]
root 25652 32.0 0.7 4168 3812 pts/0 R 16:16 0:37 awk BEGIN {for(i=0;i<100000;i++)for(j=0;j<100000;j++);}
root 25654 32.0 0.7 4168 3816 pts/0 R 16:16 0:37 awk BEGIN {for(i=0;i<100000;i++)for(j=0;j<100000;j++);}
root 25656 32.0 0.7 4168 3816 pts/0 R 16:16 0:37 awk BEGIN {for(i=0;i<100000;i++)for(j=0;j<100000;j++);}
root 25658 1.4 0.0 0 0 pts/0 T 16:16 0:01 [awk]
root 25665 0.0 0.1 3860 624 pts/0 R+ 16:18 0:00 grep awk

#到slave上top看看吧,明显看到有3个叫remoted的进程占用了cpu,这个就是迁移过来的状态吧

top - 16:19:19 up 3:10, 3 users, load average: 2.78, 1.18, 0.44
Tasks: 99 total, 5 running, 94 sleeping, 0 stopped, 0 zombie
Cpu(s): 99.3%us, 0.3%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.3%si
Mem: 515376k total, 423576k used, 91800k free, 107980k buff
Swap: 1048568k total, 0k used, 1048568k free, 234028k cach

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
16929 root 20 0 4168 3936 0 R 33.2 0.8 0:48.13 remoted
16925 root 20 0 4168 3932 0 R 32.9 0.8 0:50.57 remoted
16927 root 20 0 4168 3932 0 R 32.9 0.8 0:50.13 remoted
   1 root 20 0 2036 664 572 S 0.0 0.1 0:01.36 init
   2 root 15 -5 0 0 0 S 0.0 0.0 0:00.00 kthreadd
   3 root RT -5 0 0 0 S 0.0 0.0 0:00.00 migratio
   4 root 15 -5 0 0 0 S 0.0 0.0 0:02.00 ksoftirq

##############THE END############

MOSIX-2.24.2.2/linux-2.6.26集群(二)–配置

admin — Tue, 09 Sep 2008 08:25:01 +0000

将rhel5和slave开启,开机的时候,在grub界面按回车,然后选择2.6.26内核启动

slave启动以后,把ip地址,机器名改好(应为是由rhel5克隆得到的嘛)

[reel5]

#配置mosix

[root@rhel5 ~]# mosconf
MOSIX CONFIGURATION
===================

If this is your cluster's file-server and you want to configure MOSIX
for a set of nodes with a common root, please type their common root
directory. Otherwise, if you want to configure the node that you are
running on, just press :-

What would you like to configure?
=================================
1. Which nodes are in this cluster (ESSENTIAL)
2. Authentication (ESSENTIAL)
3. Logical node numbering (recommended)
4. Queueing policies (recommended)
5. Freezing policies
6. Miscellaneous policies
7. Become part of a multi-cluster organizational Grid

Configure what :- 1

There are no nodes in your cluster yet:
=======================================

To add a new set of nodes to your cluster, type 'n'.
To turn on advanced options, type '+'.
For help, type 'h'.
To save and exit, type 'q'. (to abandon all changes and exit, type 'Q')

Option :- n <==添加节点

Adding new node(s) to the cluster:

First host-name or IP address :- 192.168.1.5 <==节点ip
Number of nodes :- 1 <==节点数

Nodes in your cluster:
======================
1. 192.168.1.5

To add a new set of nodes to your cluster, type 'n'.
To modify an entry, type its number.
To delete an entry, type 'd' followed by that entry-number (eg. d1).
To turn on advanced options, type '+'.
For help, type 'h'.
To save and exit, type 'q'. (to abandon all changes and exit, type 'Q')

Option :- n <==添加节点

Adding new node(s) to the cluster:

First host-name or IP address :- 192.168.1.6 <==节点ip
Number of nodes :- 1 <==节点数

Nodes in your cluster:
======================
1. 192.168.1.5
2. 192.168.1.6

To add a new set of nodes to your cluster, type 'n'.
To modify an entry, type its number.
To delete an entry, type 'd' followed by that entry-number (eg. d2).
To turn on advanced options, type '+'.
For help, type 'h'.
To save and exit, type 'q'. (to abandon all changes and exit, type 'Q')

Option :- q <==保存退出

Cluster configuration was saved.

OK to also update the logical node numbers [Y/n]? y

Suggesting to assign '192.168.1.5'
as the central queue manager for the cluster
(but be cautious if you mix 32-bit and 64-bit nodes in the same cluster)
OK to update it now [Y/n]?

What would you like to configure next?
======================================
1. Which nodes are in this cluster
2. Authentication (ESSENTIAL)
3. Logical node numbering
4. Queueing policies
5. Freezing policies
6. Miscellaneous policies
7. Become part of a multi-cluster organizational Grid
q. Exit

Configure what :- 2 <==设置密码

MOSIX Authentication:
=====================

To protect your MOSIX cluster from abuse, preventing unauthorized
persons from gaining control over your computers, you need to set
up a secret cluster-protection key. This key can include any
characters, but must be identical throughout your cluster.

Your secret cluster-protection key: xxxx <==输入密码
Your key is 5 characters long.
(in the future, please consider a longer one)

To allow your users to send batch-jobs to other nodes in the cluster,
you must set up a secret batch-client key. This key can include any
characters, but must match the 'batch-server' key on the node(s) that
can receive batch-jobs from this node.

Your secret batch-client key: xxxx <==输入密码
Your key is 5 characters long.
(in the future, please consider a longer one)

For this node to accept batch jobs,
you must set up a secret batch-server key. This key can include any
characters, but must match the 'batch-client' key on the sending nodes.

To make your batch-server key the same as your batch-client key, type '+'.
Your secret batch-server key: xxxx <==输入密码
Your key is 5 characters long.
(in the future, please consider a longer one)

#保持退出

[root@rhel5 ~]# service mosix restart

[root@slave ~]# mosconf
....

#操作同rhel5一样

#重启服务

[root@slave ~]# service mosix restart

#看看状态吧

[root@slave ~]# service mosix status
This MOSIX node is: 192.168.1.6 (no features)

Nodes in cluster:
=================
192.168.1.5: proximate
192.168.1.6: proximate

Status: Running Normally (32-bits)
Load: 0.01 (equivalent to about 0.0066 CPU processes)
Speed: 6650 units
CPUS: 1
Frozen: 0
Util: 100%
Avail: YES
Procs: Running 0 MOSIX processes
Accept: Yes, will welcome processes from here
Memory: Available 461MB/503MB
Swap: Available 0.9GB/0.9GB
Daemons:
   Master Daemon: Up
   MOSIX Daemon : Up
   Queue Manager: Up
   Remote Daemon: Up
   Postal Daemon: Up
Guest processes from other clusters in the grid: 0/8

#我比较喜欢看看端口是不是起来了
#TCP/IP ports 249-253 and UDP/IP ports 249-250 must be available for MOSIX

[root@slave ~]# netstat -antu | grep -E "24|25"
tcp 0 0 0.0.0.0:2401 0.0.0.0:* LISTEN
tcp 0 0 0.0.0.0:249 0.0.0.0:* LISTEN
tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN
tcp 0 0 0.0.0.0:250 0.0.0.0:* LISTEN
tcp 0 0 0.0.0.0:251 0.0.0.0:* LISTEN
tcp 0 0 0.0.0.0:252 0.0.0.0:* LISTEN
udp 0 0 0.0.0.0:249 0.0.0.0:*
udp 0 0 0.0.0.0:250 0.0.0.0:*

#好了,装完了

MOSIX集群(一)–安装

admin — Tue, 09 Sep 2008 08:24:23 +0000

目的: 集群节点内进程能根据负载情况自动迁移

用vmware安装一台rhel5(192.168.1.5)

# 下载MOSIX和kernel代码,准备编译
# 解压到指定目录

[root@rhel5 ~]# tar xjvf MOSIX-2.24.2.2.tbz -C /usr/src/
[root@rhel5 ~]# tar xzvf linux-2.6.26.tar.gz -C /usr/src/

#进入源代码所在目录

[root@rhel5 ~]# cd /usr/src/

#由于other/patch-2.6.26的目标路径是linux-2.6.26.1,做个连接吧(可能是mosix没有为2.6.26单独写patch…,不过还是支持的)

[root@rhel5 src]# ln -s linux-2.6.26/ ./linux-2.6.26.1

#给kernel打上mosix补丁

[root@rhel5 src]# patch -p0 < /usr/src/mosix-2.24.2.2/other/patch-2.6.26

#进入源代码目录,开始编译

[root@rhel5 src]# cd linux-2.6.26

#生成配置文件

[root@rhel5 linux-2.6.26]# make menuconfig

#生成依赖关系

[root@rhel5 linux-2.6.26]# make dep

#编译内核

[root@rhel5 linux-2.6.26]# make bzImage

#编译内核模块

[root@rhel5 linux-2.6.26]# make modules

#安装内核模块

[root@rhel5 linux-2.6.26]# make modules_install

#安装内核

[root@rhel5 linux-2.6.26]# make install

#进入mosix目录

[root@rhel5 mosix-2.24.2.2]# cd ../mosix-2.24.2.2

#安装mosix,一路回车,只用安装,记得把你常用级别的mosix服务打开就可以了.配置以后再说

[root@rhel5 mosix-2.24.2.2]# ./mosix.install

关机以后,用rhel5(192.168.1.5)克隆出slave(192.168.1.6)

安装完成

iscsi+clvm+gfs2+xen+Cluster(四)–clvm+gfs2的配置

admin — Fri, 05 Sep 2008 08:57:18 +0000

刚才的iscsi将服务器上的磁盘共享出来,在cluster中,往往会使用添加共享磁盘的方式扩大共享空间,我们可以使用clvm技术,将多块共享磁盘合并成一个磁盘(卷组volume goup).也便于以后的磁盘添加
卷组的分区和硬盘的分区不一样,我们叫他为逻辑卷(logical volume)
逻辑卷用什么格式呢?我这里选择的是gfs2

现在p1上设置clvmd,用来识别lvm

[root@p2 ~]# lvmconf --enable-cluster
[root@p2 ~]# chkconfig --level 35 clvmd on

我是在iscsi这台机器上用Conga Cluster and Storage Management System对整个集群进行设置的,相当方便
那么先初始化Conga服务吧

[root@iscsi ~]# luci_admin init
Initializing the Luci server

Creating the 'admin' user

Enter password:
Confirm password:

Please wait...
The admin password has been successfully set.
Generating SSL certificates...
Luci server has been successfully initialized

Restart the Luci server for changes to take effect
eg. service luci restart

然后重启服务

[root@iscsi ~]# service luci restart
Shutting down luci: [ OK ]
Starting luci: [ OK ]

Point your web browser to https://iscsi.yubo.org:8084 to access luci

先建立一个集群吧,至于为什么这样做,请参照

点击submit后

cluster成员经历安装-重启-配置-结合4个步骤

现在应该能看到

如果还是不放心,想进一步确认是不是正常的话,好吧,点击上面那个图上的p1.yubo.org

现在总该放心了吧,那么进入正题,storage->system list->p1.yubo.org->Volume Groups->New Volume Group
将p1上的3个共享硬盘做成一个卷组,名字就叫vg1吧

添加完成之后,点击vg1上面的蓝条,建立逻辑卷

p1.yubo.org上的gfs2也就配置完成了
p2.yubo.org按道理刷新后应该跟p1一样,毕竟是一个cluster,而且3个硬盘的scsi id都是一样的
不过我做试验的时候p2看上去没有任何变化,那么硬着头皮再配置一遍吧
进入p2的配置页面,创建卷组(心想这样重新创建的vg跟p1的vg1不一致怎么办)
果然报错了
刷新以后,p1的配置更新到p2上来了….

iscsi+clvm+gfs2+xen+Cluster(三)–iscsi的配置

admin — Fri, 05 Sep 2008 06:48:38 +0000

我的目的就是让p1,p2这2台机器能使用到iscsi共享的磁盘,那么开始把

domain:yubo.org

   iscsi:192.168.1.100
   |
   __________|___________
   | |
p1:192.168.1.101 p2:192.168.1.102
(v1:192.168.1.111) (v2:192.168.1.112)

#########################服务端##########################

[iscsi.yubo.org]
在iscsi上加装一块12G的scsi硬盘,作为共享磁盘(sdb)
分3个区吧,让共享磁盘看上去多一点,到后面做clvm直观些
分区之后:

[root@iscsi ~]# fdisk -l /dev/sdb

Disk /dev/sdb: 12.8 GB, 12884901888 bytes
255 heads, 63 sectors/track, 1566 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes

Device Boot Start End Blocks Id System
/dev/sdb1 1 499 4008186 83 Linux
/dev/sdb2 500 1000 4024282+ 83 Linux
/dev/sdb3 1001 1566 4546395 83 Linux

#为了使sdb1,sdb2,sdb3共享出去
#先开启相应的tgtd(SCSI Target Administration)服务吧

[root@iscsi ~]# service tgtd start

#使用tgtadm 定义iscsi target 的qualified 名字：

[root@iscsi ~]# tgtadm --lld iscsi --op new --mode target --tid=1 --targetname org.yubo.disk1
[root@iscsi ~]# tgtadm --lld iscsi --op new --mode target --tid=2 --targetname org.yubo.disk2
[root@iscsi ~]# tgtadm --lld iscsi --op new --mode target --tid=3 --targetname org.yubo.disk3

#使用 tgtadm 为上一步创建的目标增加分区：

[root@iscsi ~]# tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 1 -b /dev/sdb1
[root@iscsi ~]# tgtadm --lld iscsi --op new --mode logicalunit --tid 2 --lun 1 -b /dev/sdb2
[root@iscsi ~]# tgtadm --lld iscsi --op new --mode logicalunit --tid 3 --lun 1 -b /dev/sdb3

#使用tgtadm 允许客户端访问这三个目标逻辑卷：

[root@iscsi ~]# tgtadm --lld iscsi --op bind --mode target --tid 1 -I ALL
[root@iscsi ~]# tgtadm --lld iscsi --op bind --mode target --tid 2 -I ALL
[root@iscsi ~]# tgtadm --lld iscsi --op bind --mode target --tid 3 -I ALL

#使用tatadm 验证所有的目标逻辑卷定义正确：

[root@iscsi ~]# tgtadm --lld iscsi --op show --mode target |grep Target

#为了使这个配置永远生效

[root@iscsi ~]# chkconfig --level 35 tgtd on
[root@iscsi ~]# cat >> /etc/rc.local <tgtadm --lld iscsi --op new --mode target --tid=1 --targetname org.yubo.disk1
tgtadm --lld iscsi --op new --mode target --tid=2 --targetname org.yubo.disk2
tgtadm --lld iscsi --op new --mode target --tid=3 --targetname org.yubo.disk3
tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 1 -b /dev/sdb1
tgtadm --lld iscsi --op new --mode logicalunit --tid 2 --lun 1 -b /dev/sdb2
tgtadm --lld iscsi --op new --mode logicalunit --tid 3 --lun 1 -b /dev/sdb3
tgtadm --lld iscsi --op bind --mode target --tid 1 -I ALL
tgtadm --lld iscsi --op bind --mode target --tid 2 -I ALL
tgtadm --lld iscsi --op bind --mode target --tid 3 -I ALL
EOF

#############################客户端#########################
[p1.yubo.org]
确认iscsid/iscsi是开启的(默认是开启的

运行下面命令，discovery iscsitarget上的逻辑卷：

[root@p1 ~]# iscsiadm -m discovery -t sendtargets -p 192.168.1.100
192.168.1.100:3260,1 org.yubo.disk1
192.168.1.100:3260,1 org.yubo.disk2
192.168.1.100:3260,1 org.yubo.disk3

说明iscsi 上共享的逻辑卷已经成功识别。

使用 iscsiadm 登录目标服务器(iscsi):

[root@p1 ~]# iscsiadm -m node -T org.yubo.disk1 -p 192.168.1.100 -l
Login session [iface: default, target: org.yubo.disk1, portal: 192.168.1.100,3260]
[root@p1 ~]# iscsiadm -m node -T org.yubo.disk2 -p 192.168.1.100 -l
Login session [iface: default, target: org.yubo.disk2, portal: 192.168.1.100,3260]
[root@p1 ~]# iscsiadm -m node -T org.yubo.disk3 -p 192.168.1.100 -l
Login session [iface: default, target: org.yubo.disk3, portal: 192.168.1.100,3260]

#用fdisk查看吧

[root@p1 ~]# fdisk -l

Disk /dev/sda: 21.4 GB, 21474836480 bytes
255 heads, 63 sectors/track, 2610 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes

Device Boot Start End Blocks Id System
/dev/sda1 * 1 2295 18434556 83 Linux
/dev/sda2 2296 2360 522112+ 82 Linux swap / Solaris

Disk /dev/sdb: 4104 MB, 4104382464 bytes
127 heads, 62 sectors/track, 1018 cylinders
Units = cylinders of 7874 * 512 = 4031488 bytes

Disk /dev/sdb doesn't contain a valid partition table

Disk /dev/sdc: 4120 MB, 4120865280 bytes
127 heads, 62 sectors/track, 1022 cylinders
Units = cylinders of 7874 * 512 = 4031488 bytes

Disk /dev/sdc doesn't contain a valid partition table

Disk /dev/sdd: 4655 MB, 4655508480 bytes
144 heads, 62 sectors/track, 1018 cylinders
Units = cylinders of 8928 * 512 = 4571136 bytes

Disk /dev/sdd doesn't contain a valid partition table

#从上面看出， iscsi.yubo.org 上的三个逻辑卷，分别被识别成了本地磁盘/dev/sdb,/dev/sdc,/dev/sdd 三个本地磁盘。
#以上用iscsiadm命令所登陆到的目标服务器信息,会被自动保存到/var/lib/iscsi/目录,所以不用担心重启丢失的问题

[root@p1 ~]# ls /var/lib/iscsi/nodes/
org.yubo.disk1 org.yubo.disk2 org.yubo.disk3

#接着在这三个磁盘上，分别建立一个分区，建立成功后的结果应为：

[root@p1 ~]# fdisk -l

Disk /dev/sda: 21.4 GB, 21474836480 bytes
255 heads, 63 sectors/track, 2610 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes

   Device Boot Start End Blocks Id System
/dev/sda1 * 1 2295 18434556 83 Linux
/dev/sda2 2296 2360 522112+ 82 Linux swap / Solaris

Disk /dev/sdb: 4104 MB, 4104382464 bytes
127 heads, 62 sectors/track, 1018 cylinders
Units = cylinders of 7874 * 512 = 4031488 bytes

   Device Boot Start End Blocks Id System
/dev/sdb1 1 1000 3936969 83 Linux

Disk /dev/sdc: 4120 MB, 4120865280 bytes
127 heads, 62 sectors/track, 1022 cylinders
Units = cylinders of 7874 * 512 = 4031488 bytes

   Device Boot Start End Blocks Id System
/dev/sdc1 1 1000 3936969 83 Linux

Disk /dev/sdd: 4655 MB, 4655508480 bytes
144 heads, 62 sectors/track, 1018 cylinders
Units = cylinders of 8928 * 512 = 4571136 bytes

   Device Boot Start End Blocks Id System
/dev/sdd1 1 1000 4463969 83 Linux

#运行下面命令，discovery iscsitarget上的逻辑卷：

[root@p1 ~]# iscsiadm -m discovery -t sendtargets -p 192.168.1.100
192.168.1.100:3260,1 org.yubo.disk1
192.168.1.100:3260,1 org.yubo.disk2
192.168.1.100:3260,1 org.yubo.disk3

说明iscsi 上共享的逻辑卷已经成功识别。

p2上也做相应的配置,只是登陆到目标服务器以后,不用分区了

iscsi配置也就完成了

iscsi+clvm+gfs2+xen+Cluster(二)–xen虚拟机的安装

admin — Fri, 05 Sep 2008 06:40:43 +0000

[dhcp.yubo.org]

#修改为用来安装虚拟机的ks文件

[root@dhcp ~]# cat /var/www/html/ks.cfg << EOF
key 4c10c9d64c6b34c9
lang en_US.UTF-8
keyboard us
xconfig --startxonboot
network --device eth0 --bootproto static --ip 192.168.1.111 --hostname v1.yubo.org --netmask 255.255.255.0 --gateway 192.168.1.254 --nameserver 202.103.24.68
rootpw mario
firewall --disabled
authconfig --enableshadow --enablemd5
selinux --disabled
timezone Asia/Shanghai
bootloader --location=mbr --driveorder=sda --append="rhgb quiet"
# The following is the partition information you requested
# Note that any partitions you deleted are not expressed
# here so unless you clear all partitions first, this is
# not guaranteed to work
clearpart --linux
part / --fstype ext3 --size=3584
part swap --size=256
%packages
@cluster-storage
@admin-tools
@editors
@text-internet
@gnome-desktop
@core
@base
@clustering
@java
@base-x
@graphics
@graphical-internet
scsi-target-utils
iscsi-initiator-utils
kexec-tools
bridge-utils
device-mapper-multipath
xorg-x11-utils
xorg-x11-server-Xnest
libsane-hpaio
-sysreport
EOF

将rhel5的iso插入dhcp.yubo.org的光驱->挂载->共享到apache的DocumentRoot目录,给v1用来远程安装

[root@dhcp ~]# mkdir /mnt/cdrom
[root@dhcp ~]# mount /dev/cdrom /mnt/cdrom
[root@dhcp ~]# ln -s /mnt/cdrom /var/www/html/cdrom

[p1.yubo.oirg]
进入图形界面,执行
[root@iscsi ~]# virt-manager
一路forward,途中填入

system Name:v1
Paravirtualized
Install Medai URL: http://192.168.1.5/cdrom
Kickstart URL: http://192.168.1.5/ks.cfg
Simple File->File Location: /var/lib/xen/images/v1.img
Shared physical device -> Device: xenbr0
VM Max Memory(MB): 252
VM Startup Memory(MB): 252
VCPUs:1

等待一段时间后,v1.yubo.org就安装完成了,下面是过程的截图

iscsi+clvm+gfs2+xen+Cluster(一)—环境的搭建

admin — Fri, 05 Sep 2008 06:33:49 +0000

根据史应生先生的<基于红帽企业版Linux RHEL5U2 GFS2＋ISCSI＋虚拟化XEN＋Cluster 的高可用性（HA）解决方案>一文的做了相应的实验,下面是自己的试验笔记,其中有些与原文有些不一致.

试验条件所限,全部试验机器在vmware(rhel5)上完成
hw: thinkpadT61/CPU:7300/RAM:3G
OS: windowXP

拓扑图

domain:yubo.org

   iscsi:192.168.1.100
   |
   __________|___________
   | |
p1:192.168.1.101 p2:192.168.1.102
(v1:192.168.1.111) (v2:192.168.1.112)

先用一个临时的192.168.1.5(dhcp.yubo.org)搭建dhcpd/apache用来以ks方式安装rehl5

[安装iscsi.yubo.org]

vmware->新建一个rhel5,ram分了500MB

在dhcp上

[root@dhcp ~]# cat /var/www/html/ks.cfg << EOF
key 4c10c9d64c6b34c9
lang en_US.UTF-8
keyboard us
xconfig --startxonboot
network --device eth0 --bootproto static --ip 192.168.1.100 --hostname iscsitarget.yubo.org --netmask 255.255.255.0 --gateway 192.168.1.254 --nameserver 202.103.24.68
rootpw mario
firewall --disabled
authconfig --enableshadow --enablemd5
selinux --disabled
timezone Asia/Shanghai
bootloader --location=mbr --driveorder=sda --append="rhgb quiet"
# The following is the partition information you requested
# Note that any partitions you deleted are not expressed
# here so unless you clear all partitions first, this is
# not guaranteed to work
clearpart --linux --drives=sda
part / --fstype ext3 --size=18000
part swap --size=512
%packages
@cluster-storage
@admin-tools
@editors
@virtualization
@gnome-desktop
@core
@base
@clustering
@base-x
@graphical-internet
scsi-target-utils
iscsi-initiator-utils
kexec-tools
bridge-utils
device-mapper-multipath
xorg-x11-utils
xorg-x11-server-Xnest
libsane-hpaio
-sysreport
EOF

开启dhcpd/httpd服务

[iscsi.yubo.org]
插入rhel5的iso,用以下命令从光盘启动

#linux fs=http://192.168.1.5/fs.conf

安装完成之后,修改hostname和hosts文件

[root@iscsi ~]# cat /var/www/html/ks.cfg << EOF
127.0.0.1 localhost.localdomain localhost
192.168.1.5 dhcp.yubo.org
192.168.1.100 iscsi.yubo.org
192.168.1.101 p1.yubo.org
192.168.1.102 p2.yubo.org
192.168.1.111 v1.yubo.org
192.168.1.112 v2.yubo.org

关机

[root@iscsi ~]# init 0

[p1.yubo.org]
用iscsi克隆出p1,p1的ram调至1G,启动后,修改hostname/ip地址/网卡的mac地址
在p1上安装用xen安装虚拟机v1.yubo.org(具体步骤请参照)

[p2.yubo.org]
用p1克隆出p2,启动后,将p2,v2的hostname/ip地址/网卡的mac地址做相应修改
将5台机器相应的ip,netmask设置正确后,将不必要的服务全部关了,运行级别都调到3,这样速度快点

保证5台机器之间通过机器名能互相访问到,到此为止,试验环境也就搭建完成

RHEL5/Cluster(TUN部分)

admin — Sat, 16 Aug 2008 13:19:23 +0000

RHEL5中的Cluster组件是基于章文嵩先生创立的LVS(Linux Virtual Server) 制作而成,

关于lvs的工作原理请参照《Linux 服务器集群系统》

Virtual Server via IP Tunneling（VS/TUN）
采用NAT技术时，由于请求和响应报文都必须经过调度器地址重写，当客户请求越来越多时，调度器的处理能力将成为瓶颈。为了解决这个问题，调度器把请求报文通过IP隧道转发至真实服务器，而真实服务器将响应直接返回给客户，所以调度器只处理请求报文。由于一般网络服务应答比请求报文大许多，采用VS/TUN技术后，集群系统的最大吞吐量可以提高10倍。

Cluster(TUN部分)实验

TUN类型的httpd负载均衡集群.网络拓扑为

eth0 192.168.1.100 eth0 192.168.1.200
tunl0 10.0.0.50 tunl0 10.0.0.50
gw 192.168.1.10 gw 192.168.1.10
______________ ______________
| | | |
| realserver 1 | | realserver 2 |
|______________| |______________|
   | |
   |___________________|
   |
   eth0 192.168.1.10
   ________
   | |
   | gw |
   |________|
   eth1 10.0.0.10
   |
   |
   |
   VIP(eth0:1)=10.0.0.50
   _ _ _ _ _
   | |
   director
   |_ _ _ _ _ |
   |
   |
   |
   ----------------------------------------
   | | |
   | | |
eth0 10.0.0.1 eth0 10.0.0.2 eth0 10.0.0.3
gw 10.0.0.10 gw 10.0.0.10 gw 10.0.0.10
______________ ______________ ______________
| | | | | |
| router | | router backup| | clinet |
|______________| |______________| |______________|

[client]
os:windwows xp

[gw]
os:rhel5
hostname:gw

[router/router backup]
os:rhel5
hostname:vs/vs_bk
software: ipvsadm piranha httpd php

[realserver1/realserver2]
os:rhel5
hostname:rs1/rs2
software:httpd

[director]
为router或者router backup中的一台虚拟出来

配置如下

[gw]

#打开路由转发
[root@gw ~]# echo 1 > /proc/sys/net/ipv4/ip_forward

[router]

#设置piranha密码
piranha-passwd

#开启服务
service piranha-gui start

#访问配置页面
http://10.0.0.1:3636/

#配置以后的文件
[root@vs ~]# cat /etc/sysconfig/ha/lvs.cf
serial_no = 53
primary = 10.0.0.1
service = lvs
backup_active = 1
backup = 10.0.0.2
heartbeat = 1
heartbeat_port = 539
keepalive = 6
deadtime = 18
network = tunnel
nat_nmask = 255.255.255.0
debug_level = NONE
monitor_links = 0
virtual HTTP {
   active = 1
   address = 10.0.0.50 eth0:1
   vip_nmask = 255.255.0.0
   port = 80
   send = "GET / HTTP/1.0\r\n\r\n"
   expect = "HTTP"
   use_regex = 0
   load_monitor = none
   scheduler = wlc
   protocol = tcp
   timeout = 6
   reentry = 15
   quiesce_server = 0
   server rs1.yubo.com {
   address = 192.168.1.100
   active = 1
   weight = 1
   }
   server rs2.yubo.com {
   address = 192.168.1.200
   active = 1
   weight = 1
   }
}

#重启服务
service pulse restart

[router backup]

#设置piranha密码
piranha-passwd

#开启服务
service piranha-gui start

#访问配置页面
http://10.0.0.2:3636/

#配置以后的文件
[root@vs_bk ~]# cat /etc/sysconfig/ha/lvs.cf
serial_no = 48
primary = 10.0.0.2
service = lvs
backup_active = 1
backup = 10.0.0.1
heartbeat = 1
heartbeat_port = 539
keepalive = 6
deadtime = 18
network = tunnel
nat_nmask = 255.255.0.0
debug_level = NONE
monitor_links = 0
virtual HTTP {
   active = 1
   address = 10.0.0.50 eth0:1
   vip_nmask = 255.255.0.0
   port = 80
   send = "GET / HTTP/1.0\r\n\r\n"
   expect = "HTTP"
   use_regex = 0
   load_monitor = none
   scheduler = wlc
   protocol = tcp
   timeout = 6
   reentry = 15
   quiesce_server = 0
   server rs1.yubo.com {
   address = 192.168.1.100
   active = 1
   weight = 1
   }
   server rs2.yubo.com {
   address = 192.168.1.200
   active = 1
   weight = 1
   }
}

#重启服务
service pulse restart

[realserver1/realserver2]
封装到达realserver的包,目的地址是vip的地址,而不是rs的ip地址,如果不做处理,会被拒绝,加上一个虚拟设备就解决问题了tunl0

ifconfig tunl0 10.0.0.50 netmask 255.255.255.255

之后开启rs1,rs2的httpd服务,为了使得便于观察

[root@rs1 ~]# echo "rs1.yubo.org" > /var/www/html/index.html
[root@rs2 ~]# echo "rs2.yubo.org" > /var/www/html/index.html

client频繁访问http://10.0.0.50时,会发现显示内容在”rs2.yubo.org”和”rs1.yubo.org”之间切换

试验完成以后,不要忘记保存配置,以免启动以后无法使用

[vs/vs_bk]

chkconfig --level 2345 piranha-gui on
chkconfig --level 2345 pulse on

[rs1/rs2]

echo "ifconfig tunl0 10.0.0.50 netmask 255.255.255.255" >> /etc/rc.local

[注意]
route 和 route backup 互为备份,没有主次之分(注意每个配置文件的backup和backup_private)
route backup 是route的备份
route 是route backup的备份
10.0.0.50个虚拟ip地址同一时刻只出现在1个router上,当前router当机以后,这个ip地址会被另一台备份机器接管

[其他]

[root@vs_bk ~]# ipvsadm --save -n
-A -t 10.0.0.50:80 -s wlc
-a -t 10.0.0.50:80 -r 192.168.1.200:80 -i -w 1
-a -t 10.0.0.50:80 -r 192.168.1.100:80 -i -w 1

RHEL5/Cluster(DR部分)

admin — Sat, 16 Aug 2008 09:18:25 +0000

RHEL5中的Cluster组件是基于章文嵩先生创立的LVS(Linux Virtual Server) 制作而成,

关于lvs的工作原理请参照《Linux 服务器集群系统》

Virtual Server via Direct Routing（VS/DR）
VS/DR通过改写请求报文的MAC地址，将请求发送到真实服务器，而真实服务器将响应直接返回给客户。同VS/TUN技术一样，VS/DR技术可极大地提高集群系统的伸缩性。这种方法没有IP隧道的开销，对集群中的真实服务器也没有必须支持IP隧道协议的要求，但是要求调度器与真实服务器都有一块网卡连在同一物理网段上。

Cluster(DR部分)实验

应为DR使基于MAC改写的,为了使试验简单明了,假设client,router,realserver都在一个网段上,
DR类型的httpd负载均衡集群.网络拓扑为
*当然,实际使用中,还需要考虑网关,路由等

#拓扑图
   ________
   | |
   | client |
   |________|
   10.0.0.10
   | eth0 10.0.0.1
   | _____________
   | | |
   VIP(eth0:1)=10.0.0.50 | router |
   _ _ _ _ _ |_____________|
   | | |
   director ---------------+
   |_ _ _ _ _ | |
   | eth0 10.0.0.2
   | ______________
   | | |
   ----------------- | router backup|
   | | |______________|
   | |
eth0 10.0.0.100 eth0 10.0.0.200
______________ ______________
| | | |
| realserver1 | | realserver2 |
|______________| |______________|

[client]
   os
   windwows xp

[router]
   os
   rhel5
   hostname
   vs
   software
   ipvsadm piranha httpd php

[router backup]
   os
   rhel5
   hostname
   vs_bk
   software
   ipvsadm piranha httpd php

[real server 1]
   os
   rhel5
   hostname
   rs1
   software
   httpd

[real server B]
   os
   rhel5
   hostname
   rs2
   software
   httpd

[director]
   为router或者router backup中的一台虚拟出来

配置如下
[router]

#设置piranha密码
piranha-passwd

#开启服务
service piranha-gui start

#访问配置页面
http://10.0.0.1:3636/

#配置以后的文件
[root@vs ~]# cat /etc/sysconfig/ha/lvs.cf
serial_no = 51
primary = 10.0.0.1
service = lvs
backup_active = 1
backup = 10.0.0.2
heartbeat = 1
heartbeat_port = 539
keepalive = 6
deadtime = 18
network = direct
nat_nmask = 255.255.255.0
debug_level = NONE
monitor_links = 0
virtual HTTP {
   active = 1
   address = 10.0.0.50 eth0:1
   vip_nmask = 255.255.0.0
   port = 80
   send = "GET / HTTP/1.0\r\n\r\n"
   expect = "HTTP"
   use_regex = 0
   load_monitor = none
   scheduler = wlc
   protocol = tcp
   timeout = 6
   reentry = 15
   quiesce_server = 0
   server rs1.yubo.com {
   address = 10.0.0.100
   active = 1
   weight = 1
   }
   server rs2.yubo.com {
   address = 10.0.0.200
   active = 1
   weight = 1
   }
}

#重启服务
service pulse restart

[router backup]

#设置piranha密码
piranha-passwd

#开启服务
service piranha-gui start

#访问配置页面
http://10.0.0.2:3636/

#配置以后的文件
[root@vs_bk ~]# cat /etc/sysconfig/ha/lvs.cf
serial_no = 46
primary = 10.0.0.2
service = lvs
backup_active = 1
backup = 10.0.0.1
heartbeat = 1
heartbeat_port = 539
keepalive = 6
deadtime = 18
network = direct
nat_nmask = 255.255.0.0
debug_level = NONE
monitor_links = 0
virtual HTTP {
   active = 1
   address = 10.0.0.50 eth0:1
   vip_nmask = 255.255.0.0
   port = 80
   send = "GET / HTTP/1.0\r\n\r\n"
   expect = "HTTP"
   use_regex = 0
   load_monitor = none
   scheduler = wlc
   protocol = tcp
   timeout = 6
   reentry = 15
   quiesce_server = 0
   server rs1.yubo.com {
   address = 10.0.0.100
   active = 1
   weight = 1
   }
   server rs2.yubo.com {
   address = 10.0.0.200
   active = 1
   weight = 1
   }
}

#重启服务
service pulse restart

[realserver1/realserver2]
修改mac地址,但是到达realserver的包,目的地址使vip的地址,而不是rs的ip地址,如果不做处理,会被拒绝,有arptables_jf/iptables 两种方法,我用的使iptables
2台rs都要做

iptables -t nat -A PREROUTING -p tcp -d 1 --dport 80 -j REDIRECT

之后开启rs1,rs2的httpd服务,为了使得便于观察

[root@rs1 ~]# echo "rs1.yubo.org" > /var/www/html/index.html
[root@rs2 ~]# echo "rs2.yubo.org" > /var/www/html/index.html

client频繁访问http://10.0.0.50时,会发现显示内容在”rs2.yubo.org”和”rs1.yubo.org”之间切换

试验完成以后,不要忘记保存配置,以免启动以后无法使用

[vs/vs_bk]

chkconfig --level 2345 piranha-gui on
chkconfig --level 2345 pulse on

[rs1/rs2]

service iptables save
chkconfig --level 2345 iptables on