准备工作

环境信息

修改dns配置

设置阿里源

root@sh-saas-cs-manager-online-01:~# mv /etc/apt/sources.list /etc/apt/sources.list.bak

root@sh-saas-cs-manager-online-01:~# cat /etc/apt/sources.list

deb http://mirrors.aliyun.com/ubuntu/ trusty main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ trusty-security main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ trusty-updates main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ trusty-proposed main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ trusty-backports main restricted universe multiverse

deb-src http://mirrors.aliyun.com/ubuntu/ trusty main restricted universe multiverse

deb-src http://mirrors.aliyun.com/ubuntu/ trusty-security main restricted universe multiverse

deb-src http://mirrors.aliyun.com/ubuntu/ trusty-updates main restricted universe multiverse

deb-src http://mirrors.aliyun.com/ubuntu/ trusty-proposed main restricted universe multiverse

deb-src http://mirrors.aliyun.com/ubuntu/ trusty-backports main restricted universe multiverse

更新系统

root@sh-saas-cs-manager-online-01:~# apt-get update #更新源信息

root@sh-saas-cs-manager-online-01:~# apt-get upgrade #更新本机应用软件

root@sh-saas-cs-manager-online-01:~# ufw status

Status: inactive

root@sh-saas-cs-manager-online-01:~# ufw disable

Firewall stopped and disabled on system startup

安装jdk1.8

root@sh-saas-cs-manager-online-01:~# add-apt-repository ppa:openjdk-r/ppa

root@sh-saas-cs-manager-online-01:~# apt-get update

root@sh-saas-cs-manager-online-01:~# apt-get install openjdk-8-jdk

添加java环境变量

在 /etc/profile和~/.bashrc中

增加

export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64

export JRE_HOME=${JAVA_HOME}/jre

export CLASSPATH=${JAVA_HOME}/lib:${JRE_HOME}/lib

export PATH=${JAVA_HOME}/bin:$PATH

root@sh-saas-cs-manager-online-01:~# java -version

openjdk version "1.8.0_162"

安装NTP

root@sh-saas-cs-manager-online-01:~# apt-get install ntp

确保在各 Ceph 节点上启动了 NTP 服务，并且要使用同一个 NTP 服务器

修改ntp.conf配置，更新为阿里云ntp服务器为优先同步服务器配置

root@sh-saas-cs-manager-online-01:~# cat /etc/ntp.conf

# ntp.conf

driftfile /var/lib/ntp/drift

pidfile /var/run/ntpd.pid

logfile /var/log/ntp.log

# Access Control Support

restrict default kod nomodify notrap nopeer noquery

restrict -6 default kod nomodify notrap nopeer noquery

restrict 127.0.0.1

# local clock

server 127.127.1.0

fudge 127.127.1.0 stratum 10

server ntp1.aliyun.com iburst minpoll 4 maxpoll 10

restrict ntp1.aliyun.com nomodify notrap nopeer noquery

server ntp1.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10

restrict ntp1.cloud.aliyuncs.com nomodify notrap nopeer noquery

server ntp10.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10

restrict ntp10.cloud.aliyuncs.com nomodify notrap nopeer noquery

server ntp11.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10

restrict ntp11.cloud.aliyuncs.com nomodify notrap nopeer noquery

server ntp12.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10

restrict ntp12.cloud.aliyuncs.com nomodify notrap nopeer noquery

server ntp2.aliyun.com iburst minpoll 4 maxpoll 10

restrict ntp2.aliyun.com nomodify notrap nopeer noquery

server ntp2.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10

restrict ntp2.cloud.aliyuncs.com nomodify notrap nopeer noquery

server ntp3.aliyun.com iburst minpoll 4 maxpoll 10

restrict ntp3.aliyun.com nomodify notrap nopeer noquery

server ntp3.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10

restrict ntp3.cloud.aliyuncs.com nomodify notrap nopeer noquery

server ntp4.aliyun.com iburst minpoll 4 maxpoll 10

restrict ntp4.aliyun.com nomodify notrap nopeer noquery

server ntp4.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10

restrict ntp4.cloud.aliyuncs.com nomodify notrap nopeer noquery

server ntp5.aliyun.com iburst minpoll 4 maxpoll 10

restrict ntp5.aliyun.com nomodify notrap nopeer noquery

server ntp5.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10

restrict ntp5.cloud.aliyuncs.com nomodify notrap nopeer noquery

server ntp6.aliyun.com iburst minpoll 4 maxpoll 10

restrict ntp6.aliyun.com nomodify notrap nopeer noquery

server ntp6.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10

restrict ntp6.cloud.aliyuncs.com nomodify notrap nopeer noquery

server ntp7.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10

restrict ntp7.cloud.aliyuncs.com nomodify notrap nopeer noquery

server ntp8.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10

restrict ntp8.cloud.aliyuncs.com nomodify notrap nopeer noquery

server ntp9.cloud.aliyuncs.com iburst minpoll 4 maxpoll 10

restrict ntp9.cloud.aliyuncs.com nomodify notrap nopeer noquery

root@sh-saas-cs-manager-online-01:~# /etc/init.d/ntp start

 

安装配置NFS

root@sh-saas-cs-manager-online-01:~# apt-get install nfs-kernel-server nfs-common -y #注：确保所有机器上都安装了nfs-common包

root@sh-saas-cs-manager-online-01:~# mkdir -p /data/{primary,secondary}

root@sh-saas-cs-manager-online-01:~# vim /etc/exports

/data/secondary *(rw,async,no_root_squash,no_subtree_check)

/data/primary *(rw,async,no_root_squash,no_subtree_check)

root@sh-saas-cs-manager-online-01:~# exportfs -a

root@sh-saas-cs-manager-online-01:~# showmount -e #查看共享目录

root@sh-saas-cs-manager-online-01:~# /etc/init.d/rpcbind restart # 重启rpcbind

root@sh-saas-cs-manager-online-01:~# /etc/init.d/nfs-kernel-server restart # 重启nfs

 

安装cloudstack

1. 添加cloudstack源(注：4.11包还没有，所以用4.10)

root@sh-saas-cs-manager-online-01:~# echo deb http://cloudstack.apt-get.eu/ubuntu trusty 4.10 | tee /etc/apt/sources.list.d/cloudstack.list

root@sh-saas-cs-manager-online-01:~# wget -O - http://cloudstack.apt-get.eu/release.asc | apt-key add -

添加cloudstack国内源

root@sh-saas-cs-manager-online-01:~# echo deb http://mirrors.cloudstack-china.com/mirror/ubuntu trusty 4.10 | tee /etc/apt/sources.list.d/cloudstack.list

root@sh-saas-cs-manager-online-01:~# wget -O - http://cloudstack.apt-get.eu/release.asc | sudo apt-key add -

安装cloudstack

root@sh-saas-cs-manager-online-01:~# sudo apt-get update

root@sh-saas-cs-manager-online-01:~# apt-get install cloudstack-management

# cloudstack-setup-databases cloud:<dbpassword>@<ip address mysql server> --deploy-as=root:<dbpassword> #mysql安装略，(cloudstack-setup-databases cloud:123456@127.0.0.1—deploy-as=root:123456)

##

sudo apt-get install mysql-server libmysqlclient-dev

参考mysql安装文档

eg:

root@sh-saas-cs-manager-online-01:~# cloudstack-setup-databases cloud:Abc12345@192.168.226.11 --deploy-as=root:Abc12345

如果以上mysql数据库不在本机，确保root@%用户有创建用户并授权的权限，可以在运行以下命令

mysql> grant all privileges on *.* to 'root'@'%'identified by 'Abc12345' with grant option;

If you are running the KVM hypervisor on the same machine with the Management Server, edit /etc/sudoers and add the following line:

Defaults:cloud !requiretty

导入系统虚拟机模板

2. 安装KVM模版

root@sh-saas-cs-manager-online-01:~# cd /data/secondary

root@sh-saas-cs-manager-online-01:~# wget http://cloudstack.apt-get.eu/systemvm/4.10/systemvm64template-4.10-4.10.0.0-kvm.qcow2.bz2

root@sh-saas-cs-manager-online-01:~# /usr/share/cloudstack-common/scripts/storage/secondary/cloud-install-sys-tmplt -m /data/secondary -f systemvm64template-4.10-4.10.0.0-kvm.qcow2.bz2 -h kvm -F

注：

1. kvm模版的版本对应cloudstack的版本号

2. /data/secondary给予777的权限，主要针对于模版文件

cloudstack4.10.x系统虚拟机默认用户名密码：root/password

安装完成, 访问

http://10.10.128.77:8080/client/

默认用户名和密码为 admin, password

 

#vim /etc/sudoers

增加

Defaults:cloud !requiretty

#　cloudstack-setup-management

停止服务

# /etc/init.d/cloudstack-management stop

If the servlet container is Tomcat7 the argument –tomcat7 must be used.

 

 

安装配置KVM agent端安装

1. 安装kvm

注：KVM 需要有 CPU 的支持（Intel vmx 或 AMD svm），在安装 KVM 之前检查一下 CPU 是否提供了虚拟技术的支持：

root@sh-saas-cs-ceph-online-01:~# egrep '^flags.*(vmx|svm)' /proc/cpuinfo

有显示, 有显示则说明处理器具有VT功能

root@sh-saas-cs-ceph-online-01:~# apt-get install qemu-kvm qemu virt-manager virt-viewer libvirt-bin bridge-utils -y

检查kvm是否成功安装：

root@sh-saas-cs-ceph-online-01:~# kvm-ok

INFO: /dev/kvm exists

KVM acceleration can be used

或者

root@sh-saas-cs-ceph-online-01:~# lsmod | grep kvm

kvm_intel 167936 0

kvm 532480 1 kvm_intel

irqbypass 16384 1 kvm

以上有输出表示成功安装

2. 修改网卡配置(修改前)

root@sh-saas-cs-ceph-online-01:~# cat /etc/network/interfaces

# The loopback network interface

auto lo

iface lo inet loopback

# The primary network interface

auto eth0

iface eth0 inet manual

auto eth1

iface eth1 inet manual

auto bond1

iface bond1 inet static

address 10.16.144.2

netmask 255.255.240.0

gateway 10.16.144.1

bond_mode 4

bond_miimon 100

bond-ad-select 1

bond_downdelay 200

bond_updelay 200

bond_xmit_hash_policy layer2+3

bond-slaves none

up /sbin/ifenslave bond1 eth0 eth1

post-up route add -net 100.64.0.0 netmask 255.192.0.0 gw 10.16.144.1

up ifconfig eth0 up

up ifconfig eth1 up

down /sbin/ifenslave -d bond1 eth0 eth1

post-down ifconfig eth0 down

post-down ifconfig eth1 down

修改网卡配置(修改后)

root@sh-saas-cs-ceph-online-01:~# cat /etc/network/interfaces

# The loopback network interface

auto lo

iface lo inet loopback

# The primary network interface

auto eth0

iface eth0 inet manual

mtu 9600

bond-master bond1

auto eth1

iface eth1 inet manual

mtu 9600

bond-master bond1

auto bond1

iface bond1 inet manual

bond_mode 4

bond_miimon 100

bond_xmit_hash_policy layer3+4

bond_downdelay 200

bond_updelay 200

mtu 9600

post-up ifenslave bond1 eth0 eth1

pre-down ifenslave -d bond1 eth0 eth1

bond-slaves none

auto cloudbr0

iface cloudbr0 inet static

bridge_ports bond1

address 10.16.144.2

netmask 255.255.240.0

gateway 10.16.144.1

dns-nameservers 10.16.253.253

dns-nameservers 10.10.253.252

post-up route add -net 10.0.0.0 netmask 255.0.0.0 gw 10.16.144.1

post-up route add -net 172.16.0.0 netmask 255.240.0.0 gw 10.16.144.1

post-up route add -net 192.168.0.0 netmask 255.255.0.0 gw 10.16.144.1

post-up route add -net 100.64.0.0 netmask 255.192.0.0 gw 10.16.144.1

root@sh-saas-cs-ceph-online-01:~# ifdown eth0 && ifup eth0 && ifup cloudbr0 #重启网卡服务

3. 修改kvm配置

root@sh-saas-cs-ceph-online-01:~# vim /etc/libvirt/qemu.conf

vnc_listen=0.0.0.0 #取消前面的注释

新增以下内容（解决权限问题）

security_driver="none"

user="root"

group="root"

root@sh-saas-cs-ceph-online-01:~# cat /etc/libvirt/qemu.conf |egrep -v "^#|^$"

vnc_listen = "0.0.0.0"

security_driver="none"

user="root"

group="root"

root@sh-saas-cs-ceph-online-01:~# cat /etc/libvirt/libvirtd.conf |egrep -v "^#|^$"

listen_tls = 0

listen_tcp = 1

tcp_port = "16509"

mdns_adv = 0

unix_sock_group = "libvirtd"

unix_sock_ro_perms = "0777"

unix_sock_rw_perms = "0770"

auth_unix_ro = "none"

auth_unix_rw = "none"

auth_tcp = "none"

root@sh-saas-cs-ceph-online-01:~# vim /etc/cloudstack/agent/agent.properties #cloudstack管理服务器地址

host=10.10.128.77

root@sh-saas-cs-ceph-online-01:~# cloudstack-setup-agent

Welcome to the CloudStack Agent Setup:

Please input the Management Server Hostname/IP-Address:[10.10.128.77]

Please input the Zone Id:[default]

Please input the Pod Id:[default]

Please input the Cluster Id:[default]

Please choose which network used to create VM:[cloudbr0]

Starting to configure your system:

Configure Apparmor ... [OK]

Configure Network ... [OK]

Configure Libvirt ... [OK]

Configure Firewall ... [OK]

Configure Nfs ... [OK]

Configure cloudAgent ... [OK]

CloudStack Agent setup is done!

root@sh-saas-cs-ceph-online-01:~#

root@sh-saas-cs-ceph-online-01:~# /etc/init.d/libvirt-bin restart

 

安装cloudstack agent端安装

安装jdk1.8

root@sh-saas-cs-ceph-online-01:~# add-apt-repository ppa:openjdk-r/ppa

root@sh-saas-cs-ceph-online-01:~# apt-get update

root@sh-saas-cs-ceph-online-01:~# apt-get install openjdk-8-jdk

添加java环境变量

在 /etc/profile和~/.bashrc中

增加

export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64

export JRE_HOME=${JAVA_HOME}/jre

export CLASSPATH=${JAVA_HOME}/lib:${JRE_HOME}/lib

export PATH=${JAVA_HOME}/bin:$PATH

root@sh-saas-cs-ceph-online-01:~# java -version

openjdk version "1.8.0_162"

添加cloudstack源站源

root@sh-saas-cs-ceph-online-01:~# echo deb http://cloudstack.apt-get.eu/ubuntu trusty 4.10 | tee /etc/apt/sources.list.d/cloudstack.list

root@sh-saas-cs-ceph-online-01:~# wget -O - http://cloudstack.apt-get.eu/release.asc | sudo apt-key add -

添加cloudstack国内源

root@sh-saas-cs-ceph-online-01:~# echo deb http://mirrors.cloudstack-china.com/mirror/ubuntu trusty 4.3 | tee /etc/apt/sources.list.d/cloudstack.list

root@sh-saas-cs-ceph-online-01:~# wget -O – 'http://cloudstack.apt-get.eu/release.asc' | apt-key add -

2. 安装cloudstack agent

root@sh-saas-cs-ceph-online-01:~# apt-get update

root@sh-saas-cs-ceph-online-01:~# sudo apt-get install cloudstack-agent -y

修改cloudstack agent配置

root@sh-saas-cs-ceph-online-01:~# vim /etc/cloudstack/agent/agent.properties #增加以下内容，cloudbr0表示网桥端口

public.network.device=cloudbr0

private.network.device=cloudbr0

root@sh-saas-cs-ceph-online-01:~# sudo /etc/init.d/cloudstack-agent restart

 

全局参数调整

secstorage.allowed.internal.sites ---> 10.16.144.0/8

network.throttling.rate ---> 0

vm.network.throttling.rate --->0

nfs主存储问题

部署好cloudstack需要删除nfs主存储，后续nfs主存储异常会导致整个机器物理机重启(切记)

 

jzGH1Sx0hjMznV6F

FWKKfDxT30NflxPg