使用免费的SSL

Standard

公司收的一大堆论坛都要加SSL,每个都要购买的话会是一笔不小的费用。
所以准备全部使用Let’s Encrypt的免费SSL。

wget -O -  https://get.acme.sh | sh
cd .acme.sh/
#确保通过域名可访问到/var/www/yemaosheng/htdocs/.well-known/下的内容
./acme.sh --issue -d yemaosheng.com -d www.yemaosheng.com -w /var/www/yemaosheng/htdocs
[Tue Mar  7 21:19:34 CST 2017] Multi domain='DNS:www.yemaosheng.com'
[Tue Mar  7 21:19:34 CST 2017] Getting domain auth token for each domain
[Tue Mar  7 21:19:34 CST 2017] Getting webroot for domain='yemaosheng.com'
[Tue Mar  7 21:19:34 CST 2017] Getting new-authz for domain='yemaosheng.com'
[Tue Mar  7 21:19:36 CST 2017] The new-authz request is ok.
[Tue Mar  7 21:19:36 CST 2017] Getting webroot for domain='www.yemaosheng.com'
[Tue Mar  7 21:19:36 CST 2017] Getting new-authz for domain='www.yemaosheng.com'
[Tue Mar  7 21:19:36 CST 2017] The new-authz request is ok.
[Tue Mar  7 21:19:37 CST 2017] yemaosheng.com is already verified, skip http-01.
[Tue Mar  7 21:19:37 CST 2017] Verifying:www.yemaosheng.com
[Tue Mar  7 21:19:39 CST 2017] Success
[Tue Mar  7 21:19:39 CST 2017] Verify finished, start to sign.
[Tue Mar  7 21:19:40 CST 2017] Cert success.
-----BEGIN CERTIFICATE-----
MIIFFzCCA............FlYV3RaDYYpw=
-----END CERTIFICATE-----
[Tue Mar  7 21:19:40 CST 2017] Your cert is in  /root/.acme.sh/yemaosheng.com/yemaosheng.com.cer 
[Tue Mar  7 21:19:40 CST 2017] Your cert key is in  /root/.acme.sh/yemaosheng.com/yemaosheng.com.key 
[Tue Mar  7 21:19:40 CST 2017] The intermediate CA cert is in  /root/.acme.sh/yemaosheng.com/ca.cer 
[Tue Mar  7 21:19:40 CST 2017] And the full chain certs is there:  /root/.acme.sh/yemaosheng.com/fullchain.cer
 
crontab -l
7 0 * * * "/root/.acme.sh"/acme.sh --cron --home "/root/.acme.sh" > /dev/null
 
vi /etc/httpd/conf.d/ssl.conf
...
<VirtualHost *:443>
        DocumentRoot "/var/www/yemaosheng/htdocs"
        ServerName yemaosheng.com
 
        SSLEngine on
        SSLProtocol all -SSLv2 -SSLv3
        SSLCipherSuite "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH EDH+aRSA !RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"
        SSLCertificateFile "/root/.acme.sh/yemaosheng.com/yemaosheng.com.cer"
        SSLCertificateKeyFile "/root/.acme.sh/yemaosheng.com/yemaosheng.com.key"
        SSLCertificateChainFile "/root/.acme.sh/yemaosheng.com/fullchain.cer"
        ...
</VirtualHost>
...

How to clone a Azure VM

Standard

run on your sample-vm

waagent -deprovision+user

run on your azure-cli env

$rgName = "VMTestGroup"
$template = "Template-test.json"
$vmName = "VMTest"
$vhdName = "VHDTest"
 
azure vm deallocate -g $rgName -n $vmName
azure vm generalize $rgName -n $vmName
azure vm capture $rgName $vmName $vhdName -t $template
 
# the $template should looks like this. and you have to change 'newvmname' before use.
...
         "storageProfile": {
          "dataDisks": [
            {
              "caching": "ReadOnly",
              "vhd": {
                "uri": "https://yourdiskname.blob.core.windows.net/vhds/dataDisk-0.newvmname.vhd"
              },
              "image": {
                "uri": "https://yourdiskname.blob.core.windows.net/system/Microsoft.Compute/Images/vhds/yourcapturedvmname-dataDisk-0.ff60129b-...3cf59bf9315a.vhd"
              },
              "createOption": "FromImage",
              "name": "yourcapturedvmname-dataDisk-0.ff60129b-4ec5-4dcd-ae97-3cf59bf9315a.vhd",
              "lun": 0
            }
          ],
          "osDisk": {
            "caching": "ReadWrite",
            "vhd": {
              "uri": "https://yourdiskname.blob.core.windows.net/vhds/osDisk.newvmname.vhd"
            },
            "image": {
              "uri": "https://yourdiskname.blob.core.windows.net/system/Microsoft.Compute/Images/vhds/yourcapturedvmname-osDisk.ff60129b-...3cf59bf9315a.vhd"
            },
            "createOption": "FromImage",
            "name": "yourcapturedvmname-osDisk.ff60129b-4ec5-4dcd-ae97-3cf59bf9315a.vhd",
            "osType": "Linux"
          }
        },
...
 
 
azure group deployment create $rgName MyDeployment -f Template-test-modified.json
    info:    Executing command group deployment create
    info:    Supply values for the following parameters
    vmName: NewVmName
    adminUserName: username
    adminPassword: password
    networkInterfaceId: /subscriptions/61719d1b-...ab74b6f77865/resourceGroups/VMTestGroup/providers/Microsoft.Network/networkInterfaces/YourNetworkInterfaceName
 
#If you do not have an existing NetworkInterface, you need create first. 
azure network nic create $rgName YourNetworkInterfaceName -k default -m YourSubnetVnetName  -l "westus2"

Using SaltStack to deploy Auto-scaling EC2

Standard

SaltStack Master: 172.66.1.100

Create AMI by default VM:

root@ip-x.x.x.x:~# cat /etc/rc.local
/root/PkgInit.sh;
/root/SaltMinionInit.sh;
/root/SaltCall.sh;
 
root@ip-x.x.x.x:~# cat /root/PkgInit.sh 
add-apt-repository ppa:saltstack/salt -y;
apt-get update;
apt-get install salt-minion -y;
apt-get install awscli -y;
 
root@ip-x.x.x.x:~# cat /root/SaltMinionInit.sh
INSTANCE_ID=$(ec2metadata --instance-id);
REGION=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep region | awk -F\" '{print $4}');
TAG=$(aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" --region=$REGION --output=text --max-items=1 | cut -f5);
/bin/echo -e "master: 172.66.1.100\ngrains:\n  roles:\n    - "$TAG > /etc/salt/minion;
service salt-minion restart;
 
root@ip-x.x.x.x:~# cat /root/SaltCall.sh
sleep 15s;
salt-call state.highstate;

Setup Saltstack Master:

root@ip-172-66-1-100:~# add-apt-repository ppa:saltstack/salt
root@ip-172-66-1-100:~# apt-get update
root@ip-172-66-1-100:~# apt-get install salt-master
root@ip-172-66-1-100:~# cat /etc/salt/master | grep -v '^#' | grep -v '^$'
file_roots:
  base:
    - /srv/salt
pillar_roots:
  base:
    - /srv/pillar
reactor:
  - 'salt/auth':
    - /srv/reactor/auth-pending.sls
 
# Automating Key Acceptance
# salt-run state.event pretty=True
root@ip-172-66-1-100:~# cat /srv/reactor/auth-pending.sls
{% if 'act' in data and data['act'] == 'pend' and data['id'].startswith('ip-172') %}
minion_add:
  wheel.key.accept:
    - match: {{ data['id'] }}
{% endif %}
 
# Get grains item
root@ip-172-66-1-100:~# salt '*' grains.item os
ip-172-66-2-214:
    ----------
    os:
        Ubuntu
ip-172-66-4-93:
    ----------
    os:
        Ubuntu
root@ip-172-66-1-100:/srv/reactor# salt '*' grains.item roles
ip-172-66-2-214:
    ----------
    roles:
        - YeMaosheng_com
ip-172-66-4-93:
    ----------
    roles:
        - YeMaosheng_com
 
root@ip-172-66-1-100:~# salt -G 'roles:YeMaosheng_com' state.highstate -t 60 test=True
root@ip-172-66-1-100:~# salt -G 'roles:YeMaosheng_com' state.highstate
root@ip-172-66-1-100:~# salt-run manage.down removekeys=True
root@ip-172-66-1-100:~# salt-run state.event pretty=True

网站所用EC2的安装及发布配置

├── pillar
│   ├── yemaosheng_com
│   │   ├── nginx.sls
│   │   ├── php56.sls
│   │   └── website.sls
│   └── top.sls
├── reactor
│   └── auth-pending.sls
└── salt
    ├── crontab
    │   └── init.sls
    ├── mysql-client
    │   └── init.sls
    ├── nginx
    │   ├── configs
    │   │   └── yemaosheng_com
    │   │       ├── blockrules.conf
    │   │       ├── nginx.conf
    │   │       └── sites-enabled
    │   │           └── yemaosheng.com
    │   └── init.sls
    ├── php56
    │   ├── configs
    │   │   └── yemaosheng_com
    │   │       └── php5-fpm
    │   │           └── www.conf
    │   └── init.sls
    ├── top.sls
    ├── website
    │   ├── configs
    │   │   └── yemaosheng_com
    │   │       ├── dhparam.pem
    │   │       └── sslkey
    │   └── init.sls
    └── websitefiles
        └── yemaosheng_com -> /var/www/yemaosheng_com
 
cat /srv/salt/top.sls 
base:
 'roles:yemaosheng_com':
 - match: grain
 - mysql-client
 - php56
 - nginx
 - website
 - crontab
 
cat /srv/pillar/top.sls 
base : 
 'roles:yemaosheng_com':
 - match: grain
 - yemaosheng_com.nginx
 - yemaosheng_com.php56
 - yemaosheng_com.website
 
cat /srv/pillar/yemaosheng_com/nginx.sls 
nginx_conf: nginx/configs/yemaosheng_com/nginx.conf
nginx_site-enable: nginx/configs/yemaosheng_com/sites-enabled
 
cat /srv/salt/nginx/init.sls 
{% set site_name = pillar['site_name'] %}
 
nginx:
  pkg:
    - name: nginx
    - installed
 
nginx_conf:
  service.running:
    - name: nginx
    - enable: True
    - reload: True
    - watch:
      - file: /etc/nginx/*
  file.managed:
    - name: /etc/nginx/nginx.conf
    - source: salt://{{ pillar['nginx_conf'] }}
    - user: root
    - group: root
    - mode: '0640'
    - require:
      - pkg: nginx
 
{% if site_name == 'yemaosheng_com' %}
upload_sslkey_to_nginx:
  file.recurse:
    - name: /srv/ssl
    - user: root
    - group: root
    - file_mode: '0644'
    - source: salt://website/configs/yemaosheng_com/sslkey
    - include_empty: True
 
upload_dhparam_to_nginx:
  file.managed:
    - name: /etc/nginx/dhparam.pem
    - source: salt://website/configs/yemaosheng_com/dhparam.pem
    - user: root
    - group: root
    - mode: '0644'
    - require:
      - pkg: nginx
{% endif %}
 
/etc/nginx/sites-enabled:
  service.running:
    - name: nginx
    - enable: True
    - reload: True
    - watch:
      - file: /etc/nginx/sites-enabled
  file.recurse:
    - name: /etc/nginx/sites-enabled
    - user: root
    - group: root
    - dir_mode: 2775
    - file_mode: '0644'
    - source: salt://{{ pillar['nginx_site-enable'] }}
    - include_empty: True
    - clean: True
    - require:
      - pkg: nginx

AWS VPC point to point with gre tunnel

Standard

related AWS VPC通过IPsec连接不同Region

AWS China EC2:

root@ip-10-33-30-103:/home/ubuntu# cat /etc/network/interfaces.d/gre1.cfg
auto gre1
iface gre1 inet tunnel
  mode gre
  netmask 255.255.255.255
  address 10.0.0.2
  dstaddr 10.0.0.1
  endpoint 52.63.189.251
  local 10.33.30.103
  ttl 255
 
root@ip-10-33-30-103:/home/ubuntu# route add -net 172.33.0.0 netmask 255.255.0.0 gw 10.0.0.2

AWS Sydney EC2:

root@ip-172-33-1-190:/home/ubuntu# cat /etc/network/interfaces.d/gre1.cfg 
auto gre1
iface gre1 inet tunnel
  mode gre
  netmask 255.255.255.255
  address 10.0.0.1
  dstaddr 10.0.0.2
  endpoint 54.222.193.171
  local 172.33.1.190
  ttl 255
 
root@ip-172-33-1-190:/home/ubuntu# route add -net 10.33.0.0 netmask 255.255.0.0 gw 10.0.0.1

网站指向不同路径的nginx配置

Standard
upstream fastcgi_backend {
    server 127.0.0.1:9000;
    server unix:/var/run/php5-fpm.sock;
    keepalive 10;
}
...
server {
  ...
  ...
  location ~ \.php {
    set $php_root /var/www/website/abc/public;
    include /etc/nginx/fastcgi_params;
 
    if ($request_uri ~ /(api/user/photos|api/user/posts) ) {
      set $php_root /var/www/website/rest/public;
    }
 
    if ($request_uri ~ /api/user/(photos|posts|links) ) {
      set $php_root /var/www/website/rest/public;
    }
 
    fastcgi_split_path_info ^(.+\.php)(/.+)$;
    fastcgi_param PATH_INFO $fastcgi_path_info;
    fastcgi_param PATH_TRANSLATED $php_root$fastcgi_path_info;
    fastcgi_param SCRIPT_NAME $fastcgi_script_name;
    fastcgi_param SCRIPT_FILENAME $php_root$fastcgi_script_name;
    fastcgi_pass fastcgi_backend;
    fastcgi_index index.php;
  }
  ...
}

Add basic HTTP access auth via HAProxy

Standard
userlist UsersForES
  user your_username insecure-password your_password
 
frontend elasticsearch_pwd
    bind *:9201
    mode http
    default_backend es-nodes_pwd
 
backend es-nodes_pwd
    acl AuthOkay_ES http_auth(UsersForES)
    http-request auth realm ES if !AuthOkay_ES
    mode http
    balance roundrobin
    option forwardfor
    server es-node1.yemaosheng.com 10.0.0.2:9200 check
    server es-node2.yemaosheng.com 10.0.0.3:9200 check
    server es-node3.yemaosheng.com 10.0.0.4:9200 check

haproxy cfg for redis sentinel

Standard
frontend redis-cluster
        mode tcp
        option tcplog
        bind *:6379
        # If at least 3 sentinels agree with the redis host that it is master, use it.
        use_backend redis-node1 if { srv_is_up(redis-node1/redis-1:10.25.0.2:6379) } { nbsrv(check_master_redis-1) ge 3 }            
        use_backend redis-node2 if { srv_is_up(redis-node2/redis-2:10.25.0.3:6379) } { nbsrv(check_master_redis-2) ge 3 }
        # If sentinel cant tell us, well, fall back to master detection
        default_backend redis-cluster
 
backend redis-node1
        mode tcp
        balance first
        option tcp-check
        tcp-check send AUTH\ password\r\n
        tcp-check expect string +OK
        tcp-check send info\ replication\r\n
        tcp-check expect string role:master
        server redis-1:10.25.0.2:6379 10.25.0.2:6379 maxconn 5000 check inter 1s
 
backend redis-node2
        mode tcp
        balance first
        option tcp-check
        tcp-check send AUTH\ password\r\n
        tcp-check expect string +OK
        tcp-check send info\ replication\r\n
        tcp-check expect string role:master
        server redis-2:10.25.0.3:6379 10.25.0.3:6379 maxconn 5000 check inter 1s
 
backend redis-cluster
        mode tcp
        balance first
        option tcp-check
        tcp-check send AUTH\ password\r\n
        tcp-check expect string +OK
        tcp-check send info\ replication\r\n
        tcp-check expect string role:master
        tcp-check send info\ persistence\r\n           
        tcp-check expect string loading:0
        server redis-1:10.25.0.2:6379 10.25.0.2:6379 maxconn 5000 check inter 1s
        server redis-2:10.25.0.3:6379 10.25.0.3:6379 maxconn 5000 check inter 1s
 
## Check 4 sentinels to see if they think redis-1 (10.25.0.2) is master
backend check_master_redis-1
        mode tcp
        option tcp-check
        tcp-check send PING\r\n
        tcp-check expect string +PONG
        tcp-check send SENTINEL\ master\ redis\r\n
        tcp-check expect string 10.25.0.2
        tcp-check send QUIT\r\n
        tcp-check expect string +OK
 
        server redis-1:10.25.0.2:26379 10.25.0.2:26379 check inter 2s
        server redis-2:10.25.0.3:26379 10.25.0.3:26379 check inter 2s
        server redis-sentinel1:10.25.0.4:26379 10.25.0.4:26379 check inter 2s
        server redis-sentinel2:10.25.0.5:26379 10.25.0.5:26379 check inter 2s
 
## Check 4 sentinels to see if they think redis-2 (10.25.0.3) is master
backend check_master_redis-2
        mode tcp
        option tcp-check
        tcp-check send PING\r\n
        tcp-check expect string +PONG
        tcp-check send SENTINEL\ master\ redis\r\n
        tcp-check expect string 10.25.0.3
        tcp-check send QUIT\r\n
        tcp-check expect string +OK
 
        server redis-1:10.25.0.2:26379 10.25.0.2:26379 check inter 2s
        server redis-2:10.25.0.3:26379 10.25.0.3:26379 check inter 2s
        server redis-sentinel1:10.25.0.4:26379 10.25.0.4:26379 check inter 2s
        server redis-sentinel2:10.25.0.5:26379 10.25.0.5:26379 check inter 2s

AWS VPC通过IPsec连接不同Region

Standard

AWS China<->AWS Sydney

AWS China VPC:

AWS China EC2:

root@ip-10-33-30-103:~# apt-get install openswan
root@ip-10-33-30-103:~# cat /etc/ipsec.conf 
config setup
    protostack=netkey
    interfaces=%defaultroute
    nat_traversal=yes
    force_keepalive=yes
    keep_alive=60
    oe=no
    nhelpers=0
conn ToAWSSydneyVPC
    left=10.33.30.103
    leftsubnets=10.33.0.0/16
    leftid=@AwsChinaGW
    right=52.63.189.251
    rightsubnets=172.33.0.0/16
    rightid=@AwsSydneyGW
    forceencaps=yes
    authby=secret
    auto=ignore
 
root@ip-10-33-30-103:~# cat /etc/ipsec.secrets 
@AwsChinaGW  @AwsSydneyGW: PSK "123321112233"
 
root@ip-10-33-30-103:~# cat /etc/sysctl.conf | grep -v '^#' | grep -v '^$'
net.ipv4.ip_forward = 1
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.default.send_redirects = 0
 
root@ip-10-33-30-103:~# ipsec verify
root@ip-10-33-30-103:~# service ipsec start
root@ip-10-33-30-103:~# ipsec auto --add ToAWSSydneyVPC
root@ip-10-33-30-103:~# ipsec auto --up ToAWSSydneyVPC
root@ip-10-33-30-103:~# service ipsec status

AWS Sydney VPC:

AWS Sydney EC2:

root@ip-172-33-1-190:~# apt-get install openswan
root@ip-172-33-1-190:~# cat /etc/ipsec.conf 
config setup
    protostack=netkey
    interfaces=%defaultroute
    nat_traversal=yes
    force_keepalive=yes
    keep_alive=60
    oe=no
    nhelpers=0
conn ToAWSCnVPC
    left=172.33.1.190
    leftsubnets=172.33.0.0/16
    leftid=@AwsSydneyGW
    right=54.222.193.171
    rightsubnets=10.33.0.0/16
    rightid=@AwsChinaGW
    forceencaps=yes
    authby=secret
    auto=ignore
 
root@ip-172-33-1-190:~# cat /etc/ipsec.secrets 
@AwsSydneyGW  @AwsChinaGW: PSK "123321112233"
 
root@ip-172-33-1-190:~# cat /etc/sysctl.conf | grep -v '^#' | grep -v '^$'
net.ipv4.ip_forward = 1
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.default.send_redirects = 0
 
root@ip-172-33-1-190:~# ipsec verify
root@ip-172-33-1-190:~# service ipsec start
root@ip-172-33-1-190:~# ipsec auto --add ToAWSCnVPC
root@ip-172-33-1-190:~# ipsec auto --up ToAWSCnVPC
root@ip-172-33-1-190:~# service ipsec status

确保两边EC2所在安全组对UDP 500, UDP 4500, TCP 50和TCP 51允许通过
关闭两边EC2上的’Source/Dest checking'(更改源/目标 检查)

Elasticsearch cluster on Azure

Standard
#3 VM on Azure
#node1 10.0.0.3
#node2 10.0.0.4
#node3 10.0.0.5
 
apt-get update;
apt-get install default-jdk;
wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -;
echo "deb http://packages.elastic.co/elasticsearch/2.x/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-2.x.list;
apt-get update && sudo apt-get install elasticsearch;
update-rc.d elasticsearch defaults 95 10;
 
apt-get install lvm2
fdisk /dev/sdc
pvcreate /dev/sdc1
pvdisplay 
vgcreate VolGroup00 /dev/sdc1
vgdisplay 
lvcreate -L 500GB -n lvData VolGroup00
lvdisplay 
mkfs -t ext4 /dev/VolGroup00/lvData
mkdir /data
mount /dev/VolGroup00/lvData /data/
blkid
  /dev/mapper/VolGroup00-lvData: UUID="b65c5a78-e078-4ca8-8119-2de94a414002" TYPE="ext4" 
cat /etc/fstab
  UUID=b65c5a78-e078-4ca8-8119-2de94a414002  /data   auto    defaults,nobootwait,nosuid,noexec,noatime,nodiratime    0 0
 
cat /etc/elasticsearch/elasticsearch.yml 
network.host: 0.0.0.0
cluster.name: es-cluster
node.name: node?
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.timeout: 10s
discovery.zen.ping.multicast.enabled: false
discovery.zen.ping.unicast.hosts: ["10.0.0.3"]
index.number_of_shards: 3
index.number_of_replicas: 1 
path.data: /data
 
mkdir /data/es-cluster
chown elasticsearch.elasticsearch /data/es-cluster
 
cat /usr/share/elasticsearch/bin/elasticsearch.in.sh
...
if [ "x$ES_MIN_MEM" = "x" ]; then
    ES_MIN_MEM=?g
fi
if [ "x$ES_MAX_MEM" = "x" ]; then
    ES_MAX_MEM=??g
fi
...
 
/usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head
/usr/share/elasticsearch/bin/plugin install analysis-smartcn
/usr/share/elasticsearch/bin/plugin install analysis-kuromoji
/usr/share/elasticsearch/bin/plugin list
 
/etc/init.d/elasticsearch start
移动(move)
把分片从一节点移动到另一个节点,可以指定索引名和分片号。
 
取消(cancel)
取消分配一个分片,可以指定索引名和分片号。
node参数可以指定在那个节点取消正在分配的分片。
allow_primary参数支持取消分配主分片。
 
分配(allocate)
分配一个未分配的分片到指定节点,可以指定索引名和分片号。
node参数指定分配到那个节点。
allow_primary参数可以强制分配主分片,不过这样可能导致数据丢失。
 
curl -XPOST 'localhost:9200/_cluster/reroute' -d '{
    "commands" : [ {
        "move" : 
            {
              "index" : "索引名称", "shard" : 分片号, 
              "from_node" : "节点名称A", "to_node" : "节点名称B"
            }
        },
       "cancel" : 
            {
              "index" : "索引名称", "shard" : 分片号, "node" : "节点名称"
            }
        },
        {
          "allocate" : {
              "index" : "索引名称", "shard" : 分片号, "node" : "节点名称"
          }
        }
    ]
}'
 
curl -XPOST localhost:9200/_aliases -d '
{
    "actions": [
        { "remove": {
            "alias": "别名",
            "index": "索引名A"
        }},
        { "add": {
            "alias": "别名",
            "index": "索引名B"
        }}
    ]
}
'
 
curl localhost:9200/_nodes/节点名称/plugins?pretty=true
 
curl -s localhost:9200/_cat/shards
 
 
Elasticsearch版本升级
https://www.elastic.co/guide/en/elasticsearch/reference/current/rolling-upgrades.html
1.
curl -XPUT 'http://localhost:9200/_cluster/settings' -d '{
  "transient": {
    "cluster.routing.allocation.enable": "none"
  }
}'
 
2.
curl -XPOST http://localhost:9200/_flush/synced
 
3.
apt-get update;
apt-get --only-upgrade install elasticsearch
 
4.
/usr/share/elasticsearch/bin/plugin remove analysis-kuromoji;
/usr/share/elasticsearch/bin/plugin remove analysis-smartcn;
/usr/share/elasticsearch/bin/plugin remove analysis-icu;
/usr/share/elasticsearch/bin/plugin remove mobz/elasticsearch-head;
 
/usr/share/elasticsearch/bin/plugin install analysis-kuromoji;
/usr/share/elasticsearch/bin/plugin install analysis-smartcn;
/usr/share/elasticsearch/bin/plugin install analysis-icu;
/usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head;
 
5.
curl -XGET http://localhost:9200/_cat/nodes
 
6.
curl -XPUT 'http://localhost:9200/_cluster/settings' -d '{
  "transient": {
    "cluster.routing.allocation.enable": "all"
  }
}'
 
7.
curl -XGET http://localhost:9200/_cat/health

Hive安装配置

Standard

Hadoop集群安装配置

wget http://mirrors.gigenet.com/apache/hive/stable-2/apache-hive-2.0.0-bin.tar.gz
 
tar zxf apache-hive-2.0.0-bin.tar.gz
mv apache-hive-2.0.0 /usr/local/
cd /usr/local
mv apache-hive-2.0.0 hive
 
cat /etc/profile
...
HIVE_HOME=/usr/local/hive
PATH=$PATH:$HIVE_HOME/bin
export HIVE_HOME PATH
...
 
cd /usr/local/hive/conf
cp hive-default.xml.template hive-default.xml
vi /usr/local/hive/conf/hive-site.xml
<configuration>
        <property>
        <name>javax.jdo.option.ConnectionURL</name>
        <value>jdbc:mysql://Master:3306/hive?createDatabaseIfNotExist=true</value>
        <description>JDBC connect string for a JDBC metastore</description>
        </property>
        <property>
        <name>javax.jdo.option.ConnectionDriverName</name>
        <value>com.mysql.jdbc.Driver</value>
        <description>Driver class name for a JDBC metastore</description>
        </property>
        <property>
        <name>javax.jdo.option.ConnectionUserName</name>
        <value>hive<value>
        <description>username to use against metastore database</description>
        </property>
        <property>
        <name>javax.jdo.option.ConnectionPassword</name>
        <value>hive</value>
        <description>password to use against metastore database</description>
        </property>
</configuration>
mysql>CREATE USER 'hive' IDENTIFIED BY 'hive';
mysql>GRANT ALL PRIVILEGES ON *.* TO 'hive'@'hadoop-master' WITH GRANT OPTION;
mysql>FLUSH privileges;
mysql>CREATE DATABASE hive;
cd ~
wget http://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.38.tar.gz
tar zxf mysql-connector-java-5.1.38.tar.gz
cd mysql-connector-java-5.1.38
cp mysql-connector-java-5.1.38-bin.jar /usr/local/hive/lib/
 
scp -r /usr/local/hive Slave1:/usr/local/
scp -r /usr/local/hive Slave2:/usr/local/
#create the schema
schematool -initSchema -dbType mysql
 
#for client on Slave1,2
hive --service metastore &
 
hive
hive> show databases;

#Slave1,2

cat /usr/local/hive/conf/hive-site.xml
<configuration>
    <property>  
        <name>hive.metastore.uris</name>  
        <value>thrift://Master:9083</value>
    </property>
</configuration>