使用条件:

• 域名接入阿里云DNS
• 已经有解析记录存在

env

• ALI_SECRET_ID=
• ALI_SECRET_KEY=

create accounts and renewal config file

• shell

$ certbot certonly --manual --preferred-challenges=dns --manual-auth-hook /usr/src/app/authenticator.sh -d hub.newops.cn

• docker

docker run -it --rm --name alidns -e ALI_SECRET_ID=xxx -e ALI_SECRET_KEY=xxx -v /data/ssl/:/etc/letsencrypt/ hub.newops.cn/base/alidns:1.0.0 update hub.newops.cn

renewal by force

• shell

certbot renew --force-renewal

• docker

docker run -it --rm --name alidns -e ALI_SECRET_ID=xxx -e ALI_SECRET_KEY=xxx -v /data/ssl/:/etc/letsencrypt/ hub.newops.cn/base/alidns:1.0.0

renewal cron

$ crontab -l

00 09 01 * * /root/cron/update_cert_cron.sh # 每月定时重新签发一次证书

VMware挂载硬盘

挂载之后，默认不会被系统发现，需要执行命令添加

ls /sys/class/scsi_host/
echo "- - -" > /sys/class/scsi_host/host0/scan
echo "- - -" > /sys/class/scsi_host/host1/scan
echo "- - -" > /sys/class/scsi_host/host2/scan

查看当前状况

• 查看物理卷

$ pvs
  PV         VG     Fmt  Attr PSize  PFree 
  /dev/sda2  centos lvm2 a--  49.51g     0 
  /dev/sdb1  centos lvm2 a--  50.00g 40.00m
$ pvdisplay 
  --- Physical volume ---
  PV Name               /dev/sda2
  VG Name               centos
  PV Size               49.51 GiB / not usable 3.00 MiB
  Allocatable           yes (but full)
  PE Size               4.00 MiB
  Total PE              12674
  Free PE               0
  Allocated PE          12674
  PV UUID               5ocR0z-ZfDF-sjE6-llnD-ilcJ-BNTE-G1MAhe
   
  --- Physical volume ---
  PV Name               /dev/sdb1
  VG Name               centos
  PV Size               50.00 GiB / not usable 3.00 MiB
  Allocatable           yes 
  PE Size               4.00 MiB
  Total PE              12799
  Free PE               10
  Allocated PE          12789
  PV UUID               nCUaT0-IPJy-zeCK-ITj1-un2I-FQIc-aYmj3V

• 查看逻辑卷组

$ vgs
  VG     #PV #LV #SN Attr   VSize  VFree 
  centos   2   2   0 wz--n- 99.50g 40.00m
$ vgdisplay 
  --- Volume group ---
  VG Name               centos
  System ID             
  Format                lvm2
  Metadata Areas        2
  Metadata Sequence No  5
  VG Access             read/write
  VG Status             resizable
  MAX LV                0
  Cur LV                2
  Open LV               2
  Max PV                0
  Cur PV                2
  Act PV                2
  VG Size               99.50 GiB
  PE Size               4.00 MiB
  Total PE              25473
  Alloc PE / Size       25463 / 99.46 GiB
  Free  PE / Size       10 / 40.00 MiB
  VG UUID               v9I5Jd-OOFk-0VUV-Yhq4-eAnC-CHKO-DQ9mlx

• 查看逻辑卷

$ lvs
  LV   VG     Attr       LSize  Pool Origin Data%  Meta%  Move Log Cpy%Sync Convert
  root centos -wi-ao---- 97.46g                                                    
  swap centos -wi-ao----  2.00g                                                    
$ lvdisplay 
  --- Logical volume ---
  LV Path                /dev/centos/swap
  LV Name                swap
  VG Name                centos
  LV UUID                QEWL0J-KDEi-A6uH-OgGl-65hZ-NZhh-dCO3YQ
  LV Write Access        read/write
  LV Creation host, time dragon.dsky, 2016-12-06 11:13:40 +0800
  LV Status              available
  # open                 2
  LV Size                2.00 GiB
  Current LE             512
  Segments               1
  Allocation             inherit
  Read ahead sectors     auto
  - currently set to     8192
  Block device           253:1
   
  --- Logical volume ---
  LV Path                /dev/centos/root
  LV Name                root
  VG Name                centos
  LV UUID                dRtNvW-Tm7Q-gkzQ-2uhR-N6Gg-0I2V-6Bh9CM
  LV Write Access        read/write
  LV Creation host, time dragon.dsky, 2016-12-06 11:13:41 +0800
  LV Status              available
  # open                 1
  LV Size                97.46 GiB
  Current LE             24951
  Segments               2
  Allocation             inherit
  Read ahead sectors     auto
  - currently set to     8192
  Block device           253:0

创建新硬盘为物理卷

$ pvcreate /dev/sdb1

扩容逻辑卷组 - 加入新物理卷

$ vgextend centos /dev/sdb1

扩容逻辑卷

$ lvextend -L +50G /dev/mapper/centos-root

刷新逻辑卷大小

• ext4 文件系统

$ resize2fs /dev/mapper/centos-root

• xfs 文件系统

$ xfs_growfs /dev/mapper/centos-root

date: 20180614

commit: b399492

• 准备golang环境

$ docker run -it --rm --name golang -v /home/docker/gitbase/falcon-plus/:/opt golang bash

#(docker) apt-get update && apt-get install -y git

• 拉取最新代码

mkdir -p $GOPATH/src/github.com/open-falcon
cd $GOPATH/src/github.com/open-falcon
git clone https://github.com/open-falcon/falcon-plus.git

编译

Compilation
cd $GOPATH/src/github.com/open-falcon/falcon-plus/

# make all modules
make all

# make specified module
make agent

# pack all modules
make pack
after make pack you will got open-falcon-vx.x.x.tar.gz
if you want to edit configure file for each module, you can edit config/xxx.json before you do make pack

部署

• db schema

无需更新

• 配置文件

在graph/config/cfg.json中新增配置

"ioWorkerNum": 64,

• 启动

$ ./open-falcon start

prepare

net.ipv6.conf.all.disable_ipv6 = 0
net.ipv6.conf.default.disable_ipv6 = 0
net.ipv6.conf.lo.disable_ipv6 = 0
net.ipv6.conf.eth0.disable_ipv6 = 0

ping6 ::1
ping6 localhost

install

export NAME=ipa-master
export IMAGE=freeipa/freeipa-server:fedora-27

docker run -ti --rm --privileged \
  -v /:/host \
  -e HOST=/host \
  -e DATADIR=/data/${NAME} \
  -e NAME=${NAME} \
  -e IMAGE=${IMAGE} ${IMAGE} \
  /bin/install.sh --hostname ipa.newops.cn

run

docker run -it -d --privileged --restart=always --name ${NAME} \
    -h ipa.newops.cn \
    -e IPA_SERVER_IP=x.x.x.x \
    -p 80:80 \
    -p 443:443 \
    -p 389:389 \
    -p 636:636 \
    -p 88:88 \
    -p 464:464 \
    -p 88:88/udp \
    -p 464:464/udp \
    -v /data/${NAME}:/data:Z \
    -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
    --tmpfs /run --tmpfs /tmp \
    -v /dev/urandom:/dev/random:ro ${IMAGE}

https://github.com/caffe2/tutorials/blob/master/Loading_Pretrained_Models.ipynb
https://github.com/caffe2/models/tree/master/squeezenet
https://caffe2.ai/docs/zoo

point

# Run the net and return prediction
#results = p.run({'data': img}) ---> 这是的数据格式，官方例子中是dict，但根据报错提示，改成list就可以了(同时去除index: data)
results = p.run([img])

master[tag] + devlop + qa(常设分支) + feature-* + bugfix-*

快速开发分支模型

master[tag] + qa + dev-* + bugfix-*

功能开发

开发新功能

$ git checkout -b dev-fed_register master
$ ./update-files.sh
$ git commit -a -m 'dev 注册功能'

完成功能开发

$ git checkout master
$ git merge --no-ff dev-fed_register
$ git branch -d dev-fed_register

$ tree 
.
├── ansible.cfg
├── test.yml
├── vars.yml
└── vault_password_file

$ cat ansible.cfg 
[defaults]
remote_user = rokadmin
forks = 15
# inventory script
#inventory = /usr/local/inventory/get_host.py

# roles path
roles_path = ./roles

# temp files
remote_tmp = /tmp/ansible_tmp_$USER/
local_tmp = /tmp/ansible_tmp_$USER/

# ssh connect
remote_port = 4399
#remote_user = wd2admin
#private_key_file = roles/common/files/id_rsa-$USER
host_key_checking = False

# retry files
retry_files_enabled = False

# ansible log
log_path = /tmp/ansible_run_$USER.log

#ask_pass      = True

# if set, always use this private key file for authentication, same as
# if passing --private-key to ansible or ansible-playbook
#private_key_file = /path/to/file

# If set, configures the path to the Vault password file as an alternative to
# specifying --vault-password-file on the command line.
vault_password_file = ./vault_password_file

deprecation_warnings=False
allow_duplicates = True # if same role exists at diffrent position, run twice

[ssh_connection]
#ssh_args = '-o ProxyCommand="ssh -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no -p 4399 -W %h:%p -q rokadmin@103.14.33.233"'
#pipelining = True

$ cat vars.yml 
ansible_become_pass: xxx

$ cat test.yml 
- hosts: rok_global_ps.gm
  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
  gather_facts: false
  vars_files:
    - vars.yml
  tasks:
    - name: enable service rok-gm
      systemd:
        name: rok-gm
        daemon_reload: yes
        state: started
        enabled: True
      become: true
      become_method: su

执行

$ ansible-playbook test.yml

版本: gitlab-ce:9.5.5
运行环境: docker + gfs

备份 for Docker or Omnibus installations

最佳实践

数据备份

$ gitlab-rake gitlab:backup:create STRATEGY=copy DIRECTORY=daily

# STRATEGY=copy 表示先进行拷贝，完成后再使用tar/gzip进行打包，避免出现因为频繁访问出来的`file changed as we read it`
# DIRECTORY=daily 将备份按天分组保存

• 注意

本例中使用了gfs作为gitlab的存储. 在进行备份时,若备份文件存储在gfs中会一直出现tar: xxx: file changed as we read it, 导致备份失败. 解决办法就是在gitlab.rb中将备份目录配置在本地存储

配置备份

主要备份gitlab.rb和gitlab-secrets.json

$ tar czf /var/opt/gitlab/backups/`date +%s`_config_gitlab_backup.tar /etc/gitlab/gitlab.rb /etc/gitlab/gitlab-secrets.json /etc/gitlab/ssl

# /etc/gitlab/gitlab.rb 配置文件
# /etc/gitlab/gitlab-secrets.json CI的secrets文件
# 本例将域名证书挂载在/etc/gitlab/ssl/, 可移除

备份的配置参数

配置文件位置/etc/gitlab/gitlab.rb

备份存储位置

gitlab_rails['backup_path'] = /var/opt/gitlab/backups

备份保留时间，超过这个时间会被删除

# limit backup lifetime to 7 days - 604800 seconds
gitlab_rails['backup_keep_time'] = 604800

恢复 for Docker or Omnibus installations

要求：

• 使用相同版本

恢复配置文件gitlab.rb到/etc/gitlab/gitlab.rb, 并重新配置

gitlab-ctl reconfigure

重启gitlab

gitlab-ctl restart

传输备份文件1512026157_2017_11_30_9.5.5_gitlab_backup.tar到gitlab.rb中配置的备份地址gitlab_rails['backup_path'], 默认为/var/opt/gitlab/backups

停止有数据库连接的进程

$ gitlab-ctl stop unicorn
$ gitlab-ctl stop sidekiq
# Verify
$ gitlab-ctl status

指定备份文件TIMESTAMP进行恢复，本例TIMESTAMP为1512026157_2017_11_30_9.5.5

gitlab-rake gitlab:backup:restore BACKUP=1512026157_2017_11_30_9.5.5

然后恢复备份的secrets文件gitlab-secrets.json到/etc/gitlab/gitlab-secrets.json

启动并测试

$ gitlab-ctl restart
$ gitlab-rake gitlab:check SANITIZE=true

crontab 例行备份

• 容器中没有启用CRON调用，这里在宿主机上进行调用定时备份

0 4 * * * docker exec gitlabce_web_1 sh -c "gitlab-rake gitlab:backup:create STRATEGY=copy DIRECTORY=daily" CRON=1

slave ssh plugin远程执行时，是non-interactive shell，同时也是login shell

让 SLAVE SSH PLUGIN 获取 SLAVE 机器环境变量的方法

针对shell模式可以有以下几种方法.

注意：SHELL=bash

1.login shell 环境变量设置

将export KEY=VALUE写入到 /etc/profile.d/xxx.sh

2.non-interactive shell 环境变量设置

将export KEY=VALUE写入到 ~/.bashrc

3.non-interactive shell 特别方法 - 开启sshd变量传递参数

有文章说需要设置UsePAM yes，实际测试并没有影响; 测试环境为centos7

non-interactive 参考

• 修改sshd参数

$ sed -i -e '$aPermitUserEnvironment yes' -e '$aUseDNS no' /etc/ssh/sshd_config
    
$ systemctl restart sshd

• 添加变量到文件

在/etc/environment或~/.ssh/environment中加入KEY=VALUE值

验证

$ ssh user@host env  #远程服务器上实际是以 bash -c env来执行的

参考文档

nebula:rpm
Gradle:RPM-Plugin wiki
Gradle:copying_files

环境准备

安装 Gradle

可以使用构建好的gradle镜像, 参考Dockerfile

• 依赖 java 1.8

下载gradle二进制包，并设置环境变量

$ VERSION=4.3.1

$ curl -O https://services.gradle.org/distributions/gradle-${VERSION}-bin.zip

$ export PATH=$PATH:/usr/local/gradle-${VERSION}/bin

验证是否安装成功

$ gradle -v

------------------------------------------------------------
Gradle 4.3.1
------------------------------------------------------------

Build time:   2017-11-08 08:59:45 UTC
Revision:     e4f4804807ef7c2829da51877861ff06e07e006d

Groovy:       2.4.12
Ant:          Apache Ant(TM) version 1.9.6 compiled on June 29 2015
JVM:          1.8.0_111 (Oracle Corporation 25.111-b14)
OS:           Linux 3.10.0-514.el7.x86_64 amd64

构建jre的rpm包

这里使用Gradle 2.1开始支持的新写法，更加简洁；老版本写法

需要创建构建文件build.gradle

Step 1: 引用依赖包 - Build script snippet

plugins {
  id "nebula.rpm" version "4.5.1"
}

Step 2: 添加task

task buildJRE(type: Rpm) {
    packageName = 'jre-8u111'
    version = '0.0.1'
    release = 'dsky'
    packageDescription = 'Digital sky - JRE Runtime Environment'
    arch = X86_64
    os = LINUX
    
    ......
}

最终生成的build.gradle

$ cat build.gradle

plugins {
  id "nebula.ospackage" version "4.5.1"
}

task buildJRE(type: Rpm) {
    packageName = 'jre-8u111'
    version = '0.0.1'
    release = 'dsky'
    packageDescription = 'Digital sky - JRE Runtime Environment'
    arch = X86_64
    os = LINUX

    /*
    preInstall file('scripts/preInstall.sh')

    requires('epel-release')
    */

    FileTree jre_files = tarTree('jre-8u111-linux-x64.tar.gz')

    from(jre_files) {
        into '/usr/local/'
    }

    from('scripts/java.sh') {
        into '/etc/profile.d/'
    }

    link('/bin/java', '/usr/local/jre1.8.0_111/bin/java')
}

println relativePath(buildJRE.archivePath)

开始构建

• 构建task: buildJRE

$ gradle clean buildJRE #buildJRE为task名称


> Configure project : 
build/distributions/jre-8u111-0.0.1-dsky.x86_64.rpm


BUILD SUCCESSFUL in 26s
2 actionable tasks: 2 executed

• 验证安装

$ rpm -ivh build/distributions/jre-8u111-0.0.1-dsky.x86_64.rpm

容器组在运行时可以绑定或者优先使用某个节点。这样就可以指定有读写要求的应用部署到集群中有SSD硬盘的节点；还有需要在一个集群中进行构建任务时，就只需要指定到性能强劲的节点中去。

下面只讨论nodeSelector方式

nodeSelector

使用时需要在PodSpec中增加nodeSelector配置，包含一个或一组key:value.

Step One: Attach label to the node

• 查询节点信息

$ kubectl get nodes --show-labels

可以返回当前所有节点信息，包括了系统默认标签.

• 给节点打标签

kubectl label nodes <node-name> <label-key>=<label-value>

$ kubectl label nodes rose nodeName=rose

Step Two: Add a nodeSelector field to your pod configuration

• 原pod配置

apiVersion: v1
kind: Pod
metadata:
  name: nginx
  labels:
    env: test
spec:
  containers:
  - name: nginx
    image: nginx

• 新pod配置-带指定标签

apiVersion: v1
kind: Pod
metadata:
  name: nginx
  labels:
    env: test
spec:
  containers:
  - name: nginx
    image: nginx
    imagePullPolicy: IfNotPresent
  nodeSelector:
    nodeName: rose

创建并查询结果

$ kubectl create -f pod.yaml
$ kubectl get pods -o wide

NAME                        READY     STATUS    RESTARTS   AGE       IP               NODE
nginx                        6/6       Running   0          3h        10.233.81.220        rose

问题：

直接编译安装python之后,使用时提示TLS/SSL不可用

$ /usr/local/python3/bin/pip3 install requests

pip is configured with locations that require TLS/SSL, however the ssl module in Python is not available.
Collecting requests

  Could not fetch URL https://pypi.python.org/simple/requests/: There was a problem confirming the ssl certificate: Can't connect to HTTPS URL because the SSL module is not available. - skipping
  Could not find a version that satisfies the requirement requests (from versions: )
No matching distribution found for requests

解决思路：

查了config.log没有openssl模块依赖, 但是系统中是安装了openssl包的，which也能找到openssl命令，应该是编译python时没有检测到openssl

解决尝试

Requires

• 涉及到编译需先安装 Xcode

$ brew install xcode

• openssl依赖, 在重新安装和更新时提示了编译时依赖ssl的使用方法

$ brew install openssl # 或者 brew update openssl

If you need to have this software first in your PATH run:
  echo 'export PATH="/usr/local/opt/openssl/bin:$PATH"' >> ~/.bash_profile

For compilers to find this software you may need to set:
    LDFLAGS:  -L/usr/local/opt/openssl/lib
    CPPFLAGS: -I/usr/local/opt/openssl/include
For pkg-config to find this software you may need to set:
    PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig

安装 python3 时，加入CFLAGS/LDFLAGS重新编译

$ wget https://www.python.org/ftp/python/3.6.2/Python-3.6.2.tar.xz

$ tar Jxvf Python-3.6.2.tar.xz

$ cd Python-3.6.2

$ CFLAGS="-I$(brew --prefix openssl)/include" LDFLAGS="-L$(brew --prefix openssl)/lib" ./configure --prefix=/usr/local/python3

$ make && make install

按以上方法重新编译安装后，pip3已经可以正常下载其它软件包，问题解决

进一步python3虚拟空间环境的安装请看Centos7 中使用 virtualenv 管理 python3

The Python Package Index is a repository of software for the Python
programming language. There are currently 102159 packages here.

使用python时，经常用到pip或easy_install安装软件包，因而一直对如何使用pip来管理自己的软件好奇.

刚好项目上有需要将腾讯公有云容器服务一些操作进行封装、简化，适合我们的部署方式.

下面就以该项目为例，将软件打包上传到PyPI, 然后通过pip安装使用.

软件包准备

1. qcloud_ccs的目录结构:

.
├── qcloud_ccs
│   └── __init__.py
├── README.md
└── setup.py

1. qcloud_ccs代码内容:

$ cat qcloud_ccs/__init__.py 
#!/usr/bin/env python3
# encoding=utf-8

def test():
    print(u'吃瓜群众')

def main():
    test()

1. 安装描述文件

$ cat setup.py 
#!/usr/bin/env python3
# coding: utf-8

from setuptools import setup

setup(
    name='qcloud_ccs',
    version='1.0.2',
    author='wanglin',
    author_email='wanglin@dbca.cn',
    url='https://newops.cn/qcloud_ccs',
    description=u'以qcloud 容器服务sdk为基础. 1, 将机器创建/服务启停及更新等封装为接口;2.将机器和服务配置抽离为配置文件',
    packages=['qcloud_ccs'],
    install_requires=[],
    entry_points={
        'console_scripts': [
            'qcloud_ccs=qcloud_ccs:test'
        ]
    }
)

• name: 软件名
• version: 版本号
• 'qcloud_ccs=qcloud_ccs:test'： 将qcloud_ccs:test封装为cmd命令，名称为qcloud_ccs

PyPI 注册并上传

帐号注册

首先在PyPI帐号注册地址申请一个帐号

帐号配置

$ cat ~/.pypirc 
[distutils]
index-servers =
    pypi

[pypi]
repository=https://pypi.python.org/pypi
username=xxxx
password=xxxx

后面上传时如何出现：403: Invalid or non-existent authentication information, 请检查用户名、密码，注意密码不能用引号、百分号、注释符号等

软件打包、上传

$ python setup.py sdist upload         
running sdist
running egg_info
writing qcloud_ccs.egg-info/PKG-INFO
writing dependency_links to qcloud_ccs.egg-info/dependency_links.txt
writing entry points to qcloud_ccs.egg-info/entry_points.txt
writing top-level names to qcloud_ccs.egg-info/top_level.txt
reading manifest file 'qcloud_ccs.egg-info/SOURCES.txt'
writing manifest file 'qcloud_ccs.egg-info/SOURCES.txt'
running check
creating qcloud_ccs-1.0.2
creating qcloud_ccs-1.0.2/qcloud_ccs
creating qcloud_ccs-1.0.2/qcloud_ccs.egg-info
copying files to qcloud_ccs-1.0.2...
copying README.md -> qcloud_ccs-1.0.2
copying setup.py -> qcloud_ccs-1.0.2
copying qcloud_ccs/__init__.py -> qcloud_ccs-1.0.2/qcloud_ccs
copying qcloud_ccs.egg-info/PKG-INFO -> qcloud_ccs-1.0.2/qcloud_ccs.egg-info
copying qcloud_ccs.egg-info/SOURCES.txt -> qcloud_ccs-1.0.2/qcloud_ccs.egg-info
copying qcloud_ccs.egg-info/dependency_links.txt -> qcloud_ccs-1.0.2/qcloud_ccs.egg-info
copying qcloud_ccs.egg-info/entry_points.txt -> qcloud_ccs-1.0.2/qcloud_ccs.egg-info
copying qcloud_ccs.egg-info/top_level.txt -> qcloud_ccs-1.0.2/qcloud_ccs.egg-info
Writing qcloud_ccs-1.0.2/setup.cfg
creating dist
Creating tar archive
removing 'qcloud_ccs-1.0.2' (and everything under it)
running upload
Submitting dist/qcloud_ccs-1.0.2.tar.gz to https://upload.pypi.org/legacy/
Server response (200): OK

安装使用

$ pip install qcloud_ccs
Collecting qcloud_ccs
  Downloading qcloud_ccs-1.0.2.tar.gz
Building wheels for collected packages: qcloud-ccs
  Running setup.py bdist_wheel for qcloud-ccs ... done
  Stored in directory: /home/k8s/.cache/pip/wheels/38/45/0d/435a007b7f6a47983ae16dd198e3864d515b42b79d31c490d6
Successfully built qcloud-ccs
Installing collected packages: qcloud-ccs
Successfully installed qcloud-ccs-1.0.2

$ qcloud_ccs
吃瓜群众

到这里，软件包已经可以在pip的软件仓库中找到并安装使用了 ^{V^}

后台管理已上传的软件包

引用依赖

如果软件依赖其它软件包，需在setup.py完善相关配置

$ cat setup.py
...
    install_requires=[
        'qcloudapi-sdk-python==2.0.7',
        'requests==2.18.4'
    ],
...

$ pip install qcloud_ccs
Collecting qcloud_ccs
  Downloading qcloud_ccs-1.0.4.tar.gz
Collecting qcloudapi-sdk-python==2.0.7 (from qcloud_ccs)
  Downloading qcloudapi-sdk-python-2.0.7.tar.gz
Collecting requests==2.18.4 (from qcloud_ccs)
  Downloading requests-2.18.4-py2.py3-none-any.whl (88kB)
    100% |████████████████████████████████| 92kB 982kB/s 
Collecting certifi>=2017.4.17 (from requests==2.18.4->qcloud_ccs)
  Downloading certifi-2017.7.27.1-py2.py3-none-any.whl (349kB)
    100% |████████████████████████████████| 358kB 783kB/s 
Collecting idna<2.7,>=2.5 (from requests==2.18.4->qcloud_ccs)
  Downloading idna-2.6-py2.py3-none-any.whl (56kB)
    100% |████████████████████████████████| 61kB 2.1MB/s 
Collecting urllib3<1.23,>=1.21.1 (from requests==2.18.4->qcloud_ccs)
  Downloading urllib3-1.22-py2.py3-none-any.whl (132kB)
    100% |████████████████████████████████| 133kB 1.2MB/s 
Collecting chardet<3.1.0,>=3.0.2 (from requests==2.18.4->qcloud_ccs)
  Downloading chardet-3.0.4-py2.py3-none-any.whl (133kB)
    100% |████████████████████████████████| 143kB 1.3MB/s 
Building wheels for collected packages: qcloud-ccs, qcloudapi-sdk-python
  Running setup.py bdist_wheel for qcloud-ccs ... done
  Stored in directory: /home/k8s/.cache/pip/wheels/cd/d8/d6/aac6a354d0fe4099b6837aceda25fc1f11de6629d4b1581cff
  Running setup.py bdist_wheel for qcloudapi-sdk-python ... done
  Stored in directory: /home/k8s/.cache/pip/wheels/55/5e/4e/e15deea2b7077e4e1838cbdbd512c4cf8e51eab3352ed8989a

在这里可以编译、运行docker镜像，十分方便

• install dependent python

$ yum install python-virtualenv

$ virtualenv -p /usr/bin/python2.7 --no-site-packages ~/pyenv/docker_py27/

• install docker compose

$ source ~/pyenv/docker_py27/bin/activate

$ pip install -U pip

$ pip install -U docker-compose #Note: pip version 6.0 or greater is required

官方安装指南

本文主要实现以下方面：

• Docker-compose安装镜像并启动Harbor
• LDAP 管理用户
• ssl 认证
• 邮箱设置

下载

国内下载源

$ wget http://harbor.orientsoft.cn/harbor-1.2.0/harbor-offline-installer-v1.2.0.tgz
$ tar zxf harbor-offline-installer-v1.2.0.tgz 
$ cd harbor

配置harbor.cfg

• 访问域名

#The IP address or hostname to access admin UI and registry service.
#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.

hostname = hub.digi-sky.com

• 访问方式 - 默认为http

#The protocol for accessing the UI and token/notification service, by default it is http.
#It can be set to https if ssl is enabled on nginx.

ui_url_protocol = https

• LDAP

#By default the auth mode is db_auth, i.e. the credentials are stored in a local database.
#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server.
#auth_mode = db_auth

auth_mode = ldap_auth

ldap_url = ldap://ipa.digi-sky.com:389
ldap_searchdn = uid=ds_login,cn=users,cn=accounts,dc=digisky,dc=com
ldap_search_pwd = xxxx
ldap_basedn = cn=users,cn=accounts,dc=digisky,dc=com
ldap_uid = uid
ldap_scope = 3
ldap_timeout = 5

• email 设置

email_identity = 

email_server = smtp.qiye.163.com
email_server_port = 994
email_username = devops@digisky.com
email_password = xxxx
email_from = dockerhub <hub.digi-sky.com>
email_ssl = true

• ssl证书

申请letsencrypt证书

#The path of cert and key files for nginx, they are applied only the protocol is set to https

ssl_cert = /data/ssl/archive/hub.digi-sky.com/fullchain1.pem
ssl_cert_key = /data/ssl/archive/hub.digi-sky.com/privkey1.pem

安装并启动

Harbor采用docker-compose的方式管理容器, 依赖docker 1.13+ 及docker-compose 1.16+

• 安装 docker-compose

• Default installation (without Notary/Clair)

$ sudo ./install.sh

Trouble

• 重新配置及管理

如果有配置修改harbor.cfg, 先停止容器，使用prepare脚本重新生成

$ cd harbor
$ docker-compose down
$ vim harbor.cfg
...
$ docker-compose up -d

• 配置文件修改

不支持双引号或单引号转义

  1.在给变量赋值的时候，不能带有引号
  
  2.变量值中不要带有#号等特殊字符

dns方式：申请验证时，会提示一个相关域名的txt记录，按提示进行解析后进行验证，验证通过后在本地目录生成证书

• docker certbot 用法帮助

docker run --rm -it certbot/certbot certbot --help all

• 获取证书 - 域名: code.digi-sky.com

docker run --rm -it -v /data/ssl:/etc/letsencrypt certbot/certbot certonly --manual --preferred-challenges dns -d code.digi-sky.com

# 证书文件生成在/data/ssl/archive/目录下
# -d 指定域名，可跟多个

shell 自动验证并获取证书 - webroot

webroot方式：先验证网站所有权，通过后发放证书

要实现自动验证需要有在网站根目录生成验证文件的脚本authenticator.sh及验证通过后清理文件的脚本cleanup.sh

• 所需脚本

# cat authenticator.sh 
#!/bin/bash
echo $CERTBOT_VALIDATION > /var/www/htdocs/newops/.well-known/acme-challenge/$CERTBOT_TOKEN

# cat cleanup.sh 
#!/bin/bash
rm -f /var/www/htdocs/newops/.well-known/acme-challenge/$CERTBOT_TOKEN

• 验证并获取证书

certbot certonly --manual --preferred-challenges=http --manual-auth-hook ./authenticator.sh --manual-cleanup-hook ./cleanup.sh -d newops.cn

# -d 指定域名，可跟多个

Docker 中国 推荐了四种安装docker ce(社区版)的方法: centos extra源安装、docker ce源安装、rpm直接安装及shell脚本安装，本文采用第二种.

安装前准备

• 初始化服务器环境

curl -s https://gitlab.com/snippets/1674279/raw?inline=false |sh -

准备镜像源

• CLEAN PACKAGES

yum remove docker \
                  docker-common \
                  docker-selinux \
                  docker-engine

• SET UP THE DOCKER CE REPOSITORY

yum install -y device-mapper-persistent-data lvm2
curl -o /etc/yum.repos.d/docker-ce.repo https://download.docker.com/linux/centos/docker-ce.repo
yum clean all

安装

• 查看可安装的docker ce版本

yum list docker-ce.x86_64  --showduplicates | sort -r

• 安装

# 安装指定版本
yum install -y docker-ce-17.12.1.ce

# 或 安装最新版
yum install -y docker-ce

安装后设置

postinstall docker

• 普通用户使用docker

默认docker以root运行，普通用户需要使用docker时就必须加sudo命令.

增加用户组docker(本文方法安装时，已经创建). 普通用户在加入到docker组后，就可以直接调用docker了

# 创建普通用户docker并加入到docker组
useradd docker -m -g docker

# 或者将新用户加入到docker组
usermod -aG docker $USER

• 镜像加速

国内使用还是用加速比较好，registry-mirror 是Docker 中国官方镜像加速站入口

# cat /etc/docker/daemon.json

{
  "registry-mirrors": ["https://registry.docker-cn.com"],
  "graph": "xxx"
}

• 开机自启动

systemd

$ sudo systemctl enable docker

# To disable this behavior, use disable instead.

$ sudo systemctl disable docker

• Specify DNS servers for Docker

还可以设置成使用指定的DNS

在申请域名证书的时候，需要提供组织信息，如公司名称等，然后用ssl工具，如openssl，使用这些信息生成公钥密钥对，其中的公钥就是证书申请文件(CSR - Cerificate Signing Request). 将CSR文件提交给证书服务商进行签发，返回证书文件(CERTIFICATE)（当然不同的格式会有不同的名称）

这样我们的域名就有了一个被外界认可的身份证明，然后在nginx/tomcat下进行配置对外, 再用浏览器访问，会发现锁标志那里是绿色的, yeah!

但是事情太顺利一般都没有好事！陆续接到反馈，在windows系统下有些工具报证书错误...持续构建服务拉代码失败...浏览器有时清理缓存后报证书不可信...

现在我知道了，这种情况一定是中了中间证书的坑

填坑

• 什么是中间证书？

普通用户的证书是由证书签发机构签发的，但他们很可能只是一个中间代理机构，也是需要有一个凭证的，这个就是中间证书. 中间证书是由上级或者根证书签发机构签发

• 为什么浏览器访问正常？

浏览器中内置了根证书，如Chrome/Firefox, 没有中间证书. 但是在访问时发现没有中间证书的话，会自动去下载或使用访问其他网站时缓存的相同证书(所以有时清理缓存会造成问题). 补齐证书后自然访问正常

• 为什么服务器工具访问失败?

因为不会自动下载中间证书...

检测证书状态

• 命令行(适合于高手，或者服务部署于内网的情况)

openssl s_client -connect app-git.ppgame.com:443

首先看depth，如果都是0, 说明在认证链(Certificate chain)的第一条就没有通过

可以看到从证书读出的信息中包含了两组认证: s表示认证对象，i表示认证机构

不能认证第一条证书，说明中间证书有问题

• 第三方工具

SSL服务器证书安装检查器

返回结果

很明显，正在缺少中间证书导致的问题

获取中间证书

• 最直接的办法当然是找中间证书代理商要啊，谁让我们给钱了

• 既然浏览器能缓存证书，那一定还在电脑里，是可以导出来的. Mac系统可以用钥匙串工具来导入导出证书

• 使用第三方工具下载中间证书

将我们的域名证书填进去会生成中间证书的下载地址. 相同的如果你要获取根证书，只要把中间证书填在这里就可以了

Let’s Encrypt

最方便的是使用Let’s Encrypt来进行证书申请及网站部署：免费！操作简单！
申请letsencrypt证书

• 不需要提供组织信息 (能证明网站或域名归你操控就可以了)
• 不需要自已生成CSR证书申请文件
• 不需要自己生成私钥

签发成功后会返还以下文件：

`privkey.pem`  : the private key for your certificate.
`fullchain.pem`: the certificate file used in most server software.
`chain.pem`    : used for OCSP stapling in Nginx >=1.3.7.
`cert.pem`     : will break many server configurations, and should not be used
                 without reading further documentation (see link below).

We recommend not moving these files. For more information, see the Certbot
User Guide at https://certbot.eff.org/docs/using.html#where-are-my-certificates.

部署的时候，使用privkey.pem/fullchain.pem